32 lines
724 B
Python
32 lines
724 B
Python
"""Dump all links found on a page.
|
|
|
|
Usage: python dump_links.py <url> [filter]
|
|
"""
|
|
import urllib.request
|
|
import re
|
|
import sys
|
|
import html as htmlmod
|
|
|
|
UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
|
|
|
|
|
|
def get(url, timeout=40):
|
|
req = urllib.request.Request(url, headers={'User-Agent': UA})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.read().decode('utf-8', 'ignore')
|
|
|
|
|
|
url = sys.argv[1]
|
|
filt = sys.argv[2] if len(sys.argv) > 2 else ''
|
|
t = get(url)
|
|
links = re.findall(r'href="([^"]+)"', t)
|
|
seen = []
|
|
for l in links:
|
|
if filt and filt not in l:
|
|
continue
|
|
if l not in seen:
|
|
seen.append(l)
|
|
print('TOTAL LINKS:', len(links), 'SHOWN:', len(seen))
|
|
for l in seen:
|
|
print(l)
|