"""Dump all links found on a page.

Usage: python dump_links.py <url> [filter]
"""
import urllib.request
import re
import sys
import html as htmlmod

UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'


def get(url, timeout=40):
    req = urllib.request.Request(url, headers={'User-Agent': UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode('utf-8', 'ignore')


url = sys.argv[1]
filt = sys.argv[2] if len(sys.argv) > 2 else ''
t = get(url)
links = re.findall(r'href="([^"]+)"', t)
seen = []
for l in links:
    if filt and filt not in l:
        continue
    if l not in seen:
        seen.append(l)
print('TOTAL LINKS:', len(links), 'SHOWN:', len(seen))
for l in seen:
    print(l)