"""Fetch pages and extract arXiv ids + nearby text.

Usage: python mine_links.py <urls.txt> <out.txt>
"""
import urllib.request
import re
import sys
import time
import html as htmlmod

UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'


def get(url, timeout=40):
    req = urllib.request.Request(url, headers={'User-Agent': UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode('utf-8', 'ignore')


def strip(s):
    s = re.sub(r'<script.*?</script>', ' ', s, flags=re.S | re.I)
    s = re.sub(r'<style.*?</style>', ' ', s, flags=re.S | re.I)
    s = re.sub(r'<[^>]+>', ' ', s)
    return ' '.join(htmlmod.unescape(s).split())


urls = [l.strip() for l in open(sys.argv[1], encoding='utf-8') if l.strip()]
out = []
for u in urls:
    try:
        t = get(u)
    except Exception as e:  # noqa: BLE001
        out.append('=== %s FAILED %s' % (u, str(e)[:100]))
        continue
    txt = strip(t)
    ids = re.findall(r'(?:arxiv[^\s]{0,20}?|arXiv:)(\d{4}\.\d{4,5})', t + ' ' + txt)
    out.append('=== %s  (%d chars, %d arxiv refs)' % (u, len(txt), len(set(ids))))
    for i in sorted(set(ids)):
        out.append('   id: ' + i)
    out.append('   TEXT: ' + txt[:2500])
    time.sleep(1)

with open(sys.argv[2], 'w', encoding='utf-8') as f:
    f.write('\n'.join(out))
print('done', len(urls))