46 lines
1.3 KiB
Python
46 lines
1.3 KiB
Python
"""Fetch pages and extract arXiv ids + nearby text.
|
|
|
|
Usage: python mine_links.py <urls.txt> <out.txt>
|
|
"""
|
|
import urllib.request
|
|
import re
|
|
import sys
|
|
import time
|
|
import html as htmlmod
|
|
|
|
UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
|
|
|
|
|
|
def get(url, timeout=40):
|
|
req = urllib.request.Request(url, headers={'User-Agent': UA})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.read().decode('utf-8', 'ignore')
|
|
|
|
|
|
def strip(s):
|
|
s = re.sub(r'<script.*?</script>', ' ', s, flags=re.S | re.I)
|
|
s = re.sub(r'<style.*?</style>', ' ', s, flags=re.S | re.I)
|
|
s = re.sub(r'<[^>]+>', ' ', s)
|
|
return ' '.join(htmlmod.unescape(s).split())
|
|
|
|
|
|
urls = [l.strip() for l in open(sys.argv[1], encoding='utf-8') if l.strip()]
|
|
out = []
|
|
for u in urls:
|
|
try:
|
|
t = get(u)
|
|
except Exception as e: # noqa: BLE001
|
|
out.append('=== %s FAILED %s' % (u, str(e)[:100]))
|
|
continue
|
|
txt = strip(t)
|
|
ids = re.findall(r'(?:arxiv[^\s]{0,20}?|arXiv:)(\d{4}\.\d{4,5})', t + ' ' + txt)
|
|
out.append('=== %s (%d chars, %d arxiv refs)' % (u, len(txt), len(set(ids))))
|
|
for i in sorted(set(ids)):
|
|
out.append(' id: ' + i)
|
|
out.append(' TEXT: ' + txt[:2500])
|
|
time.sleep(1)
|
|
|
|
with open(sys.argv[2], 'w', encoding='utf-8') as f:
|
|
f.write('\n'.join(out))
|
|
print('done', len(urls))
|