89 lines
2.7 KiB
Python
89 lines
2.7 KiB
Python
"""Verify arXiv IDs by fetching their abs pages and extracting metadata.
|
|
|
|
Usage: python verify_ids.py <ids.txt> <out.json>
|
|
ids.txt: one arXiv id per line (may include comments after '#')
|
|
"""
|
|
import urllib.request
|
|
import re
|
|
import json
|
|
import sys
|
|
import time
|
|
import html as htmlmod
|
|
|
|
UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
|
|
|
|
|
|
def get(url, timeout=40):
|
|
req = urllib.request.Request(url, headers={'User-Agent': UA})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.status, r.read().decode('utf-8', 'ignore')
|
|
|
|
|
|
def strip(s):
|
|
s = re.sub(r'<[^>]+>', ' ', s)
|
|
s = htmlmod.unescape(s)
|
|
return ' '.join(s.split())
|
|
|
|
|
|
def extract(aid, t):
|
|
def find(pat, flags=re.S):
|
|
m = re.search(pat, t, flags)
|
|
return strip(m.group(1)) if m else ''
|
|
|
|
title = find(r'<h1 class="title mathjax">(.*?)</h1>')
|
|
title = re.sub(r'^Title:\s*', '', title)
|
|
authors = find(r'<div class="authors">(.*?)</div>')
|
|
authors = re.sub(r'^Authors:\s*', '', authors)
|
|
abstract = find(r'<blockquote class="abstract mathjax">(.*?)</blockquote>')
|
|
abstract = re.sub(r'^Abstract:\s*', '', abstract)
|
|
subj = find(r'<td class="tablecell subjects">(.*?)</td>')
|
|
dateline = find(r'<div class="dateline">(.*?)</div>')
|
|
comments = find(r'<td class="tablecell comments[^"]*">(.*?)</td>')
|
|
journal = find(r'<td class="tablecell jref">(.*?)</td>')
|
|
doi = find(r'<td class="tablecell doi">(.*?)</td>')
|
|
m = re.search(r'\[v(\d+)\]', t)
|
|
ver = m.group(1) if m else ''
|
|
return {
|
|
'id': aid,
|
|
'version': ver,
|
|
'title': title,
|
|
'authors': authors,
|
|
'abstract': abstract,
|
|
'subjects': subj,
|
|
'dateline': dateline,
|
|
'comments': comments,
|
|
'journal_ref': journal,
|
|
'doi': doi,
|
|
}
|
|
|
|
|
|
def main():
|
|
ids = []
|
|
with open(sys.argv[1], encoding='utf-8') as f:
|
|
for line in f:
|
|
line = line.split('#')[0].strip()
|
|
if line:
|
|
ids.append(line)
|
|
out = []
|
|
for aid in ids:
|
|
rec = {'id': aid, 'ok': False}
|
|
for attempt in range(3):
|
|
try:
|
|
st, t = get('https://arxiv.org/abs/' + aid)
|
|
if st == 200 and 'Title:' in t:
|
|
rec.update(extract(aid, t))
|
|
rec['ok'] = True
|
|
break
|
|
except Exception as e: # noqa: BLE001
|
|
rec['error'] = str(e)[:120]
|
|
time.sleep(3)
|
|
print(('OK ' if rec['ok'] else 'MISS'), aid, rec.get('title', '')[:90], file=sys.stderr)
|
|
out.append(rec)
|
|
time.sleep(1.5)
|
|
with open(sys.argv[2], 'w', encoding='utf-8') as f:
|
|
json.dump(out, f, ensure_ascii=False, indent=1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|