"""Verify arXiv IDs by fetching their abs pages and extracting metadata.

Usage: python verify_ids.py <ids.txt> <out.json>
ids.txt: one arXiv id per line (may include comments after '#')
"""
import urllib.request
import re
import json
import sys
import time
import html as htmlmod

UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'


def get(url, timeout=40):
    req = urllib.request.Request(url, headers={'User-Agent': UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.status, r.read().decode('utf-8', 'ignore')


def strip(s):
    s = re.sub(r'<[^>]+>', ' ', s)
    s = htmlmod.unescape(s)
    return ' '.join(s.split())


def extract(aid, t):
    def find(pat, flags=re.S):
        m = re.search(pat, t, flags)
        return strip(m.group(1)) if m else ''

    title = find(r'<h1 class="title mathjax">(.*?)</h1>')
    title = re.sub(r'^Title:\s*', '', title)
    authors = find(r'<div class="authors">(.*?)</div>')
    authors = re.sub(r'^Authors:\s*', '', authors)
    abstract = find(r'<blockquote class="abstract mathjax">(.*?)</blockquote>')
    abstract = re.sub(r'^Abstract:\s*', '', abstract)
    subj = find(r'<td class="tablecell subjects">(.*?)</td>')
    dateline = find(r'<div class="dateline">(.*?)</div>')
    comments = find(r'<td class="tablecell comments[^"]*">(.*?)</td>')
    journal = find(r'<td class="tablecell jref">(.*?)</td>')
    doi = find(r'<td class="tablecell doi">(.*?)</td>')
    m = re.search(r'\[v(\d+)\]', t)
    ver = m.group(1) if m else ''
    return {
        'id': aid,
        'version': ver,
        'title': title,
        'authors': authors,
        'abstract': abstract,
        'subjects': subj,
        'dateline': dateline,
        'comments': comments,
        'journal_ref': journal,
        'doi': doi,
    }


def main():
    ids = []
    with open(sys.argv[1], encoding='utf-8') as f:
        for line in f:
            line = line.split('#')[0].strip()
            if line:
                ids.append(line)
    out = []
    for aid in ids:
        rec = {'id': aid, 'ok': False}
        for attempt in range(3):
            try:
                st, t = get('https://arxiv.org/abs/' + aid)
                if st == 200 and 'Title:' in t:
                    rec.update(extract(aid, t))
                    rec['ok'] = True
                break
            except Exception as e:  # noqa: BLE001
                rec['error'] = str(e)[:120]
                time.sleep(3)
        print(('OK  ' if rec['ok'] else 'MISS'), aid, rec.get('title', '')[:90], file=sys.stderr)
        out.append(rec)
        time.sleep(1.5)
    with open(sys.argv[2], 'w', encoding='utf-8') as f:
        json.dump(out, f, ensure_ascii=False, indent=1)


if __name__ == '__main__':
    main()