"""Scrape arXiv advanced search by author name, all pages.

Usage: python scrape_author.py <author> <out.json>
"""
import urllib.request
import urllib.parse
import re
import json
import sys
import time
import html as htmlmod

UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'


def get(url, timeout=45):
    req = urllib.request.Request(url, headers={'User-Agent': UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode('utf-8', 'ignore')


def strip(s):
    s = re.sub(r'<[^>]+>', ' ', s)
    return ' '.join(htmlmod.unescape(s).split())


def parse_page(t):
    out = []
    for blk in re.split(r'<li class="arxiv-result">', t)[1:]:
        aid = re.search(r'arxiv\.org/abs/([^"\'<]+)', blk)
        title = re.search(r'<p class="title is-5 mathjax">(.*?)</p>', blk, re.S)
        authors = re.search(r'<p class="authors">(.*?)</p>', blk, re.S)
        dates = re.search(r'<p class="is-size-7">(.*?)</p>', blk, re.S)
        if not aid:
            continue
        out.append({
            'id': aid.group(1),
            'title': strip(title.group(1)) if title else '',
            'authors': strip(authors.group(1)) if authors else '',
            'dates': strip(dates.group(1)) if dates else '',
        })
    return out


def main():
    term = sys.argv[1]
    out_path = sys.argv[2]
    size = 200
    res = []
    page = 0
    while True:
        qs = [
            ('advanced', ''),
            ('terms-0-operator', 'AND'),
            ('terms-0-term', term),
            ('terms-0-field', 'author'),
            ('classification-include_cross_list', 'include'),
            ('start', str(page * size)),
            ('size', str(size)),
            ('order', '-announced_date_first'),
        ]
        url = 'https://arxiv.org/search/advanced?' + urllib.parse.urlencode(qs)
        try:
            t = get(url)
        except Exception as e:  # noqa: BLE001
            print('page fail', page, e, file=sys.stderr)
            break
        rows = parse_page(t)
        total = re.search(r'of\s+([\d,]+)\s+results', t)
        print('page', page, 'rows', len(rows), 'total', total.group(1) if total else '?', file=sys.stderr)
        res.extend(rows)
        if len(rows) < size:
            break
        page += 1
        time.sleep(2)
        if page > 20:
            break
    seen = {}
    for r in res:
        seen.setdefault(r['id'].split('v')[0], r)
    with open(out_path, 'w', encoding='utf-8') as f:
        json.dump(list(seen.values()), f, ensure_ascii=False, indent=1)
    print('saved', len(seen), file=sys.stderr)


if __name__ == '__main__':
    main()