import urllib.request, urllib.parse, json, sys, time, re
import xml.etree.ElementTree as ET

NS = {'a':'http://www.w3.org/2005/Atom'}

def fetch(q, start=0, max_results=100, sort='submittedDate'):
    params = {
        'search_query': q,
        'start': start,
        'max_results': max_results,
        'sortBy': sort,
        'sortOrder': 'descending',
    }
    url = 'http://export.arxiv.org/api/query?' + urllib.parse.urlencode(params)
    req = urllib.request.Request(url, headers={
        'User-Agent': 'DeepSeekPaperCollector/1.0 (local research notes)',
        'Accept': 'application/atom+xml',
    })
    for attempt in range(4):
        try:
            with urllib.request.urlopen(req, timeout=60) as r:
                return r.read()
        except Exception as e:
            print('retry', attempt, e, file=sys.stderr)
            time.sleep(3)
    raise SystemExit('failed')

def parse(xml):
    root = ET.fromstring(xml)
    out = []
    for e in root.findall('a:entry', NS):
        aid = e.find('a:id', NS).text
        m = re.search(r'arxiv.org/abs/(.+)$', aid)
        out.append({
            'id': m.group(1) if m else aid,
            'title': ' '.join(e.find('a:title', NS).text.split()),
            'published': e.find('a:published', NS).text,
            'updated': e.find('a:updated', NS).text,
            'authors': [a.find('a:name', NS).text for a in e.findall('a:author', NS)],
            'summary': ' '.join(e.find('a:summary', NS).text.split()),
            'comment': (e.find('{http://arxiv.org/schemas/atom}comment').text if e.find('{http://arxiv.org/schemas/atom}comment') is not None else ''),
            'primary': (e.find('{http://arxiv.org/schemas/atom}primary_category').get('term') if e.find('{http://arxiv.org/schemas/atom}primary_category') is not None else ''),
        })
    return out

if __name__ == '__main__':
    query = sys.argv[1]
    allr = []
    for start in range(0, 400, 100):
        xml = fetch(query, start=start)
        res = parse(xml)
        allr.extend(res)
        if len(res) < 100:
            break
    print(json.dumps(allr, ensure_ascii=False, indent=1))