"""Merge verified arXiv metadata into a master paper list.

Usage: python build_list.py <out.json> <verified1.json> <verified2.json> ...
"""
import json
import sys
import re
import os

OUT = sys.argv[1]
# verified IDs that turned out NOT to be DeepSeek papers
SKIP = {'2411.08599', '2605.27765'}
recs = {}
for p in sys.argv[2:]:
    if not os.path.exists(p):
        continue
    with open(p, encoding='utf-8') as f:
        for r in json.load(f):
            if r.get('ok') and r['id'] not in SKIP:
                recs[r['id']] = r

# group/category assignment
CATS = {
    '2401.02954': '基础模型',
    '2401.06066': '基础模型',
    '2401.14196': '代码模型',
    '2402.03300': '数学模型',
    '2403.05525': '多模态',
    '2405.04434': '基础模型',
    '2405.14333': '形式化证明',
    '2406.11931': '代码模型',
    '2407.01906': '训练方法',
    '2408.08152': '形式化证明',
    '2408.14158': '基础设施',
    '2410.13848': '多模态',
    '2412.10302': '多模态',
    '2412.19437': '基础模型',
    '2501.12948': '推理模型',
    '2501.17811': '多模态',
    '2504.02495': '训练方法',
    '2504.21801': '形式化证明',
    '2505.09343': '基础设施',
    '2510.18234': '多模态',
    '2511.22570': '数学模型',
    '2512.02556': '基础模型',
    '2512.24880': '架构',
    '2601.07372': '架构',
    '2601.20552': '多模态',
    '2602.21548': '基础设施',
    '2606.19348': '基础模型',
    '2609.19969': '基础模型',
}


def slug(title, aid):
    s = title.lower()
    s = re.sub(r'[^a-z0-9]+', '-', s).strip('-')
    s = re.sub(r'-+', '-', s)
    return (s[:70] or aid) + '-' + aid.replace('.', '')


rows = []
for aid, r in recs.items():
    m = re.search(r'(\d{4})', r.get('dateline', ''))
    date = ''
    dm = re.search(r'(\d{1,2} \w+ \d{4})', r.get('dateline', ''))
    if dm:
        date = dm.group(1)
    rows.append({
        'id': aid,
        'slug': slug(r.get('title', ''), aid),
        'title': r.get('title', ''),
        'authors': r.get('authors', ''),
        'abstract': r.get('abstract', ''),
        'date': date,
        'dateline': r.get('dateline', ''),
        'comments': r.get('comments', ''),
        'category': CATS.get(aid, '其他'),
        'pdf_url': 'https://arxiv.org/pdf/' + aid,
        'abs_url': 'https://arxiv.org/abs/' + aid,
    })
rows.sort(key=lambda x: x['id'])
groups = {}
for r in rows:
    groups.setdefault(r['category'], []).append(r)
with open(OUT, 'w', encoding='utf-8') as f:
    json.dump(rows, f, ensure_ascii=False, indent=1)
print('papers:', len(rows))
for c, v in sorted(groups.items()):
    print(' ', c, len(v))