91 lines
2.6 KiB
Python
91 lines
2.6 KiB
Python
"""Merge verified arXiv metadata into a master paper list.
|
|
|
|
Usage: python build_list.py <out.json> <verified1.json> <verified2.json> ...
|
|
"""
|
|
import json
|
|
import sys
|
|
import re
|
|
import os
|
|
|
|
OUT = sys.argv[1]
|
|
# verified IDs that turned out NOT to be DeepSeek papers
|
|
SKIP = {'2411.08599', '2605.27765'}
|
|
recs = {}
|
|
for p in sys.argv[2:]:
|
|
if not os.path.exists(p):
|
|
continue
|
|
with open(p, encoding='utf-8') as f:
|
|
for r in json.load(f):
|
|
if r.get('ok') and r['id'] not in SKIP:
|
|
recs[r['id']] = r
|
|
|
|
# group/category assignment
|
|
CATS = {
|
|
'2401.02954': '基础模型',
|
|
'2401.06066': '基础模型',
|
|
'2401.14196': '代码模型',
|
|
'2402.03300': '数学模型',
|
|
'2403.05525': '多模态',
|
|
'2405.04434': '基础模型',
|
|
'2405.14333': '形式化证明',
|
|
'2406.11931': '代码模型',
|
|
'2407.01906': '训练方法',
|
|
'2408.08152': '形式化证明',
|
|
'2408.14158': '基础设施',
|
|
'2410.13848': '多模态',
|
|
'2412.10302': '多模态',
|
|
'2412.19437': '基础模型',
|
|
'2501.12948': '推理模型',
|
|
'2501.17811': '多模态',
|
|
'2504.02495': '训练方法',
|
|
'2504.21801': '形式化证明',
|
|
'2505.09343': '基础设施',
|
|
'2510.18234': '多模态',
|
|
'2511.22570': '数学模型',
|
|
'2512.02556': '基础模型',
|
|
'2512.24880': '架构',
|
|
'2601.07372': '架构',
|
|
'2601.20552': '多模态',
|
|
'2602.21548': '基础设施',
|
|
'2606.19348': '基础模型',
|
|
'2609.19969': '基础模型',
|
|
}
|
|
|
|
|
|
def slug(title, aid):
|
|
s = title.lower()
|
|
s = re.sub(r'[^a-z0-9]+', '-', s).strip('-')
|
|
s = re.sub(r'-+', '-', s)
|
|
return (s[:70] or aid) + '-' + aid.replace('.', '')
|
|
|
|
|
|
rows = []
|
|
for aid, r in recs.items():
|
|
m = re.search(r'(\d{4})', r.get('dateline', ''))
|
|
date = ''
|
|
dm = re.search(r'(\d{1,2} \w+ \d{4})', r.get('dateline', ''))
|
|
if dm:
|
|
date = dm.group(1)
|
|
rows.append({
|
|
'id': aid,
|
|
'slug': slug(r.get('title', ''), aid),
|
|
'title': r.get('title', ''),
|
|
'authors': r.get('authors', ''),
|
|
'abstract': r.get('abstract', ''),
|
|
'date': date,
|
|
'dateline': r.get('dateline', ''),
|
|
'comments': r.get('comments', ''),
|
|
'category': CATS.get(aid, '其他'),
|
|
'pdf_url': 'https://arxiv.org/pdf/' + aid,
|
|
'abs_url': 'https://arxiv.org/abs/' + aid,
|
|
})
|
|
rows.sort(key=lambda x: x['id'])
|
|
groups = {}
|
|
for r in rows:
|
|
groups.setdefault(r['category'], []).append(r)
|
|
with open(OUT, 'w', encoding='utf-8') as f:
|
|
json.dump(rows, f, ensure_ascii=False, indent=1)
|
|
print('papers:', len(rows))
|
|
for c, v in sorted(groups.items()):
|
|
print(' ', c, len(v))
|