新增: DeepSeek 全部 33 篇官方论文(原文 PDF 走 Git LFS + 中文精读笔记与索引)

2026-09-20 17:11:49 +08:00
parent f600fa5871
commit 14c280ffee
187 changed files with 94947 additions and 0 deletions
@@ -0,0 +1,88 @@
+"""Verify arXiv IDs by fetching their abs pages and extracting metadata.
+
+Usage: python verify_ids.py <ids.txt> <out.json>
+ids.txt: one arXiv id per line (may include comments after '#')
+"""
+import urllib.request
+import re
+import json
+import sys
+import time
+import html as htmlmod
+
+UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
+
+
+def get(url, timeout=40):
+    req = urllib.request.Request(url, headers={'User-Agent': UA})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.status, r.read().decode('utf-8', 'ignore')
+
+
+def strip(s):
+    s = re.sub(r'<[^>]+>', ' ', s)
+    s = htmlmod.unescape(s)
+    return ' '.join(s.split())
+
+
+def extract(aid, t):
+    def find(pat, flags=re.S):
+        m = re.search(pat, t, flags)
+        return strip(m.group(1)) if m else ''
+
+    title = find(r'<h1 class="title mathjax">(.*?)</h1>')
+    title = re.sub(r'^Title:\s*', '', title)
+    authors = find(r'<div class="authors">(.*?)</div>')
+    authors = re.sub(r'^Authors:\s*', '', authors)
+    abstract = find(r'<blockquote class="abstract mathjax">(.*?)</blockquote>')
+    abstract = re.sub(r'^Abstract:\s*', '', abstract)
+    subj = find(r'<td class="tablecell subjects">(.*?)</td>')
+    dateline = find(r'<div class="dateline">(.*?)</div>')
+    comments = find(r'<td class="tablecell comments[^"]*">(.*?)</td>')
+    journal = find(r'<td class="tablecell jref">(.*?)</td>')
+    doi = find(r'<td class="tablecell doi">(.*?)</td>')
+    m = re.search(r'\[v(\d+)\]', t)
+    ver = m.group(1) if m else ''
+    return {
+        'id': aid,
+        'version': ver,
+        'title': title,
+        'authors': authors,
+        'abstract': abstract,
+        'subjects': subj,
+        'dateline': dateline,
+        'comments': comments,
+        'journal_ref': journal,
+        'doi': doi,
+    }
+
+
+def main():
+    ids = []
+    with open(sys.argv[1], encoding='utf-8') as f:
+        for line in f:
+            line = line.split('#')[0].strip()
+            if line:
+                ids.append(line)
+    out = []
+    for aid in ids:
+        rec = {'id': aid, 'ok': False}
+        for attempt in range(3):
+            try:
+                st, t = get('https://arxiv.org/abs/' + aid)
+                if st == 200 and 'Title:' in t:
+                    rec.update(extract(aid, t))
+                    rec['ok'] = True
+                break
+            except Exception as e:  # noqa: BLE001
+                rec['error'] = str(e)[:120]
+                time.sleep(3)
+        print(('OK  ' if rec['ok'] else 'MISS'), aid, rec.get('title', '')[:90], file=sys.stderr)
+        out.append(rec)
+        time.sleep(1.5)
+    with open(sys.argv[2], 'w', encoding='utf-8') as f:
+        json.dump(out, f, ensure_ascii=False, indent=1)
+
+
+if __name__ == '__main__':
+    main()