新增: DeepSeek 全部 33 篇官方论文(原文 PDF 走 Git LFS + 中文精读笔记与索引)

2026-09-20 17:11:49 +08:00
parent f600fa5871
commit 14c280ffee
187 changed files with 94947 additions and 0 deletions
@@ -0,0 +1,89 @@
+"""Scrape arXiv advanced search (title contains term), all pages.
+
+Usage: python scrape_title.py <term> <out.json>
+"""
+import urllib.request
+import urllib.parse
+import re
+import json
+import sys
+import time
+import html as htmlmod
+
+UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
+
+
+def get(url, timeout=45):
+    req = urllib.request.Request(url, headers={'User-Agent': UA})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.read().decode('utf-8', 'ignore')
+
+
+def strip(s):
+    s = re.sub(r'<[^>]+>', ' ', s)
+    return ' '.join(htmlmod.unescape(s).split())
+
+
+def parse_page(t):
+    out = []
+    blocks = re.split(r'<li class="arxiv-result">', t)[1:]
+    for blk in blocks:
+        aid = re.search(r'arxiv\.org/abs/([^"\'<]+)', blk)
+        title = re.search(r'<p class="title is-5 mathjax">(.*?)</p>', blk, re.S)
+        authors = re.search(r'<p class="authors">(.*?)</p>', blk, re.S)
+        dates = re.search(r'<p class="is-size-7">(.*?)</p>', blk, re.S)
+        if not aid:
+            continue
+        out.append({
+            'id': aid.group(1),
+            'title': strip(title.group(1)) if title else '',
+            'authors': strip(authors.group(1)) if authors else '',
+            'dates': strip(dates.group(1)) if dates else '',
+        })
+    return out
+
+
+def main():
+    term = sys.argv[1]
+    out_path = sys.argv[2]
+    size = 200
+    res = []
+    page = 0
+    while True:
+        qs = [
+            ('advanced', ''),
+            ('terms-0-operator', 'AND'),
+            ('terms-0-term', term),
+            ('terms-0-field', 'title'),
+            ('classification-physics_archives', 'all'),
+            ('classification-include_cross_list', 'include'),
+            ('start', str(page * size)),
+            ('size', str(size)),
+            ('order', '-announced_date_first'),
+        ]
+        url = 'https://arxiv.org/search/advanced?' + urllib.parse.urlencode(qs)
+        try:
+            t = get(url)
+        except Exception as e:  # noqa: BLE001
+            print('page fail', page, e, file=sys.stderr)
+            break
+        rows = parse_page(t)
+        total = re.search(r'of\s+([\d,]+)\s+results', t)
+        print('page', page, 'rows', len(rows), 'total', total.group(1) if total else '?', file=sys.stderr)
+        res.extend(rows)
+        if len(rows) < size:
+            break
+        page += 1
+        time.sleep(2)
+        if page > 15:
+            break
+    seen = {}
+    for r in res:
+        seen.setdefault(r['id'].split('v')[0], r)
+    with open(out_path, 'w', encoding='utf-8') as f:
+        json.dump(list(seen.values()), f, ensure_ascii=False, indent=1)
+    print('saved', len(seen), file=sys.stderr)
+
+
+if __name__ == '__main__':
+    main()