新增: DeepSeek 全部 33 篇官方论文(原文 PDF 走 Git LFS + 中文精读笔记与索引)

大模型/deepseek/_scripts/mine_links.py

@@ -0,0 +1,45 @@
+"""Fetch pages and extract arXiv ids + nearby text.
+
+Usage: python mine_links.py <urls.txt> <out.txt>
+"""
+import urllib.request
+import re
+import sys
+import time
+import html as htmlmod
+
+UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
+
+
+def get(url, timeout=40):
+    req = urllib.request.Request(url, headers={'User-Agent': UA})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.read().decode('utf-8', 'ignore')
+
+
+def strip(s):
+    s = re.sub(r'<script.*?</script>', ' ', s, flags=re.S | re.I)
+    s = re.sub(r'<style.*?</style>', ' ', s, flags=re.S | re.I)
+    s = re.sub(r'<[^>]+>', ' ', s)
+    return ' '.join(htmlmod.unescape(s).split())
+
+
+urls = [l.strip() for l in open(sys.argv[1], encoding='utf-8') if l.strip()]
+out = []
+for u in urls:
+    try:
+        t = get(u)
+    except Exception as e:  # noqa: BLE001
+        out.append('=== %s FAILED %s' % (u, str(e)[:100]))
+        continue
+    txt = strip(t)
+    ids = re.findall(r'(?:arxiv[^\s]{0,20}?|arXiv:)(\d{4}\.\d{4,5})', t + ' ' + txt)
+    out.append('=== %s  (%d chars, %d arxiv refs)' % (u, len(txt), len(set(ids))))
+    for i in sorted(set(ids)):
+        out.append('   id: ' + i)
+    out.append('   TEXT: ' + txt[:2500])
+    time.sleep(1)
+
+with open(sys.argv[2], 'w', encoding='utf-8') as f:
+    f.write('\n'.join(out))
+print('done', len(urls))