新增: DeepSeek 全部 33 篇官方论文(原文 PDF 走 Git LFS + 中文精读笔记与索引)

大模型/deepseek/_scripts/page_text.py

@@ -0,0 +1,20 @@
+"""Dump full page text to a file.
+
+Usage: python page_text.py <url> <out.txt>
+"""
+import urllib.request
+import re
+import sys
+import html as htmlmod
+
+UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
+req = urllib.request.Request(sys.argv[1], headers={'User-Agent': UA})
+t = urllib.request.urlopen(req, timeout=45).read().decode('utf-8', 'ignore')
+t = re.sub(r'<script.*?</script>', ' ', t, flags=re.S | re.I)
+t = re.sub(r'<style.*?</style>', ' ', t, flags=re.S | re.I)
+t = re.sub(r'<[^>]+>', ' ', t)
+t = ' '.join(htmlmod.unescape(t).split())
+with open(sys.argv[2], 'w', encoding='utf-8') as f:
+    for i in range(0, len(t), 160):
+        f.write(t[i:i + 160] + '\n')
+print('chars', len(t))