56 lines
2.1 KiB
Python
56 lines
2.1 KiB
Python
import urllib.request, urllib.parse, json, sys, time, re
|
|
import xml.etree.ElementTree as ET
|
|
|
|
NS = {'a':'http://www.w3.org/2005/Atom'}
|
|
|
|
def fetch(q, start=0, max_results=100, sort='submittedDate'):
|
|
params = {
|
|
'search_query': q,
|
|
'start': start,
|
|
'max_results': max_results,
|
|
'sortBy': sort,
|
|
'sortOrder': 'descending',
|
|
}
|
|
url = 'http://export.arxiv.org/api/query?' + urllib.parse.urlencode(params)
|
|
req = urllib.request.Request(url, headers={
|
|
'User-Agent': 'DeepSeekPaperCollector/1.0 (local research notes)',
|
|
'Accept': 'application/atom+xml',
|
|
})
|
|
for attempt in range(4):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
return r.read()
|
|
except Exception as e:
|
|
print('retry', attempt, e, file=sys.stderr)
|
|
time.sleep(3)
|
|
raise SystemExit('failed')
|
|
|
|
def parse(xml):
|
|
root = ET.fromstring(xml)
|
|
out = []
|
|
for e in root.findall('a:entry', NS):
|
|
aid = e.find('a:id', NS).text
|
|
m = re.search(r'arxiv.org/abs/(.+)$', aid)
|
|
out.append({
|
|
'id': m.group(1) if m else aid,
|
|
'title': ' '.join(e.find('a:title', NS).text.split()),
|
|
'published': e.find('a:published', NS).text,
|
|
'updated': e.find('a:updated', NS).text,
|
|
'authors': [a.find('a:name', NS).text for a in e.findall('a:author', NS)],
|
|
'summary': ' '.join(e.find('a:summary', NS).text.split()),
|
|
'comment': (e.find('{http://arxiv.org/schemas/atom}comment').text if e.find('{http://arxiv.org/schemas/atom}comment') is not None else ''),
|
|
'primary': (e.find('{http://arxiv.org/schemas/atom}primary_category').get('term') if e.find('{http://arxiv.org/schemas/atom}primary_category') is not None else ''),
|
|
})
|
|
return out
|
|
|
|
if __name__ == '__main__':
|
|
query = sys.argv[1]
|
|
allr = []
|
|
for start in range(0, 400, 100):
|
|
xml = fetch(query, start=start)
|
|
res = parse(xml)
|
|
allr.extend(res)
|
|
if len(res) < 100:
|
|
break
|
|
print(json.dumps(allr, ensure_ascii=False, indent=1))
|