88 lines
2.6 KiB
Python
88 lines
2.6 KiB
Python
"""Scrape arXiv advanced search by author name, all pages.
|
|
|
|
Usage: python scrape_author.py <author> <out.json>
|
|
"""
|
|
import urllib.request
|
|
import urllib.parse
|
|
import re
|
|
import json
|
|
import sys
|
|
import time
|
|
import html as htmlmod
|
|
|
|
UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
|
|
|
|
|
|
def get(url, timeout=45):
|
|
req = urllib.request.Request(url, headers={'User-Agent': UA})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.read().decode('utf-8', 'ignore')
|
|
|
|
|
|
def strip(s):
|
|
s = re.sub(r'<[^>]+>', ' ', s)
|
|
return ' '.join(htmlmod.unescape(s).split())
|
|
|
|
|
|
def parse_page(t):
|
|
out = []
|
|
for blk in re.split(r'<li class="arxiv-result">', t)[1:]:
|
|
aid = re.search(r'arxiv\.org/abs/([^"\'<]+)', blk)
|
|
title = re.search(r'<p class="title is-5 mathjax">(.*?)</p>', blk, re.S)
|
|
authors = re.search(r'<p class="authors">(.*?)</p>', blk, re.S)
|
|
dates = re.search(r'<p class="is-size-7">(.*?)</p>', blk, re.S)
|
|
if not aid:
|
|
continue
|
|
out.append({
|
|
'id': aid.group(1),
|
|
'title': strip(title.group(1)) if title else '',
|
|
'authors': strip(authors.group(1)) if authors else '',
|
|
'dates': strip(dates.group(1)) if dates else '',
|
|
})
|
|
return out
|
|
|
|
|
|
def main():
|
|
term = sys.argv[1]
|
|
out_path = sys.argv[2]
|
|
size = 200
|
|
res = []
|
|
page = 0
|
|
while True:
|
|
qs = [
|
|
('advanced', ''),
|
|
('terms-0-operator', 'AND'),
|
|
('terms-0-term', term),
|
|
('terms-0-field', 'author'),
|
|
('classification-include_cross_list', 'include'),
|
|
('start', str(page * size)),
|
|
('size', str(size)),
|
|
('order', '-announced_date_first'),
|
|
]
|
|
url = 'https://arxiv.org/search/advanced?' + urllib.parse.urlencode(qs)
|
|
try:
|
|
t = get(url)
|
|
except Exception as e: # noqa: BLE001
|
|
print('page fail', page, e, file=sys.stderr)
|
|
break
|
|
rows = parse_page(t)
|
|
total = re.search(r'of\s+([\d,]+)\s+results', t)
|
|
print('page', page, 'rows', len(rows), 'total', total.group(1) if total else '?', file=sys.stderr)
|
|
res.extend(rows)
|
|
if len(rows) < size:
|
|
break
|
|
page += 1
|
|
time.sleep(2)
|
|
if page > 20:
|
|
break
|
|
seen = {}
|
|
for r in res:
|
|
seen.setdefault(r['id'].split('v')[0], r)
|
|
with open(out_path, 'w', encoding='utf-8') as f:
|
|
json.dump(list(seen.values()), f, ensure_ascii=False, indent=1)
|
|
print('saved', len(seen), file=sys.stderr)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|