21 lines
685 B
Python
21 lines
685 B
Python
"""Dump full page text to a file.
|
|
|
|
Usage: python page_text.py <url> <out.txt>
|
|
"""
|
|
import urllib.request
|
|
import re
|
|
import sys
|
|
import html as htmlmod
|
|
|
|
UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
|
|
req = urllib.request.Request(sys.argv[1], headers={'User-Agent': UA})
|
|
t = urllib.request.urlopen(req, timeout=45).read().decode('utf-8', 'ignore')
|
|
t = re.sub(r'<script.*?</script>', ' ', t, flags=re.S | re.I)
|
|
t = re.sub(r'<style.*?</style>', ' ', t, flags=re.S | re.I)
|
|
t = re.sub(r'<[^>]+>', ' ', t)
|
|
t = ' '.join(htmlmod.unescape(t).split())
|
|
with open(sys.argv[2], 'w', encoding='utf-8') as f:
|
|
for i in range(0, len(t), 160):
|
|
f.write(t[i:i + 160] + '\n')
|
|
print('chars', len(t))
|