"""Dump full page text to a file.

Usage: python page_text.py <url> <out.txt>
"""
import urllib.request
import re
import sys
import html as htmlmod

UA = 'Mozilla/5.0 (compatible; paperbot/1.0)'
req = urllib.request.Request(sys.argv[1], headers={'User-Agent': UA})
t = urllib.request.urlopen(req, timeout=45).read().decode('utf-8', 'ignore')
t = re.sub(r'<script.*?</script>', ' ', t, flags=re.S | re.I)
t = re.sub(r'<style.*?</style>', ' ', t, flags=re.S | re.I)
t = re.sub(r'<[^>]+>', ' ', t)
t = ' '.join(htmlmod.unescape(t).split())
with open(sys.argv[2], 'w', encoding='utf-8') as f:
    for i in range(0, len(t), 160):
        f.write(t[i:i + 160] + '\n')
print('chars', len(t))