#!/usr/bin/python3

# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3+


import argparse
import calendar
import datetime
import hashlib
import html5lib
import logging
import os
import re
import requests
import sys
import time
import xml.etree.ElementTree as ET
#import lxml.etree as ET

default_treebuilder = 'etree'
#default_treebuilder = 'lxml'

ans = '{http://www.w3.org/2005/Atom}'
xns = '{http://www.w3.org/1999/xhtml}'


log = logging.getLogger(__name__)

log_format = '%(asctime)s - %(levelname)-8s - %(message)s'
log_date_format = '%Y-%m-%d %H:%M:%S'


def setup_logging():
    logging.basicConfig(format=log_format,
                        datefmt=log_date_format, level=logging.DEBUG)
    logging.getLogger().handlers[0].setLevel(logging.WARNING)


def mk_arg_parser():
    p = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description='Create a LWN.net feed',
        epilog='2017, Georg Sauthoff <mail@gms.tf>, GPLv3+')
    p.add_argument('--cache',
                   help='cache directory (default: $HOME/.cache/lwn)')
    p.add_argument('--verbose', '-v', action='store_true',
                   help='turn on verbose logging')

    p.add_argument('--url', default='https://lwn.net/Articles/?offset=0',
                   metavar='URL', help='start url')
    p.add_argument('-n', default=3, type=int,
                   help='how many index pages to fetch')

    p.add_argument('input', metavar='FILE', nargs='*',
                   help='alternative to --url - when files are already loaded')

    p.add_argument('--output', '-o', metavar='FILE', default='feed.xml',
                   help='output filename (default: %(default)s)')
    p.add_argument('--force', '-f', action='store_true',
                   help="force feed writing - even if is hasn't changed")
    p.add_argument('--no-default', action='store_true',
                   help="don't write default namespace")
    p.add_argument('--all', action='store_true',
                   help="don't filter headlines")
    return p


def parse_args(*a):
    arg_parser = mk_arg_parser()
    args = arg_parser.parse_args(*a)
    if not args.no_default:
        # just for writing
        # i.e. sets the default namespace - with that the feed is created like:
        #
        #    <feed xmlns="http://www.w3.org/2005/Atom"><id>...
        #
        # and not like:
        #
        #    <feed xmlns:ns0="http://www.w3.org/2005/Atom"><ns0:id>...
        #
        # this doesn't work with lxml
        ET.register_namespace('', 'http://www.w3.org/2005/Atom')
    if args.verbose:
        logging.getLogger().handlers[0].setLevel(logging.DEBUG)
    if not args.cache:
        args.cache = os.environ['HOME'] + '/.cache/lwn'
    return args


def parse_next(root):
    link = next(filter(lambda x: x.text and x.text.startswith('Next ') and x.text.endswith('items') and x.get('href').startswith('/Articles/?offset='),
                       root.iter(tag=xns + 'a')))
    # lol, as of Python 3.6 Element objects without children are false-ish ...
    if link is None:
        r = None
    else:
        r = link.get('href')
    log.debug('Found next link: ' + r)
    return r

comment_re = re.compile('^[cC]omments.{1,20}$')


def norm_comment(e):
    for a in e.iter(tag=xns + 'a'):
        if a.text and comment_re.match(a.text):
            a.text = '(Comments)'
    return e


def parse_headlines(root):
    rs = []
    last_headline = None
    for e in root.iter():
        klasse = e.get('class')
        if not klasse:
            continue
        if klasse == 'Headline':
            if e.findall('.//' + xns + 'span[@class="Subscription"]'):
                last_headline = None
            else:
                last_headline = e
        elif e.tag == xns+'div' and klasse == 'BlurbListing' and last_headline is not None:
            headline_str = ' '.join(last_headline.itertext())
            headline_str = headline_str.strip()
            link = next(map(lambda x: x.get('href'),
                            filter(lambda x: x.text == 'Full Story' or (x.text is not None and x.text.startswith('Comments')), e.iter(xns + 'a'))), None)
            if link.endswith('#Comments'):
                link = link[:-9]
            log.debug('Full story link for {}: {}'.format(headline_str,
                                                          (link if link else 'None')))
            rs.append([headline_str, norm_comment(e), link])
    next_link = parse_next(root)
    return (rs, next_link)


def parse_headlines_s(s):
    root = html5lib.parse(s, default_treebuilder)
    return parse_headlines(root)


def parse_headlines_f(filename):
    with open(filename) as f:
        s = f.read()
        return parse_headlines_s(s)

# shared with heiser.py


def get_resource(url, session):
    log.debug('Getting: {}'.format(url))
    r = session.get(url)
    r.raise_for_status()
    return r.text

# shared with heiser.py


def clean_cache(cache, protected_days=7):
    assert cache
    for fn in os.listdir(cache):
        filename = '{}/{}'.format(cache, fn)
        # time.time() doesn't necessarily return the UTC unix epoche ...
        delta = (calendar.timegm(time.gmtime())
                 - os.path.getmtime(filename)) / 3600 / 24
        if delta > protected_days:
            log.debug('Removing cached item: ' + filename)
            os.remove(filename)

san_re = re.compile('[^A-Za-z0-9_-]')

# shared with heiser.py


def get_article(link, ident, cache, session):
    i = san_re.sub('_', ident)
    filename = '{}/{}'.format(cache, i)
    if os.path.exists(filename):
        log.debug('Found ID {} under {}'.format(ident, cache))
        return open(filename, 'r')
    else:
        s = get_resource(link, session)
        with open(filename, 'w') as f:
            f.write(s)
        return s

# shared with heiser.py


def update_urls(a, base):
    def f(e, att):
        href = e.get(att)
        if not href:
            return
        if href.startswith('//'):
            e.set(att, 'https:' + href)
        elif href.startswith('/'):
            e.set(att, base + href)

    for e in a.iter():
        if e.tag == xns + 'a':
            att = 'href'
        elif e.tag == xns + 'img' or e.tag == 'iframe':
            att = 'src'
        else:
            continue
        f(e, att)

# shared with cast.py


def gen_id(e):
    h = hashlib.sha256()
    for x in e.iter():
        if x.tag == ans + 'updated':
            continue
        if type(x.tag) is not str:
            continue
        h.update(bytes(x.tag, encoding='utf8'))
        if x.text:
            h.update(bytes(x.text, encoding='utf8'))
        for k, v in sorted(x.items()):
            h.update(bytes(k, encoding='utf8'))
            h.update(bytes(v, encoding='utf8'))
    hex = h.hexdigest()
    return 'urn:sha256:' + hex

now = datetime.datetime.now(datetime.UTC)


def updated(off=0):
    updated = ET.Element(ans + 'updated')
    updated.text = (now - datetime.timedelta(hours=off)).isoformat()[:-6] + 'Z'
    return updated


def mk_entry(row, off):
    log.debug('Making entry for: {} - {}'.format(row[0], row[2]))
    entry = ET.Element(ans + 'entry')
    ET.SubElement(entry, ans + 'title').text = row[0]
    entry.append(updated(off))
    if row[2]:
        x = row[2][1:] if row[2].startswith('/') else row[2]
        ET.SubElement(entry, ans + 'link', rel='alternate', type='text/html',
                      href='https://lwn.net/' + x)
    content = ET.SubElement(entry, ans + 'content')
    content.set('type', 'xhtml')
    content.append(row[1])
    ET.SubElement(entry, ans + 'id').text = gen_id(entry)
    return entry


def mk_feed(rows, args):
    feed = ET.Element(ans + 'feed')
    ET.SubElement(feed, ans + 'title').text = 'LWN.net'
    ET.SubElement(feed, ans + 'link', rel='alternate', type='text/html',
                  href='https://lwn.net/')
    ET.SubElement(feed, ans + 'id').text = 'lwn.net'
    feed.append(updated())
    for off, row in enumerate(rows):
        feed.append(mk_entry(row, off))
    return ET.ElementTree(feed)


def remove_header(a):
    header = a.find(xns + 'center')
    if header and header.find(xns + 'table'):
        a.remove(header)

def remove_comments(e):
    ds = []
    for x in e:
        if ds or x.tag == f'{xns}form':
            ds.append(x)
    for x in ds:
        e.remove(x)


def test_well_form_anchors():
    inp = '''<p>The discrete tuples, consisting of frequency and voltage pairs,
that the device supports are called &quot;operating performance
points&quot; (OPPs). These were explained in detail in
<a href="/Articles/718632/"</a>this article</a>.
<p>
'''
    d = html5lib.parse(inp)
    a = d.findall('./' + xns + 'body/' + xns + 'p/' + xns + 'a')[0]
    assert sorted(a.attrib.keys()) == ['<', 'a', 'href']
    sanitize_tree(d)
    a = d.findall('./' + xns + 'body/' + xns + 'p/' + xns + 'a')[0]
    assert sorted(a.attrib.keys()) == ['href']


def test_well_form_attrs():
    inp = '''           <div class="FeatureByline">
           By <b>Jonathan Corbet</b><br>July 12, 2019
           </div>
As of this writing, exactly 6,666 non-merge changesets have been pulled
into the mainline repository for the 5.3 development cycle.  The merge
window has thus just begun, there is still quite a bit in the way of
interesting changes to look at.  Read on for a list of what has been merged
so far.
<br clear="all"
<p>
'''
    d = html5lib.parse(inp)
    a = d.findall('./' + xns + 'body/' + xns + 'br')[0]
    assert sorted(a.attrib.keys()) == ['<p', 'clear']
    sanitize_tree(d)
    a = d.findall('./' + xns + 'body/' + xns + 'br')[0]
    assert sorted(a.attrib.keys()) == ['clear']


def test_well_form_attrs2():
    inp = '''
<a href="/Articles/799490/"><img
src="https://static.lwn.net/images/conf/2019/lpc/Defossez-Pillai-sm.jpg" alt="[Julien Desfossez,
Vineeth Remanan Pillai]"
title="Julien Desfossez, Vineeth Remanan Pillai", class="lthumb"></a>
'''
    d = html5lib.parse(inp)
    a = d.findall('./' + xns + 'body/' + xns + 'a/' + xns + 'img')[0]
    assert sorted(a.attrib.keys()) == [',', 'alt', 'class', 'src', 'title']
    sanitize_tree(d)
    a = d.findall('./' + xns + 'body/' + xns + 'a/' + xns + 'img')[0]
    assert sorted(a.attrib.keys()) == ['alt', 'class', 'src', 'title']


def resolve_articles(rs, args, session):
    for r in rs:
        if not r[2]:
            continue
        link = 'https://lwn.net' + r[2]
        a = get_article(link, link, args.cache, session)
        d = html5lib.parse(a, default_treebuilder)
        divs = d.findall('.//' + xns + 'div[@class="ArticleText"]')
        if divs:
            x = divs[0]
            remove_header(x)
            remove_comments(x)
            r[1] = x


def get_ids(t):
    r = [x.text for x in t.getroot().findall('.//' + ans + 'id')]
    return r


def get_ids_f(filename):
    if not os.path.exists(filename):
        return None
    log.debug('Checking {} for IDs ...'.format(filename))
    t = ET.parse(filename)
    r = get_ids(t)
    log.debug('Found existing IDs: ' + str(r))
    return r


att_name_re = re.compile('^[A-Za-z:_][A-Za-z0-9:_.-]+$')

def sanitize_tree(t):
    # NB: not necessary, as ElementTree also has iter() method
    # d = t.getroot()
    a_attrs = set(('href', 'name', 'rel', 'rev', 'urn', 'title',
                   'methods', 'id', 'download', 'hreflang', 'ping', 'referrerpolicy',
                   'target', 'type'))
    a_tag = xns + 'a'
    for e in t.iter():
        ds = []
        for k in e.attrib.keys():
            if not att_name_re.match(k):
                ds.append(k)
            elif e.tag == a_tag and k not in a_attrs:
                ds.append(k)
        for x in ds:
            del e.attrib[x]


# cf. tests above test_well_form_anchors(), test_well_form_attrs* tests
def test_img_invalid():
    inp = '''
<blockquote>
<img src="https://static.lwn.net/images/2026/2026-classical-freedoms.png" class="photo" border="0"
alt="[Slide: "Classical freedoms must remain mobile"]"
title="Slide: "Classical freedoms must remain mobile" border="0"/>
</blockquote>
'''
    d = html5lib.parse(inp)
    a = list(d.findall('.//' + xns + 'img')[0].attrib.keys())
    a.sort()
    assert a == ['alt', 'border', 'class', 'classical', 'freedoms', 'mobile"', 'mobile"]"', 'must', 'remain', 'src', 'title']
    sanitize_tree(d)
    a = list(d.findall('.//' + xns + 'img')[0].attrib.keys())
    assert a == ['src', 'class', 'border', 'alt', 'classical', 'freedoms', 'must', 'remain', 'title']


def write_feed(f, args):
    if get_ids(f) == get_ids_f(args.output) and not args.force:
        log.debug('''Don't write {} because feed IDs haven't changed'''
                  .format(args.output))
    else:
        log.debug('Writing {} ...'.format(args.output))
        f.write(args.output)


def filter_headlines(rs):
    ys = [ x for x in rs if not (x[0].startswith('Stable kernels for ') or x[0].startswith('Kernel prepatch ') or x[0].startswith('Security updates for ')) ]
    return ys


def main(args):
    rs = []
    session = requests.Session()
    if args.input:
        for i in args.input:
            rs += parse_headlines_f(i)[0]
    else:
        url = args.url
        for _ in range(args.n):
            s = get_resource(url, session)
            r = parse_headlines_s(s)
            rs += r[0]
            url = 'https://lwn.net' + r[1]
    if not args.all:
        rs = filter_headlines(rs)
    resolve_articles(rs, args, session)
    f = mk_feed(rs, args)
    sanitize_tree(f)
    ET.indent(f, space='    ')
    update_urls(f, 'https://lwn.net')
    write_feed(f, args)


if __name__ == '__main__':
    setup_logging()
    args = parse_args()
    os.makedirs(args.cache, exist_ok=True)
    clean_cache(args.cache)
    sys.exit(main(args))
