#!/usr/bin/env python3

import argparse
import bs4
from dataclasses import dataclass, field
import datetime
import dateutil
import email.utils
import logging
import os
import sys
import urllib
from xml.dom import minidom

"""
Build an RSS feed or XML sitemap from a directory
"""

logger = logging.getLogger(__name__)

long_ago = datetime.datetime.fromtimestamp(0, datetime.timezone.utc)

def error_out(msg):
    '''
    Log an error message and exit the program.

    Args:
        msg (str): The error message to log.
    '''
    logger.error(msg)
    sys.exit(1)

def dt_from_element(el):
    """
    Extract a datetime from the content of a meta element
    """
    try:
        content = el.get("content")
        found_date = dateutil.parser.parse(content)
        found_date = found_date.replace(tzinfo=datetime.timezone.utc)
        return found_date
    except:
        return None


@dataclass
class SiteInfo:
    title: str
    link: str
    description: str
    pages: list = field(default_factory=list)
    permalinks: set = field(default_factory=set)

    def __repr__(self):
        return "site %s with %d pages" % (self.title, len(self.pages))

    @classmethod
    def from_path_list(cls, dirlist, title=None, url=None, description=None):
        """
        Takes a list of directories and populates with all HTML files found
        """
        info = cls(title, url, description)
        for dir in dirlist:
            for root, dirs, files in os.walk(dir):
                for file in files:
                    if file.endswith(".html"):
                        infile = os.path.join(root, file)
                        item = FileInfo.from_pathname(info, infile)
                        if not item:
                            continue
                        if item.permalink in info.permalinks:
                            error_out("Duplicate permalink in %s" % item)
                        info.pages.append(item)
                        info.permalinks.add(item.permalink)
        info.remove_strays()
        return info

    @property
    def homepage(self):
        for page in self.pages:
            if page.urlpath == "/" and page.permalink.startswith("https://"):
                return page
        error_out("No homepage found")

    @property
    def date(self):
        return self.pages_by_date()[0].date

    @property
    def mod_date(self):
        return self.pages_by_mod_date()[0].mod_date

    def pages_by_date(self):
        return sorted(
            self.pages, key=lambda p: p.date if p.date else long_ago, reverse=True
        )

    def pages_by_mod_date(self):
        return sorted(
            self.pages,
            key=lambda p: p.mod_date if p.mod_date else long_ago,
            reverse=True,
        )

    def remove_strays(self):
        """
        Remove any pages that have a permalink that's not https and the same site as the homepage.
        """
        here = self.homepage.netloc
        pagelist = []
        for p in self.pages:
            if p.permalink.startswith("https://") and p.netloc == here and p.date:
                pagelist.append(p)
                continue
            if p is self.homepage:
                if not p.date:
                    p.date = self.date
                if not p.mod_date:
                    p.mod_date = self.mod_date
                pagelist.append(p)
        self.pages = pagelist

    def to_rss(self, count=15, use_fulltext=False, use_mod_date=False):
        if not self.title:
            error_out("Can\'t make RSS: no title")
        if not self.link:
            error_out("Can\'t make RSS: no link")
        if not self.description:
            self.description = "This is an RSS 2.0 feed."
        doc = minidom.Document()
        rss = doc.createElement("rss")
        rss.setAttribute("version", "2.0")
        rss.setAttribute("xmlns:atom", "http://www.w3.org/2005/Atom")
        doc.appendChild(rss)
        channel = doc.createElement("channel")
        atomlink = doc.createElement("atom:link")
        atomlink.setAttribute("href", self.link)
        atomlink.setAttribute("rel", "self")
        atomlink.setAttribute("type", "application/rss+xml")
        channel.appendChild(atomlink)
        rss.appendChild(channel)
        for channeltag in [
            ["title", self.title],
            ["link", self.link],
            ["description", self.description],
        ]:
            el = doc.createElement(channeltag[0])
            el.appendChild(doc.createTextNode(channeltag[1]))
            channel.appendChild(el)
        i = 0
        pages = self.pages_by_mod_date() if use_mod_date else self.pages_by_date()
        for page in pages:
            if page is self.homepage:
                continue
            if i < count:
                item_el = page.to_rss_item(doc, use_fulltext, use_mod_date)
                if item_el:
                    channel.appendChild(item_el)
                    i += 1
            else:
                break
        return doc.toprettyxml()

    def to_sitemap(self):
        # always uses mod date, not pub date
        doc = minidom.Document()
        urlset = doc.createElement("urlset")
        urlset.setAttribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")
        doc.appendChild(urlset)
        for item in self.pages_by_mod_date():
            el = item.to_sitemap_entry(doc)
            if el:
                urlset.appendChild(el)
        return doc.toprettyxml()


@dataclass
class FileInfo:
    site: SiteInfo
    title: str
    permalink: str
    date: datetime.datetime
    mod_date: datetime.datetime
    description: str
    body_copy: str
    pathname: str

    def __repr__(self):
        return "%s for %s with title %s published: %s modified: %s" % (
            self.pathname,
            self.permalink,
            self.title,
            self.date,
            self.mod_date,
        )

    @property
    def rss_date(self):
        return email.utils.format_datetime(self.date)

    @property
    def rss_mod_date(self):
        return email.utils.format_datetime(self.mod_date)

    @property
    def webdate(self):
        return self.date.strftime("%Y-%m-%d")

    @property
    def web_mod_date(self):
        return self.mod_date.strftime("%Y-%m-%d")

    @property
    def urlpath(self):
        u = urllib.parse.urlsplit(self.permalink)
        return u.path

    @property
    def netloc(self):
        u = urllib.parse.urlsplit(self.permalink)
        return u.netloc

    def to_rss_item(self, doc, use_fulltext=True, use_mod_date=False):
        rssdate = self.rss_mod_date if use_mod_date else self.rss_date
        item_el = doc.createElement("item")
        for itemtag in [
            ["title", self.title],
            ["link", self.permalink],
            ["guid", self.permalink],
            ["pubDate", rssdate],
        ]:
            el = doc.createElement(itemtag[0])
            el.appendChild(doc.createTextNode(itemtag[1]))
            item_el.appendChild(el)
        description = self.body_copy if use_fulltext else self.description
        if description:
            dtag = doc.createElement("description")
            cdata = doc.createCDATASection(description)
            dtag.appendChild(cdata)
            item_el.appendChild(dtag)
        return item_el

    def to_sitemap_entry(self, doc):
        if not self.permalink:
            return None
        url_el = doc.createElement("url")
        for urltag in [["loc", self.permalink], ["lastmod", self.web_mod_date]]:
            el = doc.createElement(urltag[0])
            el.appendChild(doc.createTextNode(urltag[1]))
            url_el.appendChild(el)
        return url_el

    @classmethod
    def from_pathname(cls, site, pathname):
        with open(pathname, "r") as fh:
            try:
                html_content = fh.read()
                soup = bs4.BeautifulSoup(html_content, "html.parser")
            except Exception as e:
                logger.debug("couldn\'t parse HTML from %s: %s" % (pathname, e))
                return False
            robots_el = soup.find("meta", attrs={"name": "robots"})
            if robots_el:
                robots = robots_el.get("content", "")
                for robot_entry in robots.split(","):
                    robot_entry = robot_entry.strip()
                    if robot_entry == "noindex":
                        return None

            canon_el = soup.find("link", attrs={"rel": "canonical"})
            found_permalink = canon_el.get("href") if canon_el else None
            if not found_permalink:
                logger.debug("No permalink found in %s" % pathname)
                return None

            try:
                el = soup.find("title")
                found_title = "".join(str(txt) for txt in el.contents)
                found_title = " ".join(found_title.split())
            except Exception as e:
                logger.debug("Can\'t get title element from %s: %s" % (pathname, e))
                return None

            feed_el = soup.find(
                "link", attrs={"rel": "alternate", "type": "application/rss+xml"}
            )
            found_rss_link = feed_el.get("href") if feed_el else None
            found_feed_title = feed_el.get("title") if feed_el else found_title

            # Original date (mostly useful for RSS)
            found_date = None
            date_el = soup.find("meta", attrs={"name": "dcterms.date"})
            if date_el:
                found_date = dt_from_element(date_el)

            # We don't trust mtime of built files but if we did
            #    ts = os.path.getmtime(pathname)
            #    found_date = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)

            # Modification date (mostly useful for sitemaps)
            found_mod_date = None
            mod_el = soup.find("meta", attrs={"property": "article:modified_time"})
            if mod_el:
                found_mod_date = dt_from_element(mod_el)

            desc_el = soup.find("meta", attrs={"name": "description"})
            if desc_el:
                found_description = desc_el.get("content", "")
                found_description = " ".join(found_description.split())
            else:
                found_description = None

            b_el = soup.find(id="body_copy")
            found_body = "".join(str(item) for item in b_el.contents) if b_el else None

        if found_mod_date is None and found_date is not None:
            found_mod_date = found_date
        if found_date is None and found_mod_date is not None:
            found_date = found_mod_date

        # If pub date is after mod date, set pub date to mod date
        if found_date and found_date > found_mod_date:
            found_date = found_mod_date

        new_obj = cls(
            site,
            found_title,
            found_permalink,
            found_date,
            found_mod_date,
            found_description,
            found_body,
            pathname,
        )

        # Use title and meta tags from home page to populate feed
        if new_obj.urlpath == "/":
            if found_feed_title and not new_obj.site.title:
                new_obj.site.title = found_feed_title
            if found_rss_link and not new_obj.site.link:
                new_obj.site.link = found_rss_link
            if found_description and not new_obj.site.description:
                new_obj.site.description = found_description
        return new_obj


if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("--format", help="RSS or sitemap", default="rss")
    parser.add_argument("--title", help="Feed title", default=None)
    parser.add_argument("--description", default=None)
    parser.add_argument("--url", default=None)
    parser.add_argument("--fulltext", action="store_true")
    parser.add_argument(
        "--mod", action="store_true", help="use article:modified time meta tag"
    )
    parser.add_argument("--pub", action="store_true", help="use published date")
    parser.add_argument("--debug", action="store_true", help="Enable debug logging")
    parser.add_argument("dirs", nargs=argparse.REMAINDER)
    args = parser.parse_args()

    log_level = logging.DEBUG if args.debug else logging.INFO
    logging.basicConfig(level=log_level, stream=sys.stderr, format='%(levelname)s: %(message)s')

    info = SiteInfo.from_path_list(args.dirs, args.title, args.url, args.description)
    if args.format == 'sitemap':
        print(info.to_sitemap())
    else:
        print(info.to_rss(15, args.fulltext, args.mod))

# vim: autoindent textwidth=100 tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=python