#!/usr/bin/env python3

"""
uriel

A static site generator for the small web.

Named for the archangel Uriel in the novel Unsong, whose job was to perform
the fantastic and mundane work necessary to keep the world functioning.

Copyright 2021-2026 Nathan Rosenquist

https://uriel.foo/

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.

"""

# imports
import datetime
import os
import re
import shutil
import sys
import heapq

# deferred imports
#import ast
#import hashlib
#import gzip
#import traceback

# type hinting imports
from types import ModuleType
from typing import Any
from typing import Dict
from typing import List
from typing import Set
from typing import Tuple
from typing import Optional

# the name of the program
PROGRAM_NAME = "uriel"

# program version
VERSION = "1.6.1"

# exit codes
EXIT_OK = 0
EXIT_FAIL = 1

# sub-directories under the project root directory
STATIC_ROOT = "static"
NODES_ROOT = "nodes"
TEMPLATES_ROOT = "templates"
LIB_ROOT = "lib"
PUBLIC_ROOT = "public"

# node index filename
NODE_INDEX = "index"

# html index filename
HTML_INDEX = "index.html"

# default template to use when rendering nodes
# (can be overridden via Template header)
DEFAULT_TEMPLATE = "default.html"

# default maximum number of RSS feed entries
# (can be overridden via RSS-Max-Entries header)
RSS_MAX_ENTRIES = 20

# default maximum number of sitemap files
# (can be overridden via Sitemap-Max-Files header)
SITEMAP_MAX_FILES = 500

# default maximum number of sitemap entries
# (can be overridden via Sitemap-Max-Entries header)
SITEMAP_MAX_ENTRIES = 50000

# performance optimization
# static file path -> hashed filename
static_hash_filenames: Dict[str, str] = {}

# performance optimization
# (node destination directory, static url) -> absolute path on disk
static_url_abspaths: Dict[Tuple[str, str], str] = {}

# performance optimization
# template name -> template file path
template_file_paths: Dict[str, str] = {}

# performance optimization
# template file path -> list of (template line, tokens parsed from that line)
# the tokens in the tuple are optional, many lines don't have parameters
template_tokens: Dict[str, List[Tuple[str, Optional[List["Token"]]]]] = {}

class UrielError(Exception):
    """
    General purpose exception that will result in a program error, but
    without showing a stack trace to the user.

    """

    pass


class SojuError(Exception):
    """
    Represents an error from user-defined soju code that should not throw
    a stack trace.

    """

    pass


class HandlerError(Exception):
    """
    Represents an error from user-defined handler code that should not throw
    a stack trace.

    """

    pass


class _ImpossibleError(Exception):
    """
    An exception that should never actually be thrown in practice.

    If this does get thrown, it means there was a unexpected bug in the
    program.

    This is also used to provide a hint to the type checking machinery that
    a method will not return a value, since a function or method it called
    will throw an exception.

    """

    pass


class Token:
    """
    Represents a single token, either a literal or a substitution parameter.

    Lines are parsed one at a time. Each line can be divided into one or
    more tokens. A token is either a literal, or a substitution parameter,
    e.g. {{foo:bar}}

    """

    # possible types
    # {{$TOKEN_TYPE_CONSTANT:*}}
    # all are lvalues for substitution parameters, except for "literal",
    # which is just a literal string
    LITERAL = "literal"
    VALUE = "value"
    VALUE_UNESCAPED = "value-unescaped"
    INCLUDE = "include"
    CREATED = "created"
    MODIFIED = "modified"
    OLDEST = "oldest"
    LATEST = "latest"
    BREADCRUMBS = "breadcrumbs"
    BREADCRUMBS_ROOT = "breadcrumbs-root"
    STATIC_URL = "static-url"
    STATIC_HASH_URL = "static-hash-url"
    RSS = "rss"
    NODE = "node"
    NODE_URL = "node-url"
    NODE_NAME = "node-name"
    NODE_TITLE = "node-title"
    NODE_LINK = "node-link"
    NODE_LIST = "node-list"
    TAG_LIST = "tag-list"
    SOJU = "soju"

    # allowed lvalue keywords inside of a {{ parameter }}
    # (e.g. {{some_keyword:*}}
    # (all are allowed except "literal")
    KEYWORDS = set([
        VALUE,
        VALUE_UNESCAPED,
        INCLUDE,
        CREATED,
        MODIFIED,
        OLDEST,
        LATEST,
        BREADCRUMBS,
        BREADCRUMBS_ROOT,
        STATIC_URL,
        STATIC_HASH_URL,
        RSS,
        NODE,
        NODE_URL,
        NODE_NAME,
        NODE_TITLE,
        NODE_LINK,
        NODE_LIST,
        TAG_LIST,
        SOJU,
    ])

    def __init__(self, s: str) -> None:
        """
        Accepts a fragment of text, which may or may not be a parameter.

        Sets the "type" and "value" fields according to the string contents.

        """

        # remember the original string that was used to make the token
        self.original_string = s

        # does it look like this was supposed to be a parameter,
        # but we don't understand it?
        self.unidentified_parameter = False

        # does it look like this was supposed to be a parameter, but
        # the closing "}}" is missing?
        self.unterminated_parameter = False

        # parameter
        if s.startswith("{{") and s.endswith("}}"):
            # strip off the {{ and }}
            content = s[2:-2]

            # parse (key, value) inside of "{{ key: value }}"
            try:
                (keyword, value) = content.split(":", maxsplit=1)
                keyword = keyword.strip()
                value = value.strip()
            except ValueError:
                keyword = None
                value = s

            # turn the parameter key into a token type
            self.type = None
            if keyword in Token.KEYWORDS:
                self.type = keyword

            # if we didn't find a token type that we understood,
            # make a note of it, but pass the entire value on as a literal
            if self.type is None:
                self.unidentified_parameter = True
                self.type = Token.LITERAL
                value = self.original_string

            self.value = value

        # a parameter that was opened, but never closed
        elif s.startswith("{{") and ("}}" not in s):
            self.unterminated_parameter = True
            self.type = Token.LITERAL
            self.value = s

        # literal
        else:
            self.type = Token.LITERAL
            self.value = s

    def has_unidentified_parameter(self) -> bool:
        """
        Does this token contain something that looks syntactically like a
        substitution parameter, but not one that we recognize?

        """

        return self.unidentified_parameter

    def has_unterminated_parameter(self) -> bool:
        """
        Does this token start something that looks like a substitution
        parameter, without ever closing it?

        """

        return self.unterminated_parameter

    def __eq__(self, other: Any) -> bool:
        if not isinstance(other, Token):
            return NotImplemented

        if self.type == other.type:
            if self.value == other.value:
                if self.original_string == other.original_string:
                    return True

        return False

    def __str__(self) -> str:
        return "[token type=" + \
            str(self.type) + ", value='" + \
            str(self.value) + "']"

    def __repr__(self) -> str:
        return self.__str__()


class TemplateStack:
    """
    Nodes and templates can include other templates.

    This class represents the stack of templates, in the order they were
    included.

    """

    def __init__(self) -> None:
        """
        Constructor.

        """

        self.templates: List[str] = []

    def push(self, template: str, node_path: str, in_node_body: int) -> None:
        """
        Add a template to the stack.

        Accepts the template to add to the stack, the node path,
        and a node body semaphore.

        """

        # if an include loop is detected, show a stack trace and raise an error
        if template in self.templates:
            log("include loop error:")

            # indent errors as we work our way down
            # from the node, through the templates, to wherever the problem is
            indent = 1

            # node
            log("%s%s/%s" % (indent_spaces(indent), NODES_ROOT, node_path))

            # highlight the range of the loop
            loop_started = False

            indent += 1
            triple_arrow = ">>>" + (((indent * 2) - 3) * " ")
            single_arrow = "  >" + (((indent * 2) - 3) * " ")

            while self.has_more_elements():
                t = self.shift()

                if t == template:
                    loop_started = True
                    log("%s%s/%s <<< LOOP STARTS HERE" %
                        (triple_arrow, TEMPLATES_ROOT, t))
                elif loop_started:
                    log("%s%s/%s" % (single_arrow, TEMPLATES_ROOT, t))
                else:
                    log("%s%s/%s" % (indent_spaces(indent), TEMPLATES_ROOT, t))

            # if we're back in the node body, indicate that with a parameter,
            # followed by the node path
            if in_node_body:
                # {{node:body}}
                indent += 1
                triple_arrow = ">>>" + (((indent * 2) - 3) * " ")
                single_arrow = "  >" + (((indent * 2) - 3) * " ")
                log("%s{{node:body}}" % (single_arrow))

                # node path
                indent += 1
                triple_arrow = ">>>" + (((indent * 2) - 3) * " ")
                single_arrow = "  >" + (((indent * 2) - 3) * " ")
                log("%s%s/%s" % (single_arrow, NODES_ROOT, node_path))

                # {{include:template}}
                indent += 1
                triple_arrow = ">>>" + (((indent * 2) - 3) * " ")
                single_arrow = "  >" + (((indent * 2) - 3) * " ")
                log("%s{{include:%s}}" % (single_arrow, template))

                # indent the final template that comes after this
                indent += 1
                triple_arrow = ">>>" + (((indent * 2) - 3) * " ")
                single_arrow = "  >" + (((indent * 2) - 3) * " ")

            # log the first instance of the repeated template loop
            log("%s%s/%s <<< WOULD REPEAT FOREVER" %
                (triple_arrow, TEMPLATES_ROOT, template))

            raise UrielError(
                "include loop detected in: '%s/%s'" %
                (TEMPLATES_ROOT, template))

        self.templates.append(template)

    def pop(self) -> str:
        """
        Remove the most recent template from the stack and return it.

        """

        return self.templates.pop()

    def shift(self) -> str:
        """
        Remove the first template from the stack and return it.

        """

        template = self.templates[0]

        self.templates = self.templates[1:]

        return template

    def get_templates(self) -> List[str]:
        """
        Get the templates on this stack, in the order they were included.

        Returns a copy of the list, so that the caller can read the stack
        without removing anything from it.

        """

        return list(self.templates)

    def has_more_elements(self) -> bool:
        """
        Does this stack still have more templates that can be removed?

        """

        if len(self.templates) > 0:
            return True

        return False


class Node:
    """
    Represents a dynamic page, either virtual or backed by a file.

    This class is intended to be treated as abstract, and not
    instantiated directly.

    """

    def __init__(self,
                 project_root: str,
                 path: str,
                 parent_node=None) -> None:

        """
        Accepts the project root directory.

        Accepts the path (as a subdirectory of the node root).

        Optionally accepts a parent Node instance.

        """

        self.project_root: str = project_root
        self.nodes_root: str = os.path.join(project_root, NODES_ROOT)
        self.path: str = path
        self.parent_node: Optional[Node] = parent_node
        self.body: Optional[str] = None
        self.rendered_body: Optional[str] = None
        self.created: Optional[datetime.datetime] = None
        self.modified: Optional[datetime.datetime] = None

        # list of child Node instances
        self.children: List["Node"] = []
        self.child_nodes_presorted: bool = False

        # header -> value
        self.headers: Dict[str, str] = {}

        # tag -> set of Node matching that tag
        self.tag_node_index: Optional[Dict[str, Set[Node]]] = None

        #
        # cache performance optimizations
        #
        # url
        self.url_cache: Optional[str] = None
        # name
        self.name_cache: Optional[str] = None
        # title
        self.title_cache: Optional[str] = None
        # escaped title
        self.escaped_title_cache: Optional[str] = None
        # link
        self.link_cache: Optional[str] = None
        # canonical URL
        self.canonical_url_base_cache: Optional[str] = None
        #
        # root node cache (always points to the root node, on every node)
        self.root_node_cache = self
        if parent_node is not None:
            self.root_node_cache = parent_node.root_node_cache
        #
        # node path cache
        # (only exists on the root node, all child nodes have references)
        # path -> Node
        self.node_path_cache: Dict[str, Node] = {}
        if parent_node is None:
            # if this is the root node, add ourselves to the cache
            self.node_path_cache[NODE_INDEX] = self
        else:
            self.node_path_cache = self.root_node_cache.node_path_cache
        #
        # tag node cache
        self.tag_node_cache: Optional[Node] = None
        #
        # sorted list of tags
        self.sorted_tags_cache: Optional[List[str]] = None
        #
        # map of tag -> Node representing that tag (e.g. /tag/<SOME_TAG>)
        self.tag_vnode_cache: Optional[Dict[str, Node]] = None
        if parent_node is None:
            self.tag_vnode_cache = {}

        # check for duplicate node path elsewhere in the tree
        if parent_node is not None:
            root_node = parent_node.get_root_node()
            dupe = root_node.find_node_by_path(path, False)
            if dupe is not None:
                err = "can not create node, another node already exists " + \
                      "with this path: '%s'"
                raise UrielError(err % (path))

        # import headers from parent node
        if self.parent_node:
            # to start, copy all parent headers into this node.
            # we intentionally bypass sorting here for performance.
            # the key order doesn't matter during this step.
            for (key, value) in list(self.parent_node.headers.items()):
                self.set_header(key, value)

            # N.B. the order of header processing in this block is important,
            #      to make sure the literal headers, + and - variants all
            #      operate correctly

            # disable inheritance on fields that should be unique

            # Title
            if self.has_header("title"):
                self.delete_header("title")

            # Created
            if self.has_header("created"):
                self.delete_header("created")

            # Modified
            if self.has_header("modified"):
                self.delete_header("modified")

            # N.B. we reverse the header keys because ++foo sorts after +foo,
            #      which prevents ++foo from overwriting +foo before being set

            # +
            # go through each header, and put the + headers in a separate list
            plus_keys = []
            for key in self.headers:
                if key.startswith("+"):
                    plus_keys.append(key)

            # if we have any + headers, go through them in sorted order,
            # and apply them to this node. this two step pass with partial
            # sorting is a performance optimization.
            for key in reversed(sorted(plus_keys)):
                plus_key = key
                stripped_key = key[1:]
                value = self.get_header(plus_key)
                self.set_header(stripped_key, value)
                self.delete_header(plus_key)

            # -
            self._process_minus_headers()

    def _process_minus_headers(self) -> None:
        """
        Process the "-" headers that are currently set on this Node.

        A header with a leading "-" and a value of "*" is an instruction to
        remove the header with the same name, but without the leading "-",
        from this Node (e.g. "-Foo: *" removes the "Foo" header).

        The "-" header itself is also removed, once it has been acted on.

        Raises a UrielError if a "-" header has a value other than "*".

        This method is only intended to be called from inside of the
        Node class family of constructors.

        """

        # go through each "-" key, in reverse sorted order
        #
        # N.B. --foo naturally sorts before -foo, but when reversed,
        #      --foo will be processed after -foo.
        #      this is important, because -foo removes foo,
        #      --foo removes -foo, etc.

        # pick out only the "-" keys
        minus_keys = []
        for key in self.headers:
            if key.startswith("-"):
                minus_keys.append(key)

        # go through only the "-" keys in reverse sorted order
        # this two-step pass and partial sorting is a performance optimization
        for key in reversed(sorted(minus_keys)):
            # if a header starts with a "-", and has a value of "*",
            # it's an instruction to remove that header from this Node
            # (e.g. -Foo: *)

            # -Foo
            minus_key = key

            # Foo
            remove_key = key[1:]

            # value of -Foo (ideally *)
            value = self.get_header(minus_key)

            if "*" != value:
                err = "invalid header in node '%s': '%s': '%s' " + \
                      "(value must be '*')"
                raise UrielError(
                    err % (self.get_path(), minus_key, value))

            # remove the header this key refers to
            if self.has_header(remove_key):
                self.delete_header(remove_key)

            # remove the "-" key itself, now that it has been acted on
            self.delete_header(minus_key)

    def get_parent_node(self) -> Optional["Node"]:
        """
        Get the parent Node, or None if this is the root.

        """

        return self.parent_node

    def get_path(self) -> str:
        """
        Get the node path (e.g. "index", "foo/bar").

        """

        return self.path

    def get_node_type(self) -> str:
        """
        Get the Node type as a string (e.g. "file", "virtual").

        Subclasses need to implement this, and return a string describing
        the type of node they represent.

        """

        raise Exception("not implemented in abstract parent class")

    def add_child(self, node: "Node") -> None:
        """
        Add the given Node as a child of this Node.

        """

        # add the child node
        self.children.append(node)

        # invalidate the pre-sorted flag
        self.child_nodes_presorted = False

        # update the node path cache on the root node
        self.root_node_cache.node_path_cache[node.get_path()] = node

    def get_children(self) -> List["Node"]:
        """
        Get a list of child Node entries immediately under this Node.

        """

        # if the child nodes have not already been pre-sorted,
        # sort them now, in place, and remember that we did
        if not self.child_nodes_presorted:
            self.children = sorted(self.children, key=Node.get_sort_key)
            self.child_nodes_presorted = True

        # return the (pre-sorted) child nodes
        return self.children

    def get_url(self) -> str:
        """
        Get the URL path for this Node.

        URL paths are relative to the root of the site.

        Example URL paths:
            /
            /foo/
            /foo/bar/

        This method is the authoritative source of where a node will be
        placed in the rendered files.

        This method never returns canonical URLs, only URL paths.

        """

        # cache hit
        if self.url_cache is not None:
            return self.url_cache

        # set node_dir to the directory we want to use in our URL
        # if it's an index node, strip off the "index" part
        #
        # is this the root node?
        if self.get_path() == NODE_INDEX:
            # the root node is represented as a /
            # (e.g. index becomes /)
            #
            # we return early here, because all other nodes have both
            # leading and trailing slashes added later
            self.url_cache = "/"
            return self.url_cache

        # is this an index node in a subdirectory?
        elif self.get_path().endswith("/" + NODE_INDEX):
            # strip off the "/index" portion for the node directory
            # (e.g. foo/index becomes foo)
            end_index = (len(NODE_INDEX) * -1) -1
            node_dir = self.get_path()[:end_index]

        # this is a non-index node
        else:
            # the node directory is equal to the node path
            # (e.g. foo/bar remains foo/bar)
            node_dir = self.get_path()

        # preserve node hierarchy by default
        target_dir = node_dir

        # flat URL (e.g. just the leaf node name, or directory if index)
        flat_url = node_dir.split("/")[-1:][0]

        # optionally flatten the URL to only the most specific node directory
        if self.get_boolean_header_value("flat-url", False):
            target_dir = flat_url

        # if this node doesn't have a flat URL, one of its parents might
        else:
            parent_node = self.get_parent_node()
            if parent_node is not None:
                # get the parent URL (without the slash)
                parent_url = parent_node.get_url()[1:]

                # set our target dir to the parent URL plus the flat URL
                target_dir = parent_url + flat_url

        # return the target dir, with a leading and trailing slash
        self.url_cache = "/" + target_dir + "/"

        return self.url_cache

    def get_canonical_url_base(self) -> str:
        """
        Get the base URL of the web site, as set by the Canonical-URL
        header (e.g. "https://example.com").

        Raises a UrielError if the Canonical-URL header is not set, or
        if it is set to a value that ends with a "/".

        """

        # return the canonical URL from cache, if possible
        if self.canonical_url_base_cache is not None:
            return self.canonical_url_base_cache

        # make sure the Canonical-URL header is set
        if not self.has_header("canonical-url"):
            raise UrielError(
                "Canonical-URL not set, but required by node '%s'" %
                (self.get_path()))

        canonical_url_base = self.get_header("canonical-url")

        # make sure the canonical URL doesn't end with a (redundant) slash
        if canonical_url_base.endswith("/"):
            err = "invalid Canonical-URL header value in node " + \
                  "'%s': '%s' (can not end with a '/')"
            raise UrielError(err % (self.get_path(), canonical_url_base))

        # cache the results for next time
        self.canonical_url_base_cache = canonical_url_base

        return canonical_url_base

    def get_canonical_url(self) -> str:
        """
        Get the canonical URL for this Node.

        This method only returns canonical URLs. If the Canonical-URL header
        is not set, it will raise an error.

        """

        return self.get_canonical_url_base() + self.get_url()

    def get_name(self) -> str:
        """
        Get the name of this Node (e.g. "index", "bar").

        This is generally the most specific dirent from the URL path.
        For example, a node with a node path of "foo/index" and a URL
        path of "/foo/" will have a node name of "foo".

        As a special case, the name of the top-level index node is "index".

        """

        if self.name_cache is not None:
            return self.name_cache

        url = self.get_url()

        leaf = NODE_INDEX

        url_parts = url.split("/")
        for part in url_parts:
            if "" != part:
                leaf = part

        self.name_cache = leaf

        return self.name_cache

    def get_display_name(self) -> str:
        """
        Get the display name of the node.

        This is constructed from the base node name, but is formatted to be
        more human readable. For example, if the base node name is
        "hello-world", then this method will return "Hello World".

        It is used as a fallback plan in case a Title header is not set.

        """

        # get the node name
        name = self.get_name()

        # we're going to split the node name into a bunch of parts,
        # and recombine them separated by spaces. we split on several
        # different characters, and subdivide the parts further and further.
        # to start, put the entire node name in the parts list as one element.
        parts = [name]

        # go through each single-character pattern we're splitting on
        for pattern in "-_ ":
            # temporarily save the parts that come out of the split
            # for this one character
            future_parts = []

            # go through all of the parts
            for part in parts:
                # further split this part on the current pattern
                tmp_parts = part.split(pattern)

                # for each part that came out of this split,
                # add them to what will become the new parts list
                for tmp_part in tmp_parts:
                    # by default, we pass the part straight through
                    future_part = tmp_part

                    # if this part doesn't have any capital letters,
                    # give it initial caps
                    if not re.match(r"[A-Z]", future_part):
                        future_part = future_part.capitalize()

                    # add it to the parts list
                    future_parts.append(future_part)

            # replace the higher-level parts list with the local one,
            # which is now further subdivided
            parts = future_parts

        # return all of the subdivided parts, separated by spaces
        return " ".join(parts)

    def get_title(self) -> str:
        """
        Get the title of the node, as set by the Title header.

        If that doesn't work, fall back on using the node display name.

        """

        # if the title is already cached, return it
        if self.title_cache is not None:
            return self.title_cache

        # use the Title header, if we have it
        if self.has_header("title"):
            self.title_cache = self.get_header("title")
            return self.title_cache

        # otherwise fall back on the node name
        self.title_cache = self.get_display_name()
        return self.title_cache

    def get_escaped_title(self) -> str:
        """
        Get the escaped title of the node, as set by the Title header.

        If the Escape-Title: false header is set, return the Title
        without escaping.

        If that doesn't work, fall back on using the node display name.

        """

        # if the escaped title is already cached, return it
        if self.escaped_title_cache is not None:
            return self.escaped_title_cache

        # if the Escape-Title header is set, escape the title
        if self.get_boolean_header_value("escape-title", True):
            self.escaped_title_cache = escape(self.get_title())
        else:
            self.escaped_title_cache = self.get_title()

        # return the cached value
        return self.escaped_title_cache

    def get_link(self) -> str:
        """
        Get a link to this node, using its title as the link text.

        This method never returns canonical URL links.

        """

        # if the link is already cached, return it
        if self.link_cache is not None:
            return self.link_cache

        # cache the link
        self.link_cache = "<a href=\"" + self.get_url() + "\">" + \
               self.get_escaped_title() + "</a>"

        # return the cached link
        return self.link_cache

    def get_canonical_link(self) -> str:
        """
        Get a canonical link to this node, using its title as the link text.

        This method only returns canonical URL links. If the Canonical-URL
        header is not set, it will raise an error.

        """

        return "<a href=\"" + self.get_canonical_url() + "\">" + \
               self.get_escaped_title() + "</a>"

    def get_link_prefix(self) -> str:
        """
        Get the HTML prefix to use in generated lists of links.

        Returns the unescaped value of the Link-Prefix header,
        or a default of <p>

        """

        if self.has_header("link-prefix"):
            return self.get_header("link-prefix")

        return "<p>"

    def get_link_suffix(self) -> str:
        """
        Get the HTML suffix to use in generated lists of links.

        Returns the unescaped value of the Link-Suffix header,
        or a default of </p>

        """

        if self.has_header("link-suffix"):
            return self.get_header("link-suffix")

        return "</p>"

    def get_tags(self) -> List[str]:
        """
        Get the list of tags associated with this Node.

        Returns a list of tags.

        """

        # return from cache if possible
        if self.sorted_tags_cache is not None:
            return self.sorted_tags_cache

        tag_set = set()

        if self.has_header("tags"):
            tag_list = self.get_header("tags").split(",")
            for tag in tag_list:
                # strip whitespace from the tag
                tag = tag.strip()

                # empty tag
                if "" == tag:
                    errstr = "invalid Tags header value in node '%s': '%s'"
                    header_value = self.get_header("tags")
                    raise UrielError(errstr % (self.get_path(), header_value))

                # make sure the tag can be used as a directory name
                if "/" in tag:
                    errstr = "invalid tag '%s' in node '%s', " + \
                             "can not contain '/'"
                    raise UrielError(errstr % (tag, self.get_path()))

                # make sure the tag isn't a relative directory reference
                if ("." == tag) or (".." == tag):
                    errstr = "invalid tag '%s' in node '%s', " + \
                             "can not be '.' or '..'"
                    raise UrielError(errstr % (tag, self.get_path()))

                # make sure tags don't contain whitespace
                if re.match(r"^.*?\s+.*?$", tag):
                    errstr = "invalid tag '%s' in node '%s', " + \
                             "can not contain whitespace"
                    raise UrielError(errstr % (tag, self.get_path()))

                # make sure special URL characters aren't used in tags
                if ("#" in tag) or ("?" in tag) or ("%" in tag):
                    errstr = "invalid tag '%s' in node '%s', " + \
                             "can not contain '#', '?' or '%%'"
                    raise UrielError(errstr % (tag, self.get_path()))

                tag_set.add(tag)

        self.sorted_tags_cache = sorted(tag_set)

        return self.sorted_tags_cache

    def get_dest_dir(self) -> str:
        """
        Get the destination directory in the public website that will
        contain this page.

        """

        # get the URL path
        url = self.get_url()

        # root index node
        if "/" == url:
            return os.path.join(self.project_root, PUBLIC_ROOT)

        # strip leading and trailing slashes from URL
        target_url = url[1:-1]

        return os.path.join(self.project_root, PUBLIC_ROOT, target_url)

    def get_dest_file(self) -> str:
        """
        Get the destination file in the public website that will
        contain this page.

        """

        return os.path.join(self.get_dest_dir(), HTML_INDEX)

    def get_boolean_header_value(self, header: str, default: bool) -> bool:
        """
        Get the boolean value for a header as True or False.

        For example, if the value of the header is "true", then this method
        will return True.

        If the header is not set, return the provided default value.

        Raises an UrielError if any other value is found.

        """

        # canonicalize the header name to lowercase
        #
        # performance optimization: look it up directly first in lowercase,
        # and only convert to lowercase if we don't find a match
        lc_header = header
        value = self.headers.get(lc_header)
        if value is None:
            lc_header = header.lower()
            value = self.headers.get(lc_header)

        # if we found a value for this header, interpret it
        if value is not None:
            if "true" == value:
                return True
            elif "false" == value:
                return False
            else:
                err = "invalid value for '%s' " + \
                      "header in node '%s': '%s'"
                raise UrielError(err % (header, self.get_path(), value))

        # if we didn't have a value for this header,
        # return the requested default value instead
        return default

    def get_format(self) -> str:
        """
        Get the node body format for this node.

        Returns "html" or "text".

        If the Format header is not set, then "html" is used as the default
        value.

        Raises a UrielError if the Format header is set to an invalid value.

        """

        # if we don't have a format header, return the default value
        if not self.has_header("format"):
            return "html"

        # get the value of the format header
        node_format = self.get_header("format")

        # if the value is invalid, raise an error
        if ("html" != node_format) and ("text" != node_format):
            err = "invalid Format header value in node '%s': '%s'"
            raise UrielError(err % (self.get_path(), node_format))

        # return the format value
        return node_format

    def get_breadcrumb_separator(self) -> str:
        """
        Get the unescaped breadcrumb separator for this node, possibly
        surrounded by spaces.

        If the Breadcrumb-Separator header is not set, then "&raquo;"
        is used as the default value.

        If the Breadcrumb-Separator-Spaces header is not explicitly set to
        "false", then the return value will contain a space before and after
        the breadcrumb separator.

        The default return value, if neither of these headers are set, is
        " &raquo; "

        """

        separator = "&raquo;"
        maybe_space = " "

        # look at headers to see if we should change the separator
        if self.has_header("breadcrumb-separator"):
            separator = self.get_header("breadcrumb-separator")

        # should we surround the separator with spaces?
        if not self.get_boolean_header_value("breadcrumb-separator-spaces",
                                             True):
            maybe_space = ""

        return maybe_space + separator + maybe_space

    def find_node_by_path(self,
                          path:str,
                          raise_exceptions: bool = True) -> Optional["Node"]:

        """
        Find the Node that matches the given node path.

        Returns the given node.

        Raises a UrielError if the node can not be found.

        If the optional raise_exceptions parameter is set to False,
        then instead of raising an exception, it will return None instead.

        """

        # if this node matches, return it
        if self.get_path() == path:
            return self

        # return the path from the cache
        # this should always be accurate, since the cache is built
        # as child nodes are added to the tree
        if self.root_node_cache:
            if path in self.root_node_cache.node_path_cache:
                return self.root_node_cache.node_path_cache[path]

        if raise_exceptions:
            raise UrielError("node not found: '%s'" % (path))

        return None

    def get_root_node(self) -> "Node":
        """
        Get the root Node.

        """

        return self.root_node_cache

    def get_tag_node(self) -> Optional["Node"]:
        """
        Get the tag node, by looking at the value of the Tag-Node header,
        and then returning the matching node.

        There is only one tag node, and the Tag-Node header should only be
        set on the root node.

        Returns the tag node, or None if the tag node is not configured.

        Raises a UrielError if the tag node is configured, but can not be
        found.

        """

        # return the tag node from the cache, if possible
        if self.tag_node_cache is not None:
            return self.tag_node_cache

        # find and return the tag node
        if self.has_header("tag-node"):
            # get the root node
            root = self.get_root_node()

            # find the tag node and cache it
            self.tag_node_cache = \
                root.find_node_by_path(self.get_header("tag-node"))

            # return the tag node
            return self.tag_node_cache

        return None

    def set_tag_node_index(self,
            tag_node_index: Optional[Dict[str, Set["Node"]]]) -> None:

        """
        Set (or replace) the tag node index on the root Node.

        """

        # find the root node
        root = self.get_root_node()

        # overwrite the tag node index on the root node
        root.tag_node_index = tag_node_index

    def get_tag_node_index(self) -> Dict[str, Set["Node"]]:
        """
        Get the tag node index.

        The tag node index is a dict of tags, with sets of Node as values.

        Returns the tag node index, or raises a UrielError if the tag node
        index has not already been created (via create_tag_node_index()).

        """

        # whatever node this is, pull the tag index out of the root node
        tag_node_index = self.get_root_node().tag_node_index

        if tag_node_index is None:
            raise UrielError("tag node index not set")

        return tag_node_index

    def create_tag_node_index(self,
            node: Optional["Node"] = None,
            tag_node_index: Optional[Dict[str, Set["Node"]]] = None) \
                -> Dict[str, Set["Node"]]:

        """
        Create tag / node index.

        The tag node index is a dict of tag -> set of node

        Optionally accepts a node, and a tag node index.

        Returns a tag to node index.

        """

        is_top_level_method_call = False

        # if we didn't have a Node passed in, start at the root
        if node is None:
            node = self.get_root_node()
            is_top_level_method_call = True

        # if we don't have a tag index, create it
        if tag_node_index is None:
            # tag -> set of Node matching that tag
            tag_node_index = {}

        # go through the tags for this node
        for tag in node.get_tags():
            # if this is a new tag we haven't seen before,
            # create a set for it and add it to the index
            if tag not in tag_node_index:
                tag_node_index[tag] = set()

            # add the node to the index for this tag
            tag_node_index[tag].add(node)

        # recurse
        for child in node.get_children():
            self.create_tag_node_index(child, tag_node_index)

        # if this is the top-level method call,
        # set the tag node index on the root node
        if is_top_level_method_call:
            self.set_tag_node_index(tag_node_index)

        return tag_node_index

    def get_vnode_for_tag(self, tag: str) -> Optional["Node"]:
        """
        Get the virtual node that represents the given tag
        (e.g. the node for /tag/$SOME_TAG/)

        """

        # performance optimization:
        # check the global tag vnode cache first,
        # and just return the node from there if possible
        if self.root_node_cache.tag_vnode_cache is not None:
            if tag in self.root_node_cache.tag_vnode_cache:
                return self.root_node_cache.tag_vnode_cache[tag]

        # search for the vnode by tag
        tag_root = self.get_tag_node()
        if tag_root is not None:
            for vnode in tag_root.get_children():
                if vnode.get_name() == tag:
                    # performance optimization:
                    # add the vnode to the tag vnode cache
                    if self.root_node_cache.tag_vnode_cache is not None:
                        self.root_node_cache.tag_vnode_cache[tag] = vnode

                    return vnode

        return None

    def get_body(self) -> Optional[str]:
        """
        Get the unrendered Node body as a multi-line string.

        """

        return self.body

    def get_rendered_body(self) -> Optional[str]:
        """
        Get the rendered Node body as a multi-line string.

        """

        return self.rendered_body

    def set_body(self, body: Optional[str]) -> None:
        """
        Set the unrendered Node body. Accepts a multi-line string.

        """

        self.body = body

    def set_rendered_body(self, rendered_body: Optional[str]) -> None:
        """
        Replace the contents of the Node body (e.g. with a rendered version).

        """

        self.rendered_body = rendered_body

    def invalidate_cache_by_header(self, header: str) -> None:
        """
        Invalidate the cache for certain performance optimizations.

        """

        # canonicalize to lowercase
        header = header.lower()

        # headers affecting title-related fields
        if "title" == header:
            self.title_cache = None
            self.escaped_title_cache = None
            self.link_cache = None
            return
        if "escape-title" == header:
            self.title_cache = None
            self.escaped_title_cache = None
            self.link_cache = None
            return

        # headers affecting canonical URL
        if "canonical-url" == header:
            self.canonical_url_base_cache = None
            return

        # headers affecting sorted_tags_cache field
        if "tags" == header:
            self.sorted_tags_cache = None
            return

        # headers affecting tag_node_cache and tag_vnode_cache fields
        if "tag-node" == header:
            self.tag_node_cache = None

            # N.B. the tag vnode cache only lives on the root node
            if self.root_node_cache.tag_vnode_cache is not None:
                self.root_node_cache.tag_vnode_cache = {}

            return

        # headers affecting url_cache field
        if "flat-url" == header:
            self.invalidate_url_cache()
            return

    def invalidate_url_cache(self) -> None:
        """
        Invalidate the cached URL of this Node, and of every Node
        underneath it in the tree.

        """

        # invalidate the URL cache for this node
        self.url_cache = None

        # invalidate cached link (which contains the URL)
        self.link_cache = None

        # invalidate the URL cache for our child nodes recursively
        #
        # child nodes of this node have URLs that are based on this node,
        # so we have to invalidate all child nodes of this node to be safe.
        #
        # we iterate through the child nodes directly here, because we are
        # invalidating cache elements, and get_children() sorts the child
        # nodes (and caches the results). this would be fine, except that
        # sorting the nodes calls Node __lt__(), which calls other methods,
        # that repopulate the cache we're trying to clear before we're ready
        for child in self.children:
            child.invalidate_url_cache()

    def has_header(self, header: str) -> bool:
        """
        Does the given header exist on this node?

        Inside the program, headers should all be lowercase.

        """

        # check to see if the header is set first, without lowercasing it
        if header in self.headers:
            return True

        # canonicalize to lowercase, and return a boolean value
        # indicating whether the header is set on this node
        return header.lower() in self.headers

    def get_header(self, header: str) -> str:
        """
        Get the value associated with the given header.

        Inside the program, headers should all be lowercase.

        Returns the value associated with this header, or raises a KeyError
        if the header is not set.

        """

        # check to see if we have the header first, without lowercasing it
        try:
            return self.headers[header]
        except KeyError:
            pass

        # canonicalize to lowercase, and return the value of the header,
        # or raise a KeyError if the header is not found.
        return self.headers[header.lower()]

    def set_header(self, header: str, value: str) -> None:
        """
        Set the given header to the specified value.

        Inside the program, headers should all be lowercase.

        """

        # canonicalize to lowercase
        header = header.lower()

        self.invalidate_cache_by_header(header)

        self.headers[header] = value

    def delete_header(self, header: str) -> None:
        """
        Delete the given header.

        Inside the program, headers should all be lowercase.

        """

        # canonicalize to lowercase
        header = header.lower()

        self.invalidate_cache_by_header(header)

        del(self.headers[header])

    def get_header_keys(self) -> List[str]:
        """
        Get each of the headers for this node.

        """

        return sorted(self.headers.keys())

    def get_header_key_values(self) -> List[Tuple[str, str]]:
        """
        Get each key/value element of the headers and their values,
        as a list of two-element tuples.

        """

        return sorted(self.headers.items())

    def get_datetime_from_date_str(self, date_str: str) -> datetime.datetime:
        """
        Get a datetime.datetime instance from the provided date string.

        The date string must be in the ISO 8601 date format.

        Raises a UrielError if the date string conversion fails.

        """

        try:
            # turn the date string into a datetime instance
            dt = datetime.datetime.fromisoformat(date_str)

            # if the datetime instance doesn't have a time zone, assume
            # that it is in the local time zone
            #
            # N.B. astimezone() attaches the UTC offset that was in effect
            #      locally at that date and time, rather than the offset
            #      that happens to be in effect right now. this keeps the
            #      date/time that was written in the header intact when the
            #      site is built at another time of year, on the other side
            #      of a daylight saving time change.
            if dt.tzinfo is None:
                dt = dt.astimezone()

            # return the datetime with time zone
            return dt

        except Exception as e:
            err = "invalid date string conversion in node " + \
                  "'%s': '%s'"
            raise UrielError(err % (self.get_path(), date_str))

    def get_sort_key(self) -> Tuple[float, str, str]:
        """
        Get a Node sort key.

        Node instances are sorted by (in order):
            date descending (created if available, otherwise modified)
            title ascending
            URL ascending

        N.B. The exact format and type returned by this method is
             subject to change in the future. The intended use of this
             method is to use it to compare the relative sort ordering
             of two different nodes, e.g.:

                if node1.get_sort_key() < node2.get_sort_key():
                    # do something

             Since Node __lt__() references this method directly, it
             is usually a better choice to just sort Node, FileNode,
             and VirtualObjects together, and let __lt__() handle it.

        """

        # self and other candidate dates
        # created isn't always set, but modified is always available
        date = self.modified
        if self.created:
            date = self.created

        # every Node should always at least have a modified time set
        if date is None:
            raise Exception("node '%s' does not have a modified date" % \
                            (self.get_path()))

        # sort by (in order):
        #
        # date (descending, newest to oldest)
        # title (ascending)
        # URL (ascending, guaranteed to be unique)
        return (-date.timestamp(), self.get_title(), self.get_url())

    def __lt__(self, other):
        """
        Sort Node instances, according to get_sort_key().

        """

        # Node classes and subclasses can only be compared to each other
        if not isinstance(other, Node):
            raise TypeError(
                "'<' not supported between instances of '%s' and '%s'" %
                (type(self).__name__, type(other).__name__))

        # ordering is defined by Node sort keys
        return self.get_sort_key() < other.get_sort_key()

    def __str__(self) -> str:
        return self.get_path()


class FileNode(Node):
    """
    Represents a dynamic page backed by a node file.

    Contains headers and a body (format similar to HTTP, email, etc.)

    Headers are inherited from the parent node.

    Nodes have a parent/child tree relationship.

    """

    def __init__(self,
                 project_root: str,
                 path: str,
                 parent_node: Optional[Node] = None) -> None:

        """
        Accepts the project root directory.

        Accepts the path to the node file (as a sub path of NODES_ROOT).

        Optionally accepts a parent Node instance.

        """

        # call the superconstructor
        super().__init__(project_root, path, parent_node=parent_node)

        # node file path
        node_path = os.path.join(self.nodes_root, self.get_path())

        try:
            # set the default date based on the node modified time
            node_modified = os.path.getmtime(node_path)
            self.modified = \
                datetime.datetime.fromtimestamp(
                    node_modified,
                    datetime.timezone.utc).astimezone()

            # read dynamic node file, and parse out headers and body
            body_lines = []

            with open(os.path.join(self.nodes_root, path),
                      encoding="UTF-8") as f:

                parsing_headers = True

                # read each line in the node file
                for line in f:
                    line = line.rstrip()

                    # if we're still parsing headers...
                    if parsing_headers:
                        # if we find a line without a ":", stop parsing headers
                        # this lets us have a body-only template
                        if ":" not in line:
                            parsing_headers = False

                        # if we hit a blank line, it could either be the blank
                        # line separating the headers from the body, or it
                        # could be a node without any headers at all
                        if "" == line:
                            # either way, we're not parsing headers anymore
                            parsing_headers = False

                            # if we already had any headers before this,
                            # skip the blank separating line
                            # (if not, we'll continue on and treat it as the
                            # first line of the page body)
                            if len(self.headers) > 0:
                                continue

                    # if we're parsing headers...
                    if parsing_headers:
                        # parse headers
                        (key, value) = line.split(":", maxsplit=1)

                        # canonicalize header key to lowercase
                        key = key.strip().lower()

                        # strip leading/trailing whitespace from value
                        value = value.strip()

                        # set the header
                        self.set_header(key, value)

                    # if we're parsing the body...
                    else:
                        body_lines.append(line)

        except Exception as e:
            err = "could not read node file '%s': '%s'"
            raise UrielError(err % (node_path, str(e)))

        # process "-" headers that were set in this node file
        self._process_minus_headers()

        #
        # process other special headers
        #

        # get our own local reference to the headers data structure for
        # convenience, since we refer to it extensively in the stanzas below
        headers = self.headers

        # Canonical-URL
        if "canonical-url" in headers:
            # called for fail-fast validation side effects
            self.get_canonical_url_base()

        # Created
        if "created" in headers:
            value = headers["created"]
            try:
                self.created = self.get_datetime_from_date_str(value)
            except UrielError as e:
                err = "invalid Created date header value in node " + \
                      "'%s': '%s'"
                raise UrielError(err % (self.get_path(), value))

        # Format
        if "format" in headers:
            # called for fail-fast validation side effects
            self.get_format()

        # Modified
        if "modified" in headers:
            value = headers["modified"]
            try:
                self.modified = self.get_datetime_from_date_str(value)
            except UrielError as e:
                err = "invalid Modified date header value in node " + \
                      "'%s': '%s'"
                raise UrielError(err % (self.get_path(), value))

        # remember the body as a string
        self.body = "\n".join(body_lines)

    def get_node_type(self) -> str:
        return "file"


class VirtualNode(Node):
    """
    Represents a virtual node, not backed by a file.

    To use this class:
      - call set_body() to set the node body (which will be merged with
        template later)
      - set headers (although it inherits headers from the parent node)
      - add children (if necessary)

    """

    def __init__(self,
                 project_root: str,
                 path: str,
                 parent_node: Optional[Node] = None) -> None:

        """
        Accepts the project root directory.

        Accepts the path (as a subdirectory of the node root).

        Optionally accepts a parent Node instance.

        """

        # call the superconstructor
        super().__init__(project_root, path, parent_node=parent_node)

        # set the date, either to the parent node value, or to now
        parent_node = self.get_parent_node()
        if parent_node:
            self.created = parent_node.created
            self.modified = parent_node.modified
        else:
            # use the current date/time/timezone
            now = datetime.datetime.now(datetime.timezone.utc).astimezone()

            self.created = now
            self.modified = now

    def get_node_type(self) -> str:
        return "virtual"


class Page:
    """
    A Page is a combination of a Node and a template.

    """

    def __init__(self,
                 project_root: str,
                 node: Node,
                 use_canonical_url: bool = False) -> None:

        """
        Accepts the project root, the template, and a Node.

        Optionally accepts use_canonical_url, a boolean that influences
        whether substitution parameter URLs are rewritten to use the
        Canonical-URL. The default is False.

        """

        self.project_root = project_root
        self.node: Node = node

        # it is possible for line_error() to be called more than once
        # during the exception handling process. however, we only
        # want to print the first error message
        self.line_error_reason: Optional[str] = None

        # use the template from the node header, or default if not set
        if node.has_header("template"):
            self.template = node.get_header("template")
        else:
            self.template = DEFAULT_TEMPLATE

        # templates can include other templates
        # keep track of which one we're using right now, in case we
        # have to show an error message
        self.template_stack = TemplateStack()

        # make sure we don't get into a {{node:body}} inclusion loop
        self.node_body_semaphore = 0

        # the {{soju:*}} parameters for user-defined code currently executing
        self.soju_tokens: List[Token] = []

        # remember whether we want to include the canonical URL in links
        self.use_canonical_url = use_canonical_url

    def line_error(self,
                   token: Token,
                   reason: str,
                   raise_exception: bool = True) -> None:

        """
        Log a parameter error stack trace.

        Accepts a Token, the reason for the error, and whether or not
        this method should raise an exception (by default it does).

        We only show an error message for the first error that was
        encountered, which may or may not be the error that was triggered by
        this method call. It's possible that something else called this
        method earlier, and we haven't had a chance to report that error yet.
        If an earlier error got here first, this method will raise a
        UrielError for the first error that we encountered.

        """

        # if an error has already been reported for this page, then that
        # is the error the build fails on. don't print a second error, and
        # don't let rendering carry on past the first one, even if
        # user-defined code caught the earlier exception.
        if self.line_error_reason is not None:
            raise UrielError(self.line_error_reason)

        # if this is the first error, remember the error reason, so that
        # we can report it later if necessary
        self.line_error_reason = reason

        # print out debugging information to highlight the precise
        # location where the user can troubleshoot the error

        # indent errors as we work our way down
        # from the node, through the templates, to wherever the problem is
        indent = 1

        # are we using a template? (or is the template "null"?)
        has_template = self.template_stack.has_more_elements()

        log("parameter error:")

        # node
        log("%s%s/%s" %
            (indent_spaces(indent),
             NODES_ROOT,
             self.node.get_path()))

        # templates
        if has_template:
            indent += 1
            for template in self.template_stack.get_templates():
                log("%s%s/%s" %
                    (indent_spaces(indent),
                     TEMPLATES_ROOT,
                     template))

        # {{node:body}}
        #
        # if we ran into an error while we were processing the contents
        # of the node itself (e.g. {{node:body}}), list the node again,
        # since it is the file the user should go look at to find the
        # error. however, we only want to print the node twice if we
        # are using a template, otherwise it's just redundant.
        if has_template and (self.node_body_semaphore > 0):
            indent += 1
            log("%s%s/%s" %
                (indent_spaces(indent),
                 NODES_ROOT,
                 self.node.get_path()))

        # user-defined soju code (pathological cases only)
        #
        # most projects will never hit this stanza at all
        #
        # if we got here, it was from a complex web of ill-advised calls from
        # soju to other parts of the uriel API and back, and we might need to
        # show the best form of a soju call stack that we can piece together.
        # this does not cover simple cases, like soju function foo() raising
        # an exception, or soju function a() calling soju function b(). it
        # only really comes into play in more unusual circumstances, like soju
        # manipulating elements of the uriel API directly (especially the Page
        # class), running into errors, ignoring the exceptions, and going on
        # to do other things like directly include templates using Page
        # merge_line(), etc.
        #
        # really, if you hit this, you might have been asking for trouble
        #
        # this is added here in the hopes that it will be helpful to navigate
        # really bizarre situations.
        #
        # for your normal, garden variety soju function errors, the regular
        # token stanza below will deal with those cases, and this won't log
        # anything at all for the vast majority of soju function errors.
        if len(self.soju_tokens) > 0:
            indent += 1
            for soju_token in self.soju_tokens:
                log("%s'%s'" %
                    (indent_spaces(indent), soju_token.original_string))

        # token
        # (including 99.999% of soju errors)
        indent += 1
        log("%s'%s'" % (indent_spaces(indent), token.original_string))

        # if we are not raising an exception, log the reason for the error
        if not raise_exception:
            indent += 1
            log("%s%s" % (indent_spaces(indent), reason))

        # if we were asked to raise an exception, raise it now
        if raise_exception:
            raise UrielError(reason)

    def node_body_loop_error(self, token: Token) -> None:
        """
        Log a {{node:body}} inclusion loop error stack trace.

        Accepts a Token.

        """

        # we don't have all of the information we need here to highlight
        # exactly where the loop started and stopped, but we can at least
        # print out the node, templates, and token that were involved.
        #
        # to highlight exactly where the loop started and stopped, we
        # would need more information about what included what, and where,
        # and not just a node and a template stack.
        #
        # includes can come from templates or the node body.
        # a typical template stack will include the node body somewhere in
        # one of the templates.
        #
        # it's only a loop if the node itself eventually includes the node
        # body again, but without adding a lot more tracking information
        # just for error reporting on this one case, this is the best we can
        # reasonably do with the information available here.

        log("node body include loop error:")

        # indent errors as we work our way down
        # from the node, through the templates, to wherever the problem is
        indent = 1

        # keep track of whether we have any templates at all
        has_templates = self.template_stack.has_more_elements()

        # node path
        log("%s%s/%s" %
            (indent_spaces(indent), NODES_ROOT, self.node.get_path()))

        # templates
        if has_templates:
            indent += 1
            while self.template_stack.has_more_elements():
                template = self.template_stack.shift()
                log("%s%s/%s" %
                    (indent_spaces(indent), TEMPLATES_ROOT, template))

        # node path again (if we had any templates, otherwise it's redundant)
        if has_templates:
            indent += 1
            log("%s%s/%s" %
                (indent_spaces(indent), NODES_ROOT, self.node.get_path()))

        # {{node:body}}
        indent += 1
        log("%s'%s'" % (indent_spaces(indent), token.original_string))

        raise UrielError(
            "node body include loop detected in node: '%s'" %
            (self.node.get_path()))

    def tokenize(self, line: str) -> List[Token]:
        """
        Accepts a single line of text as input.

        Turns the input into one or more Token instances.

        Returns a list of Token.

        """

        # Token instances
        tokens = []

        # split the tokens
        index = 0
        while True:
            # N.B. find() is given a starting index, rather than being
            #      called on a slice of the line, so that a copy of the
            #      rest of the line is not made on every pass
            open_index = line.find("{{", index)

            # there's a parameter coming up, but we're not there yet
            if open_index > index:
                # add this section, up to right before the next parameter
                tokens.append(Token(line[index:open_index]))
                index = open_index

            # we're at the parameter now
            elif open_index == index:
                close_index = line.find("}}", index)

                # we have an opening {{, but no }}
                if -1 == close_index:
                    # add the rest of the line as a token
                    tokens.append(Token(line[index:]))
                    break

                # scoop up the "{{ ... }}" part and make it into a token
                next_index = close_index + 2
                tokens.append(Token(line[index:next_index]))
                index = next_index

            # no more parameters
            else:
                # if we're already at the end of the line,
                # don't create an empty token
                if index == len(line):
                    break

                # add the rest of the line as a token
                tokens.append(Token(line[index:]))
                break

        return tokens

    def text_to_html(self, text: str) -> str:
        """
        Turn a multi-line text string into a multi-line string with
        <br> tags at the end of each line.

        """

        lines = text.split("\n")

        return "<br>\n".join(lines)

    def get_node_by_path(self, token: Token, path: str) -> Node:
        """
        Get the Node that corresponds with the given path.

        Accepts a token, and a path (e.g. index, foo/index)

        Returns the requested Node.

        Raises a UrielError if the node can not be found.

        """

        # get the root node to start our search
        root = self.node.get_root_node()

        # find the node that matches the path, and return it
        node = root.find_node_by_path(path, raise_exceptions=False)
        if node is None:
            self.line_error(token, "node not found: '%s'" % (path))
            raise _ImpossibleError()

        return node

    def create_breadcrumbs(self, include_root: bool = False) -> str:
        """
        Create a breadcrumbs HTML fragment for the Node associated with
        this Page.

        Accepts a boolean parameter indicating whether the root node should be
        included in the breadcrumbs.

        Return the HTML fragment as a string.

        """

        # list of nodes, starting from the leaf node and
        # going back up to the root
        nodes = []

        # list of breadcrumb HTML fragments, in the order we want
        breadcrumbs = []

        # build a list of the nodes from the leaf up to the root
        cur = self.node
        while True:
            nodes.append(cur)

            if cur.parent_node is None:
                break

            cur = cur.parent_node

        # if we don't want to include the root node in the breadcrumbs
        if not include_root:
            # remove the last element from the node list (the root)
            nodes = nodes[:-1]

        # go through the nodes from (almost) the root to the bottom,
        # in the order that we want the breadcrumb HTML to appear
        first_breadcrumb = True
        for cur_node in reversed(nodes):
            if not first_breadcrumb:
                breadcrumbs.append(self.node.get_breadcrumb_separator())

            breadcrumbs.append(self.get_node_link(cur_node))

            first_breadcrumb = False

        return "".join(breadcrumbs)

    def get_soju_result(self, code: str) -> str:
        """
        Run the given code in the soju module, and return the result.

        """

        # make sure the soju module has been imported
        if "soju" not in globals():
            soju_file = os.path.join(self.project_root, LIB_ROOT, "soju.py")
            raise UrielError("soju module not found, '%s' does not exist" %
                             (soju_file))

        # set up the local environment so the code fragment passed in can
        # refer to these variables without qualifying them
        page = self
        node = self.node
        project_root = self.project_root
        use_canonical_url = self.use_canonical_url

        # don't export the "self" reference from this method into eval code
        del(self)

        # deferred import
        import ast

        # check whether the syntax of the code inside of the soju token value
        # is syntactically valid python. this does not execute the code yet,
        # and the symbols don't necessarily have to exist. it just checks
        # for things like mismatched parenthesis, and that sort of thing,
        # so we can report them without a full stack trace.
        #
        # by explicitly checking this before we run the code, and not
        # just catching SyntaxError when the code is executed, we still
        # allow SyntaxError exceptions within the user-defined code to bubble
        # up, in case they eval code of their own in a function, etc.
        try:
            ast.parse(code)
        except SyntaxError as e:
            raise SojuError("syntax error: " + e.msg)

        # execute the user-defined code, and save the result
        result = eval("soju." + code)

        # if the user-defined code didn't return a string, report the error
        if type(result) is not str:
            err = "%s returned %s instead of string"
            raise SojuError(err % (code, str(type(result))))

        return result

    def get_node_url(self, node: Node) -> str:
        """
        Get the URL for the node.

        """

        # canonical
        if self.use_canonical_url:
            return node.get_canonical_url()

        # non-canonical
        return node.get_url()

    def get_node_link(self, node: Node) -> str:
        """
        Get a link to the node, using the node title as the link text.

        """

        # canonical
        if self.use_canonical_url:
            return node.get_canonical_link()

        # non-canonical
        return node.get_link()

    def get_static_url(self, node: Node, url: str) -> str:
        """
        Get the static URL, in relation to the node.

        """

        # canonical
        if self.use_canonical_url:
            if url.startswith("/"):
                root_node = self.node.get_root_node()
                url = url[1:]
                return root_node.get_canonical_url() + url

            return node.get_canonical_url() + url

        # non-canonical
        return url

    def get_maybe_canonical_html_fragment(self,
                                          token: Token,
                                          node: Node,
                                          header_basename: str) -> str:

        """
        Get the canonical or non-canonical HTML fragment associated with the
        given header basename.

        Accepts a Token, a Node, and a header basename (e.g. __node-list-html)

        Returns the value from the given header, or the version of the
        header that has canonical URLs.

        Raises a UrielError if the fragment was never created for this
        node.

        """

        # canonical
        if self.use_canonical_url:
            header = header_basename + "-canonical"

        # non-canonical
        else:
            header = header_basename

        # in normal operation, the headers being returned from this method
        # (with double underscores) are created earlier, when the node
        # tree is augmented. if for some reason the program is requesting
        # an augmented header that is not available (possibly due to
        # user-defined handler interference), show a suitable error message.
        if not node.has_header(header):
            if self.use_canonical_url:
                self.line_error(
                    token,
                    "Canonical-URL header not set on node 'index'" +
                    " when node tree was created")
            else:
                errstr = "node '%s' was not part of the node tree " + \
                         "when it was created"
                self.line_error(token, errstr % (node.get_path()))

            raise _ImpossibleError()

        return node.get_header(header)

    def merge_token_literal(self, token: Token) -> str:
        """
        Merge a literal token and return the results.

        """

        # a token literal is sort of the last resort, as far as parsing goes.
        # anything that doesn't match some other known pattern ends up here.

        # was this actually supposed to be a parameter,
        # but it's an invalid parameter?
        if token.has_unidentified_parameter():
            self.line_error(token,
                            "unidentified parameter: '%s'" % (token.value))

        # was this a half-baked attempt at starting a parameter,
        # but it never got closed out? (e.g. "{{foo:bar" or "{{foo:bar}")
        if token.has_unterminated_parameter():
            self.line_error(token,
                            "parameter is missing a closing '}}': '%s'" %
                            (token.value))

        # return the literal string value from this token
        return token.value

    def merge_token_value(self, token: Token) -> str:
        """
        Merge a {{value:*}} token and return the results.

        """

        # defer to merge_token_value_unescaped() to get the unescaped value
        unescaped_value = self.merge_token_value_unescaped(token)

        # return the escaped results
        return escape(unescaped_value)

    def merge_token_value_unescaped(self, token: Token) -> str:
        """
        Merge a {{value-unescaped:*}} token and return the results.

        """

        try:
            return self.node.get_header(token.value)
        except KeyError:
            self.line_error(token,
                            "header '%s' not set on this node" %
                            (token.value))
            raise _ImpossibleError()

    def merge_token_include(self, token: Token) -> str:
        """
        Merge an {{include:*}} token and return the results.

        """

        if "null" == token.value:
            self.line_error(
                token,
                "can not include 'null' template: reserved word")

        # prevent directory traversal attempts
        self.validate_template_name(token, token.value)

        # get the path to the template file
        template_file = template_file_paths.get(token.value)
        if template_file is None:
            template_file = os.path.join(self.project_root,
                                         TEMPLATES_ROOT,
                                         token.value)
            template_file_paths[token.value] = template_file

        # make sure the template exists
        #
        # N.B. we have to check the cache, and the filesystem.
        #      in that order. if it's in the cache, we don't need to check
        #      the filesystem again.
        if (template_file not in template_tokens) and \
                (not os.path.exists(template_file)):
            self.line_error(
                token,
                "include file not found: '%s'" %
                (os.path.join(TEMPLATES_ROOT, token.value)))

        # return the merged template contents
        return self.merge_template(token.value)

    def merge_token_created(self, token: Token) -> str:
        """
        Merge a {{created:*}} token and return the results.

        """

        if self.node.created is None:
            self.line_error(token, "'Created' header not set")
            raise _ImpossibleError()

        try:
            formatted_date = self.node.created.strftime(token.value)
            return escape(formatted_date)
        except ValueError:
            self.line_error(token,
                            "invalid date format string: '%s'" %
                            (token.value))
            raise _ImpossibleError()

    def merge_token_modified(self, token: Token) -> str:
        """
        Merge a {{modified:*}} token and return the results.

        """

        if self.node.modified is None:
            self.line_error(token, "'Modified' header not set")
            raise _ImpossibleError()

        try:
            formatted_date = self.node.modified.strftime(token.value)
            return escape(formatted_date)
        except ValueError:
            self.line_error(token,
                            "invalid date format string: '%s'" %
                            (token.value))
            raise _ImpossibleError()

    # helper method for oldest/latest
    # recursively traverse this and all child nodes,
    # find all the created/modified dates,
    # and stick them in the dates set
    def __find_all_dates_in_child_nodes(self,
                                        node: Node,
                                        dates: Set[datetime.datetime]):

        # add created/modified dates for this node
        if node.created is not None:
            dates.add(node.created)
        if node.modified is not None:
            dates.add(node.modified)

        # recurse
        for child in node.get_children():
            self.__find_all_dates_in_child_nodes(child, dates)

    def merge_token_oldest(self, token: Token) -> str:
        """
        Merge a {{oldest:*}} token and return the results.

        """

        # call the helper method
        # to find all of the node created/modified dates
        dates: Set[datetime.datetime] = set()
        self.__find_all_dates_in_child_nodes(self.node, dates)

        # pick the oldest date out of the set
        # we will only handle the first (oldest) entry in this set,
        # which represents the oldest date from this and all child nodes
        for latest in sorted(dates):
            # format the latest date with the strftime string from the token
            try:
                formatted_date = latest.strftime(token.value)
                return escape(formatted_date)
            except ValueError:
                self.line_error(token,
                                "invalid date format string: '%s'" %
                                (token.value))
                raise _ImpossibleError()

        # we couldn't find even a single date in any of the nodes
        self.line_error(token, "'Created' and 'Modified' headers not set")
        raise _ImpossibleError()

    def merge_token_latest(self, token: Token) -> str:
        """
        Merge a {{latest:*}} token and return the results.

        """

        # call the helper method
        # to find all of the node created/modified dates
        dates: Set[datetime.datetime] = set()
        self.__find_all_dates_in_child_nodes(self.node, dates)

        # pick the latest date out of the set
        # we will only handle the first (most recent) entry in this set,
        # which represents the latest date from this and all child nodes
        for latest in reversed(sorted(dates)):
            # format the latest date with the strftime string from the token
            try:
                formatted_date = latest.strftime(token.value)
                return escape(formatted_date)
            except ValueError:
                self.line_error(token,
                                "invalid date format string: '%s'" %
                                (token.value))
                raise _ImpossibleError()

        # we couldn't find even a single date in any of the nodes
        self.line_error(token, "'Created' and 'Modified' headers not set")
        raise _ImpossibleError()

    def merge_token_breadcrumbs(self, token: Token) -> str:
        """
        Merge a {{breadcrumbs:*}} token and return the results.

        """

        if "*" == token.value:
            return self.create_breadcrumbs(False)

        self.line_error(
            token,
            "invalid value for breadcrumbs parameter: '%s'" %
            (token.value))
        raise _ImpossibleError()

    def merge_token_breadcrumbs_root(self, token: Token) -> str:
        """
        Merge a {{breadcrumbs-root:*}} token and return the results.

        """

        if "*" == token.value:
            return self.create_breadcrumbs(True)

        self.line_error(
            token,
            "invalid value for breadcrumbs parameter: '%s'" %
            (token.value))
        raise _ImpossibleError()

    def get_static_url_abspath(self, token: Token, url: str) -> str:
        """
        Get the absolute path on disk to the given static URL path.

        """

        # create a cache key to look up the cache entry for this path
        if url.startswith("/"):
            cache_key = ("", url)
        else:
            cache_key = (self.node.get_dest_dir(), url)

        # if this path has already been cached, return it
        cached_abspath = static_url_abspaths.get(cache_key)
        if cached_abspath is not None:
            return cached_abspath

        # prevent directory traversal links for static URLs
        if is_directory_traversal_attempt(url):
            self.line_error(
                token,
                "directory traversal not allowed in static URL: '%s'" %
                (url))

        # if the static url path starts with a slash, and is
        # relative to the root of the site, start the directory path
        # from PUBLIC_ROOT
        if url.startswith("/"):
            root_node = self.node.get_root_node()
            public_root_dir = root_node.get_dest_dir()
            static_url_dirent = public_root_dir + "/" + url

        # if the static url path does not start with a slash,
        # start the directory path relative to the node that
        # included the static url reference
        else:
            static_url_dirent = self.node.get_dest_dir() + "/" + url

        # figure out the absolute filesystem path to the file
        abspath = os.path.abspath(static_url_dirent)

        # cache the results
        static_url_abspaths[cache_key] = abspath

        # return the absolute path
        return abspath

    def merge_token_static_url(self, token: Token) -> str:
        """
        Merge a {{static-url:*}} token and return the results.

        """

        url = token.value

        # get the path on disk to the URL path
        static_url_abspath = self.get_static_url_abspath(token, url)

        # make sure the file or directory exists on the filesystem
        if not os.path.exists(static_url_abspath):
            self.line_error(token,
                            "file not found: '%s'" %
                            (os.path.join(STATIC_ROOT, url)))

        return self.get_static_url(self.node, url)

    def merge_token_static_hash_url(self, token: Token) -> str:
        """
        Merge a {{static-hash-url:*}} token and return the results.

        """

        url = token.value

        # get the full filesystem path to the file referenced by the URL
        static_url_abspath = self.get_static_url_abspath(token, url)

        # if the original file has already been hashed,
        # pull the hashed filename out of the cache
        if static_url_abspath in static_hash_filenames:
            hash_filename = static_hash_filenames[static_url_abspath]

        # this is the first time this file has been referenced as a
        # {{static-hash-url:*}}, so we need to hash it now
        else:
            # make sure the file exists
            if not os.path.exists(static_url_abspath):
                self.line_error(token,
                                "file not found: '%s'" %
                                (os.path.join(STATIC_ROOT, url)))

            # make sure the path is actually a file
            if not os.path.isfile(static_url_abspath):
                self.line_error(token,
                                "path exists, but is not a file: '%s'" %
                                (os.path.join(STATIC_ROOT, url)))

            # get the source filename by itself
            src_base_filename = os.path.basename(static_url_abspath)

            # split the source filename into a base name and a file
            # extension
            (basename, file_ext) = os.path.splitext(src_base_filename)

            # deferred import
            import hashlib

            # hash the file contents
            md5_hash = hashlib.md5()
            try:
                with open(static_url_abspath, "rb") as f:
                    contents = f.read()
                    md5_hash.update(contents)
            except Exception as e:
                self.line_error(token,
                                "could not read file: '%s'" %
                                (static_url_abspath))

            # construct the base hash filename
            hash_filename = md5_hash.hexdigest() + file_ext

            # get the destination directory we want to write the hash
            # file into
            dest_dir = os.path.dirname(static_url_abspath)

            # construct the absolute path to the hashed file
            dest_hash_abspath = os.path.join(dest_dir, hash_filename)

            # copy source file to dest hash file
            copy_file(static_url_abspath, dest_hash_abspath)

            # cache the hashed filename
            static_hash_filenames[static_url_abspath] = hash_filename

        # create the hashed URL, which can either be a local or
        # absolute URL path (e.g. <hash>.css, /path/to/<hash>.css),
        # depending on how it was referenced from the node and parameter
        dest_hash_url = os.path.join(os.path.dirname(url), hash_filename)

        # return the URL in relation to the node, in the same way that a
        # {{static-url:*}} parameter would
        return self.get_static_url(self.node, dest_hash_url)

    def merge_token_rss(self, token: Token) -> str:
        """
        Merge an {{rss:*}} token and return the results.

        """

        # only support {{rss:url}}
        if "url" != token.value:
            self.line_error(token,
                            "invalid value for rss parameter: '%s'" %
                            (token.value))

        # make sure required headers are present
        if not self.node.has_header("rss-url"):
            self.line_error(token, "RSS-URL header not set")
        if not self.node.has_header("canonical-url"):
            self.line_error(token, "Canonical-URL header not set")

        # get relative RSS URL
        rss_url = self.node.get_header("rss-url")
        if rss_url.startswith("/"):
            rss_url = rss_url[1:]

        # get canonical RSS URL
        root_node = self.node.get_root_node()
        rss_canonical_url = root_node.get_canonical_url() + rss_url

        return rss_canonical_url

    def merge_token_node_body(self, token: Token) -> str:
        """
        Merge a {{node:body}} token and return the results.

        """

        # if we're in a {{node:body}} loop, show an error
        if self.node_body_semaphore != 0:
            self.node_body_loop_error(token)

        # entering {{node:body}} merge
        self.node_body_semaphore += 1

        # get node body
        body = self.node.get_body()
        if body is None:
            raise UrielError("node body can not be null in node '%s'" % \
                             (self.node.get_path()))

        # if the format is set to text, add HTML line breaks
        if "text" == self.node.get_format():
            body = self.text_to_html(body)

        # merge node body contents
        merged_body = self.merge_multiline(body)

        # leaving {{node:body}} merge
        self.node_body_semaphore -= 1

        return merged_body

    def merge_token_node(self, token: Token) -> str:
        """
        Merge a {{node:*}} token and return the results.

        """

        # url
        if "url" == token.value:
            return self.get_node_url(self.node)
        # name
        elif "name" == token.value:
            return escape(self.node.get_name())
        # title
        elif "title" == token.value:
            return self.node.get_escaped_title()
        # link
        elif "link" == token.value:
            return self.get_node_link(self.node)
        # body
        elif "body" == token.value:
            return self.merge_token_node_body(token)
        else:
            self.line_error(token,
                            "invalid value for node parameter: '%s'" %
                            (token.value))
            raise _ImpossibleError()

    def merge_token_node_url(self, token: Token) -> str:
        """
        Merge a {{node-url:*}} token and return the results.

        """

        try:
            target_node = self.get_node_by_path(token, token.value)
        except UrielError as e:
            self.line_error(token, "node not found: '%s'" % (token.value))

        return self.get_node_url(target_node)

    def merge_token_node_name(self, token: Token) -> str:
        """
        Merge a {{node-name:*}} token and return the results.

        """

        try:
            target_node = self.get_node_by_path(token, token.value)
        except UrielError as e:
            self.line_error(token, "node not found: '%s'" % (token.value))

        return escape(target_node.get_name())

    def merge_token_node_title(self, token: Token) -> str:
        """
        Merge a {{node-title:*}} token and return the results.

        """

        try:
            target_node = self.get_node_by_path(token, token.value)
        except UrielError as e:
            self.line_error(token, "node not found: '%s'" % (token.value))

        return target_node.get_escaped_title()

    def merge_token_node_link(self, token: Token) -> str:
        """
        Merge a {{node-link:*}} token and return the results.

        """

        try:
            target_node = self.get_node_by_path(token, token.value)
        except UrielError as e:
            self.line_error(token, "node not found: '%s'" % (token.value))

        return self.get_node_link(target_node)

    def merge_token_node_list(self, token: Token) -> str:
        """
        Merge a {{node-list:*}} token and return the results.

        """

        if "*" == token.value:
            return self.get_maybe_canonical_html_fragment(
                token, self.node, "__node-list-html")

        self.line_error(token,
                        "invalid value for node-list parameter: '%s'" %
                        (token.value))
        raise _ImpossibleError()

    def merge_token_tag_list(self, token: Token) -> str:
        """
        Merge a {{tag-list:*}} token and return the results.

        """

        if "*" == token.value:
            if self.node.has_header("__tag-list-html"):
                return self.get_maybe_canonical_html_fragment(
                    token, self.node, "__tag-list-html")
            else:
                # __tag-list-html is an internal header, that is set
                # elsewhere on nodes with tags defined. show a useful
                # error here instead of something obtuse.
                self.line_error(token, "Tags header not set on this node")

        self.line_error(token,
                        "invalid value for tag-list parameter: '%s'" %
                        (token.value))
        raise _ImpossibleError()

    def merge_token_soju(self, token: Token) -> str:
        """
        Merge a {{soju:*}} token and return the results.

        """

        # try to run the user-defined soju code, and return the value
        try:
            # remember the token for the soju code we are running.
            # this is only necessary for pathological cases, like if
            # the soju function is calling Page merge_line() directly
            # to include other templates, or something similarly misguided
            self.soju_tokens.append(token)

            # execute the user-defined soju function, and get the result
            try:
                result = self.get_soju_result(token.value)
            finally:
                self.soju_tokens.pop()

        # if we get a SojuError, log it, and raise a new SojuError
        # that summarizes where the user should look to start debugging
        except SojuError as e:
            # prepare the error message we want to log
            # if we have a user-provided error message from a SojuError
            # exception, surround it with quotes
            # otherwise, use the name of the exception class, without quotes
            soju_error_string = str(e)
            if "" == soju_error_string:
                soju_error_string = get_exception_reason(e)
            else:
                soju_error_string = "'" + soju_error_string + "'"

            # log the error within the context of how we got here
            self.line_error(token, soju_error_string, False)

            # create summary error message
            summary_error = "error in function call to 'soju." + \
                            token.value + "': " + soju_error_string

            # raise new chained exception with summary error message
            raise SojuError(summary_error) from e

        # for unexpected exceptions originating from user-defined soju code,
        # show the subset of the stack trace that applies to their code,
        # and raise a chained summary exception
        except Exception as e:
            # get a suitable error string to represent this error
            # this will either be the error string from the exception,
            # wrapped in quotes, or the name of the exception type itself
            error_string = str(e)
            if "" == error_string:
                error_string = get_exception_reason(e)
            else:
                error_string = "'" + error_string + "'"

            # log the error within the context of how we got here
            self.line_error(token, error_string, False)

            # log filtered traceback
            log_filtered_traceback(e, "soju.py")

            # create summary error message
            summary_error = "error in function call to 'soju." + \
                            token.value + "': " + error_string

            # raise a new chained exception with summary error message
            raise SojuError(summary_error) from e

        # if the user-defined soju code encountered another error inside of
        # the Page class for some reason, but ignored the exception, check
        # to see if we should raise the exception their code ignored now,
        if self.line_error_reason is not None:
            raise UrielError(self.line_error_reason)

        # return the output of the soju function call
        return result

    def merge_token(self, token: Token) -> str:
        """
        Merge the contents of a single token, and return the results.

        """
        # literal
        if Token.LITERAL == token.type:
            return self.merge_token_literal(token)

        # substitution parameter
        handler = _PAGE_MERGE_TOKEN_DISPATCH.get(token.type)
        if handler is not None:
            return handler(self, token)

        # unknown
        self.line_error(
            token,
            "invalid parameter or value: '%s'" %
            (token.original_string))
        raise _ImpossibleError()

    def merge_line(self, line: str) -> str:
        """
        Accepts a single line of text as input.

        Renders any substitution parameters in the line into their
        corresponding values (including the entire contents of other
        included templates, and their substitution parameters, recursively).

        Returns the (possibly multiline) string of rendered text.

        Raises a UrielError if the line is null.

        """

        # make sure we actually have a line to work with
        if line is None:
            raise UrielError("line can not be null")

        # if the line doesn't have anything that could be the start of a
        # substitution parameter, just return it as a literal
        if "{{" not in line:
            return line

        # tokenize the line
        tokens = self.tokenize(line)

        #################################
        # PARSE SUBSTITUTION PARAMETERS #
        #################################

        # render the token, and place it in a list of rendered tokens
        rendered = []
        for token in tokens:
            merged_token = self.merge_token(token)
            rendered.append(merged_token)

        # return the rendered tokens as a multi-line string
        return "".join(rendered)

    def merge_lines(self, lines: List[str]) -> str:
        """
        Accepts a list of string.

        Merges the template and node for each line of string.

        Returns a multi-line string with the merged contents.

        N.B. this is the same as merge_tokenized_lines(), for lines that
             have not already been split into tokens.

        """

        # rendered lines to return
        merged_lines = []

        # go through each input line
        for line in lines:
            # strip trailing newlines
            line = line.rstrip()

            # render the line
            merged_line = self.merge_line(line)

            # add the rendered line to the list
            merged_lines.append(merged_line)

        # return all of the merged lines as a string
        return "\n".join(merged_lines)

    def merge_tokenized_lines(
            self,
            tokenized: List[Tuple[str, Optional[List[Token]]]]) -> str:

        """
        Accepts a list of (line, tokens) pairs, as they are cached for a
        template.

        Merges the tokens for each line with the Node values.

        Returns a multi-line string with the merged contents.

        N.B. this is the same as merge_lines(), for lines that have
             already been split into tokens.

        """

        # rendered lines to return
        merged_lines = []

        # go through each (line, tokens) tuple
        for (line, tokens) in tokenized:
            # this line doesn't have any substitution parameters
            if tokens is None:
                # add the literal line
                merged_lines.append(line)

            # this line has tokenized substitution parameters
            else:
                # go through each token and render them
                merged_tokens = []
                for token in tokens:
                    merged_token = self.merge_token(token)
                    merged_tokens.append(merged_token)

                # join the merged tokens into a line
                merged_line = "".join(merged_tokens)

                # add the rendered line
                merged_lines.append(merged_line)

        # return all of the merged lines as a string
        return "\n".join(merged_lines)

    def merge_multiline(self, multiline: str) -> str:
        """
        Accepts a multi-line string.

        Returns a multi-line string with merged values.

        """

        lines = multiline.split("\n")

        return self.merge_lines(lines)

    def validate_template_name(self, token: Token, template: str) -> None:
        """
        Make sure the given template name doesn't contain a directory
        traversal attempt.

        Accepts a Token, and a template name.

        Raises a UrielError if the template name is invalid.

        """

        # prevent directory traversal
        if is_directory_traversal_attempt(template):
            self.line_error(
                token,
                "directory traversal not allowed in template name: '%s'" %
                (template))

        # prevent absolute filesystem paths
        if template.startswith("/"):
            self.line_error(
                token,
                "template name can not start with a '/': '%s'" %
                (template))

    def merge_template(self, template: str) -> str:
        """
        Accepts a template as input.

        Merges the template with the Node values, and returns the
        rendered output.

        """

        # prevent directory traversal attempts
        self.validate_template_name(Token("Template: " + template), template)

        # get the path to the template file
        template_file = template_file_paths.get(template)
        if template_file is None:
            template_file = os.path.join(self.project_root,
                                         TEMPLATES_ROOT,
                                         template)
            template_file_paths[template] = template_file

        # if Template is set to "null"
        if "null" == self.template:
            # and we are being called based on the node template,
            # instead of an {{include:*}} for a different template
            if "null" == template:
                # return a minimal template that just includes the node
                # contents, but allows for parameter substitution
                return self.merge_lines(["{{node:body}}"])

        # get the tokens for this template, if it has been used before
        tokenized = template_tokens.get(template_file)

        # if it hasn't already been cached,
        # read all of the lines from the template
        if tokenized is None:
            # keep track of the lines we read from the template file
            lines = []

            # open the template file
            try:
                with open(template_file, encoding="UTF-8") as f:
                    for line in f:
                        lines.append(line)

            except FileNotFoundError:
                token = Token("Template: " + template)
                self.line_error(token,
                                "template not found: '" + template_file + "'")

            except Exception as e:
                token = Token("Template: " + template)
                self.line_error(token,
                                "could not read template file: '" +
                                template_file + "'")

            # split the lines from the template into tokens
            tokenized = []
            for line in lines:
                # strip trailing newlines
                line = line.rstrip()

                # if this line doesn't have any substitution parameters,
                # just cache the line without any tokens
                if "{{" not in line:
                    tokenized.append((line, None))

                # otherwise, cache the line along with the parsed tokens
                else:
                    tokenized.append((line, self.tokenize(line)))

            # cache the results of the template parsing for next time
            template_tokens[template_file] = tokenized

        # add the template to the stack
        self.template_stack.push(template,
                                 self.node.get_path(),
                                 self.node_body_semaphore)

        # merge the template
        result = self.merge_tokenized_lines(tokenized)

        # remove the current template from the stack
        self.template_stack.pop()

        # return the result
        return result

    def render(self) -> str:
        """
        Render the combination of this Page, using its template and Node.

        """

        # validate Require-Tags / Tags header combination
        if self.node.get_boolean_header_value("require-tags", False):
            if not self.node.has_header("tags"):
                err = "Require-Tags: true, but Tags header not set " + \
                      "in node '%s'" % (self.node.get_path())
                raise UrielError(err)

        # validate Format header value (if defined)
        self.node.get_format()

        # render the page, starting with the template defined in the node
        rendered = self.merge_template(self.template)

        # make sure the last character in the rendered HTML file is a newline
        if "" != rendered:
            if "\n" != rendered[-1]:
                rendered += "\n"

        return rendered

    def __str__(self) -> str:
        return str(self.node) + " [" + self.template + "]"


# performance optimization:
# Token type -> Page merge_token_* method dispatch
_PAGE_MERGE_TOKEN_DISPATCH: Dict[Optional[str], Any] = {
    Token.VALUE: Page.merge_token_value,
    Token.VALUE_UNESCAPED: Page.merge_token_value_unescaped,
    Token.INCLUDE: Page.merge_token_include,
    Token.CREATED: Page.merge_token_created,
    Token.MODIFIED: Page.merge_token_modified,
    Token.OLDEST: Page.merge_token_oldest,
    Token.LATEST: Page.merge_token_latest,
    Token.BREADCRUMBS: Page.merge_token_breadcrumbs,
    Token.BREADCRUMBS_ROOT: Page.merge_token_breadcrumbs_root,
    Token.STATIC_URL: Page.merge_token_static_url,
    Token.STATIC_HASH_URL: Page.merge_token_static_hash_url,
    Token.RSS: Page.merge_token_rss,
    Token.NODE: Page.merge_token_node,
    Token.NODE_URL: Page.merge_token_node_url,
    Token.NODE_NAME: Page.merge_token_node_name,
    Token.NODE_TITLE: Page.merge_token_node_title,
    Token.NODE_LINK: Page.merge_token_node_link,
    Token.NODE_LIST: Page.merge_token_node_list,
    Token.TAG_LIST: Page.merge_token_tag_list,
    Token.SOJU: Page.merge_token_soju,
}


class FileWriter:
    """
    Writes an individual file to disk.

    """

    def __init__(self, path: str, mode: str = "w") -> None:
        """
        Accepts the path to the file, and an optional write mode.

        """

        self.path = path
        self.mode = mode

        try:
            # open in binary mode
            if "b" in mode:
                self.f = open(path, mode)

            # open in text mode
            else:
                self.f = open(path, mode, encoding="UTF-8")

        except FileNotFoundError as e:
            # get the base directory
            base_dir = os.path.split(path)[0]

            # base directory doesn't exist
            if not os.path.exists(base_dir):
                err = "could not create file '%s', " + \
                      "because directory '%s' does not exist"
                raise UrielError(err % (path, base_dir))

            # some other reason
            err = "could not create file '%s': '%s'"
            raise UrielError(err % (path, str(e)))

        except Exception as e:
            err = "could not create file '%s': '%s'"
            raise UrielError(err % (path, str(e)))

    def write(self, content: str) -> None:
        """
        Writes the content to the file.

        """

        try:
            self.f.write(content)
        except Exception as e:
            err = "error writing file '%s': '%s'"
            raise UrielError(err % (self.path, str(e)))

    def close(self) -> None:
        """
        Closes the file.

        """

        try:
            self.f.flush()
            self.f.close()
        except Exception as e:
            err = "error writing file '%s': '%s'"
            raise UrielError(err % (self.path, str(e)))


class GzipFileWriter:
    """
    Writes an individual file to disk with gzip compression.

    """

    def __init__(self, path: str, mode: str = "wt") -> None:
        """
        Accepts the path to the file, and an optional write mode.

        """

        # deferred import
        import gzip

        self.path = path
        self.mode = mode
        self.f: Any = None

        try:
            # open in binary mode
            if "b" in mode:
                self.f = gzip.open(path, mode)

            # open in text mode
            else:
                self.f = gzip.open(path, mode, encoding="UTF-8")

        except FileNotFoundError as e:
            # get the base directory
            base_dir = os.path.split(path)[0]

            # base directory doesn't exist
            if not os.path.exists(base_dir):
                err = "could not create file '%s', " + \
                      "because directory '%s' does not exist"
                raise UrielError(err % (path, base_dir))

            # some other reason
            err = "could not create file '%s': '%s'"
            raise UrielError(err % (path, str(e)))

        except Exception as e:
            err = "could not create file '%s': '%s'"
            raise UrielError(err % (path, str(e)))

    def write(self, content: str) -> None:
        """
        Writes the content to the file.

        """

        try:
            self.f.write(content)
        except Exception as e:
            err = "error writing file '%s': '%s'"
            raise UrielError(err % (self.path, str(e)))

    def close(self) -> None:
        """
        Closes the file.

        """

        try:
            self.f.flush()
            self.f.close()
        except Exception as e:
            err = "error writing file '%s': '%s'"
            raise UrielError(err % (self.path, str(e)))


def sys_exit(exit_code: int) -> None:
    """
    Exit the program, with the given exit code.

    """

    sys.exit(exit_code)

def log(s: str) -> None:
    """
    Log the given string to stderr.

    """

    sys.stderr.write(str(s) + "\n")
    sys.stderr.flush()

def warn(s: str) -> None:
    """
    Log the program name and the given string as a warning to stderr.

    """

    log(PROGRAM_NAME + ": " + str(s))

def die(s: str) -> None:
    """
    Log the program name and the given string as an error to stderr, and exit
    the program with a non-zero exit code.

    """

    warn(s)
    sys_exit(EXIT_FAIL)

def print_impossible_error() -> None:
    """
    Log a message confirming that the error the user encountered is most
    definitely a uriel bug.

    """

    warn("-" * 71)
    warn("CONGRATULATIONS!")
    warn("")
    warn("You found a bug in the uriel program!")
    warn("")
    warn("Please consider reporting it, so it can be fixed.")
    warn("")
    warn("Sorry for the inconvenience.")
    warn("-" * 71)

def show_usage() -> None:
    """
    Show command-line usage information for the program, and exit.

    """

    log(PROGRAM_NAME + " " + VERSION)
    log("Usage: " + PROGRAM_NAME + " <project-root>")
    log("See the uriel(1) man page, " + \
        "or https://uriel.foo/ for more information")
    sys_exit(EXIT_FAIL)

def printable_path(path: str) -> str:
    """
    Makes a filesystem path look a bit nicer in log messages.

    Accepts a file path

    Returns a nicer looking but equivalent file path

    """

    # special case, if we were given a /, just return it
    # (never mind that we should never be dealing with the root filesystem)
    if "/" == path:
        return path

    tmp = path

    # strip leading ./
    # .///test///static/// -> test///static///
    tmp = re.sub(r'^\.\/*', r'', tmp)

    # collapse multiple / in a row
    # test///static/// -> test/static/
    tmp = re.sub(r'\/{2,}', r'/', tmp)

    # strip trailing /
    # test/static/ -> test/static
    tmp = re.sub(r'\/*$', r'', tmp)

    return tmp

def indent_spaces(indent: int) -> str:
    """
    Get a string with blank spaces, according to the specified indenting.

    Accepts a number of indentations.

    Returns a string with two spaces for each indentation.

    """

    return "  " * indent

def get_exception_reason(e: Exception) -> str:
    """
    Get the reason associated with an exception, or if that isn't present,
    default to the type of the exception.

    Accepts an Exception (or a subclass of Exception)

    Returns a non-blank reason string derived from the exception

    """

    reason = str(e)
    if "" == reason:
        reason = e.__class__.__name__

    return reason

def log_filtered_traceback(e: Exception, filter_string: str) -> None:
    """
    Log a filtered Python traceback to stderr.

    The traceback will be filtered, so that it omits the first part of the
    traceback, and only starts being printed when the filter_string is
    encountered.

    The idea here is that we can print out the part of the traceback that
    a user is actually interested in (in their user-defined code), and
    filter out the parts that come from the calling chain in the uriel
    program itself.

    Accepts an exception object, and a filter string (e.g. "soju.py")

    Prints a filtered traceback to stderr, starting with the line that
    matches the filter string.

    """

    # deferred import
    import traceback

    # extract a subset of the traceback from the exception we
    # caught, starting with the entry point into the user-defined
    # code
    filtered_traceback_lines = []
    started_user_traceback = False
    for traceback_line in traceback.format_tb(e.__traceback__):
        if filter_string in traceback_line:
            started_user_traceback = True

        if started_user_traceback:
            filtered_traceback_lines.append(traceback_line.rstrip())

    # print out a custom stack trace starting with the user-defined code,
    # so the user can debug their code with a minimum of
    # unnecessary noise from the machinery of the uriel program
    if len(filtered_traceback_lines) > 0:
        log("Traceback (most recent call last):")
        for line in filtered_traceback_lines:
            log(line)

def copy_file(src: str, dest: str) -> None:
    """
    Copy the source file to the destination file.

    Raises a UrielError if an error occurs.

    """

    try:
        # a symlink is never copied into the public directory
        if os.path.islink(src):
            raise UrielError(
                "error copying '%s' to '%s': source path is a symlink" %
                (src, dest))

        # if the destination file is a symlink, delete it first, so that
        # the copy is not written through it
        if os.path.islink(dest):
            os.unlink(dest)

        # if the destination path is a directory, we don't want
        # to copy the source file into a directory with the same name
        if os.path.isdir(dest):
            errstr = "error copying '%s' to '%s': destination path " + \
                     "is a directory"
            raise UrielError(errstr % (src, dest))

        # copy the file
        shutil.copy2(src, dest)

    except UrielError as e:
        raise

    except FileNotFoundError as e:
        raise UrielError("error copying '%s' to '%s': file not found" %
                         (src, dest))

    except PermissionError as e:
        raise UrielError("error copying '%s' to '%s': permission denied" %
                         (src, dest))

    except IsADirectoryError as e:
        # was the source path a directory?
        if os.path.isdir(src):
            raise UrielError(
                "error copying '%s' to '%s': source path is a directory" %
                (src, dest))

        # was the destination path a directory?
        if os.path.isdir(dest):
            raise UrielError(
                "error copying '%s' to '%s': destination path is a directory" %
                (src, dest))

        # in case the checks above both fail
        raise UrielError(
            "error copying '%s' to '%s': '%s'" %
            (src, dest, get_exception_reason(e)))

    except Exception as e:
        raise UrielError(
            "error copying '%s' to '%s': '%s'" %
            (src, dest, get_exception_reason(e)))

def copy_file_if_different(src: str, dest: str) -> None:
    """
    Copy the source file to the destination file, if they are different.

    Raises a UrielError if an error occurs.

    """

    # if the source file doesn't exist, or is a directory, raise an error
    if (not os.path.isfile(src)) and (not os.path.islink(src)):
        if not os.path.exists(src):
            raise UrielError("source file does not exist: '%s'" % (src))
        else:
            raise UrielError("source file can not be a directory: '%s'" %
                             (src))

    # if the destination file doesn't exist, just copy src to dest
    if not os.path.exists(dest):
        copy_file(src, dest)

    # we have both source and destination files
    # decide whether we really want to expend the energy copying the file
    else:
        src_stat = os.stat(src)
        dest_stat = os.stat(dest)

        # try to skip copying the file, if we can get away with it
        skip_copy = True

        # compare symlink status
        if skip_copy:
            if os.path.islink(src) != os.path.islink(dest):
                skip_copy = False

        # compare file sizes
        if skip_copy:
            if src_stat.st_size != dest_stat.st_size:
                skip_copy = False

        # compare file permissions
        if skip_copy:
            if src_stat.st_mode != dest_stat.st_mode:
                skip_copy = False

        # compare file modification times
        if skip_copy:
            if src_stat.st_mtime != dest_stat.st_mtime:
                skip_copy = False

        # if anything is different, copy the file
        if not skip_copy:
            copy_file(src, dest)

def copy_files_recursive(src: str, dest: str) -> None:
    """
    Recursively copy the source directory into the destination directory.

    Raises a UrielError if an error occurs.

    """

    try:
        # don't follow symlinks
        if os.path.islink(src):
            raise UrielError("symlink not allowed: '%s'" % (src))

        # if the source path is a directory
        if os.path.isdir(src):
            # if the destination path exists
            if os.path.exists(dest) or os.path.islink(dest):
                # if the destination path is a symlink or a file,
                # remove it and create the destination directory
                if os.path.islink(dest) or not os.path.isdir(dest):
                    os.unlink(dest)
                    os.mkdir(dest)

            # the destination path does not exist, create it
            else:
                os.mkdir(dest)

            # get the permissions and timestamps from src,
            # and apply them to dest
            st = os.stat(src)
            os.chmod(dest, st.st_mode)
            os.utime(dest, times=(st.st_atime, st.st_mtime))

            # go through the files and directories in the source
            for dirent in os.listdir(src):
                # calculate subdir paths
                src_path = os.path.join(src, dirent)
                dest_path = os.path.join(dest, dirent)

                # don't copy or follow symlinks
                if os.path.islink(src_path):
                    raise UrielError("symlink not allowed: '%s'" %
                                     (src_path))

                # if this is a directory, recurse
                if os.path.isdir(src_path):
                    copy_files_recursive(src_path, dest_path)

                # if this is a file, copy it from src to dest
                else:
                    copy_file_if_different(src_path, dest_path)

        # if we got called with files as arguments,
        # just copy the source file to the dest file
        else:
            copy_file_if_different(src, dest)

    # if a UrielError bubbled up from somewhere in the try block,
    # let it bubble up from this function
    except UrielError as e:
        raise

    # something else went wrong
    except Exception as e:
        raise UrielError("error copying '%s' to '%s': '%s'" %
                         (src, dest, get_exception_reason(e)))

def copy_files_recursive_overwrite(src: str, dest: str) -> None:
    """
    Recursively copy the source directory into the destination directory,
    deleting anything in the destination directory that is not present
    in the source directory.

    Raises a UrielError if an error occurs.

    """

    try:
        # a uriel project is self contained, so the source path itself is
        # not allowed to be a symlink either
        if os.path.islink(src):
            raise UrielError("symlink not allowed: '%s'" % (src))

        # if the source path is a directory
        if os.path.isdir(src):
            # if the destination path exists
            if os.path.exists(dest) or os.path.islink(dest):
                # if the destination path is a symlink or a file,
                # remove it and create the destination directory
                if os.path.islink(dest) or not os.path.isdir(dest):
                    os.unlink(dest)
                    os.mkdir(dest)

            # the destination path does not exist, create it
            else:
                os.mkdir(dest)

            # get the permissions and timestamps from src,
            # and apply them to dest
            st = os.stat(src)
            os.chmod(dest, st.st_mode)
            os.utime(dest, times=(st.st_atime, st.st_mtime))

            # get the list of entries in the source directory
            # these will be copied over from src to dest
            src_copy_list = os.listdir(src)

            # get the list of entries in the destination directory
            # these will be deleted from dest, unless they are
            # removed from this list due to also appearing in src
            dest_delete_list = os.listdir(dest)

            # remove entries from the delete list if they are found in src
            for src_file in src_copy_list:
                if src_file in dest_delete_list:
                    dest_delete_list.remove(src_file)

            # copy everything on the copy list
            for dirent in src_copy_list:
                # calculate subdir paths
                src_path = os.path.join(src, dirent)
                dest_path = os.path.join(dest, dirent)

                # symlinks are not copied into the public directory
                if os.path.islink(src_path):
                    raise UrielError("symlink not allowed: '%s'" %
                                     (src_path))

                # if this is a directory, recurse
                if os.path.isdir(src_path):
                    copy_files_recursive_overwrite(src_path, dest_path)

                # if this is a file, copy it from src to dest
                else:
                    copy_file_if_different(src_path, dest_path)

            # delete everything on the delete list
            for dirent in dest_delete_list:
                dest_delete_path = os.path.join(dest, dirent)

                # delete path is a directory
                if os.path.isdir(dest_delete_path):
                    # if it's actually a symlink in disguise, unlink it
                    if os.path.islink(dest_delete_path):
                        os.unlink(dest_delete_path)

                    # otherwise, recursively delete the dest delete directory
                    else:
                        shutil.rmtree(dest_delete_path)

                # delete path is a file
                else:
                    os.unlink(dest_delete_path)

        # if we got called with files as arguments,
        # just copy the source file to the dest file
        else:
            copy_file_if_different(src, dest)

    # N.B. a UrielError already says what went wrong, and which paths
    #      were involved. it is re-raised as it is, so that it does not
    #      get wrapped again by every level of the recursion.
    except UrielError as e:
        raise

    except Exception as e:
        raise UrielError("error copying '%s' to '%s': '%s'" %
                         (src, dest, get_exception_reason(e)))

def delete_directory_recursive(path: str) -> None:
    """
    Recursively delete the given directory.

    Raises a UrielError if an error occurs.

    """

    try:
        shutil.rmtree(path)
    except Exception as e:
        raise UrielError(
            "could not delete directory '%s': '%s'" %
            (path, get_exception_reason(e)))

def is_directory_traversal_attempt(path: str) -> bool:
    """
    Checks the given path for directory traversal attempts.

    Looks for patterns related to . and .., but does NOT validate
    absoulte paths starting with /, since these are frequently used
    as URL paths relative to the root of the site.

    In general, this is useful for validating that a file or directory
    path doesn't contain any relative path components that could turn
    a file into a directory, or that could traverse up the directory
    tree.

    Multi-directory paths (e.g. foo/bar, /foo/bar/) are allowed.
    Paths like ., .., foo/../bar, foo/bar/.., are not.

    Returns True if a directory traversal attempt was detected,
    or False otherwise.

    Raises UrielError if the path is null.

    """

    # make sure path is not null
    if path is None:
        raise UrielError("path can not be null")

    # current directory
    if "." == path:
        return True

    # parent directory
    if ".." == path:
        return True

    # parent directory
    if "/../" in path:
        return True

    # current directory
    if "/./" in path:
        return True

    # parent directory
    if path.startswith("../"):
        return True

    # current directory
    if path.endswith("/."):
        return True

    # parent directory
    if path.endswith("/.."):
        return True

    # no directory traversal detected
    return False

def escape(text: str) -> str:
    """
    Escape HTML.

    Accepts unescaped text.

    Returns an HTML escaped string.

    """

    if text is None:
        return ""

    # if the text doesn't need to be escaped, it's cheaper to detect
    # that up front if it lets us avoid the escape transformations altogether
    if ("&" not in text) and \
       ('"' not in text) and \
       ("'" not in text) and \
       (">" not in text) and \
       ("<" not in text):

        return text

    return text.replace("&", "&amp;") \
               .replace('"', "&quot;") \
               .replace("'", "&apos;") \
               .replace(">", "&gt;") \
               .replace("<", "&lt;")

def escape_xml(text: str) -> str:
    """
    Escape XML.

    Accepts unescaped text.

    Return an XML escaped string.

    The main purpose of this function is to make sure that no <![CDATA[ ]]>
    tags slip through and break the XML formatting in things like RSS feeds.

    """

    unescaped_lines = text.split("\n")
    escaped_lines = []

    for unescaped_line in unescaped_lines:
        escaped_line = unescaped_line.replace("<![CDATA[", "&lt;![CDATA[")
        escaped_line = escaped_line.replace("]]>", "]]&gt;")
        escaped_lines.append(escaped_line)

    return "\n".join(escaped_lines)

def create_file_node_tree(project_root: str,
                          parent_path: Optional[str] = None,
                          parent_node: Optional[Node] = None) -> FileNode:

    """
    Create Node entries for all of the dynamic pages.

    Accepts project_root

    Optionally accepts the following arguments:
        parent_path - subdirectory of nodes_root containing this node
        parent_node - Node instance to use as the parent of discovered nodes

    Returns the top Node in the tree (the top-level dynamic index page)

    """

    # there are a lot of dir/path variables in here.
    # here is what they all mean:
    #
    # project_root  top-level project root directory
    # parent_path   parent directory path above this part of the tree
    #               (without the NODES_ROOT container directory)
    # dir_path      project_root + NODES_ROOT + parent_path
    # dirent        directory entry in this directory (e.g. "foo")
    # dirent_path   project_root + NODES_ROOT + parent_path + dirent
    # node_path     parent_path + dirent

    # create the index node for this directory first
    # if this is not the root node, create the index for this subdirectory
    if parent_path:
        dir_path = os.path.join(project_root, NODES_ROOT, parent_path)
        node_path = os.path.join(parent_path, NODE_INDEX)

        # we have a directory, and we hope that an index file is inside
        # if the index file isn't there, show a descriptive error
        dirent_path = os.path.join(dir_path, NODE_INDEX)
        if not os.path.exists(dirent_path):
            # make sure this isn't a permissions problem on the directory
            # before we report an error about the index file not existing
            try:
                dirents = os.listdir(dir_path)
            except Exception as e:
                err = "could not read directory '%s': '%s'"
                raise UrielError(err % (dir_path, str(e)))

            # index file not found
            raise UrielError(
                "directory '%s' exists, but '%s/%s' node not found" %
                (dir_path, dir_path, NODE_INDEX))

        index = FileNode(project_root,
                         node_path,
                         parent_node)

    # special case: create the root node
    else:
        # unlike the index nodes in subdirectories,
        # uriel will recreate the root index file if it goes missing
        dir_path = os.path.join(project_root, NODES_ROOT)
        index = FileNode(project_root, NODE_INDEX)
        is_root_node = True

    # make sure we can read this directory in the node tree
    try:
        dirents = sorted(os.listdir(dir_path))
    except Exception as e:
        err = "could not read directory '%s': '%s'"
        raise UrielError(err % (dir_path, str(e)))

    # create all the rest of the nodes in memory
    for dirent in dirents:
        # skip hidden files
        if dirent.startswith("."):
            continue

        # skip emacs backup files
        if dirent.endswith("~"):
            continue

        # we already got the index file, don't get it again
        if NODE_INDEX == dirent:
            continue

        # get full path to child node
        dirent_path = os.path.join(dir_path, dirent)

        # create relative path from below NODES_ROOT
        if parent_path:
            node_path = os.path.join(parent_path, dirent)
        else:
            node_path = dirent

        # file: create node
        if os.path.isfile(dirent_path):
            node = FileNode(project_root, node_path, index)
            index.add_child(node)

        # directory: recurse
        elif os.path.isdir(dirent_path):
            node = create_file_node_tree(project_root,
                                         node_path,
                                         index)
            index.add_child(node)

    return index

def create_tag_node_tree(project_root: str,
                         root: Node,
                         use_canonical_url: bool) -> None:

    """
    Create the tag node tree.

    Accepts project_root directory, the root node, and a boolean indicating
    whether we should pre-compute canonical URLs.

    Looks for the Tag-Node header in the root node, to determine where to
    place the tag root.

    Builds an index of tags and nodes.

    Creates a virtual node hierarchy under the tag root.

    """

    # get tag node
    tag_node = root.get_tag_node()
    if tag_node is None:
        return

    # build tag node index
    root.create_tag_node_index()
    tag_node_index = root.get_tag_node_index()

    # get link prefix/suffix
    link_prefix = tag_node.get_link_prefix()
    link_suffix = tag_node.get_link_suffix()

    ############
    # TAG ROOT #
    ############

    # __Tag-List-HTML / __Tag-List-HTML-Canonical
    lines = []
    canonical_lines = []
    for tag in sorted(tag_node_index.keys()):
        escaped_tag = escape(tag)

        line = link_prefix + \
            "<a href=\"" + \
            tag_node.get_url() + escaped_tag + "/" + "\">" + \
            escaped_tag + "</a>" + \
            link_suffix
        lines.append(line)

        if use_canonical_url:
            c_line = link_prefix + \
                "<a href=\"" + \
                tag_node.get_canonical_url() + escaped_tag + "/" + "\">" + \
                escaped_tag + "</a>" + \
                link_suffix
            canonical_lines.append(c_line)

    tag_node.set_header("__tag-list-html",
                        "\n".join(lines))

    if use_canonical_url:
        tag_node.set_header("__tag-list-html-canonical",
                            "\n".join(canonical_lines))

    # RSS-Include: false
    # (omit tag root and child vnodes from RSS feed)
    tag_node.set_header("rss-include", "false")

    ################
    # CHILD VNODES #
    ################

    # set the base path for the vnodes (without trailing slash)
    vnode_base_path = tag_node.get_path()
    if vnode_base_path.endswith(NODE_INDEX):
        end_index = (len(NODE_INDEX) * -1) - 1
        vnode_base_path = vnode_base_path[:end_index]

    # create child vnodes for each tag
    for tag in sorted(tag_node_index.keys()):
        # create the path for the child vnode
        vnode_path = vnode_base_path + "/" + tag

        # create the vnode
        vnode = VirtualNode(project_root, vnode_path, tag_node)

        # copy the tag node body into the child vnode
        vnode.set_body(tag_node.get_body())

        # Title
        vnode.set_header("title", vnode.get_name())

        # Flat-URL
        vnode.set_header("flat-url", "false")

        # __Tag-List-HTML
        lines = []
        canonical_lines = []
        for node in sorted(tag_node_index[tag], key=Node.get_sort_key):
            line = link_prefix + node.get_link() + link_suffix
            lines.append(line)

            if use_canonical_url:
                c_line = link_prefix + node.get_canonical_link() + link_suffix
                canonical_lines.append(c_line)

        vnode.set_header("__tag-list-html",
                         "\n".join(lines))

        if use_canonical_url:
            vnode.set_header("__tag-list-html-canonical",
                             "\n".join(canonical_lines))

        tag_node.add_child(vnode)

def create_tag_links(node: Node, use_canonical_url: bool) -> str:
    """
    Create HTML to link to all tags associated with this node, as defined
    in the Tags header for the given node.

    Accepts a Node, and a boolean indicating whether we should
    pre-compute canonical URLs.

    """

    links: List[str] = []

    # go through all of the tags for this node
    for tag in sorted(node.get_tags()):
        # separate tags with commas
        if len(links) > 0:
            links.append(", ")

        # get the vnode for this tag
        tag_vnode = node.get_vnode_for_tag(tag)
        if tag_vnode is None:
            # this is not expected to happen in normal operation,
            # but we provide a detailed error message and traceback
            # in case user-defined code triggers it somehow
            errstr = "tag vnode can not be null for tag '%s' on node '%s'"
            raise Exception(errstr % (tag, node.get_path()))

        if use_canonical_url:
            url = tag_vnode.get_canonical_url()
        else:
            url = tag_vnode.get_url()

        # construct a link to the vnode
        link = "<a href=\"" + url + "\">" + escape(tag) + "</a>"
        links.append(link)

    return "".join(links)

def create_tag_links_recursive(node: Node, use_canonical_url: bool) -> None:
    """
    Walk through the node tree, from the root, and create tag links for
    each node that does not already have something for the {{tag-list:*}}
    parameter.

    Accepts a Node, and a boolean indicating whether we should pre-compute
    canonical URL links.

    """

    # leaf node
    if not node.has_header("__tag-list-html"):
        node.set_header("__tag-list-html", create_tag_links(node, False))

    if use_canonical_url:
        if not node.has_header("__tag-list-html-canonical"):
            node.set_header("__tag-list-html-canonical",
                            create_tag_links(node, True))

    # recurse
    for child in node.get_children():
        create_tag_links_recursive(child, use_canonical_url)

def create_child_node_list_html(node: Node, use_canonical_url: bool) -> None:
    """
    Create an HTML fragment of links to all of the child nodes of the given
    Node, and makes it available for use with the {{node-list:*}} parameter.

    Accepts a Node, and a boolean indicating whether we should pre-compute
    canonical URL links.

    """

    # __Node-List-HTML / __Node-List-HTML-Canonical
    lines = []
    canonical_lines = []
    for child in node.get_children():
        # get link prefix/suffix
        link_prefix = node.get_link_prefix()
        link_suffix = node.get_link_suffix()

        # create the link to the child node
        line = link_prefix + child.get_link() + link_suffix
        lines.append(line)

        if use_canonical_url:
            c_line = link_prefix + child.get_canonical_link() + link_suffix
            canonical_lines.append(c_line)

    # add the HTML fragment with all the links to the node headers
    node.set_header("__node-list-html",
                    "\n".join(lines))

    # maybe add another HTML fragment with all the canonical links
    # to a different node header
    if use_canonical_url:
        node.set_header("__node-list-html-canonical",
                        "\n".join(canonical_lines))

    # recurse
    for child in node.get_children():
        create_child_node_list_html(child, use_canonical_url)

def augment_node_tree(project_root: str, root_node: Node) -> None:
    """
    Augment the node tree with additional computed values before rendering.

    Accepts the project root directory, and the root Node.

    """

    # check to see if we should pre-compute canonical URL links
    use_canonical_url = False
    if root_node.has_header("canonical-url"):
        use_canonical_url = True

    # create tag-related things, if Tag-Node is set
    tag_node = root_node.get_tag_node()
    if tag_node is not None:
        # if the tag node doesn't have explicit created or modified times set,
        # default to now, because tags are dynamically generated anyway
        if not tag_node.has_header("created"):
            if not tag_node.has_header("modified"):
                now = \
                    datetime.datetime.now(datetime.timezone.utc).astimezone()

                tag_node.created = now
                tag_node.modified = now

        # create tag nodes
        create_tag_node_tree(project_root, root_node, use_canonical_url)

        # create links to the tag pages, on all the nodes that have tags
        create_tag_links_recursive(root_node, use_canonical_url)

    # create {{list-node:*}} HTML fragment for each node
    create_child_node_list_html(root_node, use_canonical_url)

def render_node_tree(project_root: str, node: Node) -> None:
    """
    Walk the tree of Node entries and render their contents in place.

    """

    # create a Page for this Node
    page = Page(project_root, node)

    # render the contents in place
    node.set_rendered_body(page.render())

    # recurse through the child nodes
    for child in node.get_children():
        render_node_tree(project_root, child)

def get_max_url_path_len(node: Node,
                         max_url_len: int = 0,
                         max_path_len: int = 0) -> Tuple[int, int]:

    """
    Get the maximum URL and path lengths from all of the nodes,
    and return them as a two-element tuple.

    """

    # get the maximum length of the url and path for this node
    url_len = len(node.get_url())
    path_len = len(node.get_path())

    # raise the max values, if the new lengths are higher
    max_url_len = max(url_len, max_url_len)
    max_path_len = max(path_len, max_path_len)

    # recurse
    for child in node.get_children():
        (max_url_len, max_path_len) = \
            get_max_url_path_len(child, max_url_len, max_path_len)

    return (max_url_len, max_path_len)

def write_dynamic_nodes(project_root: str, node: Node) -> None:
    """
    Write the dynamic node/template merged data into files under PUBLIC_ROOT.

    """

    # get the maximum URL and path string lengths
    (max_url_len, max_path_len) = get_max_url_path_len(node, 3, 4)

    # not a metasyntactic variable, an actual horizontal display bar
    bar = ("-" * 7) + "-+-" + \
          ("-" * max_path_len) + "-+-" + \
          ("-" * max_url_len)

    # write the nodes
    log("creating pages in '%s' from nodes and templates" %
        (printable_path(os.path.join(project_root, PUBLIC_ROOT))))

    log(bar)
    log("type    | " + "node".ljust(max_path_len) + " | " + "url")
    log(bar)

    unique_urls: Set[str] = set()

    # call the recursive function to write the nodes
    (file_nodes_written, virtual_nodes_written) = write_nodes(project_root,
                                                              node,
                                                              max_url_len,
                                                              max_path_len,
                                                              unique_urls)

    log(bar)

    # calculate the total number of nodes written
    total_nodes_written = file_nodes_written + virtual_nodes_written

    # log the number of nodes written
    if 1 == total_nodes_written:
        log("created %d page (%d file, %d virtual)" %
            (total_nodes_written,
             file_nodes_written,
             virtual_nodes_written))
    else:
        log("created %d pages (%d file, %d virtual)" %
            (total_nodes_written,
             file_nodes_written,
             virtual_nodes_written))

def write_nodes(project_root: str,
                node: Node,
                max_url_len: int,
                max_path_len: int,
                unique_urls: Set[str]) -> Tuple[int, int]:

    """
    Recursive function to write the nodes.

    Accepts the following arguments:
        project_root - project root directory
        node         - node to write (recursively)
        max_url_len  - max URL length (for pretty printing)
        max_path_len - max path length (for pretty printing)
        unique_urls  - mutable set of unique URLs to detect duplicates

    Returns a two-element tuple with the number of file and virtual nodes
    that were written.

    """

    # count how many nodes we have written
    file_nodes_written = 0
    virtual_nodes_written = 0

    # get the directory and file to use for this node
    directory = node.get_dest_dir()
    file = node.get_dest_file()

    # create the directory, if it doesn't exist already
    os.makedirs(directory, exist_ok=True)

    # log this node being written
    #
    # node types in the first column are "file" or "virtual",
    # so a maximum of 7 characters
    log("%s | %s | %s" %
        (node.get_node_type().ljust(7),
         node.get_path().ljust(max_path_len),
         node.get_url()))

    # make sure the URL is unique before we write the node
    url = node.get_url()
    if url in unique_urls:
        err = "duplicate node URL '%s' in node '%s'"
        raise UrielError(err % (url, node.get_path()))
    unique_urls.add(url)

    # write the rendered file
    fw = FileWriter(file)
    body = node.get_rendered_body()
    if body is None:
        raise UrielError("node body can not be null in node '%s'" % \
                         (node.get_path()))
    fw.write(body)
    fw.close()

    # count the node we just wrote
    node_type = node.get_node_type()
    if "file" == node_type:
        file_nodes_written += 1
    elif "virtual" == node_type:
        virtual_nodes_written += 1

    # recurse into the node children
    for child in node.get_children():
        (child_fnodes, child_vnodes) = write_nodes(project_root,
                                                   child,
                                                   max_url_len,
                                                   max_path_len,
                                                   unique_urls)

        file_nodes_written += child_fnodes
        virtual_nodes_written += child_vnodes

    return (file_nodes_written, virtual_nodes_written)

def get_eligible_nodes(node: Node,
                       boolean_header: str,
                       default: bool,
                       node_set: Set[Node]) -> None:

    """
    Get nodes that have the given header set to true.

    Accepts a Node, the name of the header to check for true status,
    and a node set to store the eligible nodes.

    Accepts the following arguments:
        node            - root node to search
        boolean_header  - header name to check for boolean value
        default         - default boolean value to use if header not set
        node_set        - set the node will be added to if eligible

    """

    # leaf
    if node.get_boolean_header_value(boolean_header, default):
        node_set.add(node)

    # recurse
    for child in node.get_children():
        get_eligible_nodes(child, boolean_header, default, node_set)

def get_rfc_2822_date(dt: datetime.datetime) -> str:
    """
    Gets a date string in RFC 2822 format.

    Accepts a datetime.datetime instance.

    If the datetime instance has a UTC offset, it will be used in the
    string conversion. If it does not have a time zone, then the local
    time zone will be used by default.

    Returns a date string.

    """

    # default to the local time zone if not specified
    if dt.tzinfo is None:
        dt = dt.astimezone()

    return dt.strftime("%a, %d %b %Y %H:%M:%S %z")

def get_w3c_datetime(dt: datetime.datetime) -> str:
    """
    Gets a date string in W3C Datetime format.

    Accepts a datetime.datetime instance.

    If the datetime instance has a UTC offset, it will be used in the
    string conversion. If it does not have a time zone, then the local
    time zone will be used by default.

    Returns a date string.

    """

    # default to the local time zone if not specified
    if dt.tzinfo is None:
        dt = dt.astimezone()

    return dt.isoformat(timespec="seconds")

def get_rss_url(root_node: Node) -> Optional[str]:
    """
    Get the RSS URL (without a leading slash).

    Returns None if RSS-URL is not enabled.

    """

    # if RSS is not enabled, stop here
    if not root_node.has_header("rss-url"):
        return None

    # get the full path to the RSS file we're going to write on disk
    rss_url = root_node.get_header("rss-url")

    # prevent directory traversal attempts
    if is_directory_traversal_attempt(rss_url):
        errstr = "invalid RSS-URL header value in node '%s' " + \
                 "(directory traversal): '%s'"
        raise UrielError(errstr % (root_node.get_path(), rss_url))

    # if the RSS URL starts with a leading slash,
    # normalize it to the root of the site
    if rss_url.startswith("/"):
        rss_url = rss_url[1:]

    return rss_url

def write_rss(project_root: str, root_node: Node) -> None:
    """
    Write RSS feed to disk (if RSS-URL is enabled).

    Accepts the project root, and the root Node.

    """

    # get the RSS URL
    rss_url = get_rss_url(root_node)

    # if RSS is not enabled, stop here
    if rss_url is None:
        return

    # get the path to the RSS file to write
    rss_file = os.path.join(root_node.get_dest_dir(), rss_url)

    # in normal operation, the root node passed into this function
    # will be the actual root node of the site
    #
    # however, if a user-defined handler calls this function with an
    # alternate node as an argument, then we will honor that as well,
    # and take care to fix up the RSS-URL path to start at the root of
    # the site if it is specified as an absolute path starting with a "/"
    actual_root_node = root_node.get_root_node()
    if root_node is not actual_root_node:
        # get_rss_url() strips leading slashes, so we need to check directly
        raw_rss_url = root_node.get_header("rss-url")

        # if we are actually writing an RSS feed for a node below the root,
        # but the RSS-URL was specified with a leading slash, then start the
        # path to write the RSS file from the actual root node
        if raw_rss_url.startswith("/"):
            rss_file_dest_dir = actual_root_node.get_dest_dir()
            rss_file = os.path.join(rss_file_dest_dir, rss_url)

    # skip creating the file if there's a static file with the same name
    if os.path.exists(rss_file):
        log("skipping creation of '%s': file exists" %
            (printable_path(rss_file)))
        return

    log("creating '%s'" % (printable_path(rss_file)))

    # Canonical-URL
    if not root_node.has_header("canonical-url"):
        err = "Canonical-URL must be set on node '%s' to write RSS feed"
        raise UrielError(err % (root_node.get_path()))

    # RSS-Title / Title
    if root_node.has_header("rss-title"):
        title = root_node.get_header("rss-title")
    elif root_node.has_header("title"):
        title = root_node.get_header("title")
    else:
        err = "RSS-Title or Title must be set on node '%s' to write RSS feed"
        raise UrielError(err % (root_node.get_path()))

    # RSS-Description
    if root_node.has_header("rss-description"):
        channel_description = root_node.get_header("rss-description")
    else:
        err = "RSS-Description must be set on node '%s' to write RSS feed"
        raise UrielError(err % (root_node.get_path()))

    # RSS-Max-Entries
    rss_max_entries = RSS_MAX_ENTRIES
    if root_node.has_header("rss-max-entries"):
        value = root_node.get_header("rss-max-entries")
        try:
            rss_max_entries = int(value)
            if rss_max_entries < 0:
                raise UrielError()

        except Exception:
            err = "RSS-Max-Entries must be a valid unsigned integer " + \
                  "on node '%s'"
            raise UrielError(err % (root_node.get_path()))

    # RSS-Image-URL
    canonical_rss_image_url: Optional[str] = None
    if root_node.has_header("rss-image-url"):
        rss_image_url = root_node.get_header("rss-image-url")

        # remote RSS image URL
        if rss_image_url.startswith(("http://", "https://")):
            canonical_rss_image_url = rss_image_url

        # local RSS image static file reference
        else:
            # leading slash implies the root of the web site
            if rss_image_url.startswith("/"):
                rss_image_url = rss_image_url[1:]

            # prevent directory traversal
            if is_directory_traversal_attempt(rss_image_url):
                errstr = "invalid RSS-Image-URL header value in " + \
                         "node '%s' (directory traversal): '%s'"
                raise UrielError(
                    errstr % (root_node.get_path(), rss_image_url))

            # get the path to the RSS image on the filesystem
            rss_image_file = os.path.join(root_node.get_dest_dir(),
                                          rss_image_url)

            # if this is a local static file reference, make sure it exists
            if not os.path.exists(rss_image_file):
                errstr = "RSS-Image-URL on node '%s' file not found: '%s'"
                raise UrielError(
                    errstr % (root_node.get_path(), rss_image_url))

            # create canonical RSS image URL
            canonical_rss_image_url = \
                root_node.get_canonical_url() + rss_image_url

    # get all of the RSS-eligible nodes
    eligible_node_set: Set[Node] = set()
    get_eligible_nodes(root_node, "rss-include", False, eligible_node_set)

    # keep track of which nodes we want to include
    nodes_to_write = []

    # pick out the most recent eligible nodes for inclusion
    for node in heapq.nsmallest(rss_max_entries,
                                eligible_node_set,
                                key=Node.get_sort_key):
        nodes_to_write.append(node)

    # write the RSS file
    fw = FileWriter(rss_file)

    fw.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
    fw.write("<rss version=\"2.0\">\n")

    # channel
    fw.write("<channel>\n")

    # title
    fw.write("    <title>" + escape(title) + "</title>\n")

    # link
    fw.write("    <link>" + escape(root_node.get_canonical_url()) + \
              "</link>\n")

    # description
    fw.write("    <description>" + escape(channel_description) + \
        "</description>\n")

    # lastBuildDate
    fw.write("    <lastBuildDate>" + \
        escape(get_rfc_2822_date(datetime.datetime.now())) + \
        "</lastBuildDate>\n")

    # image
    if canonical_rss_image_url is not None:
        fw.write("    <image>\n")

        # url
        fw.write("        <url>" + escape(canonical_rss_image_url) + \
            "</url>\n")

        # title
        fw.write("        <title>" + escape(title) + "</title>\n")

        # link
        fw.write("        <link>" + \
            escape(root_node.get_canonical_url()) + "</link>\n")

        # width
        if root_node.has_header("rss-image-width"):
            fw.write("        <width>" + \
                escape(root_node.get_header("rss-image-width")) + \
                "</width>\n")

        # height
        if root_node.has_header("rss-image-height"):
            fw.write("        <height>" + \
                escape(root_node.get_header("rss-image-height")) + \
                "</height>\n")

        fw.write("    </image>\n")

    # nodes
    for node in nodes_to_write:
        # render the node for RSS
        # this involves setting the template for the node to null
        # before rendering it, and then resetting the template back
        # to its original value
        template = None
        if node.has_header("template"):
            template = node.get_header("template")
        node.set_header("template", "null")

        # RSS-Add-Node-Title-Header
        add_node_title_header = \
            node.get_boolean_header_value("rss-add-node-title-header", True)

        # create page
        page = Page(project_root, node, use_canonical_url=True)

        # get node body
        rendered_node_body = page.render().rstrip()
        if add_node_title_header:
            item_description = \
                "<h1>" + escape(node.get_title()) + "</h1>\n\n" + \
                rendered_node_body
        else:
            item_description = rendered_node_body

        # set template back to its previous value (or lack thereof)
        if template is not None:
            node.set_header("template", template)
        else:
            node.delete_header("template")

        # item
        fw.write("    <item>\n")

        # title
        # (always escape the title, XML has stricter requirements)
        fw.write("        <title>" + escape(node.get_title()) + \
            "</title>\n")

        # link
        fw.write("        <link>" + escape(node.get_canonical_url()) + \
              "</link>\n")

        # description
        fw.write("        <description>")
        fw.write("<![CDATA[" + escape_xml(item_description) + "]]>")
        fw.write("</description>\n")

        # category
        for tag in sorted(node.get_tags()):
            fw.write("        <category>" + escape(tag) + \
                "</category>\n")

        # pubDate
        node_date = node.modified
        if node.created:
            node_date = node.created
        if node_date is None:
            # this is not expected to happen in normal operation,
            # but we provide a detailed error message and traceback
            # in case user-defined code triggers it somehow
            raise Exception(
                "node '%s' has no created or modified time" %
                (node.get_path()))
        fw.write("        <pubDate>" + \
            escape(get_rfc_2822_date(node_date)) + \
            "</pubDate>\n")

        fw.write("    </item>\n")

    fw.write("</channel>\n")
    fw.write("</rss>\n")

    fw.close()

def get_sitemap_url(root_node: Node) -> Optional[str]:
    """
    Get the sitemap URL (without a leading slash).

    Returns None if Sitemap-URL is not enabled.

    """

    # if sitemap is not enabled, stop here
    if not root_node.has_header("sitemap-url"):
        return None

    # get the full path to the sitemap file we're going to write on disk
    sitemap_url = root_node.get_header("sitemap-url")
    if is_directory_traversal_attempt(sitemap_url):
        errstr = "invalid Sitemap-URL header value in node '%s' " + \
                 "(directory traversal): '%s'"
        raise UrielError(errstr % (root_node.get_path(), sitemap_url))

    # if the sitemap URL starts with a slash,
    # normalize it to the root of the site
    if sitemap_url.startswith("/"):
        sitemap_url = sitemap_url[1:]

    return sitemap_url

def handle_write_sitemaps(project_root: str, root_node: Node) -> None:
    """
    Handle writing a sitemap, or a sitemap index and multiple sitemaps
    (if Sitemap-URL is enabled).

    Accepts the project root, and the root Node

    """

    # is the Sitemap-Index header set to true on the root node?
    # if not, just try to write one sitemap (if Sitemap-URL is enabled),
    # and return early
    index_enabled = root_node.get_boolean_header_value("sitemap-index", False)
    if index_enabled:
        write_sitemap_index_and_sitemap_files(project_root, root_node)
    else:
        write_sitemap(project_root, root_node)

def get_numbered_sitemap_url(sitemap_url: str, number: int) -> str:
    """
    Get a numbered sitemap URL varation on the given sitemap URL.

    For example, if "sitemap.xml.gz" is passed in with a number of 1,
    this will return "sitemap-001.xml.gz"

    Accepts the primary sitemap URL used for the sitemap index, and the
    number associated with this numbered sitemap variant.

    Returns a numbered sitemap URL, based on the original sitemap URL and
    number.

    """

    # sitemap_url is going to be something like:
    # sitemap.xml, sitemap.xml.gz, /sitemap.xml, /foo/sitemap.xml.gz, etc.

    # get the portion before the filename (if any)
    dirname = os.path.dirname(sitemap_url)

    # get the filename, without any directory portion
    basename = os.path.basename(sitemap_url)

    # get the portion of the filename before the first dot
    # for example, if we have sitemap.xml.gz, this will become
    # ["sitemap", "xml.gz"]
    # filename_parts will end up having either 1 or 2 elements
    filename_parts = basename.split(".", 1)

    # reassemble the URL
    url_parts = []

    # directory/path portion before the filename
    if "" != dirname:
        url_parts.append(dirname)

    # base filename, without extension
    url_parts.append(filename_parts[0])

    # -
    url_parts.append("-")

    # 001 (sitemap number)
    url_parts.append("%03d" % number)

    # file extension (xml.gz, etc.)
    if len(filename_parts) > 1:
        url_parts.append(".")
        url_parts.append(filename_parts[1])

    # return numbered sitemap URL
    return "".join(url_parts)

def write_sitemap_index_and_sitemap_files(project_root: str,
                                          root_node: Node) -> None:

    """
    Write a sitemap index, and multiple linked sitemaps.

    Accepts the project root, and the root Node

    """

    # get sitemap URL (we will use this as the sitemap index)
    sitemap_index_url = get_sitemap_url(root_node)
    if sitemap_index_url is None:
        return

    # get the path to the sitemap index file to write
    sitemap_index_file = os.path.join(root_node.get_dest_dir(),
                                      sitemap_index_url)

    # skip creating the file if there's a static file with the same name
    if os.path.exists(sitemap_index_file):
        log("skipping creation of '%s': file exists" %
            (printable_path(sitemap_index_file)))
    else:
        log("creating '%s'" % (printable_path(sitemap_index_file)))

    # get all of the sitemap-eligible nodes
    eligible_node_set: Set[Node] = set()
    get_eligible_nodes(root_node, "sitemap-include", True, eligible_node_set)

    # Canonical-URL
    if not root_node.has_header("canonical-url"):
        err = "Canonical-URL must be set on node '%s' to write sitemap"
        raise UrielError(err % (root_node.get_path()))
    canonical_url = root_node.get_canonical_url_base()

    # Sitemap-Max-Files
    sitemap_max_files = SITEMAP_MAX_FILES
    if root_node.has_header("sitemap-max-files"):
        value = root_node.get_header("sitemap-max-files")
        try:
            sitemap_max_files = int(value)
            if sitemap_max_files < 0:
                raise UrielError()

        except Exception:
            err = "Sitemap-Max-Files must be a valid unsigned integer " + \
                  "on node '%s'"
            raise UrielError(err % (root_node.get_path()))

    # Sitemap-Max-Entries
    sitemap_max_entries = SITEMAP_MAX_ENTRIES
    if root_node.has_header("sitemap-max-entries"):
        value = root_node.get_header("sitemap-max-entries")
        try:
            sitemap_max_entries = int(value)
            if sitemap_max_entries < 0:
                raise UrielError()

        except Exception:
            err = "Sitemap-Max-Entries must be a valid unsigned integer " + \
                  "on node '%s'"
            raise UrielError(err % (root_node.get_path()))

    # figure out how many nodes we want to include across all sitemap files
    total_node_count = len(eligible_node_set)
    max_capacity = (sitemap_max_files * sitemap_max_entries)

    # raise an error if we have more nodes than we can fit across all sitemaps
    if total_node_count > max_capacity:
        err = "sitemap capacity exceeded " + \
            "(eligible nodes: %d, sitemap total capacity: %d)\n" % \
            (total_node_count, max_capacity)

        maybe_default_files_str = ""
        if not root_node.has_header("sitemap-max-files"):
            maybe_default_files_str = " (default)"
        err += "       Sitemap-Max-Files: %s%s\n" % \
            (sitemap_max_files, maybe_default_files_str)

        maybe_default_entries_str = ""
        if not root_node.has_header("sitemap-max-entries"):
            maybe_default_entries_str = " (default)"
        err += "       Sitemap-Max-Entries: %s%s\n" % \
            (sitemap_max_entries, maybe_default_entries_str)

        err += "       sitemap creation failed"

        raise UrielError(err)

    # sitemap number -> list of nodes
    sitemap_number_to_nodes_map: Dict[int, List[Node]] = {}

    # sitemap number -> sitemap URL path (without canonical portion)
    sitemap_number_to_sitemap_url: Dict[int, str] = {}

    # start sitemap files with a 1-based number by convention
    cur_sitemap = 1

    # keep track of how many records we have added to the current sitemap
    cur_sitemap_records = 0

    # if we are going to write any sitemap files at all,
    # figure out what we want to include in them
    if sitemap_max_entries > 0:
        # go through all of the nodes we want to write
        for node in sorted(eligible_node_set, key=lambda n: n.get_url()):
            # if the current sitemap is at capacity, switch to a new one
            if cur_sitemap_records >= sitemap_max_entries:
                cur_sitemap += 1
                cur_sitemap_records = 0

            # if this is a new sitemap, and it isn't in our maps
            if cur_sitemap not in sitemap_number_to_nodes_map:
                # create an empty node list for this sitemap
                sitemap_number_to_nodes_map[cur_sitemap] = []

                # create a URL for this sitemap
                sitemap_number_to_sitemap_url[cur_sitemap] = \
                    get_numbered_sitemap_url(sitemap_index_url, cur_sitemap)

            # add this node to the current sitemap
            sitemap_number_to_nodes_map[cur_sitemap].append(node)
            cur_sitemap_records += 1

    # figure out the canonical URL for each sitemap we want to write
    sitemap_urls = []
    for sitemap_number in sorted(sitemap_number_to_sitemap_url):
        sitemap_url = sitemap_number_to_sitemap_url[sitemap_number]
        sitemap_canonical_url = canonical_url + "/" + sitemap_url
        sitemap_urls.append(sitemap_canonical_url)

    # write sitemap index file
    # (unless there is a static file with the same name)
    if not os.path.exists(sitemap_index_file):
        write_sitemap_index_file(sitemap_index_file, sitemap_urls)

    # write sitemap files
    for sitemap_number in sorted(sitemap_number_to_nodes_map):
        nodes_to_write = sitemap_number_to_nodes_map[sitemap_number]
        sitemap_url = sitemap_number_to_sitemap_url[sitemap_number]
        sitemap_file = os.path.join(root_node.get_dest_dir(), sitemap_url)

        # skip creating the file if there's a static file with the same name
        if os.path.exists(sitemap_file):
            log("skipping creation of '%s': file exists" %
                (printable_path(sitemap_file)))
            continue

        log("creating '%s'" % (printable_path(sitemap_file)))
        write_sitemap_file(sitemap_file, nodes_to_write)

def write_sitemap_index_file(sitemap_index_file: str,
                             sitemap_urls: List[str]) -> None:

    """
    Actually write the specified sitemap index file to disk, using the list of
    sitemap URLs to include in the sitemap index.

    Accepts the path to the sitemap index file to write, and a list of sitemap
    URLs to include in the sitemap index file.

    """

    # get the current date/time
    now = datetime.datetime.now()

    # open the sitemap index file for writing
    # how we do this depends on whether the file has a .gz extension
    fw: Any = None
    if sitemap_index_file.lower().endswith(".gz"):
        fw = GzipFileWriter(sitemap_index_file)
    else:
        fw = FileWriter(sitemap_index_file)

    # write the sitemap index

    fw.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
    fw.write("<sitemapindex " + \
        "xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n")

    # sitemaps
    for url in sitemap_urls:
        # url
        fw.write("    <sitemap>\n")

        # loc
        fw.write("        <loc>" + escape(url) + "</loc>\n")

        # lastmod
        fw.write("        <lastmod>" + escape(get_w3c_datetime(now)) + \
            "</lastmod>\n")

        fw.write("    </sitemap>\n")

    fw.write("</sitemapindex>\n")
    fw.close()

def write_sitemap_file(sitemap_file: str, nodes_to_write: List[Node]) -> None:
    """
    Actually write the specified sitemap file to disk, using the list of nodes
    to include in the sitemap.

    Accepts the path to the sitemap file to write, and a list of nodes to
    include in the sitemap file.

    This is a lower-level function than write_sitemap(). We assume that
    everything has been decided and validated already, and now we're just
    going to actually write the file, using whatever nodes were provided.

    """

    # open the sitemap file for writing
    # how we do this depends on whether the file has a .gz extension
    fw: Any = None
    if sitemap_file.lower().endswith(".gz"):
        fw = GzipFileWriter(sitemap_file)
    else:
        fw = FileWriter(sitemap_file)

    # write the sitemap

    fw.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
    fw.write("<urlset " + \
        "xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n")

    # nodes
    for node in nodes_to_write:
        # url
        fw.write("    <url>\n")

        # loc
        fw.write("        <loc>" + escape(node.get_canonical_url()) + \
            "</loc>\n")

        # lastmod
        if node.modified is None:
            # this is not expected to happen in normal operation,
            # but we provide a detailed error message and traceback
            # in case user-defined code triggers it somehow
            errstr = "node '%s' modified date can not be null"
            raise Exception(errstr % (node.get_path()))
        fw.write("        <lastmod>" + \
            escape(get_w3c_datetime(node.modified)) + \
            "</lastmod>\n")

        fw.write("    </url>\n")

    fw.write("</urlset>\n")
    fw.close()

def write_sitemap(project_root: str, root_node: Node) -> None:
    """
    Write a single sitemap file to disk (if Sitemap-URL enabled).

    Accepts the project root, and the root Node.

    """

    # get sitemap URL
    sitemap_url = get_sitemap_url(root_node)
    if sitemap_url is None:
        return

    # get the path to the sitemap file to write
    sitemap_file = os.path.join(root_node.get_dest_dir(), sitemap_url)

    # skip creating the file if there's a static file with the same name
    if os.path.exists(sitemap_file):
        log("skipping creation of '%s': file exists" %
            (printable_path(sitemap_file)))
        return

    # get all of the sitemap-eligible nodes
    eligible_node_set: Set[Node] = set()
    get_eligible_nodes(root_node, "sitemap-include", True, eligible_node_set)

    log("creating '%s'" % (printable_path(sitemap_file)))

    # Canonical-URL
    if not root_node.has_header("canonical-url"):
        err = "Canonical-URL must be set on node '%s' to write sitemap"
        raise UrielError(err % (root_node.get_path()))

    # Sitemap-Max-Entries
    sitemap_max_entries = SITEMAP_MAX_ENTRIES
    if root_node.has_header("sitemap-max-entries"):
        value = root_node.get_header("sitemap-max-entries")
        try:
            sitemap_max_entries = int(value)
            if sitemap_max_entries < 0:
                raise UrielError()

        except Exception:
            err = "Sitemap-Max-Entries must be a valid unsigned integer " + \
                  "on node '%s'"
            raise UrielError(err % (root_node.get_path()))

    # keep track of which nodes we want to include
    nodes_to_write = []

    # gather the nodes to add to the sitemap
    item_count = 0
    for node in sorted(eligible_node_set, key=lambda n: n.get_url()):
        item_count += 1
        if item_count > sitemap_max_entries:
            err = "sitemap capacity exceeded " + \
                "(eligible nodes: %d, sitemap total capacity: %d)\n" % \
                (len(eligible_node_set), sitemap_max_entries)

            if root_node.has_header("sitemap-index"):
                err += "       Sitemap-Index: false\n"
            else:
                err += "       Sitemap-Index: false (default)\n"

            maybe_default_entries_str = ""
            if not root_node.has_header("sitemap-max-entries"):
                maybe_default_entries_str = " (default)"
            err += "       Sitemap-Max-Entries: %s%s\n" % \
                (sitemap_max_entries, maybe_default_entries_str)

            err += "       sitemap creation failed"

            raise UrielError(err)

        nodes_to_write.append(node)

    # actually write the sitemap file to disk
    write_sitemap_file(sitemap_file, nodes_to_write)

def write_robots_txt(project_root: str, root_node: Node) -> None:
    """
    Write robots.txt file.

    Accepts the project_root, and the root Node.

    Only writes robots.txt if Sitemap-URL is enabled.

    """

    # get the path to the robots.txt file
    robots_txt_file = os.path.join(root_node.get_dest_dir(), "robots.txt")
    if os.path.exists(robots_txt_file):
        log("skipping creation of '%s': file exists" %
            (printable_path(robots_txt_file)))
        return

    # get the sitemap URL (without a leading slash)
    sitemap_url = get_sitemap_url(root_node)

    # if we don't have a sitemap, there's no point to creating this
    # particular form of robots.txt that only includes a Sitemap entry
    if sitemap_url is None:
        return

    # write robots.txt
    #
    # this is just enough to get the sitemap listed,
    # a real robots.txt on a production website might be better off
    # using this as a starting point and copying it into static/
    if not os.path.exists(robots_txt_file):
        log("creating '%s'" % (printable_path(robots_txt_file)))
        fw = FileWriter(robots_txt_file)
        fw.write("Sitemap: " + \
                root_node.get_canonical_url() + sitemap_url + "\n")
        fw.close()

def get_default_template_contents() -> str:
    """
    Get the contents of the initial default template, as a multi-line string.

    """

    lines = []

    lines.append("<!DOCTYPE html>")
    lines.append("<html lang=\"en-US\">")
    lines.append("<head>")
    lines.append("    <meta charset=\"UTF-8\">")
    lines.append("    <title>{{node:title}}</title>")
    lines.append("</head>")
    lines.append("")
    lines.append("<body>")
    lines.append("")
    lines.append("<h1>{{node:title}}</h1>")
    lines.append("")
    lines.append("<p>The value of the <i>Foo</i> header is: ")
    lines.append("<b>{{value:foo}}</b></p>")
    lines.append("")
    lines.append("{{node:body}}")
    lines.append("")
    lines.append(
        "<p>This page was generated by " + \
        "<a href=\"https://uriel.foo/\">Uriel</a></p>")
    lines.append("")
    lines.append("</body>")
    lines.append("</html>")
    lines.append("")

    return "\n".join(lines)

def get_default_index_node_contents() -> str:
    """
    Get the contents of the default index node, as a multi-line string.

    """

    lines = []

    lines.append("Title: Hello World")
    lines.append("Foo: bar")
    lines.append("")
    lines.append("<p>This page is generated from a combination of the ")
    lines.append("<i>index</i> node and the <i>default.html</i> ")
    lines.append("template.</p>")
    lines.append("")
    lines.append("<p>Replace this with your own content, etc.</p>")
    lines.append("")

    return "\n".join(lines)

def get_default_soju_contents() -> str:
    """
    Get the contents of the default soju.py file, as a multi-line string.

    """

    lines = []

    lines.append("#" * 78)
    lines.append("# soju.py" + (" " * 68) + "#")
    lines.append("#" * 78)
    lines.append("")
    lines.append("# The following symbols are imported using magic:")
    lines.append("#")
    lines.append("# import uriel")
    lines.append("# from uriel import SojuError")
    lines.append("# from uriel import log")
    lines.append("# from uriel import escape")
    lines.append("")
    lines.append("# The following variables are available to pass to " + \
                 "functions:")
    lines.append("#")
    lines.append("# page")
    lines.append("# node")
    lines.append("# project_root")
    lines.append("# use_canonical_url")
    lines.append("")
    lines.append("# {{soju:node_title(node)}}")
    lines.append("def node_title(node):")
    lines.append("    return escape(node.get_title())")
    lines.append("")
    lines.append("")

    return "\n".join(lines)

def get_default_handlers_contents() -> str:
    """
    Get the contents of the default handlers.py file, as a multi-line string.

    """

    lines = []

    lines.append("#" * 78)
    lines.append("# handlers.py" + (" " * 64) + "#")
    lines.append("#" * 78)
    lines.append("")
    lines.append("# The following symbols are imported using magic:")
    lines.append("#")
    lines.append("# import uriel")
    lines.append("# from uriel import Page")
    lines.append("# from uriel import Node")
    lines.append("# from uriel import FileNode")
    lines.append("# from uriel import VirtualNode")
    lines.append("# from uriel import HandlerError")
    lines.append("# from uriel import log")
    lines.append("# from uriel import escape")
    lines.append("")
    lines.append("#def init(project_root):")
    lines.append("#    pass")
    lines.append("")
    lines.append("#def before_render_node_tree(project_root, root_node):")
    lines.append("#    pass")
    lines.append("")
    lines.append("#def after_render_node_tree(project_root, root_node):")
    lines.append("#    pass")
    lines.append("")
    lines.append("#def cleanup(project_root, root_node):")
    lines.append("#    pass")
    lines.append("")
    lines.append("")

    return "\n".join(lines)

def get_default_makefile_contents() -> str:
    """
    Get the contents of the default Makefile, as a multi-line string.

    """

    lines = []

    lines.append("#" * 78)
    lines.append("# uriel project Makefile " + (" " * 52) + "#")
    lines.append("#" * 78)
    lines.append("")
    lines.append("# path to uriel")
    lines.append("URIEL=" + os.path.abspath(__file__))
    lines.append("")
    lines.append("# uriel project subdirectories")
    lines.append("STATIC=static")
    lines.append("NODES=nodes")
    lines.append("TEMPLATES=templates")
    lines.append("PUBLIC=public")
    lines.append("LIB=lib")
    lines.append("")
    lines.append("#" * 78)
    lines.append("# targets " + (" " * 67) + "#")
    lines.append("#" * 78)
    lines.append("")
    lines.append(".PHONY: site clean preview")
    lines.append("")
    lines.append("site:")
    lines.append("\t${URIEL} .")
    lines.append("")
    lines.append("clean:")
    lines.append("\trm -rf ${PUBLIC}/")
    lines.append("\trm -rf ${LIB}/__pycache__")
    lines.append("")
    lines.append("preview: site")
    lines.append("\tcd ${PUBLIC}/ && python3 -m http.server")
    lines.append("")
    lines.append("")

    return "\n".join(lines)

def init_project_root(project_root: str,
                      project_root_just_created: bool) -> None:

    """
    (Re)initialize the rendered web site public directory with only the
    static content, deleting everything else.

    Accepts the project root directory, and a boolean indicating whether the
    project root directory was just created for the first time.

    Create any missing directories and files required for the project:
        static      - static files to include in the website
        nodes       - dynamic node pages
        templates   - templates to merge with dynamic nodes
        public      - rendered website

    If the project root was just created (and therefore this is the initial
    creation of this project), then some additional non-required but helpful
    files will be created as well. If these files are deleted, they will
    not be recreated when uriel is run against the project again.

    """

    # section root directories
    static = os.path.join(project_root, STATIC_ROOT)
    nodes = os.path.join(project_root, NODES_ROOT)
    templates = os.path.join(project_root, TEMPLATES_ROOT)
    lib = os.path.join(project_root, LIB_ROOT)
    public = os.path.join(project_root, PUBLIC_ROOT)

    # create all of the required project directories, if they don't exist
    for dirname in [nodes, templates]:
        if not os.path.isdir(dirname):
            log("creating '%s'" % (printable_path(dirname)))
            try:
                os.mkdir(dirname)
            except Exception as e:
                err = "could not create directory '%s': '%s'"
                raise UrielError(err % (dir, str(e)))

    # create all of the optional project directories, if they don't exist
    if project_root_just_created:
        for dirname in [static, lib]:
            if not os.path.isdir(dirname):
                log("creating '%s'" % (printable_path(dirname)))
                try:
                    os.mkdir(dirname)
                except Exception as e:
                    err = "could not create directory '%s': '%s'"
                    raise UrielError(err % (dirname, str(e)))

    # if we have a static directory, copy it over the public directory,
    # deleting any previously generated contents in public
    if os.path.exists(static):
        log("copying '%s' to '%s', overwriting previous contents" %
            (printable_path(static), printable_path(public)))

        # (re)initialize public directory with only the static content
        copy_files_recursive_overwrite(static, public)

    # if we don't have static/, we need to get public/ into a good known
    # state, by deleting it and recreating it
    else:
        log("skipping static file copy, '%s' not found" %
            (printable_path(static)))

        # if we have a public directory, but no static directory, it means
        # that we effectively want to have an empty public directory, since
        # everything that could exist in there would have been created from
        # a previous run. the first step is to delete the public directory.
        if os.path.exists(public):
            log("deleting '%s'" % (printable_path(public)))

            # delete the directory
            delete_directory_recursive(public)

        log("creating '%s'" % (printable_path(public)))

        # create a new public directory
        try:
            os.mkdir(public)
        except Exception as e:
            err = "could not create directory '%s': '%s'"
            raise UrielError(err % (public, str(e)))

    # create a simple default template, if it doesn't already exist
    default_template = os.path.join(templates, DEFAULT_TEMPLATE)
    if not os.path.exists(default_template):
        log("creating '" + printable_path(default_template) + "'")
        fw = FileWriter(default_template)
        fw.write(get_default_template_contents())
        fw.close()

    # create a simple default home page node, if it doesn't already exist
    default_node = os.path.join(nodes, NODE_INDEX)
    if not os.path.exists(default_node):
        log("creating '" + printable_path(default_node) + "'")
        fw = FileWriter(default_node)
        fw.write(get_default_index_node_contents())
        fw.close()

    # optional files
    if project_root_just_created:
        # create a simple default soju file
        default_soju = os.path.join(lib, "soju.py")
        if not os.path.exists(default_soju):
            log("creating '" + printable_path(default_soju) + "'")
            fw = FileWriter(default_soju)
            fw.write(get_default_soju_contents())
            fw.close()

        # create a simple default handlers file
        default_handlers = os.path.join(lib, "handlers.py")
        if not os.path.exists(default_handlers):
            log("creating '" + printable_path(default_handlers) + "'")
            fw = FileWriter(default_handlers)
            fw.write(get_default_handlers_contents())
            fw.close()

        # create a simple Makefile
        default_makefile = os.path.join(project_root, "Makefile")
        if not os.path.exists(default_makefile):
            log("creating '" + printable_path(default_makefile) + "'")
            fw = FileWriter(default_makefile)
            fw.write(get_default_makefile_contents())
            fw.close()

def get_uriel_module() -> ModuleType:
    """
    Get a reference to ourselves as a module.

    """

    return sys.modules[__name__]

def init_modules(project_root: str) -> None:
    """
    Initialize the modules we can use under LIB_ROOT.

    """

    # get the path to the lib directory
    lib_dir = os.path.join(project_root, LIB_ROOT)

    # if the lib directory was deleted, skip initializing modules
    if not os.path.exists(lib_dir):
        log("skipping module initialization, '%s' not found" %
            (printable_path(lib_dir)))
        return

    # add the project lib dir to our module import path
    sys.path.append(lib_dir)

    # initialize soju module
    lib_soju_file = os.path.join(lib_dir, "soju.py")
    if os.path.exists(lib_soju_file):
        log("initializing soju")
        import soju  # type: ignore

        # export relevant symbols to soju
        soju.uriel = get_uriel_module()
        soju.SojuError = SojuError
        def soju_log(s):
            return log("soju: " + str(s))
        soju.log = soju_log
        soju.escape = escape

        # make this module available to the rest of the program
        globals()["soju"] = soju

    else:
        log("skipping module initialization, '%s' not found" %
            (printable_path(lib_soju_file)))

    # initialize handlers module
    lib_handlers_file = os.path.join(lib_dir, "handlers.py")
    if os.path.exists(lib_handlers_file):
        log("initializing handlers")
        import handlers  # type: ignore

        # export relevant symbols to handlers
        handlers.uriel = get_uriel_module()
        handlers.Page = Page
        handlers.Node = Node
        handlers.FileNode = FileNode
        handlers.VirtualNode = VirtualNode
        handlers.HandlerError = HandlerError
        def handlers_log(s):
            return log("handler: " + str(s))
        handlers.log = handlers_log
        handlers.escape = escape

        # make this module available to the rest of the program
        globals()["handlers"] = handlers

    else:
        log("skipping module initialization, '%s' not found" %
            (printable_path(lib_handlers_file)))

def write_additional_files(project_root: str, node: Node) -> None:
    """
    Write additional files after the nodes have been generated.

    """

    # write RSS feed out to disk (if enabled)
    write_rss(project_root, node)

    # write sitemap (or multiple sitemaps) out to disk (if enabled)
    handle_write_sitemaps(project_root, node)

    # write robots.txt (if sitemap is enabled)
    write_robots_txt(project_root, node)

def copy_static_files(project_root: str) -> None:
    """
    Copy the static files into the public website, without modification.

    """

    # create path to src directory
    src = os.path.join(project_root, STATIC_ROOT)

    # if we don't have a static directory, just skip this step
    if not os.path.exists(src):
        # if we got here, this would be the second time the user has
        # been notified that this directory doesn't exist, so skip
        # the log message as well
        return

    # create path to dest directory
    dest = os.path.join(project_root, PUBLIC_ROOT)

    log("copying '%s' to '%s'" %
        (printable_path(src), printable_path(dest)))

    # copy the files
    copy_files_recursive(src, dest)

def create_project_root(project_root: str) -> bool:
    """
    Create the project root directory, if it doesn't exist.

    Returns True if the project root directory didn't exist before and was
    newly created.

    Returns False if the project root directory already existed.

    Raises a UrielError if we tried to create the project root, but failed.

    """

    if not os.path.exists(project_root):
        try:
            log("creating '%s'" % (printable_path(project_root)))
            os.mkdir(project_root)
        except Exception as e:
            raise UrielError("could not create project root '%s'" %
                             (project_root))

        # we just created the project root directory
        return True

    # the project root directory already existed
    return False

def call_handler(handler_name: str, *args: Any) -> Any:
    """
    Run a user-defined handler.

    Accepts the handler name, and variable arguments to the handler function.

    """

    # is the handlers module loaded?
    if "handlers" not in globals():
        return

    # is the specified handler defined in the handlers module?
    if handler_name not in dir(globals()["handlers"]):
        return

    # create a reference to the user-defined handler function we want to call
    handler_function = eval("globals()['handlers']." + handler_name)

    log("running handler: %s" % (handler_name))

    # try to run the user-defined handler function
    try:
        handler_function(*args)

    # if we get a HandlerError, log it, and raise the exception
    except HandlerError as e:
        # get a suitable error string to represent this error
        # this will either be the error string from the exception,
        # wrapped in quotes, or the name of the exception type itself
        error_string = str(e)
        if "" == error_string:
            error_string = get_exception_reason(e)
        else:
            error_string = "'" + error_string + "'"

        # create summary error message
        summary_error = "error in handler function %s(): %s" % \
                        (handler_name, error_string)

        # raise new chained exception with summary error message
        raise HandlerError(summary_error) from e

    # for unexpected exceptions originating from user-defined handler code,
    # show the subset of the stack trace that applies to their code,
    # and raise a chained summary exception
    except Exception as e:
        # get a suitable error string to represent this error
        # this will either be the error string from the exception,
        # wrapped in quotes, or the name of the exception type itself
        error_string = str(e)
        if "" == error_string:
            error_string = get_exception_reason(e)
        else:
            error_string = "'" + error_string + "'"

        # create summary error message
        summary_error = "error in handler function %s(): %s" % \
                        (handler_name, error_string)

        # log filtered traceback
        log_filtered_traceback(e, "handlers.py")

        # log the exception type (and the error string, if available)
        error_string_rvalue = str(e)
        if "" == error_string_rvalue:
            log(type(e).__name__)
        else:
            log(type(e).__name__ + ": '" + error_string_rvalue + "'")

        # raise the original exception with the summary error
        raise HandlerError(summary_error) from e

def handle_project(project_root: str) -> None:
    """
    Handle whatever needs to be done for the project.

    If the project_root does not exist, then create a new project.

    If the project_root does exist, then build it.

    Exits the program upon completion, with a 0 exit code for
    success, and a 1 exit code on failure.

    """

    try:
        # create project root directory (if necessary)
        project_root_just_created = create_project_root(project_root)

        # initialize the project root
        init_project_root(project_root, project_root_just_created)

        # initialize modules under lib/
        init_modules(project_root)

        # handler: init
        call_handler("init", project_root)

        # create a tree of all the file-based nodes (in memory)
        log("reading node files")
        node = create_file_node_tree(project_root)

        # handler: before_render_node_tree
        call_handler("before_render_node_tree",
                     project_root,
                     node)

        # augment nodes with additional values before rendering
        augment_node_tree(project_root, node)

        # render nodes using templates (in memory)
        log("rendering node content")
        render_node_tree(project_root, node)

        # handler: after_render_node_tree
        call_handler("after_render_node_tree",
                     project_root,
                     node)

        # write rendered node pages out to disk
        write_dynamic_nodes(project_root, node)

        # write additional files
        write_additional_files(project_root, node)

        # copy static files into the public directory
        copy_static_files(project_root)

        # handler: cleanup
        call_handler("cleanup", project_root, node)

    except SojuError as e:
        log("soju: " + get_exception_reason(e))
        sys_exit(EXIT_FAIL)

    except HandlerError as e:
        die(get_exception_reason(e))

    except UrielError as e:
        die(get_exception_reason(e))

    except _ImpossibleError as e:
        print_impossible_error()
        raise

    except KeyboardInterrupt:
        sys_exit(EXIT_FAIL)

    sys_exit(EXIT_OK)

def main() -> None:
    """
    Main entry point into the program.

    """

    # show usage information if we didn't get exactly one argument
    if 2 != len(sys.argv):
        show_usage()

    # get the project root
    project_root = sys.argv[1]

    # short and long options are not supported, only the project root
    # if the user passes in -h or --help or similar, show usage and exit
    if project_root.startswith("-"):
        show_usage()

    # now that we have the project root, handle this run for the project
    handle_project(project_root)

if __name__ == "__main__":
    main()
