checkhtml.py
Strict HTML parse, dead links, feed and sitemap
156 lines. This is the file the build actually runs, copied verbatim at build time.
"""Parse every generated page with html5lib in strict mode and report problems.
Also does a few checks a parser won't: internal links resolve to real files,
referenced assets exist, and no page accidentally references a third-party host.
"""
import os
import re
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "pylib"))
ROOT = os.path.abspath(os.path.join(HERE, ".."))
import html5lib # noqa: E402
PAGES = ["index.html", "tokens/index.html", "vocabulary/index.html",
"predict/index.html", "cost/index.html", "learn/index.html",
"about/index.html"]
# Hosts a page is allowed to link to in href/src. Anything else is a surprise.
ALLOWED_EXTERNAL = {"github.com"}
SITE = "https://sweedworks.com"
def local_target(url):
"""Map a site-absolute URL to a file on disk, or None if not local.
Absolute URLs to our own domain count as local: og:image, og:url and
canonical all have to be absolute, and they still need to resolve.
"""
if url.startswith(SITE):
url = url[len(SITE):] or "/"
if url.startswith(("http://", "https://", "mailto:", "#", "data:")):
return None
path = url.split("#")[0].split("?")[0]
if not path.startswith("/"):
return None
fs = os.path.join(ROOT, path.lstrip("/"))
if path.endswith("/"):
fs = os.path.join(fs, "index.html")
return fs
def main():
problems = []
for page in PAGES:
full = os.path.join(ROOT, page)
src = open(full, encoding="utf-8").read()
parser = html5lib.HTMLParser(strict=True)
try:
parser.parse(src)
print(f"PASS {page:<20} parses clean")
except Exception as e:
problems.append(f"{page}: parse error: {e}")
print(f"FAIL {page:<20} {str(e)[:120]}")
continue
# Links and assets
for attr in ("href", "src", "content"):
for url in re.findall(rf'{attr}="([^"]+)"', src):
if url.startswith(("http://", "https://")) and \
not url.startswith(SITE):
host = url.split("/")[2]
if host not in ALLOWED_EXTERNAL:
problems.append(f"{page}: unexpected external host {host}")
continue
fs = local_target(url)
if fs and not os.path.isfile(fs):
problems.append(f"{page}: dead local link {url} -> {fs}")
# Accessibility / metadata basics
if "<h1" not in src:
problems.append(f"{page}: no <h1>")
if 'name="description"' not in src:
problems.append(f"{page}: no meta description")
if src.count("<title>") != 1:
problems.append(f"{page}: expected exactly one <title>")
if "lang=" not in src.split(">")[1]:
problems.append(f"{page}: <html> missing lang")
# No page should ship a stray template artefact. Patterns are anchored so
# ordinary prose ("None of this is mysterious…") doesn't trip them.
ARTEFACTS = [
(r">None<", "bare None rendered into markup"),
(r"=\"None\"", "None in an attribute"),
(r"\bNaN\b", "NaN"),
(r"\bundefined\b", "undefined"),
(r"\{d\[", "unexpanded f-string"),
(r"\{DATA", "unexpanded f-string"),
(r"\{[a-z_]+\[[\"']", "unexpanded f-string"),
(r"\{[A-Z][A-Z_]{2,}\}", "unexpanded template placeholder"),
]
for page in PAGES:
src = open(os.path.join(ROOT, page), encoding="utf-8").read()
for pattern, label in ARTEFACTS:
if re.search(pattern, src):
problems.append(f"{page}: {label} in output")
# Feed and sitemap: well-formed, and pointing only at pages that exist.
import xml.etree.ElementTree as ET
atom = "{http://www.w3.org/2005/Atom}"
sm = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
try:
feed = ET.parse(os.path.join(ROOT, "feed.xml")).getroot()
entries = feed.findall(atom + "entry")
if not entries:
problems.append("feed.xml: no entries")
for e in entries:
link = e.find(atom + "link")
href = link.get("href") if link is not None else ""
path = href.replace("https://sweedworks.com", "")
target = local_target(path)
if not target or not os.path.isfile(target):
problems.append(f"feed.xml: entry points at missing page {href}")
print(f"PASS feed.xml {len(entries)} entries, all resolve")
except Exception as e:
problems.append(f"feed.xml: {e}")
try:
smap = ET.parse(os.path.join(ROOT, "sitemap.xml")).getroot()
locs = [u.text for u in smap.iter(sm + "loc")]
for loc in locs:
target = local_target(loc.replace("https://sweedworks.com", ""))
if not target or not os.path.isfile(target):
problems.append(f"sitemap.xml: missing page {loc}")
print(f"PASS sitemap.xml {len(locs)} urls, all resolve")
except Exception as e:
problems.append(f"sitemap.xml: {e}")
# Every published page should advertise the feed.
for page in PAGES:
src = open(os.path.join(ROOT, page), encoding="utf-8").read()
if 'type="application/atom+xml"' not in src:
problems.append(f"{page}: no feed discovery link")
# /.build/ is denied by the web server, so a link into it is a dead link
# that this checker would otherwise pass, since the file exists on disk.
if 'href="/.build/' in src or 'src="/.build/' in src:
problems.append(f"{page}: links into /.build/, which returns 403")
print()
if problems:
print(f"{len(problems)} problem(s):")
for p in problems:
print(f" - {p}")
return 1
print("All pages valid, all local links resolve, no third-party assets.")
return 0
if __name__ == "__main__":
sys.exit(main())