mithril-vndb/scripts/lint-docs.js

#!/usr/bin/env node
"use strict"

const {promises: fs} = require("fs")
const path = require("path")
const {Glob} = require("glob")
const marked = require("marked")
// Accept just about anything
const babelParser = require("@babel/parser")
// Peer dependency on `request`
const request = require("request-promise-native")

// lint rules
class LintRenderer extends marked.Renderer {
	constructor(file) {
		super()
		this._dir = path.dirname(file)
		this._context = undefined
		this._code = undefined
		this._lang = undefined
		this._error = undefined
		this._awaiting = []
		this._warnings = []
		this._errors = []
	}

	_addWarning(...data) {
		this._warnings.push(formatMessage(...data))
	}

	_addError(...data) {
		this._errors.push(formatMessage(...data))
	}

	_block() {
		return `\`\`\`${this._lang || ""}\n${this._code}\n\`\`\``
	}

	link(href) {
		// Don't fail if something byzantine shows up - it's the freaking
		// internet. Just log it and move on.
		const httpError = (e) =>
			this._addWarning(`http error for ${href}`, e.message)

		// Prefer https: > http: where possible, but allow http: when https: is
		// inaccessible.
		if ((/^https?:\/\//).test(href)) {
			const url = href.replace(/#.*$/, "")
			const isHTTPS = href.startsWith("https:")
			// pass along realistic headers, some sites (i.e. the IETF) return a 403 otherwise.
			const headers = {
				"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:71.0) Gecko/20100101 Firefox/71.0",
			}
			// some more headers if more were ever needed (from my local Firefox)

			// "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
			// "Accept-Language": "en-US,en;q=0.5",
			// "Accept-Encoding": "gzip, deflate, br",
			// "DNT": "1",
			// "Connection": "keep-alive",
			// "Upgrade-Insecure-Requests": "1",
			// "Pragma": "no-cache",
			// "Cache-Control": "no-cache"

			this._awaiting.push(request.head(url, {headers}).then(() => {
				if (!isHTTPS) {
					return request.head(`https:${url.slice(7)}`, {headers}).then(
						() => this._addError("change http: to https:"),
						() => { /* ignore inner errors */ }
					)
				}
			}, (e) => {
				if (e.statusCode === 404) {
					this._addError(`broken external link: ${href}`)
				}
				else {
					if (
						isHTTPS && (
							e.error.code === "ERR_TLS_CERT_ALTNAME_INVALID" ||
							(/ssl/i).test(e.message)
						)
					) {
						return request.head(`http:${url.slice(6)}`, {headers}).then(
							() => this._addError(`change ${href} to use http:`),
							// ignore inner errors
							() => httpError(e)
						)
					}
					httpError(e)
				}
			}))
		}
		else {
			const exec = (/^([^#?]*\.md)(?:$|\?|#)/).exec(href)
			if (exec != null) {
				const resolved = path.resolve(this._dir, exec[1])
				this._awaiting.push(fs.access(resolved).catch(() => {
					this._addError(`broken internal link: ${href}`)
				}))
			}
		}
	}

	code(code, lang) {
		this._code = code
		this._lang = lang
		this._error = null

		if (!lang || lang === "js" || lang === "javascript") {
			try {
				// Could be within any production.
				babelParser.parse(code, {
					sourceType: "unambiguous",
					allowReturnOutsideFunction: true,
					allowAwaitOutsideFunction: true,
					allowSuperOutsideMethod: true,
					allowUndeclaredExports: true,
					plugins: ["dynamicImport"],
				})
			}
			catch (e) {
				this._error = e
			}
		}
		this._ensureCodeIsHighlightable()
		this._ensureCodeIsSyntaticallyValid()
		this._ensureCommentStyle()
	}

	_ensureCodeIsHighlightable() {
		// We only care about what's not tagged here.
		if (!this._lang) {
			// TODO: ensure all code blocks have tags, and check this in CI.
			if (this._error == null) {
				this._addError(
					"Code block possibly missing `javascript` language tag",
					this._block(),
				)
			}

			try {
				JSON.parse(this._code)
				this._addError(
					"Code block possibly missing `json` language tag",
					this._block(),
				)
			}
			catch (_) {
				// ignore
			}
		}
	}

	_ensureCodeIsSyntaticallyValid() {
		if (!this._lang || !(/^js$|^javascript$/).test(this._lang)) return
		if (this._error != null) {
			this._addError(
				"JS code block has invalid syntax", this._error.message,
				this._block()
			)
		}
	}

	_ensureCommentStyle() {
		if (!this._lang || !(/^js$|^javascript$/).test(this._lang)) return
		if ((/(^|\s)\/\/[\S]/).test(this._code)) {
			this._addError("Comment is missing a preceding space", this._block())
		}
	}
}

async function getFileInfo(file) {
	const {size, mtime} = await fs.stat(file)
	const timestamp = Number(mtime)
	return {size, timestamp}
}

function report(file, data, totals, nextCache) {
	const {_warnings, _errors} = data;
	if (_warnings.length + _errors.length > 0) {
		console.log("- ".repeat(file.length/2 + 1))
		console.log(file)
		console.log("- ".repeat(file.length/2 + 1) + "\n")
		if (_errors.length > 0) {
			process.exitCode = 1
			const s = _errors.length > 1 ? "s " : " -"
			console.log(`-- ${_errors.length} Error${s}----------`)
			_errors.forEach((msg) => console.log(`\n${msg}`))
			console.log("\n")
		}
		if (_warnings.length > 0) {
			const s = _warnings.length > 1 ? "s " : " -"
			console.log(`-- ${_warnings.length} Warning${s}--------`)
			_warnings.forEach((msg) => console.log(`\n${msg}`))
			console.log("\n")
		}
		if (totals != null) {
			totals.errors += _errors.length
			totals.warnings += _warnings.length
		}
	}
	if (nextCache != null) nextCache[file] = data
}

function formatMessage(...data) {
	let str = data.join("\n")
	if (str.endsWith("\n")) str = str.slice(0, -1)
	return str
}

exports.lintOne = lintOne
// `cache` and `nextCache` are only passed from `lintAll()`, not when watching
async function lintOne(file, totals, cache, nextCache) {
	const contents = await fs.readFile(file, "utf-8")
	// check for nextCache, because cache will be undefined the first time the linter runs
	const {size, timestamp} = (nextCache != null) ? await getFileInfo(file) : {}
	if (cache != null && cache[file] != null) {
		const cached = cache[file]
		if (
			size === cached.size &&
			timestamp === cached.timestamp &&
			cached._errors.length + cached._warnings.length === 0
		) {
			report(file, cached, totals, nextCache)
			return
		}
	}
	const renderer = new LintRenderer(file)
	marked(contents, {renderer})
	return Promise.all(renderer._awaiting).then(() => {
		const {_warnings, _errors} = renderer
		report(file, {_warnings, _errors, size, timestamp}, totals, nextCache)
	})
}


const cachePath = path.join(process.cwd(), ".lint-docs-cache")

async function loadCache() {
	try {
		const source = await fs.readFile(cachePath, "utf-8")
		try {
			return JSON.parse(source)
		} catch (e) {
			console.error(e)
			return
		}
	} catch (e) {
		return
	}
}

function saveCache(nextCache) {
	return fs.writeFile(cachePath, JSON.stringify(nextCache), "utf-8")
}

function finalReport(totals) {
	const buffer = []
	if (totals.errors > 0) {
		buffer.push(`${totals.errors} error${totals.errors > 1 ? "s" : ""}`)
	}
	if (totals.warnings > 0) {
		buffer.push(`${totals.warnings} warning${totals.warnings > 1 ? "s" : ""}`)
	}
	if (buffer.length > 0) console.log(`\n${buffer.join(", ")} found in the docs\n`)
	else console.log("The docs are in good shape!\n")
}

exports.lintAll = lintAll
async function lintAll({useCache}) {
	const cache = useCache ? await loadCache() : null
	const totals = {
		errors: 0,
		warnings: 0,
	}
	// always populate the cache, even if we don't read from it
	const nextCache = {}
	await new Promise((resolve, reject) => {
		const glob = new Glob(path.resolve(__dirname, "../**/*.md"), {
			ignore: [
				"**/change-log.md",
				"**/migration-v02x.md",
				"**/node_modules/**",
			],
			nodir: true,
		})
		const awaiting = []

		glob.on("match", (file) => {
			awaiting.push(lintOne(file, totals, cache, nextCache))
		})

		glob.on("error", reject)
		glob.on("end", () => resolve(Promise.all(awaiting)))
	})
	finalReport(totals)
	await saveCache(nextCache)
	// don't return anything so that _command.js picks up the errorCode.
}

/* eslint-disable global-require */
if (require.main === module) {
	require("./_command")({
		exec: lintAll,
		watch() {
			require("chokidar")
				.watch(path.resolve(__dirname, "../**/*.md"), {
					ignored: [
						"**/change-log.md",
						"**/migration-v02x.md",
						"**/node_modules/**",
					],
				})
				.on("add", lintOne)
				.on("change", lintOne)
		},
	})
}