Pimp the docs linter (and assorted changes) (#2553)
### Pimp the docs linter (and assorted changes) #### `scripts/lint-docs.js` - Add an optional cache for faster runs - Add a final report - Don't return anything from `exec()` - Cover more files #### `scripts/_command.js` - Look for a "--cache" option #### `package.json` scripts - Added `watch:lint-docs` - Added `cleanup:lint` to remove the eslint and lint-docs cache files - Changed `lint:docs` to use the `--cache` option - Added `test:js` so that we can run the test suite without the linter - Changed `test` to defer to `test:js` #### Actual lint fixes: - Bad link in a migration guide - The unicode dashes in the "https://en.wikipedia.org/wiki/Subject–verb–object" are not escaped by marked ### Some more lint-docs pimping #### `scripts/lint-docs.js` - some code reorg and cleanup (take a hint from the local coding conventions) - fix misc bugs - pass a User-Agent header to the requests - even nicer reporting #### `package.json` - bump the @babel/parser dep to the latest #### Docs - tweaks based on lints missed due to previous bugs ### Docs: use the github page for velocity.js, the home page has too many errors. Co-Authored-By: Isiah Meadows <contact@isiahmeadows.com>
This commit is contained in:
Pierre-Yves Gérardy
GitHub
parent
d257025253
commit
4a3a486d80
11 changed files with 182 additions and 62 deletions
|
|
@ -18,6 +18,7 @@ process.on("unhandledRejection", function (e) {
|
|||
|
||||
module.exports = ({exec, watch}) => {
|
||||
const index = process.argv.indexOf("--watch")
|
||||
const useCache = process.argv.indexOf("--cache") >= 0
|
||||
if (index >= 0) {
|
||||
process.argv.splice(index, 1)
|
||||
|
||||
|
|
@ -29,7 +30,7 @@ module.exports = ({exec, watch}) => {
|
|||
|
||||
watch()
|
||||
} else {
|
||||
Promise.resolve(exec()).then((code) => {
|
||||
Promise.resolve(exec({useCache})).then((code) => {
|
||||
if (code != null) process.exitCode = code
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,24 +14,22 @@ const request = require("request-promise-native")
|
|||
class LintRenderer extends marked.Renderer {
|
||||
constructor(file) {
|
||||
super()
|
||||
this._file = file
|
||||
this._dir = path.dirname(file)
|
||||
this._context = undefined
|
||||
this._code = undefined
|
||||
this._lang = undefined
|
||||
this._error = undefined
|
||||
this._awaiting = []
|
||||
this._warnings = []
|
||||
this._errors = []
|
||||
}
|
||||
|
||||
_emitTolerate(...data) {
|
||||
let str = data.join("\n")
|
||||
if (str.endsWith("\n")) str = str.slice(0, -1)
|
||||
console.log(`${this._file} - ${str}\n${"-".repeat(60)}`)
|
||||
_addWarning(...data) {
|
||||
this._warnings.push(formatMessage(...data))
|
||||
}
|
||||
|
||||
_emit(...data) {
|
||||
this._emitTolerate(...data)
|
||||
process.exitCode = 1
|
||||
_addError(...data) {
|
||||
this._errors.push(formatMessage(...data))
|
||||
}
|
||||
|
||||
_block() {
|
||||
|
|
@ -42,31 +40,48 @@ class LintRenderer extends marked.Renderer {
|
|||
// Don't fail if something byzantine shows up - it's the freaking
|
||||
// internet. Just log it and move on.
|
||||
const httpError = (e) =>
|
||||
this._emitTolerate(`http error for ${href}`, e.message)
|
||||
this._addWarning(`http error for ${href}`, e.message)
|
||||
|
||||
// Prefer https: > http: where possible, but allow http: when https: is
|
||||
// inaccessible.
|
||||
if ((/^https?:\/\//).test(href)) {
|
||||
const url = href.replace(/#.*$/, "")
|
||||
this._awaiting.push(request.head(url).then(() => {
|
||||
const isHTTPS = href.startsWith("https:")
|
||||
const isHTTPS = href.startsWith("https:")
|
||||
// pass along realistic headers, some sites (i.e. the IETF) return a 403 otherwise.
|
||||
const headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:71.0) Gecko/20100101 Firefox/71.0",
|
||||
}
|
||||
// some more headers if more were ever needed (from my local Firefox)
|
||||
|
||||
// "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
// "Accept-Language": "en-US,en;q=0.5",
|
||||
// "Accept-Encoding": "gzip, deflate, br",
|
||||
// "DNT": "1",
|
||||
// "Connection": "keep-alive",
|
||||
// "Upgrade-Insecure-Requests": "1",
|
||||
// "Pragma": "no-cache",
|
||||
// "Cache-Control": "no-cache"
|
||||
|
||||
this._awaiting.push(request.head(url, {headers}).then(() => {
|
||||
if (!isHTTPS) {
|
||||
return request.head(`https:${url.slice(7)}`).then(
|
||||
() => this._emit("change http: to https:"),
|
||||
return request.head(`https:${url.slice(7)}`, {headers}).then(
|
||||
() => this._addError("change http: to https:"),
|
||||
() => { /* ignore inner errors */ }
|
||||
)
|
||||
}
|
||||
}, (e) => {
|
||||
if (e.statusCode === 404) {
|
||||
this._emit(`broken external link: ${href}`)
|
||||
this._addError(`broken external link: ${href}`)
|
||||
}
|
||||
else {
|
||||
if (
|
||||
e.error.code === "ERR_TLS_CERT_ALTNAME_INVALID" &&
|
||||
href.startsWith("https://")
|
||||
isHTTPS && (
|
||||
e.error.code === "ERR_TLS_CERT_ALTNAME_INVALID" ||
|
||||
(/ssl/i).test(e.message)
|
||||
)
|
||||
) {
|
||||
return request.head(`http:${url.slice(6)}`).then(
|
||||
() => this._emit(`change ${href} to use http:`),
|
||||
return request.head(`http:${url.slice(6)}`, {headers}).then(
|
||||
() => this._addError(`change ${href} to use http:`),
|
||||
// ignore inner errors
|
||||
() => httpError(e)
|
||||
)
|
||||
|
|
@ -80,7 +95,7 @@ class LintRenderer extends marked.Renderer {
|
|||
if (exec != null) {
|
||||
const resolved = path.resolve(this._dir, exec[1])
|
||||
this._awaiting.push(fs.access(resolved).catch(() => {
|
||||
this._emit(`broken internal link: ${href}`)
|
||||
this._addError(`broken internal link: ${href}`)
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
|
@ -89,6 +104,8 @@ class LintRenderer extends marked.Renderer {
|
|||
code(code, lang) {
|
||||
this._code = code
|
||||
this._lang = lang
|
||||
this._error = null
|
||||
|
||||
if (!lang || lang === "js" || lang === "javascript") {
|
||||
try {
|
||||
// Could be within any production.
|
||||
|
|
@ -115,7 +132,7 @@ class LintRenderer extends marked.Renderer {
|
|||
if (!this._lang) {
|
||||
// TODO: ensure all code blocks have tags, and check this in CI.
|
||||
if (this._error == null) {
|
||||
this._emit(
|
||||
this._addError(
|
||||
"Code block possibly missing `javascript` language tag",
|
||||
this._block(),
|
||||
)
|
||||
|
|
@ -123,7 +140,7 @@ class LintRenderer extends marked.Renderer {
|
|||
|
||||
try {
|
||||
JSON.parse(this._code)
|
||||
this._emit(
|
||||
this._addError(
|
||||
"Code block possibly missing `json` language tag",
|
||||
this._block(),
|
||||
)
|
||||
|
|
@ -135,9 +152,9 @@ class LintRenderer extends marked.Renderer {
|
|||
}
|
||||
|
||||
_ensureCodeIsSyntaticallyValid() {
|
||||
if (!this.lang || !(/^js$|^javascript$/).test(this._lang)) return
|
||||
if (!this._lang || !(/^js$|^javascript$/).test(this._lang)) return
|
||||
if (this._error != null) {
|
||||
this._emit(
|
||||
this._addError(
|
||||
"JS code block has invalid syntax", this._error.message,
|
||||
this._block()
|
||||
)
|
||||
|
|
@ -145,28 +162,124 @@ class LintRenderer extends marked.Renderer {
|
|||
}
|
||||
|
||||
_ensureCommentStyle() {
|
||||
if (!this.lang || !(/^js$|^javascript$/).test(this._lang)) return
|
||||
if (!this._lang || !(/^js$|^javascript$/).test(this._lang)) return
|
||||
if ((/(^|\s)\/\/[\S]/).test(this._code)) {
|
||||
this._emit("Comment is missing a preceding space", this._block())
|
||||
this._addError("Comment is missing a preceding space", this._block())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getFileInfo(file) {
|
||||
const {size, mtime} = await fs.stat(file)
|
||||
const timestamp = Number(mtime)
|
||||
return {size, timestamp}
|
||||
}
|
||||
|
||||
function report(file, data, totals, nextCache) {
|
||||
const {_warnings, _errors} = data;
|
||||
if (_warnings.length + _errors.length > 0) {
|
||||
console.log("- ".repeat(file.length/2 + 1))
|
||||
console.log(file)
|
||||
console.log("- ".repeat(file.length/2 + 1) + "\n")
|
||||
if (_errors.length > 0) {
|
||||
process.exitCode = 1
|
||||
const s = _errors.length > 1 ? "s " : " -"
|
||||
console.log(`-- ${_errors.length} Error${s}----------`)
|
||||
_errors.forEach((msg) => console.log(`\n${msg}`))
|
||||
console.log("\n")
|
||||
}
|
||||
if (_warnings.length > 0) {
|
||||
const s = _warnings.length > 1 ? "s " : " -"
|
||||
console.log(`-- ${_warnings.length} Warning${s}--------`)
|
||||
_warnings.forEach((msg) => console.log(`\n${msg}`))
|
||||
console.log("\n")
|
||||
}
|
||||
if (totals != null) {
|
||||
totals.errors += _errors.length
|
||||
totals.warnings += _warnings.length
|
||||
}
|
||||
}
|
||||
if (nextCache != null) nextCache[file] = data
|
||||
}
|
||||
|
||||
function formatMessage(...data) {
|
||||
let str = data.join("\n")
|
||||
if (str.endsWith("\n")) str = str.slice(0, -1)
|
||||
return str
|
||||
}
|
||||
|
||||
exports.lintOne = lintOne
|
||||
async function lintOne(file) {
|
||||
// `cache` and `nextCache` are only passed from `lintAll()`, not when watching
|
||||
async function lintOne(file, totals, cache, nextCache) {
|
||||
const contents = await fs.readFile(file, "utf-8")
|
||||
// check for nextCache, because cache will be undefined the first time the linter runs
|
||||
const {size, timestamp} = (nextCache != null) ? await getFileInfo(file) : {}
|
||||
if (cache != null && cache[file] != null) {
|
||||
const cached = cache[file]
|
||||
if (
|
||||
size === cached.size &&
|
||||
timestamp === cached.timestamp &&
|
||||
cached._errors.length + cached._warnings.length === 0
|
||||
) {
|
||||
report(file, cached, totals, nextCache)
|
||||
return
|
||||
}
|
||||
}
|
||||
const renderer = new LintRenderer(file)
|
||||
marked(contents, {renderer})
|
||||
return Promise.all(renderer._awaiting)
|
||||
return Promise.all(renderer._awaiting).then(() => {
|
||||
const {_warnings, _errors} = renderer
|
||||
report(file, {_warnings, _errors, size, timestamp}, totals, nextCache)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
const cachePath = path.join(process.cwd(), ".lint-docs-cache")
|
||||
|
||||
async function loadCache() {
|
||||
try {
|
||||
const source = await fs.readFile(cachePath, "utf-8")
|
||||
try {
|
||||
return JSON.parse(source)
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
return
|
||||
}
|
||||
} catch (e) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
function saveCache(nextCache) {
|
||||
return fs.writeFile(cachePath, JSON.stringify(nextCache), "utf-8")
|
||||
}
|
||||
|
||||
function finalReport(totals) {
|
||||
const buffer = []
|
||||
if (totals.errors > 0) {
|
||||
buffer.push(`${totals.errors} error${totals.errors > 1 ? "s" : ""}`)
|
||||
}
|
||||
if (totals.warnings > 0) {
|
||||
buffer.push(`${totals.warnings} warning${totals.warnings > 1 ? "s" : ""}`)
|
||||
}
|
||||
if (buffer.length > 0) console.log(`\n${buffer.join(", ")} found in the docs\n`)
|
||||
else console.log("The docs are in good shape!\n")
|
||||
}
|
||||
|
||||
exports.lintAll = lintAll
|
||||
function lintAll() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const glob = new Glob(path.resolve(__dirname, "../docs/**/*.md"), {
|
||||
async function lintAll({useCache}) {
|
||||
const cache = useCache ? await loadCache() : null
|
||||
const totals = {
|
||||
errors: 0,
|
||||
warnings: 0,
|
||||
}
|
||||
// always populate the cache, even if we don't read from it
|
||||
const nextCache = {}
|
||||
await new Promise((resolve, reject) => {
|
||||
const glob = new Glob(path.resolve(__dirname, "../**/*.md"), {
|
||||
ignore: [
|
||||
"**/change-log.md",
|
||||
"**/migration-*.md",
|
||||
"**/migration-v02x.md",
|
||||
"**/node_modules/**",
|
||||
],
|
||||
nodir: true,
|
||||
|
|
@ -174,12 +287,15 @@ function lintAll() {
|
|||
const awaiting = []
|
||||
|
||||
glob.on("match", (file) => {
|
||||
awaiting.push(lintOne(file))
|
||||
awaiting.push(lintOne(file, totals, cache, nextCache))
|
||||
})
|
||||
|
||||
glob.on("error", reject)
|
||||
glob.on("end", () => resolve(Promise.all(awaiting)))
|
||||
})
|
||||
finalReport(totals)
|
||||
await saveCache(nextCache)
|
||||
// don't return anything so that _command.js picks up the errorCode.
|
||||
}
|
||||
|
||||
/* eslint-disable global-require */
|
||||
|
|
@ -188,10 +304,10 @@ if (require.main === module) {
|
|||
exec: lintAll,
|
||||
watch() {
|
||||
require("chokidar")
|
||||
.watch(path.resolve(__dirname, "../docs/**/*.md"), {
|
||||
ignore: [
|
||||
.watch(path.resolve(__dirname, "../**/*.md"), {
|
||||
ignored: [
|
||||
"**/change-log.md",
|
||||
"**/migration-*.md",
|
||||
"**/migration-v02x.md",
|
||||
"**/node_modules/**",
|
||||
],
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue