fix(docutils): better parsing/updating of nav tree in mkdocs.yml

After we build the reference docs, we need to update the `nav` prop of the given `mkdocs.yml` before building the site.  This was only _sorta_ working before; it did not take into account the myriad ways in which the data structure could be expressed (particularly, it did not understand "custom names").

I think I've done this in such a way that if a custom name is provided (they must be provided manually by hand-editing `mkdocs.yml`), it retains them _unless_ the file in question disappears.  Or that's the idea. Can't really be too sure; needs tests.

This change makes an attempt to parse the `nav` prop into something "normalized", then the data is processed and recomplexified before writing out to `mkdocs.yml`.  However, I've disabled the ability to define a custom header for command docs and/or omit the header entirely, as the latter especially was causing extra complexity and it's already bad enough.  I think we can re-enable "custom header" somehow, if needed.

Also added an `--all` flag which causes the nav to be updated with _all_ the TypeDoc-generated content--not just command docs.
This commit is contained in:
Christopher Hiller
2023-02-06 14:02:50 -08:00
parent 2b3e576393
commit 42dd6785c3
5 changed files with 333 additions and 206 deletions
+1
View File
@@ -1,3 +1,4 @@
export * from './deploy';
export * from './site';
export * from './reference';
export * from './nav';
+320
View File
@@ -0,0 +1,320 @@
/**
* Handles updating/adding the `nav` property of `mkdocs.yml`, based on the output of `typedoc`;
* specifically, the command documentation generated by `@appium/typedoc-plugin-appium`.
*
* @module
*/
import {fs} from '@appium/support';
import _ from 'lodash';
import path from 'node:path';
import {
DEFAULT_NAV_HEADER,
DEFAULT_REL_TYPEDOC_OUT_PATH,
NAME_BIN,
NAME_MKDOCS_YML,
NAME_TYPEDOC_JSON,
} from '../constants';
import {DocutilsError} from '../error';
import {
findDirsIn,
findMkDocsYml,
findTypeDocJsonPath,
readTypedocJson,
readYaml,
safeWriteFile,
stringifyYaml,
} from '../fs';
import logger from '../logger';
import {MkDocsYml, MkDocsYmlNav} from '../model';
import {relative} from '../util';
const log = logger.withTag('builder:nav');
/**
* Gets a list of `.md` files relative to `docs_dir`
* @param targetDir Directory ostensibly containing Markdown files; must be absolute
* @param mkDocsDocsDir The path to the `docs_dir` in the `mkdocs.yml` file; must be absolute
* @returns List of Markdown files relative to the `docs_dir` in the `mkdocs.yml` file
*/
async function findRelativeMarkdownFiles(
targetDir: string,
mkDocsDocsDir: string
): Promise<string[]> {
if (!path.isAbsolute(targetDir)) {
throw new DocutilsError(`Expected absolute path, got '${targetDir}'`);
}
if (!path.isAbsolute(mkDocsDocsDir)) {
throw new DocutilsError(`Expected absolute path, got '${mkDocsDocsDir}'`);
}
const relDir = path.relative(mkDocsDocsDir, targetDir);
const dirEnts = await fs.readdir(targetDir, {withFileTypes: true});
return dirEnts
.filter((ent) => ent.isFile() && ent.name.endsWith('.md'))
.map((ent) => path.join(relDir, ent.name));
}
/**
* Because the `nav` property of `mkdocs.yml` is both a recursive type and a kind of awful one, it's
* easier to work with it if we rewrite the data into a flat array of objects. We keep a `keypath`
* prop which represents the deep/nested location within the `nav` object.
*
* @privateRemarks This function is not recursive; instead it loops over a queue of items to process
* data, and we append to that queue while processing if needed.
* @param nav Contents of the `nav` prop of `mkdocs.yml`
* @returns A list of objects, each with a `keypath` property and a `fileOrUrl` property (and maybe
* a `name` property)
*/
export function parseNav(nav: MkDocsYmlNav): ParsedNavData[] {
let parsedNav: ParsedNavData[] = [];
const entries = Object.entries(nav);
type QueueItem = {
entries: typeof entries;
keypath: string;
};
const queue: QueueItem[] = [{entries, keypath: ''}];
while (queue.length) {
const {entries, keypath} = queue.shift()!;
for (const [key, item] of entries) {
if (_.isString(item)) {
const navData: ParsedNavData = {
keypath: keypath ? `${keypath}.${key}` : key,
fileOrUrl: item,
};
// if the key is not convertible to a number, it's a name
// which was manually put there by somebody.
if (isNaN(Number(key))) {
navData.name = key;
}
parsedNav = [...parsedNav, navData];
} else if (_.isObject(item)) {
const subEntries = Object.entries(item);
queue.push({entries: subEntries, keypath: keypath ? `${keypath}.${key}` : key});
}
}
}
return parsedNav;
}
/**
* Finds all items within the list of parsed nav data which correpsond to the header. This is
* imperfect, as it's possible for the header string to appear in multiple places in the nav, but let's just ignore that until we can't.
* @param navData Some parsed nav data
* @param header Header name
*/
function itemsForHeader(navData: ParsedNavData[], header: string) {
return _.filter(navData, (item) => _.toPath(item.keypath).includes(header));
}
/**
* Update the `nav` property of `mkdocs.yml` with a list of "command" files generated by TypeDoc via
* `@appium/typedoc-plugin-appium`.
*
* To be clear, this function **modifies the MkDocs config file (`mkdocs.yml`) in place**; it is
* typically under version control, so if this function makes any changes, you'll want to commit them.
* @param opts - Options
* @todo implement `dryRun` option
*/
export async function updateNav({
cwd = process.cwd(),
mkdocsYml: mkDocsYmlPath,
typedocJson: typeDocJsonPath,
all = false,
dryRun = false,
}: UpdateNavOpts = {}) {
// we need `mkdocs.yml` to update
// and we need `typedoc.json` to know where to look for the command docs
[mkDocsYmlPath, typeDocJsonPath] = await Promise.all([
mkDocsYmlPath ?? findMkDocsYml(cwd),
typeDocJsonPath ?? findTypeDocJsonPath(cwd),
]);
if (!mkDocsYmlPath) {
throw new DocutilsError(
`Could not find ${NAME_MKDOCS_YML} from ${cwd}; run "${NAME_BIN} init" to create it`
);
}
if (!typeDocJsonPath) {
throw new DocutilsError(
`Could not find ${NAME_TYPEDOC_JSON} from ${cwd}; run "${NAME_BIN} init" to create it`
);
}
const relativePath = relative(cwd);
const relMkDocsYmlPath = relativePath(mkDocsYmlPath);
const typeDocJson = readTypedocJson(typeDocJsonPath);
const mkDocsYml = (await readYaml(mkDocsYmlPath)) as MkDocsYml;
/**
* Absolute path to `typedoc.json`
*/
const absTypeDocJsonPath = path.isAbsolute(typeDocJsonPath)
? typeDocJsonPath
: path.resolve(cwd, typeDocJsonPath);
/**
* Absolute path to TypeDoc's output directory (`out`)
*/
const typeDocOutDir = path.resolve(
path.dirname(absTypeDocJsonPath),
typeDocJson.out ? typeDocJson.out : DEFAULT_REL_TYPEDOC_OUT_PATH
);
/**
* Absolute path to `mkdocs.yml`
*/
const absMkdocsYmlPath = path.isAbsolute(mkDocsYmlPath)
? mkDocsYmlPath
: path.resolve(cwd, mkDocsYmlPath);
const {docs_dir: docsDir, nav = []} = mkDocsYml;
/**
* Absolute path to the directory containing MkDocs input docs
*/
const mkDocsDocsDir = path.resolve(path.dirname(absMkdocsYmlPath), docsDir ?? 'docs');
/**
* @todo
* `commands` is a dirname configurable via the `commandsDir` option added by
* `@appium/typedoc-plugin-appium`. this lives in `typedoc.json`, but in order for it to be parsed
* using TypeDoc's facilities, we have to load plugins before reading `typedoc.json`, which is
* slow. we will probably have to support this in the future, but for now, we can just hardcode it
*/
const dirs = all ? await findDirsIn(typeDocOutDir) : [path.join(typeDocOutDir, 'commands')];
let shouldWriteMkDocsYml = false;
const navData = parseNav(nav);
// this is the thing which will be assigned to the `nav` prop
// and thus written to `mkdocs.yml` if there were any changes.
// we don't need the `name` prop, since the name is already present in the keypath.
const newData: Omit<ParsedNavData, 'name'>[] = [];
for (const dir of dirs) {
const newDataForDir: Omit<ParsedNavData, 'name'>[] = [];
const newRefFilepaths = await findRelativeMarkdownFiles(dir, mkDocsDocsDir);
const header = all ? _.startCase(path.basename(dir)) : DEFAULT_NAV_HEADER;
const headerItems = itemsForHeader(navData, header);
// if we found items with this header already, we are going
// to replace them all wholesale
if (headerItems.length) {
shouldWriteMkDocsYml = true;
// these are the parts of the keypath of the first item, which will contain the header string.
const rootHeaderKeypathParts = _.toPath(_.first(headerItems)!.keypath);
// this is the keypath up to the header string, inclusive.
// we append indices or names to this keypath
const rootHeaderKeypath = rootHeaderKeypathParts
.slice(0, rootHeaderKeypathParts.indexOf(header) + 1)
.join('.');
for (const [offset, newRefFilepath] of newRefFilepaths.entries()) {
const data = headerItems.find((item) => item.fileOrUrl === newRefFilepath);
if (data?.name) {
newDataForDir.push({
keypath: `${rootHeaderKeypath}.${offset}.${data.name}`,
fileOrUrl: newRefFilepath,
});
} else {
newDataForDir.push({
keypath: `${rootHeaderKeypath}.${offset}`,
fileOrUrl: newRefFilepath,
});
}
}
// look for any differences between what we have and what's in the file
if (_.xor(newDataForDir, _.map(headerItems, _.partial(_.omit, _, 'name')))) {
newData.push(...newDataForDir);
shouldWriteMkDocsYml = true;
log.debug('Will write new nav data for header %s', header);
} else {
log.debug('No changes for header %s', header);
}
} else {
let navOffset = nav.length;
for (const [idx, newRefFilepath] of newRefFilepaths.entries()) {
newDataForDir.push({
keypath: `${navOffset}.${header}.${idx}`,
fileOrUrl: newRefFilepath,
});
}
newData.push(...newDataForDir);
shouldWriteMkDocsYml = true;
log.debug('Will create nav data for header %s', header);
}
}
for (const {keypath, fileOrUrl} of newData) {
_.set(mkDocsYml, `nav.${keypath}`, fileOrUrl);
}
if (shouldWriteMkDocsYml) {
const yaml = stringifyYaml(mkDocsYml);
log.debug('Writing to %s:\n%s', mkDocsYmlPath, yaml);
await safeWriteFile(mkDocsYmlPath, yaml, true);
log.success(
'Updated MkDocs navigation config for reference docs; please run "git add -A %s" and commit this change',
relMkDocsYmlPath
);
} else {
log.info('No changes needed for MkDocs config at %s', relMkDocsYmlPath);
}
}
/**
* Options for {@linkcode updateNav}
*/
export interface UpdateNavOpts {
/**
* Current working directory
*/
cwd?: string;
/**
* Path to `mkdocs.yml`
*/
mkdocsYml?: string;
/**
* Path to `package.json`
*/
packageJson?: string;
/**
* Path to `typedoc.json`
*/
typedocJson?: string;
/**
* If `true`, do not write any files
* @remarks Not yet implemented
*/
dryRun?: boolean;
/**
* If `true`, add _all_ reference documentation to the navigation config (not just commands)
*/
all?: boolean;
}
/**
* Used internally by {@linkcode updatedNav}
* @see {@linkcode parseNav}
*/
interface ParsedNavData {
/**
* Keypath within `nav` for some file or URL
*/
keypath: string;
/**
* A filepath (usually) or a URL.
* This is considered the "index" of the data, and should be unique within its parent. If it's not
* unique, then it will probably end up that way after updating...
*/
fileOrUrl: string;
/**
* If this file or url has a proper name, this would be it. Most don't.
*/
name?: string;
}
+1 -2
View File
@@ -1,8 +1,7 @@
import {CommandModule, InferredOptionTypes, Options} from 'yargs';
import {buildReferenceDocs, buildSite, deploy} from '../../builder';
import {buildReferenceDocs, buildSite, deploy, updateNav} from '../../builder';
import {NAME_BIN} from '../../constants';
import logger from '../../logger';
import {updateNav} from '../../nav';
import {stopwatch} from '../../util';
const log = logger.withTag('build');
+11
View File
@@ -258,3 +258,14 @@ export const readMkDocsYml = _.memoize(
return mkDocsYml;
}
);
/**
* Given an abs path to a directory, return a list of all abs paths of all directories in it
*/
export const findDirsIn = _.memoize(async (dirpath: string): Promise<string[]> => {
if (!path.isAbsolute(dirpath)) {
throw new DocutilsError(`Expected absolute path, got '${dirpath}'`);
}
const dirEnts = await fs.readdir(dirpath, {withFileTypes: true});
return dirEnts.filter((ent) => ent.isDirectory()).map((ent) => path.join(dirpath, ent.name));
});
-204
View File
@@ -1,204 +0,0 @@
/**
* Handles updating/adding the `nav` property of `mkdocs.yml`, based on the output of `typedoc`;
* specifically, the command documentation generated by `@appium/typedoc-plugin-appium`.
*
* @module
*/
import {fs} from '@appium/support';
import _ from 'lodash';
import path from 'node:path';
import {
DEFAULT_REL_TYPEDOC_OUT_PATH,
NAME_BIN,
NAME_MKDOCS_YML,
NAME_TYPEDOC_JSON,
} from './constants';
import {DocutilsError} from './error';
import {
findMkDocsYml,
findTypeDocJsonPath,
readTypedocJson,
readYaml,
safeWriteFile,
stringifyYaml,
} from './fs';
import logger from './logger';
import {MkDocsYml} from './model';
import {relative, isStringArray} from './util';
const DEFAULT_REFERENCE_HEADER = 'Reference';
const log = logger.withTag('mkdocs-nav');
/**
* Update the `nav` property of `mkdocs.yml` with a list of "command" files generated by TypeDoc via `@appium/typedoc-plugin-appium`
* @param opts - Options
* @todo implement `dryRun` option
*/
export async function updateNav<S extends string>({
cwd = process.cwd(),
mkdocsYml: mkDocsYmlPath,
referenceHeader = <S>DEFAULT_REFERENCE_HEADER,
noReferenceHeader = false,
typedocJson: typeDocJsonPath,
dryRun = false,
}: UpdateNavOpts<S> = {}) {
[mkDocsYmlPath, typeDocJsonPath] = await Promise.all([
mkDocsYmlPath ?? findMkDocsYml(cwd),
typeDocJsonPath ?? findTypeDocJsonPath(cwd),
]);
if (!mkDocsYmlPath) {
throw new DocutilsError(
`Could not find ${NAME_MKDOCS_YML} from ${cwd}; run "${NAME_BIN} init" to create it`
);
}
if (!typeDocJsonPath) {
throw new DocutilsError(
`Could not find ${NAME_TYPEDOC_JSON} from ${cwd}; run "${NAME_BIN} init" to create it`
);
}
const relativePath = relative(cwd);
const relMkDocsYmlPath = relativePath(mkDocsYmlPath);
const typeDocJson = readTypedocJson(typeDocJsonPath);
const mkDocsYml = (await readYaml(mkDocsYmlPath)) as MkDocsYml;
const findRefDictIndex: (nav: MkDocsYml['nav']) => number = _.partial(
_.findIndex,
_,
_.overEvery([_.isObject, _.partial(_.has, _, referenceHeader)])
);
/**
* Absolute path to `typedoc.json`
*/
const absTypeDocJsonPath = path.isAbsolute(typeDocJsonPath)
? typeDocJsonPath
: path.resolve(cwd, typeDocJsonPath);
/**
* Absolute path to TypeDoc's output directory (`out`)
*/
const typeDocOutDir = path.resolve(
path.dirname(absTypeDocJsonPath),
typeDocJson.out ? typeDocJson.out : DEFAULT_REL_TYPEDOC_OUT_PATH
);
/**
* Absolute path to `mkdocs.yml`
*/
const absMkdocsYmlPath = path.isAbsolute(mkDocsYmlPath)
? mkDocsYmlPath
: path.resolve(cwd, mkDocsYmlPath);
const {docs_dir: docsDir, nav = []} = mkDocsYml;
/**
* Absolute path to the directory containing MkDocs input docs
*/
const mkDocsDocsDir = path.resolve(path.dirname(absMkdocsYmlPath), docsDir ?? 'docs');
/**
* The dir we need to prepend to all entries within `nav`
*/
const relReferenceDir = path.relative(mkDocsDocsDir, typeDocOutDir);
const partitionRefArray: <T, U extends T>(arr: T[]) => [U[], Array<Exclude<T, U>>] =
_.partialRight(_.partition, _.partialRight(_.startsWith, `${relReferenceDir}/`));
const newRefFilepaths: string[] = [];
// TODO: this doesn't respect the 'commandsDir' option for typedoc-plugin-appium. in fact,
// typeDocJson does not even include it, because it's unknown. I suppose that will mean we need
// to load plugins, but that means bootstrapping TypeDoc entirely just to read a `typedoc.json`
// file, which is slow.
const commandDir = path.join(typeDocOutDir, 'commands');
const relCommandDir = path.relative(mkDocsDocsDir, commandDir);
const commandDocFileEnts = await fs.readdir(commandDir, {withFileTypes: true});
if (!commandDocFileEnts.length) {
log.warn('No reference API docs were found in %s; skipping navigation update', commandDir);
return;
}
for (const ent of commandDocFileEnts) {
if (ent.isFile() && ent.name.endsWith('.md')) {
newRefFilepaths.push(path.join(relCommandDir, ent.name));
}
}
log.debug('New reference filepaths: %O', newRefFilepaths);
const navUsesHeaders = noReferenceHeader || !isStringArray(nav);
let shouldWriteMkDocsYml = false;
let refFilepaths: string[];
let nonRefFilepaths: string[];
const refDictIdx = findRefDictIndex(nav);
if (refDictIdx >= 0) {
const refDict = nav[refDictIdx] as Record<S, string[]>;
const refArray = refDict[referenceHeader];
[refFilepaths, nonRefFilepaths] = partitionRefArray(refArray);
} else {
[refFilepaths, nonRefFilepaths] = partitionRefArray(<string[]>nav);
}
const symmetricDiff = _.xor(newRefFilepaths, refFilepaths);
if (symmetricDiff.length) {
log.debug('Difference in old nav vs new: %O', symmetricDiff);
shouldWriteMkDocsYml = true;
if (navUsesHeaders) {
if (refDictIdx >= 0) {
const res = [...nonRefFilepaths, ...newRefFilepaths];
(mkDocsYml.nav![refDictIdx] as Record<S, string[]>)[referenceHeader] = res;
log.debug('Replaced "%s" section with %O', referenceHeader, res);
} else {
mkDocsYml.nav = [...nonRefFilepaths, {[referenceHeader]: newRefFilepaths}];
log.debug('Added "%s" section with %O', referenceHeader, newRefFilepaths);
}
} else {
mkDocsYml.nav = [...nonRefFilepaths, ...newRefFilepaths];
log.debug('Replaced nav with %O', mkDocsYml.nav);
}
}
if (shouldWriteMkDocsYml) {
const yaml = stringifyYaml(mkDocsYml);
log.debug(yaml);
await safeWriteFile(mkDocsYmlPath, yaml, true);
log.success('Updated navigation for reference documents in %s', relMkDocsYmlPath);
} else {
log.info('No changes to navigation for reference documents in %s', relMkDocsYmlPath);
}
}
/**
* Options for {@linkcode updateNav}
*/
export interface UpdateNavOpts<S extends string> {
/**
* Current working directory
*/
cwd?: string;
/**
* Path to `mkdocs.yml`
*/
mkdocsYml?: string;
/**
* Path to `package.json`
*/
packageJson?: string;
/**
* The name of the header in `nav` to use for reference docs.
*/
referenceHeader?: S;
/**
* If `true`, do not add a reference header to `nav` if one does not exist
*/
noReferenceHeader?: boolean;
/**
* Path to `typedoc.json`
*/
typedocJson?: string;
/**
* If `true`, do not write any files
* @remarks Not yet implemented
*/
dryRun?: boolean;
}