diff --git a/.gitignore b/.gitignore index c9ef4754b..a3e79a0bd 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ typescript # Github actions RELEASE_NOTES.md + +# Binary files +bin \ No newline at end of file diff --git a/app/supervisor.ts b/app/supervisor.ts new file mode 100644 index 000000000..61a1705b9 --- /dev/null +++ b/app/supervisor.ts @@ -0,0 +1,160 @@ +import { join as joinPath, resolve as resolveToAbsolutePath } from 'path'; +import { createWriteStream, mkdirSync, existsSync } from 'fs'; +import { spawn as spawnProcess, ChildProcess } from 'child_process'; +import locatePath from 'locate-path'; +import psList from 'ps-list'; +import { cyan, green, yellow, red } from 'nanocolors'; +import intervalToHuman from 'interval-to-human'; + +const createLogger = (namespace: string) => { + const ns = namespace.toUpperCase(); + return { + info(message: string, ...args: any[]) { + console.info(`${cyan(`[${ns}]`)} ${message}`, ...args); + }, + debug(message: string, ...args: any[]) { + if (!isDebug) { + return; + } + + console.debug(`${green(`[${ns}]`)} ${message}`, ...args); + }, + warning(message: string, ...args: any[]) { + console.warn(`${yellow(`[${ns}][WARNING]`)} ${message}`, ...args); + }, + error(message: string | Error, ...args: any[]) { + console.error(`${red(`[${ns}][ERROR]`)} ${message}`, ...args); + }, + print(message: string, ...args: any[]) { + console.log(message, ...args); + } + }; +}; + +const isDebug = process.env.DEBUG !== undefined; +const appName = 'unraid-api'; +const logsPath = process.env.API_LOGS_PATH ?? '/var/log/unraid-api/'; +const instances = 1; +const maxRestarts = 100; +const logger = createLogger('supervisor'); + +let apiPid: number; +const isApiRunning = async () => { + const list = await psList(); + const api = list.find(process => { + return process.name.endsWith('unraid-api') || (process.name === 'node' && process.cmd?.split(' ')[1]?.endsWith('unraid-api')); + }); + if (api) { + apiPid = api.pid; + } + + return api !== undefined; +}; + +const sleep = async (ms: number) => new Promise(resolve => { + setTimeout(() => { + resolve(); + }, ms); +}); + +export const startApi = async (restarts = 0, shouldRestart = true) => { + const isRunning = await isApiRunning(); + if (isRunning) { + logger.debug('%s process is running with pid %s', appName, apiPid); + return; + } + + const apiPath = resolveToAbsolutePath(process.env.UNRAID_API_BINARY_LOCATION ?? await locatePath([ + // Global + 'unraid-api', + // Local + './unraid-api' + ]) ?? '') ?? undefined; + + // Save the child process outside of the race + // This is to allow us to kill it if the timeout is quicker + let childProcess: ChildProcess; + + logger.debug('Starting %s.', appName); + + // Ensure the directories exist for the log files + const logDirectory = joinPath(logsPath, 'apps'); + if (!existsSync(logDirectory)) { + logger.debug('Creating log directory %s', logDirectory); + mkdirSync(logDirectory, { recursive: true }); + } + + // Either the new process will spawn + // or it'll timeout/disconnect/exit + await Promise.race([ + new Promise((resolve, reject) => { + logger.debug('Spawning %s instance%s of %s from %s', instances, instances === 1 ? '' : 's', appName, apiPath); + + // Fork the child process + childProcess = spawnProcess(apiPath, ['start', '--debug'], { + stdio: 'pipe' + }); + + // Create stdout and stderr log files + const logConsoleStream = createWriteStream(joinPath(logDirectory, `${appName}.stdout.log`), { flags: 'a' }); + const logErrorStream = createWriteStream(joinPath(logDirectory, `${appName}.stderr.log`), { flags: 'a' }); + + // Redirect stdout and stderr to log files + childProcess.stdout?.pipe(logConsoleStream); + childProcess.stderr?.pipe(logErrorStream); + + // App has started + childProcess.stdout?.once('data', () => { + logger.debug('%s has started', appName); + resolve(); + }); + + // App has thrown an error + childProcess.stderr?.once('data', (data: Buffer) => { + logger.debug('%s threw an error %s', appName, data); + reject(new Error(data.toString())); + }); + + // App has exited + childProcess.once('exit', async code => { + const exitCode = code ?? 0; + logger.debug('%s exited with code %s', appName, exitCode); + + // Increase timeout by 30s for every restart + const initialTimeoutLength = restarts === 0 ? 30_000 : 30_000 * restarts; + // Reset the timeout to 30s if it gets above 5 minutes + const timeoutLength = initialTimeoutLength >= (60_000 * 5) ? 30_000 : initialTimeoutLength; + + // Wait for timer before restarting + logger.info('Restarting %s in %s %s/%s', appName, intervalToHuman(timeoutLength), restarts + 1, maxRestarts); + await sleep(timeoutLength); + + // Restart the app + if (shouldRestart && restarts < maxRestarts) { + logger.info('Restarting %s in %s/%s', appName, restarts + 1, maxRestarts); + await startApi(restarts + 1).catch(error => { + logger.error('Failed restarting %s with %s', appName, error); + }); + } + }); + + logger.debug('Waiting for %s to start', appName); + }), + new Promise((_resolve, reject) => { + // Increase timeout by 30s for every restart + const initialTimeoutLength = restarts === 0 ? 30_000 : 30_000 * restarts; + // Reset the timeout to 30s if it gets above 5 minutes + const timeoutLength = initialTimeoutLength >= (60_000 * 5) ? 30_000 : initialTimeoutLength; + setTimeout(() => { + reject(new Error(`Timed-out starting \`${appName}\`.`)); + }, timeoutLength); + }) + ]).catch((error: unknown) => { + logger.error('Failed spawning %s with %s', appName, error); + childProcess.kill(); + }); +}; + +startApi().catch(error => { + logger.error('Failed starting %s with %s', appName, error); +}); diff --git a/package-lock.json b/package-lock.json index dfd0a23be..9f7431d36 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1447,14 +1447,6 @@ "@types/node": "*" } }, - "@types/fs-extra": { - "version": "9.0.12", - "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz", - "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==", - "requires": { - "@types/node": "*" - } - }, "@types/http-assert": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.1.tgz", @@ -1548,27 +1540,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==" }, - "@types/node-fetch": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz", - "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==", - "requires": { - "@types/node": "*", - "form-data": "^3.0.0" - }, - "dependencies": { - "form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } - } - } - }, "@types/normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -5810,7 +5781,8 @@ "duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", - "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==" + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "dev": true }, "duplexer3": { "version": "0.1.4", @@ -6346,30 +6318,6 @@ "es5-ext": "~0.10.14" } }, - "event-stream": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", - "integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=", - "requires": { - "duplexer": "~0.1.1", - "from": "~0", - "map-stream": "~0.1.0", - "pause-stream": "0.0.11", - "split": "0.3", - "stream-combiner": "~0.0.4", - "through": "~2.3.1" - }, - "dependencies": { - "split": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", - "integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=", - "requires": { - "through": "2" - } - } - } - }, "eventemitter3": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", @@ -6939,11 +6887,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, - "from": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", - "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=" - }, "fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", @@ -8131,6 +8074,12 @@ "kind-of": "^6.0.2" } }, + "interval-to-human": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/interval-to-human/-/interval-to-human-0.1.1.tgz", + "integrity": "sha1-TF/91qV1oMUvAcqm2oz+sDJoXFA=", + "dev": true + }, "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -9268,11 +9217,6 @@ "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.2.1.tgz", "integrity": "sha512-+WA2/1sPmDj1dlvvJmB5G6JKfY9dpn7EVBUL06+y6PoljPkh+6V1QihwxNkbcGxCRjt2b0F9K0taiCuo7MbdFQ==" }, - "map-stream": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", - "integrity": "sha1-5WqpTEyAVaFkBKBnS3jyFffI4ZQ=" - }, "map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", @@ -9858,6 +9802,12 @@ "remove-array-items": "^1.0.0" } }, + "nanocolors": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/nanocolors/-/nanocolors-0.2.12.tgz", + "integrity": "sha512-SFNdALvzW+rVlzqexid6epYdt8H9Zol7xDoQarioEFcFN0JHo4CYNztAxmtfgGTVRCmFlEOqqhBpoFGKqSAMug==", + "dev": true + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -10721,14 +10671,6 @@ "integrity": "sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg==", "dev": true }, - "pause-stream": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", - "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", - "requires": { - "through": "~2.3" - } - }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -10928,13 +10870,11 @@ "ipaddr.js": "1.9.1" } }, - "ps-tree": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz", - "integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==", - "requires": { - "event-stream": "=3.3.4" - } + "ps-list": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ps-list/-/ps-list-7.2.0.tgz", + "integrity": "sha512-v4Bl6I3f2kJfr5o80ShABNHAokIgY+wFDTQfE+X3zWYgSGQOCBeYptLZUpoOALBqO5EawmDN/tjTldJesd0ujQ==", + "dev": true }, "psl": { "version": "1.8.0", @@ -12473,14 +12413,6 @@ "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==" }, - "stream-combiner": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", - "integrity": "sha1-TV5DPBhSYd3mI8o/RMWGvPXErRQ=", - "requires": { - "duplexer": "~0.1.1" - } - }, "stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", @@ -12955,7 +12887,8 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "dev": true }, "through2": { "version": "2.0.5", @@ -13905,107 +13838,6 @@ "tslib": "^1.9.3", "zen-observable": "^0.8.0" } - }, - "zx": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/zx/-/zx-4.2.0.tgz", - "integrity": "sha512-/4f7FaJecA9I655KXKXIHO3CFNYjAz2uSmTz6v2eNlKdrQKyz4VyF3RjqFuP6nQG+Hd3+NjOvrVNBkv8Ne9d4Q==", - "requires": { - "@types/fs-extra": "^9.0.12", - "@types/minimist": "^1.2.2", - "@types/node": "^16.6", - "@types/node-fetch": "^2.5.12", - "chalk": "^4.1.2", - "fs-extra": "^10.0.0", - "globby": "^12.0.1", - "minimist": "^1.2.5", - "node-fetch": "^2.6.1", - "ps-tree": "^1.2.0", - "which": "^2.0.2" - }, - "dependencies": { - "@types/minimist": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", - "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==" - }, - "array-union": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-3.0.1.tgz", - "integrity": "sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw==" - }, - "chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - } - }, - "fast-glob": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.7.tgz", - "integrity": "sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==", - "requires": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - } - }, - "fs-extra": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", - "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", - "requires": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - } - }, - "globby": { - "version": "12.0.2", - "resolved": "https://registry.npmjs.org/globby/-/globby-12.0.2.tgz", - "integrity": "sha512-lAsmb/5Lww4r7MM9nCCliDZVIKbZTavrsunAsHLr9oHthrZP1qi7/gAnHOsUs9bLvEt2vKVJhHmxuL7QbDuPdQ==", - "requires": { - "array-union": "^3.0.1", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.7", - "ignore": "^5.1.8", - "merge2": "^1.4.1", - "slash": "^4.0.0" - } - }, - "jsonfile": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "requires": { - "graceful-fs": "^4.1.6", - "universalify": "^2.0.0" - } - }, - "slash": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", - "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==" - }, - "universalify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" - }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "requires": { - "isexe": "^2.0.0" - } - } - } } } } diff --git a/package.json b/package.json index 39a949a78..e0dc8d5fc 100644 --- a/package.json +++ b/package.json @@ -6,11 +6,13 @@ "author": "Alexis Tyler (https://wvvw.me/)", "license": "UNLICENSED", "scripts": { - "build": "npm run build-app && npm run build-cli && npm run copy-schemas", + "build": "npm run build-app && npm run build-cli && npm run build-supervisor && npm run copy-schemas", "build-app": "npx tsup ./app/index.ts", "build-cli": "npx tsup ./app/cli.ts", - "build-binary-step-1": "nexe ./dist/cli.js --make=\"-j$(nproc 2> /dev/null || echo 1)\" -r './dist/**/*' -r './node_modules' && mv ./cli ./unraid-api && echo '✔ Binary built: ./unraid-api'", - "build-binary-step-2": "rm -rf ./node_modules && rm -rf ./dist && echo '✔ Source files deleted'", + "build-supervisor": "npx tsup ./app/supervisor.ts", + "build-binary-step-1": "nexe ./dist/cli.js --make=\"-j$(nproc 2> /dev/null || echo 1)\" -r './dist/**/*' -r './node_modules' && mv ./cli ./bin/unraid-api && echo '✔ Binary built: ./bin/unraid-api'", + "build-binary-step-2": "nexe ./dist/supervisor.js --make=\"-j$(nproc 2> /dev/null || echo 1)\" -r './dist/**/*' -r './node_modules' && mv ./supervisor ./bin/unraid-supervisor && echo '✔ Binary built: ./bin/unraid-supervisor'", + "build-binary-step-3": "rm -rf ./node_modules && rm -rf ./dist && echo '✔ Source files deleted'", "build-binary": "npm run build-binary-step-1 && npm run build-binary-step-2", "copy-schemas": "cpx app/**/*.graphql dist/types", "clean": "modclean --no-progress --run --path .", @@ -148,12 +150,15 @@ "cpx": "1.5.0", "cz-conventional-changelog": "3.3.0", "eslint": "^7.32.0", + "interval-to-human": "^0.1.1", "modclean": "^3.0.0-beta.1", + "nanocolors": "^0.2.12", "node-env-run": "^4.0.2", "nyc": "^15.1.0", "p-each-series": "^3.0.0", "p-props": "^5.0.0", "path-type": "^5.0.0", + "ps-list": "^7.2.0", "source-map-support": "0.5.19", "standard-version": "^9.1.1", "supertest": "^6.1.3", @@ -264,4 +269,4 @@ "uuid-apikey", "xhr2" ] -} +} \ No newline at end of file