feat: add supervisor

This commit is contained in:
Alexis
2021-09-29 14:49:42 +09:30
parent 7d24bd24c5
commit 42051849d7
4 changed files with 193 additions and 193 deletions

3
.gitignore vendored
View File

@@ -69,3 +69,6 @@ typescript
# Github actions
RELEASE_NOTES.md
# Binary files
bin

160
app/supervisor.ts Normal file
View File

@@ -0,0 +1,160 @@
import { join as joinPath, resolve as resolveToAbsolutePath } from 'path';
import { createWriteStream, mkdirSync, existsSync } from 'fs';
import { spawn as spawnProcess, ChildProcess } from 'child_process';
import locatePath from 'locate-path';
import psList from 'ps-list';
import { cyan, green, yellow, red } from 'nanocolors';
import intervalToHuman from 'interval-to-human';
const createLogger = (namespace: string) => {
const ns = namespace.toUpperCase();
return {
info(message: string, ...args: any[]) {
console.info(`${cyan(`[${ns}]`)} ${message}`, ...args);
},
debug(message: string, ...args: any[]) {
if (!isDebug) {
return;
}
console.debug(`${green(`[${ns}]`)} ${message}`, ...args);
},
warning(message: string, ...args: any[]) {
console.warn(`${yellow(`[${ns}][WARNING]`)} ${message}`, ...args);
},
error(message: string | Error, ...args: any[]) {
console.error(`${red(`[${ns}][ERROR]`)} ${message}`, ...args);
},
print(message: string, ...args: any[]) {
console.log(message, ...args);
}
};
};
const isDebug = process.env.DEBUG !== undefined;
const appName = 'unraid-api';
const logsPath = process.env.API_LOGS_PATH ?? '/var/log/unraid-api/';
const instances = 1;
const maxRestarts = 100;
const logger = createLogger('supervisor');
let apiPid: number;
const isApiRunning = async () => {
const list = await psList();
const api = list.find(process => {
return process.name.endsWith('unraid-api') || (process.name === 'node' && process.cmd?.split(' ')[1]?.endsWith('unraid-api'));
});
if (api) {
apiPid = api.pid;
}
return api !== undefined;
};
const sleep = async (ms: number) => new Promise<void>(resolve => {
setTimeout(() => {
resolve();
}, ms);
});
export const startApi = async (restarts = 0, shouldRestart = true) => {
const isRunning = await isApiRunning();
if (isRunning) {
logger.debug('%s process is running with pid %s', appName, apiPid);
return;
}
const apiPath = resolveToAbsolutePath(process.env.UNRAID_API_BINARY_LOCATION ?? await locatePath([
// Global
'unraid-api',
// Local
'./unraid-api'
]) ?? '') ?? undefined;
// Save the child process outside of the race
// This is to allow us to kill it if the timeout is quicker
let childProcess: ChildProcess;
logger.debug('Starting %s.', appName);
// Ensure the directories exist for the log files
const logDirectory = joinPath(logsPath, 'apps');
if (!existsSync(logDirectory)) {
logger.debug('Creating log directory %s', logDirectory);
mkdirSync(logDirectory, { recursive: true });
}
// Either the new process will spawn
// or it'll timeout/disconnect/exit
await Promise.race([
new Promise<void>((resolve, reject) => {
logger.debug('Spawning %s instance%s of %s from %s', instances, instances === 1 ? '' : 's', appName, apiPath);
// Fork the child process
childProcess = spawnProcess(apiPath, ['start', '--debug'], {
stdio: 'pipe'
});
// Create stdout and stderr log files
const logConsoleStream = createWriteStream(joinPath(logDirectory, `${appName}.stdout.log`), { flags: 'a' });
const logErrorStream = createWriteStream(joinPath(logDirectory, `${appName}.stderr.log`), { flags: 'a' });
// Redirect stdout and stderr to log files
childProcess.stdout?.pipe(logConsoleStream);
childProcess.stderr?.pipe(logErrorStream);
// App has started
childProcess.stdout?.once('data', () => {
logger.debug('%s has started', appName);
resolve();
});
// App has thrown an error
childProcess.stderr?.once('data', (data: Buffer) => {
logger.debug('%s threw an error %s', appName, data);
reject(new Error(data.toString()));
});
// App has exited
childProcess.once('exit', async code => {
const exitCode = code ?? 0;
logger.debug('%s exited with code %s', appName, exitCode);
// Increase timeout by 30s for every restart
const initialTimeoutLength = restarts === 0 ? 30_000 : 30_000 * restarts;
// Reset the timeout to 30s if it gets above 5 minutes
const timeoutLength = initialTimeoutLength >= (60_000 * 5) ? 30_000 : initialTimeoutLength;
// Wait for timer before restarting
logger.info('Restarting %s in %s %s/%s', appName, intervalToHuman(timeoutLength), restarts + 1, maxRestarts);
await sleep(timeoutLength);
// Restart the app
if (shouldRestart && restarts < maxRestarts) {
logger.info('Restarting %s in %s/%s', appName, restarts + 1, maxRestarts);
await startApi(restarts + 1).catch(error => {
logger.error('Failed restarting %s with %s', appName, error);
});
}
});
logger.debug('Waiting for %s to start', appName);
}),
new Promise<void>((_resolve, reject) => {
// Increase timeout by 30s for every restart
const initialTimeoutLength = restarts === 0 ? 30_000 : 30_000 * restarts;
// Reset the timeout to 30s if it gets above 5 minutes
const timeoutLength = initialTimeoutLength >= (60_000 * 5) ? 30_000 : initialTimeoutLength;
setTimeout(() => {
reject(new Error(`Timed-out starting \`${appName}\`.`));
}, timeoutLength);
})
]).catch((error: unknown) => {
logger.error('Failed spawning %s with %s', appName, error);
childProcess.kill();
});
};
startApi().catch(error => {
logger.error('Failed starting %s with %s', appName, error);
});

210
package-lock.json generated
View File

@@ -1447,14 +1447,6 @@
"@types/node": "*"
}
},
"@types/fs-extra": {
"version": "9.0.12",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
"integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
"requires": {
"@types/node": "*"
}
},
"@types/http-assert": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.1.tgz",
@@ -1548,27 +1540,6 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",
"integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="
},
"@types/node-fetch": {
"version": "2.5.12",
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
"integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
"requires": {
"@types/node": "*",
"form-data": "^3.0.0"
},
"dependencies": {
"form-data": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
}
}
}
},
"@types/normalize-package-data": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
@@ -5810,7 +5781,8 @@
"duplexer": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
"integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg=="
"integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==",
"dev": true
},
"duplexer3": {
"version": "0.1.4",
@@ -6346,30 +6318,6 @@
"es5-ext": "~0.10.14"
}
},
"event-stream": {
"version": "3.3.4",
"resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz",
"integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=",
"requires": {
"duplexer": "~0.1.1",
"from": "~0",
"map-stream": "~0.1.0",
"pause-stream": "0.0.11",
"split": "0.3",
"stream-combiner": "~0.0.4",
"through": "~2.3.1"
},
"dependencies": {
"split": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz",
"integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=",
"requires": {
"through": "2"
}
}
}
},
"eventemitter3": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz",
@@ -6939,11 +6887,6 @@
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
},
"from": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
"integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4="
},
"fromentries": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz",
@@ -8131,6 +8074,12 @@
"kind-of": "^6.0.2"
}
},
"interval-to-human": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/interval-to-human/-/interval-to-human-0.1.1.tgz",
"integrity": "sha1-TF/91qV1oMUvAcqm2oz+sDJoXFA=",
"dev": true
},
"ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
@@ -9268,11 +9217,6 @@
"resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.2.1.tgz",
"integrity": "sha512-+WA2/1sPmDj1dlvvJmB5G6JKfY9dpn7EVBUL06+y6PoljPkh+6V1QihwxNkbcGxCRjt2b0F9K0taiCuo7MbdFQ=="
},
"map-stream": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz",
"integrity": "sha1-5WqpTEyAVaFkBKBnS3jyFffI4ZQ="
},
"map-visit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz",
@@ -9858,6 +9802,12 @@
"remove-array-items": "^1.0.0"
}
},
"nanocolors": {
"version": "0.2.12",
"resolved": "https://registry.npmjs.org/nanocolors/-/nanocolors-0.2.12.tgz",
"integrity": "sha512-SFNdALvzW+rVlzqexid6epYdt8H9Zol7xDoQarioEFcFN0JHo4CYNztAxmtfgGTVRCmFlEOqqhBpoFGKqSAMug==",
"dev": true
},
"nanomatch": {
"version": "1.2.13",
"resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
@@ -10721,14 +10671,6 @@
"integrity": "sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg==",
"dev": true
},
"pause-stream": {
"version": "0.0.11",
"resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
"integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=",
"requires": {
"through": "~2.3"
}
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@@ -10928,13 +10870,11 @@
"ipaddr.js": "1.9.1"
}
},
"ps-tree": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz",
"integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==",
"requires": {
"event-stream": "=3.3.4"
}
"ps-list": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/ps-list/-/ps-list-7.2.0.tgz",
"integrity": "sha512-v4Bl6I3f2kJfr5o80ShABNHAokIgY+wFDTQfE+X3zWYgSGQOCBeYptLZUpoOALBqO5EawmDN/tjTldJesd0ujQ==",
"dev": true
},
"psl": {
"version": "1.8.0",
@@ -12473,14 +12413,6 @@
"resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz",
"integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw=="
},
"stream-combiner": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz",
"integrity": "sha1-TV5DPBhSYd3mI8o/RMWGvPXErRQ=",
"requires": {
"duplexer": "~0.1.1"
}
},
"stream-shift": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
@@ -12955,7 +12887,8 @@
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
"dev": true
},
"through2": {
"version": "2.0.5",
@@ -13905,107 +13838,6 @@
"tslib": "^1.9.3",
"zen-observable": "^0.8.0"
}
},
"zx": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/zx/-/zx-4.2.0.tgz",
"integrity": "sha512-/4f7FaJecA9I655KXKXIHO3CFNYjAz2uSmTz6v2eNlKdrQKyz4VyF3RjqFuP6nQG+Hd3+NjOvrVNBkv8Ne9d4Q==",
"requires": {
"@types/fs-extra": "^9.0.12",
"@types/minimist": "^1.2.2",
"@types/node": "^16.6",
"@types/node-fetch": "^2.5.12",
"chalk": "^4.1.2",
"fs-extra": "^10.0.0",
"globby": "^12.0.1",
"minimist": "^1.2.5",
"node-fetch": "^2.6.1",
"ps-tree": "^1.2.0",
"which": "^2.0.2"
},
"dependencies": {
"@types/minimist": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz",
"integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ=="
},
"array-union": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/array-union/-/array-union-3.0.1.tgz",
"integrity": "sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw=="
},
"chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"requires": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
}
},
"fast-glob": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.7.tgz",
"integrity": "sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==",
"requires": {
"@nodelib/fs.stat": "^2.0.2",
"@nodelib/fs.walk": "^1.2.3",
"glob-parent": "^5.1.2",
"merge2": "^1.3.0",
"micromatch": "^4.0.4"
}
},
"fs-extra": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
"integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
"requires": {
"graceful-fs": "^4.2.0",
"jsonfile": "^6.0.1",
"universalify": "^2.0.0"
}
},
"globby": {
"version": "12.0.2",
"resolved": "https://registry.npmjs.org/globby/-/globby-12.0.2.tgz",
"integrity": "sha512-lAsmb/5Lww4r7MM9nCCliDZVIKbZTavrsunAsHLr9oHthrZP1qi7/gAnHOsUs9bLvEt2vKVJhHmxuL7QbDuPdQ==",
"requires": {
"array-union": "^3.0.1",
"dir-glob": "^3.0.1",
"fast-glob": "^3.2.7",
"ignore": "^5.1.8",
"merge2": "^1.4.1",
"slash": "^4.0.0"
}
},
"jsonfile": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
"requires": {
"graceful-fs": "^4.1.6",
"universalify": "^2.0.0"
}
},
"slash": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz",
"integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew=="
},
"universalify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
},
"which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
"requires": {
"isexe": "^2.0.0"
}
}
}
}
}
}

View File

@@ -6,11 +6,13 @@
"author": "Alexis Tyler <xo@wvvw.me> (https://wvvw.me/)",
"license": "UNLICENSED",
"scripts": {
"build": "npm run build-app && npm run build-cli && npm run copy-schemas",
"build": "npm run build-app && npm run build-cli && npm run build-supervisor && npm run copy-schemas",
"build-app": "npx tsup ./app/index.ts",
"build-cli": "npx tsup ./app/cli.ts",
"build-binary-step-1": "nexe ./dist/cli.js --make=\"-j$(nproc 2> /dev/null || echo 1)\" -r './dist/**/*' -r './node_modules' && mv ./cli ./unraid-api && echo '✔ Binary built: ./unraid-api'",
"build-binary-step-2": "rm -rf ./node_modules && rm -rf ./dist && echo '✔ Source files deleted'",
"build-supervisor": "npx tsup ./app/supervisor.ts",
"build-binary-step-1": "nexe ./dist/cli.js --make=\"-j$(nproc 2> /dev/null || echo 1)\" -r './dist/**/*' -r './node_modules' && mv ./cli ./bin/unraid-api && echo '✔ Binary built: ./bin/unraid-api'",
"build-binary-step-2": "nexe ./dist/supervisor.js --make=\"-j$(nproc 2> /dev/null || echo 1)\" -r './dist/**/*' -r './node_modules' && mv ./supervisor ./bin/unraid-supervisor && echo '✔ Binary built: ./bin/unraid-supervisor'",
"build-binary-step-3": "rm -rf ./node_modules && rm -rf ./dist && echo '✔ Source files deleted'",
"build-binary": "npm run build-binary-step-1 && npm run build-binary-step-2",
"copy-schemas": "cpx app/**/*.graphql dist/types",
"clean": "modclean --no-progress --run --path .",
@@ -148,12 +150,15 @@
"cpx": "1.5.0",
"cz-conventional-changelog": "3.3.0",
"eslint": "^7.32.0",
"interval-to-human": "^0.1.1",
"modclean": "^3.0.0-beta.1",
"nanocolors": "^0.2.12",
"node-env-run": "^4.0.2",
"nyc": "^15.1.0",
"p-each-series": "^3.0.0",
"p-props": "^5.0.0",
"path-type": "^5.0.0",
"ps-list": "^7.2.0",
"source-map-support": "0.5.19",
"standard-version": "^9.1.1",
"supertest": "^6.1.3",
@@ -264,4 +269,4 @@
"uuid-apikey",
"xhr2"
]
}
}