fix: add ratelimiting + update systeminformation dep

app/graphql/index.ts

@@ -6,7 +6,7 @@
 import get from 'lodash.get';
 import { v4 as uuid } from 'uuid';
 import * as core from '../core';
-import { bus, apiManager, graphqlLogger, config, pluginManager, modules } from '../core';
+import { bus, apiManager, graphqlLogger, config, pluginManager, modules, coreLogger } from '../core';
 import { AppError, FatalAppError, PluginError } from '../core/errors';
 import { usersState } from '../core/states';
 import { makeExecutableSchema, SchemaDirectiveVisitor } from 'graphql-tools';
@@ -292,9 +292,11 @@ const apiKeyToUser = (apiKey: string) => {
 
 // Update array values when slots change
 bus.on('slots', async () => {
+	coreLogger.silly('slots updated: loading user');
 	// @todo: Create a system user for this
 	const user = usersState.findOne({ name: 'root' });
 
+	coreLogger.silly('slots updated: running getArray');
 	await run('array', 'UPDATED', {
 		moduleToRun: modules.getArray,
 		context: {

app/mothership/index.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import WebSocket from 'ws';
 import * as Sentry from '@sentry/node';
 import { Mutex, MutexInterface } from 'async-mutex';
-import { MOTHERSHIP_RELAY_WS_LINK, INTERNAL_WS_LINK, ONE_MINUTE } from '../consts';
+import { MOTHERSHIP_RELAY_WS_LINK, INTERNAL_WS_LINK, ONE_MINUTE, ONE_SECOND } from '../consts';
 import { mothershipLogger, apiManager } from '../core';
 import { getMachineId } from '../core/utils';
 import { varState, networkState } from '../core/states';
@@ -89,6 +89,7 @@ class MothershipService {
 	}
 
 	public async connect(wsServer: WebSocket.Server, currentRetryAttempt: number = 0): Promise<void> {
+		const mothership = this;
 		this.connectionAttempt++;
 		if (currentRetryAttempt >= 1) {
 			mothershipLogger.debug('connection attempt %s', currentRetryAttempt);
@@ -128,7 +129,7 @@ class MothershipService {
 				mothershipLogger.debug('Connected to mothership\'s relay via %s.', this.relayWebsocketLink);
 		
 				// Reset connection attempts
-				this.connectionAttempt = 0;
+				mothership.connectionAttempt = 0;
 		
 				// Connect to the internal graphql server
 				this.localGraphqlApi = new WebSocket(this.internalWsLink, ['graphql-ws']);
@@ -184,9 +185,8 @@ class MothershipService {
 				// Sub to /servers on mothership
 				this.mothershipServersEndpoint = subscribeToServers(apiKey);
 			});
-		
+
 			// Relay is closed
-			const mothership = this;
 			this.relay.on('close', async function (this: WebSocketWithHeartBeat, code, _message) {
 				try {
 					mothershipLogger.debug('Connection closed with code %s.', code);
@@ -212,6 +212,16 @@ class MothershipService {
 							mothershipLogger.debug('Invalid API key, waiting for new key...');
 							return;
 						}
+
+						// Rate limited
+						if (code === 4429) {
+							try {
+								const message = JSON.parse(_message);
+								const retryAfter = message['Retry-After'];
+								mothershipLogger.debug('Rate limited, retrying after %ss', retryAfter);
+								await sleep(ONE_SECOND * retryAfter);
+							} catch {};
+						}
 					}
 		
 					// We likely closed this
@@ -221,12 +231,18 @@ class MothershipService {
 						mothership.connect(wsServer);
 						return;
 					}
+
+					// Something went wrong on mothership
+					// Let's wait an extra bit
+					if (code === 4500) {
+						await sleep(ONE_SECOND * 5);
+					}
 					
 					// Wait a few seconds
 					await sleep(backoff(mothership.connectionAttempt, ONE_MINUTE, 5));
 		
 					// Reconnect
-					await mothership.connect(wsServer, currentRetryAttempt + 1);
+					await mothership.connect(wsServer, mothership.connectionAttempt + 1);
 				} catch (error) {
 					mothershipLogger.error('close error', error);
 				}

app/run.ts

@@ -41,6 +41,7 @@ export const run = async (channel: string, mutation: string, options: RunOptions
     } = options;
 
     if (!moduleToRun) {
+        coreLogger.silly('Tried to run but has no "moduleToRun"');
         return publish(channel, mutation, node);
     }
 
@@ -51,6 +52,12 @@ export const run = async (channel: string, mutation: string, options: RunOptions
             return resolve(moduleToRun(context));
         });
 
+        // Services
+        if (moduleToRun.name !== 'Gg') {
+            // Log result
+            coreLogger.silly(`run:${moduleToRun.name}`, JSON.stringify(result, null, 2));
+        }
+
         // Save result
         publish(channel, mutation, result.json);
     } catch (error) {

app/server.ts

@@ -194,6 +194,8 @@ export const server = {
 	httpServer,
 	server: stoppableServer,
 	async start() {
+		let connectedAtleastOnce = false;
+
 		// If key is in an invalid format disconnect
 		apiManager.on('expire', async () => {
 			await mothership.disconnect();
@@ -201,6 +203,10 @@ export const server = {
 
 		// If the key changes try to (re)connect to Mothership
 		apiManager.on('replace', async () => {
+			if (!connectedAtleastOnce) {
+				connectedAtleastOnce = true;
+				await sleep(1000);
+			}
 			await mothership.connect(wsServer);
 		});
 

package-lock.json

@@ -14120,9 +14120,9 @@
       "integrity": "sha512-sQV7phh2WCYAn81oAkakC5qjq2Ml0g8ozqz03wOGnx9dDlG1de6yrF+0RAzSJD8fPUow3PTSMf2SAbOGxb93BA=="
     },
     "systeminformation": {
-      "version": "4.30.1",
-      "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-4.30.1.tgz",
-      "integrity": "sha512-FrZISqs8G/oZfnzodPU/zCk41JIfa0os2WY6CJKuk9FwIVc9UWMJfRlgP307AkFu8IRjuVI/HiCNiP4dAnURTA=="
+      "version": "4.31.0",
+      "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-4.31.0.tgz",
+      "integrity": "sha512-j1eNsuHxpW00RpxSvLy2IJHXpH54TyzZGQRTSFM5flD+dl83qmZ7TWIPnVkACMgHFABkL95I4KTf6S7aRsGUWg=="
     },
     "table": {
       "version": "5.4.6",

package.json

@@ -6,7 +6,7 @@
   "author": "Alexis Tyler <xo@wvvw.me> (https://wvvw.me/)",
   "license": "UNLICENSED",
   "scripts": {
-    "build": "npm run bundle-minify && npm run copy-schemas",
+    "build": "npm run bundle && npm run copy-schemas",
     "bundle": "npx tsup ./app/index.ts",
     "bundle-minify": "npx tsup ./app/index.ts --minify",
     "bundle-dependencies": "bundle-dependencies update",
@@ -113,7 +113,7 @@
     "spread-the-word": "^0.8.4",
     "stoppable": "^1.1.0",
     "subscriptions-transport-ws": "^0.9.18",
-    "systeminformation": "^4.29.3",
+    "systeminformation": "^4.31.0",
     "tracer": "^1.1.4",
     "unix-dgram": "^2.0.3",
     "upcast": "^4.0.0",