Merge remote-tracking branch 'origin/feat/split-cfg' into plg/improvements-2021-01-21

.env.test

@@ -3,5 +3,6 @@ PATHS_STATES=$(pwd)/../core/test/fixtures/states
 PATHS_PLUGINS=$(pwd)/../core/test/fixtures/plugins
 PATHS_UNRAID_VERSION=$(pwd)/../core/test/fixtures/etc/unraid-version
 PATHS_DYNAMIX_CONFIG=$(pwd)/../core/test/fixtures/boot/config/plugins/dynamix/dynamix.cfg
+PATHS_MY_SERVERS_CONFIG=$(pwd)/../core/test/fixtures/boot/config/plugins/Unraid.net/myservers.cfg
 NCHAN=false
 PORT=5000

README.md

@@ -29,7 +29,7 @@ Debug logs can be enabled via stdout while running with `start-debug`.
 ## Playground
 
 The playground can be access via `http://tower.local/graphql` while `PLAYGROUND=true` and `INTROSPECTION=true`. These values can be set in the `ecosystem.config.js` file in `/usr/local/bin/node/unraid-api`.
-To get your API key open a terminal on your server and run `cat /boot/config/plugins/dynamix/dynamix.cfg | grep apikey= | cut -d '"' -f2`. Add that api key in the "HTTP headers" panel of the playground.
+To get your API key open a terminal on your server and run `cat /boot/config/plugins/Unraid.net/myservers.cfg | grep apikey= | cut -d '"' -f2`. Add that api key in the "HTTP headers" panel of the playground.
 
 ```json
 {

app/core/api-manager.ts

@@ -3,6 +3,7 @@
  * Written by: Alexis Tyler
  */
 
+import path from 'path';
 import chokidar from 'chokidar';
 import { EventEmitter } from 'events';
 import toMillisecond from 'ms';
@@ -116,12 +117,11 @@ export class ApiManager extends EventEmitter {
 		// Create singleton
 		ApiManager.instance = this;
 
-		// Watch for changes to the dynamix.cfg file
+		// Watch for changes to the myservers.cfg file
 		// @todo Move API keys to their own file
-		const basePath = paths.get('dynamix-base')!;
-		const configPath = paths.get('dynamix-config')!;
+		const configPath = paths.get('myservers-config')!;
 		if (options.watch) {
-			chokidar.watch(basePath, {
+			chokidar.watch(path.basename(configPath), {
 				ignoreInitial: true
 			}).on('all', async (_eventName, filePath) => {
 				if (filePath === configPath) {

app/core/log.ts

@@ -5,9 +5,9 @@
 
 import { Logger } from 'logger';
 
-export const log = new Logger();
-export const coreLogger = log.createChild({ prefix: '[@unraid/core]: '});
-export const mothershipLogger = log.createChild({ prefix: '[@unraid/mothership]: '});
-export const graphqlLogger = log.createChild({ prefix: '[@unraid/graphql]: '});
-export const relayLogger = log.createChild({ prefix: '[@unraid/relay]: '});
-export const discoveryLogger = log.createChild({ prefix: '[@unraid/discovery]: '});
+export const log = new Logger({ prefix: '@unraid' });
+export const coreLogger = log.createChild({ prefix: 'core' });
+export const mothershipLogger = log.createChild({ prefix: 'mothership'});
+export const graphqlLogger = log.createChild({ prefix: 'graphql'});
+export const relayLogger = log.createChild({ prefix: 'relay'});
+export const discoveryLogger = log.createChild({ prefix: 'discovery'});

app/core/paths.ts

@@ -16,6 +16,7 @@ export interface Paths {
 	'emhttpd-socket': string;
 	'dynamix-base': string;
 	'dynamix-config': string;
+	'myservers-config': string;
 	'nginx-origin': string;
 	'machine-id': string;
 }
@@ -46,6 +47,7 @@ export const defaultPaths = new Map<keyof Paths, string>([
 	['states', '/usr/local/emhttp/state/'],
 	['dynamix-base', '/boot/config/plugins/dynamix/'],
 	['dynamix-config', '/boot/config/plugins/dynamix/dynamix.cfg'],
+	['myservers-config', '/boot/config/plugins/Unraid.net/myservers.cfg'],
 	['nginx-origin', '/var/run/nginx.origin'],
 	['machine-id', '/etc/machine-id']
 ]);

app/core/states/var.ts

@@ -252,7 +252,8 @@ const parse = (state: VarIni): Var => {
 		shareNfsCount: toNumber(state.shareNfsCount),
 		shareNfsEnabled: iniBooleanToJsBoolean(state.shareNfsEnabled),
 		shareSmbCount: toNumber(state.shareSmbCount),
-		shareSmbEnabled: iniBooleanToJsBoolean(state.shareSmbEnabled),
+		shareSmbEnabled: ['yes', 'ads'].includes(state.shareSmbEnabled),
+		shareSmbMode: state.shareSmbEnabled === 'ads' ? 'active-directory' : 'workgroup',
 		shutdownTimeout: toNumber(state.shutdownTimeout),
 		spindownDelay: toNumber(state.spindownDelay),
 		spinupGroups: iniBooleanToJsBoolean(state.spinupGroups),

app/core/types/states/var.ts

@@ -158,7 +158,10 @@ export interface Var {
 	shareNfsEnabled: boolean;
 	/** Total number of SMB shares. */
 	shareSmbCount: number;
+	/** Is smb enabled */
 	shareSmbEnabled: boolean;
+	/** Which mode is smb running in? active-directory | workgroup */
+	shareSmbMode: string;
 	shareUser: string;
 	// shareUserExclude
 	shutdownTimeout: number;

app/mothership/custom-socket.ts

@@ -0,0 +1,305 @@
+import WebSocket, { Server as WebsocketServer } from 'ws';
+import { Mutex, MutexInterface } from 'async-mutex';
+import { ONE_SECOND, ONE_MINUTE } from '../consts';
+import { log } from '../core';
+import { AppError } from '../core/errors';
+import { sleep } from '../core/utils';
+import { backoff } from './utils';
+
+export interface WebSocketWithHeartBeat extends WebSocket {
+	heartbeat?: NodeJS.Timeout
+}
+
+function heartbeat(this: WebSocketWithHeartBeat) {
+	if (this.heartbeat) {
+		clearTimeout(this.heartbeat);
+	}
+
+	// Use `WebSocket#terminate()`, which immediately destroys the connection,
+	// instead of `WebSocket#close()`, which waits for the close timer.
+	// Delay should be equal to the interval at which your server
+	// sends out pings plus a conservative assumption of the latency.
+	this.heartbeat = setTimeout(() => {
+		this.terminate();
+	}, 30000 + 1000);
+};
+
+interface Options {
+    name: string;
+    uri: string;
+    apiKey: string;
+    logger: typeof log;
+	lazy: boolean;
+	wsServer: WebsocketServer
+}
+
+export class CustomSocket {
+    public name: string;
+	public uri: string;
+	public connection?: WebSocketWithHeartBeat;
+
+    protected apiKey: string;
+    protected logger: typeof log;
+    protected connectionAttempts = 0;
+
+	private lock?: MutexInterface;
+
+    constructor(public options: Partial<Options> = {}) {
+        this.name = options.name ?? 'CustomSocket';
+        this.uri = options.uri ?? 'localhost';
+        this.apiKey = options.apiKey ?? '';
+        this.logger = options.logger ?? log;
+
+        // Connect right away
+        if (!options.lazy) {
+            this.connect();
+        }
+    }
+
+    public isConnected() {
+        return this.connection && (this.connection.readyState === this.connection.OPEN);
+    }
+    
+    public isConnecting() {
+        return this.connection && (this.connection.readyState === this.connection.CONNECTING);
+    }
+
+    public onError() {
+		return (error: NodeJS.ErrnoException) => {
+			this.logger.error(error);
+		};
+    }
+
+    public onConnect() {
+        const customSocket = this;
+		return async function(this: WebSocketWithHeartBeat) {
+            try {
+                const apiKey = customSocket.apiKey;
+                if (!apiKey || (typeof apiKey === 'string' && apiKey.length === 0)) {
+                    throw new AppError('Missing key', 4422);
+                }
+
+                customSocket.logger.debug('Connected via %s.', customSocket.connection?.url);
+
+                // Reset connection attempts
+                customSocket.connectionAttempts = 0;
+            } catch (error) {
+                this.close(error.code.length === 4 ? error.code : `4${error.code}`, JSON.stringify({
+				    message: error.message ?? 'Internal Server Error'
+                }));
+            }
+        };
+    }
+
+    protected onDisconnect() {
+		const customSocket = this;
+		return async function (this: WebSocketWithHeartBeat, code: number, _message: string) {
+			try {
+				const message = _message.trim() === '' ? { message: '' } : JSON.parse(_message);
+				customSocket.logger.debug('Connection closed with code=%s reason="%s"', code, code === 1006 ? 'Terminated' : message.message);
+	
+				// Stop ws heartbeat
+				if (this.heartbeat) {
+					clearTimeout(this.heartbeat);
+				}
+	
+				// Http 4XX error
+				if (code >= 4400 && code <= 4499) {
+					// Unauthorized - Invalid/missing API key.
+					if (code === 4401) {
+						customSocket.logger.debug('Invalid API key, waiting for new key...');
+						return;
+					}
+	
+					// Rate limited
+					if (code === 4429) {
+						try {
+							let interval: NodeJS.Timeout | undefined;
+							const retryAfter = parseInt(message['Retry-After'], 10) || 30;
+							customSocket.logger.debug('Rate limited, retrying after %ss', retryAfter);
+	
+							// Less than 30s
+							if (retryAfter <= 30) {
+								let seconds = retryAfter;
+	
+								// Print retry once per second
+								interval = setInterval(() => {
+									seconds--;
+									customSocket.logger.debug('Retrying connection in %ss', seconds);
+								}, ONE_SECOND);
+							}
+	
+							if (retryAfter >= 1) {
+								await sleep(ONE_SECOND * retryAfter);
+							}
+	
+							if (interval) {
+								clearInterval(interval);
+							}
+						} catch {};
+					}
+				}
+	
+				// We likely closed this
+				// This is usually because the API key is updated
+				if (code === 4200) {
+					// Reconnect
+					customSocket.connect();
+					return;
+				}
+	
+				// Something went wrong on the connection
+				// Let's wait an extra bit
+				if (code === 4500) {
+					await sleep(ONE_SECOND * 5);
+				}
+			} catch (error) {
+				customSocket.logger.debug('Connection closed with code=%s reason="%s"', code, error.message);
+			}
+	
+			try {
+				// Wait a few seconds
+				await sleep(backoff(customSocket.connectionAttempts, ONE_MINUTE, 5));
+	
+				// Reconnect
+				await customSocket.connect(customSocket.connectionAttempts + 1);
+			} catch (error) {
+				customSocket.logger.debug('Failed reconnecting to %s reason="%s"', customSocket.uri, error.message);
+			}
+		};
+	}
+
+    public onMessage() {
+		const customSocket = this;
+        return async function(message: string, ...args) {
+			customSocket.logger.silly('message="%s" args="%s"', message, ...args);
+		};
+    }
+
+    protected async cleanup() {
+        // Kill existing socket connection
+		if (this.connection) {
+			this.connection.close(4200, JSON.stringify({
+				message: 'Reconnecting'
+			}));
+		}
+    }
+
+    protected async getApiKey() {
+        return '';
+    }
+
+    protected async getHeaders() {
+        return {};
+    }
+
+    protected async isConnectionAllowed() {
+        return true;
+    }
+
+    protected async sendMessage(client?: WebSocketWithHeartBeat, message?: string, timeout = 1000) {
+		try {
+			if (!client || client.readyState === 0 || client.readyState === 3) {
+				// Wait for $timeout seconds
+				await sleep(timeout);
+	
+				// Retry sending
+				await this.sendMessage(client, message, timeout);
+				return;
+			}
+	
+			// Only send when socket is open
+			if (client.readyState === client.OPEN) {
+				client.send(message);
+				this.logger.silly('Message sent to %s.', message, client?.url);
+				return;
+			}
+	
+			// Failed replying as socket isn't open
+			this.logger.error('Failed replying to %s. state=%s message="%s"', client?.url, client.readyState, message);
+		} catch (error) {
+			this.logger.error('Failed replying to %s.', client?.url, error);
+		};
+	};
+
+    private async getLock() {
+		if (!this.lock) {
+			this.lock = new Mutex();
+		}
+
+		const release = await this.lock.acquire();
+		return {
+			release
+		};
+    }
+
+    private async setRetryAttempt(currentRetryAttempt = 0) {
+        this.connectionAttempts += 1;
+		if (currentRetryAttempt >= 1) {
+			this.logger.debug('Connection attempt %s', currentRetryAttempt);
+		}
+    }
+
+    private async _connect() {
+        this.connection = new WebSocket(this.uri, ['graphql-ws'], {
+            headers: await this.getHeaders()
+        });
+		this.connection.on('ping', heartbeat.bind(this.connection));
+		this.connection.on('error', this.onError());
+		this.connection.on('close', this.onDisconnect());
+		this.connection.on('open', this.onConnect());
+		this.connection.on('message', this.onMessage());
+		// this.connection.on('ping', console.log);
+		// this.connection.on('error', console.log);
+		// this.connection.on('close', console.log);
+		// this.connection.on('open', console.log);
+		// this.connection.on('message', console.log);
+    }
+
+    public async connect(retryAttempt: number = 0) {
+        const lock = await this.getLock();
+		try {
+			// Set retry attempt count
+            await this.setRetryAttempt(retryAttempt);
+            
+            // Get the current apiKey
+            this.apiKey = await this.getApiKey();
+
+			// Check the connection is allowed
+			await this.isConnectionAllowed();
+
+			// Cleanup old connections
+			await this.cleanup();
+
+			// Connect to endpoint
+			await this._connect();
+
+			// Log we connected
+			this.logger.debug('Connected to %s', this.uri);
+		} catch (error) {
+			this.logger.error('Failed connecting reason=%s', error.message);
+		} finally {
+			lock.release();
+		}
+	}
+	
+	public async disconnect() {
+		const lock = await this.getLock();
+		try {
+			if (this.connection && (this.connection.readyState !== this.connection.CLOSED)) {
+				// 4200 === ok
+				this.connection.close(4200);
+			}
+		} catch(error) {
+			this.logger.error('Failed disconnecting reason=%s', error.message);
+		} finally {
+			lock.release();
+		}
+	}
+
+	public async reconnect() {
+		await this.disconnect();
+		await sleep(1000);
+		await this.connect();
+	}
+};

app/mothership/sockets/mothership.ts

@@ -0,0 +1,155 @@
+import { MOTHERSHIP_RELAY_WS_LINK, ONE_MINUTE } from '../../consts';
+import { mothershipLogger, apiManager } from '../../core';
+import { getMachineId, sleep } from '../../core/utils';
+import { varState, networkState } from '../../core/states';
+import { subscribeToServers } from '../subscribe-to-servers';
+import { AppError } from '../../core/errors';
+import { readFileIfExists } from '../utils';
+import { CustomSocket, WebSocketWithHeartBeat } from '../custom-socket';
+import { InternalGraphql } from './internal-graphql';
+
+export class MothershipSocket extends CustomSocket {
+    private internalGraphqlSocket?: CustomSocket;
+    private mothershipServersEndpoint?: {
+        unsubscribe: () => void;
+    };
+
+	constructor(options: CustomSocket['options'] = {}) {
+        super({
+            name: 'Mothership',
+            uri: MOTHERSHIP_RELAY_WS_LINK,
+            logger: mothershipLogger,
+            lazy: false,
+            ...options
+        });
+    }
+
+    private connectToInternalGraphql(options: InternalGraphql['options'] = {}) {
+        this.internalGraphqlSocket = new InternalGraphql(options);
+    }
+
+    private async connectToMothershipsGraphql() {
+        this.mothershipServersEndpoint = await subscribeToServers(this.apiKey);
+    }
+
+    private async disconnectFromMothershipsGraphql() {
+        this.mothershipServersEndpoint?.unsubscribe();
+    }
+
+	protected async getApiKey() {
+		const key = apiManager.getKey('my_servers');
+		if (!key) {
+			throw new AppError('No API key found.');
+		}
+
+		return key.key;
+    }
+    
+    protected async getHeaders() {
+		const apiKey = apiManager.getKey('my_servers')?.key!;
+		const keyFile = varState.data?.regFile ? readFileIfExists(varState.data?.regFile).toString('base64') : '';
+		const serverName = `${varState.data?.name}`;
+		const lanIp = networkState.data.find(network => network.ipaddr[0]).ipaddr[0] || '';
+		const machineId = `${await getMachineId()}`;
+
+		return {
+            'x-api-key': apiKey,
+            'x-flash-guid': varState.data?.flashGuid ?? '',
+            'x-key-file': keyFile ?? '',
+            'x-server-name': serverName,
+            'x-lan-ip': lanIp,
+            'x-machine-id': machineId
+        };
+    }
+
+	onConnect() {
+        const mothership = this;
+        const onConnect = super.onConnect;
+		return async function(this: WebSocketWithHeartBeat) {
+            try {
+                // Run super
+                onConnect();
+
+                // Connect to local graphql
+                mothership.connectToInternalGraphql();
+
+                // Sub to /servers on mothership
+                mothership.connectToMothershipsGraphql();
+            } catch (error) {
+                this.close(error.code.length === 4 ? error.code : `4${error.code}`, JSON.stringify({
+				    message: error.message ?? 'Internal Server Error'
+                }));
+            }
+		};
+    }
+
+    protected onDisconnect() {
+        const mothership = this;
+        const onDisconnect = super.onDisconnect;
+		return async function (this: WebSocketWithHeartBeat, code: number, _message: string) {
+			try {
+                // Close connection to local graphql endpoint
+                mothership.internalGraphqlSocket?.connection?.close(200);
+                
+                // Close connection to motherships's server's endpoint
+                mothership.disconnectFromMothershipsGraphql();
+
+                // Process disconnection
+                onDisconnect();
+			} catch (error) {
+				mothership.logger.debug('Connection closed with code=%s reason="%s"', code, error.message);
+			}
+		};
+	}
+
+	// When we get a message from relay send it through to our local graphql instance
+	protected onMessage() {
+		const mothership = this;
+		return async function (this: WebSocketWithHeartBeat, data: string) {
+			try {
+				await mothership.sendMessage(mothership.internalGraphqlSocket?.connection, data);
+			} catch (error) {
+				// Something weird happened while processing the message
+				// This is likely a malformed message
+				mothership.logger.error('Failed sending message to relay.', error);
+			}
+		};
+	}
+
+	onError() {
+		const mothership = this;
+		return async function(this: WebSocketWithHeartBeat, error: NodeJS.ErrnoException) {
+            try {
+                mothership.logger.error(error);
+
+                // The relay is down
+                if (error.message.includes('502')) {
+                    // Sleep for 30 seconds
+                    await sleep(ONE_MINUTE / 2);
+                }
+
+                // Connection refused, aka couldn't connect
+                // This is usually because the address is wrong or offline
+                if (error.code === 'ECONNREFUSED') {
+                    // @ts-expect-error
+                    mothership.logger.debug(`Couldn't connect to %s:%s`, error.address, error.port);
+                    return;
+                }
+
+                // Closed before connection started
+                if (error.toString().includes('WebSocket was closed before the connection was established')) {
+                    mothership.logger.debug(error.message);
+                    return;
+                }
+
+                throw error;
+            } catch {
+                // Unknown error
+                mothership.logger.error('socket error', error);
+            } finally {
+                // Kick the connection
+                this.close(4500, JSON.stringify({ message: error.message }));
+            }
+        };
+    }
+};

app/mothership/subscribe-to-servers.ts

@@ -4,7 +4,7 @@ import { MOTHERSHIP_GRAPHQL_LINK } from '../consts';
 import { userCache, CachedServers } from '../cache';
 import { log as logger } from '../core';
 
-const log = logger.createChild({ prefix: '[@unraid/subscribe-to-servers]: '});
+const log = logger.createChild({ prefix: 'subscribe-to-servers'});
 const client = new SubscriptionClient(MOTHERSHIP_GRAPHQL_LINK, {
     reconnect: true,
     lazy: true, // only connect when there is a query

ava.config.cjs

@@ -8,6 +8,7 @@ const config = {
         PATHS_STATES: path.resolve(__dirname, './dev/states'),
         PATHS_DYNAMIX_BASE: path.resolve(__dirname, './dev/dynamix'),
         PATHS_DYNAMIX_CONFIG: path.resolve(__dirname, './dev/dynamix/dynamix.cfg'),
+        PATHS_MY_SERVERS_CONFIG: path.resolve(__dirname, './dev/Unraid.net/myservers.cfg'),
         API_KEY: 'TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST'
     },
     files: [

dev/Unraid.net/myservers.cfg

@@ -0,0 +1,11 @@
+[remote]
+apikey="_______________________BIG_API_KEY_HERE_________________________"
+email="test@example.com"
+wanaccess="no"
+wanport="0"
+username="zspearmint"
+avatar="https://via.placeholder.com/200"
+regWizTime="1611175408732_0951-1653-3509-FBA155FA23C0"
+event="REG_WIZARD"
+keyfile="_____________________EVEN_BIGGER_KEY_HERE_________________________"
+license=""

dev/dynamix/dynamix.cfg

@@ -18,20 +18,6 @@ critical="90"
 hot="45"
 max="55"
 sysinfo="/Tools/SystemProfiler"
-[remote]
-apikey="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST"
-wanaccess="yes"
-wanport="0"
-sshprivkey="-----BEGIN OPENSSH PRIVATE KEY-----
-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
------END OPENSSH PRIVATE KEY-----
-"
-sshpubkey="ssh-ed25519 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-"
 [notify]
 entity="1"
 normal="1"
@@ -46,6 +32,4 @@ date="d-m-Y"
 time="H:i"
 position="top-right"
 path="/tmp/notifications"
-system="*/1 * * * *"
-[wizard]
-hideWizard="1"
+system="*/1 * * * *"