diff --git a/api/dev/Unraid.net/myservers.cfg b/api/dev/Unraid.net/myservers.cfg index 83ce8c53c..05e2d2db4 100644 --- a/api/dev/Unraid.net/myservers.cfg +++ b/api/dev/Unraid.net/myservers.cfg @@ -1,5 +1,5 @@ [api] -version="3.0.1+b26ff388" +version="3.1.0" [local] [notifier] apikey="unnotify_30994bfaccf839c65bae75f7fa12dd5ee16e69389f754c3b98ed7d5" diff --git a/api/dev/states/myservers.cfg b/api/dev/states/myservers.cfg index 5df26fd86..e66482b9c 100644 --- a/api/dev/states/myservers.cfg +++ b/api/dev/states/myservers.cfg @@ -1,5 +1,5 @@ [api] -version="3.0.1+b26ff388" +version="3.1.0" [local] [notifier] apikey="unnotify_30994bfaccf839c65bae75f7fa12dd5ee16e69389f754c3b98ed7d5" diff --git a/api/package-lock.json b/api/package-lock.json index 4cfd1a948..e3a4cb013 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -16,7 +16,6 @@ "@graphql-tools/merge": "^8.4.0", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^9.2.1", - "@gridplus/docker-events": "github:unraid/docker-events", "@reduxjs/toolkit": "^1.9.5", "@reflet/cron": "^1.3.1", "@runonflux/nat-upnp": "^1.0.2", @@ -35,6 +34,7 @@ "convert": "^4.10.0", "cors": "^2.8.5", "cross-fetch": "^3.1.5", + "docker-event-emitter": "^0.3.0", "dockerode": "^3.3.5", "dotenv": "^16.0.3", "express": "^4.18.2", @@ -2857,22 +2857,6 @@ "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" } }, - "node_modules/@gridplus/docker-events": { - "version": "1.0.0", - "resolved": "git+ssh://git@github.com/unraid/docker-events.git#5eb1e71044d9f60e8227b1022bdd99139c82a57f", - "license": "ISC", - "dependencies": { - "debug": "^3.1.0" - } - }, - "node_modules/@gridplus/docker-events/node_modules/debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "dependencies": { - "ms": "^2.1.1" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -6790,6 +6774,17 @@ "node": ">=8" } }, + "node_modules/docker-event-emitter": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/docker-event-emitter/-/docker-event-emitter-0.3.0.tgz", + "integrity": "sha512-QWpJsTOcLOiOctbCTH3T+w34Aw+zK6JzTh8xOqD/5/dDEhPhnCFmR8VzsCvTYAlTmkgxMUkRMTlBz1sGNZB5vg==", + "dependencies": { + "debug": "^4.1.1" + }, + "peerDependencies": { + "dockerode": "^3.0.2" + } + }, "node_modules/docker-modem": { "version": "3.0.8", "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-3.0.8.tgz", @@ -18249,23 +18244,6 @@ "integrity": "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==", "requires": {} }, - "@gridplus/docker-events": { - "version": "git+ssh://git@github.com/unraid/docker-events.git#5eb1e71044d9f60e8227b1022bdd99139c82a57f", - "from": "@gridplus/docker-events@github:unraid/docker-events", - "requires": { - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, "@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -21224,6 +21202,14 @@ } } }, + "docker-event-emitter": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/docker-event-emitter/-/docker-event-emitter-0.3.0.tgz", + "integrity": "sha512-QWpJsTOcLOiOctbCTH3T+w34Aw+zK6JzTh8xOqD/5/dDEhPhnCFmR8VzsCvTYAlTmkgxMUkRMTlBz1sGNZB5vg==", + "requires": { + "debug": "^4.1.1" + } + }, "docker-modem": { "version": "3.0.8", "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-3.0.8.tgz", diff --git a/api/package.json b/api/package.json index e8bd80400..8fac3071f 100644 --- a/api/package.json +++ b/api/package.json @@ -64,7 +64,6 @@ "@graphql-tools/merge": "^8.4.0", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^9.2.1", - "@gridplus/docker-events": "github:unraid/docker-events", "@reduxjs/toolkit": "^1.9.5", "@reflet/cron": "^1.3.1", "@runonflux/nat-upnp": "^1.0.2", @@ -83,6 +82,7 @@ "convert": "^4.10.0", "cors": "^2.8.5", "cross-fetch": "^3.1.5", + "docker-event-emitter": "^0.3.0", "dockerode": "^3.3.5", "dotenv": "^16.0.3", "express": "^4.18.2", diff --git a/api/src/__test__/store/modules/paths.test.ts b/api/src/__test__/store/modules/paths.test.ts index 05cfd32e9..20f0cc0a5 100644 --- a/api/src/__test__/store/modules/paths.test.ts +++ b/api/src/__test__/store/modules/paths.test.ts @@ -24,6 +24,7 @@ test('Returns paths', async () => { "keyfile-base", "machine-id", "log-base", + "var-run", ] `); }); diff --git a/api/src/common/dashboard/generate-data.ts b/api/src/common/dashboard/generate-data.ts new file mode 100644 index 000000000..af21d469c --- /dev/null +++ b/api/src/common/dashboard/generate-data.ts @@ -0,0 +1,141 @@ +import { ConnectListAllDomainsFlags } from '@vmngr/libvirt'; +import { getHypervisor } from '@app/core/utils/vms/get-hypervisor'; +import display from '@app/graphql/resolvers/query/display'; +import { docker } from '@app/core/utils/clients/docker'; +import { getUnraidVersion } from '@app/common/dashboard/get-unraid-version'; +import { getArray } from '@app/common/dashboard/get-array'; +import { bootTimestamp } from '@app/common/dashboard/boot-timestamp'; +import { dashboardLogger } from '@app/core/log'; +import { getters, store } from '@app/store'; +import { type DashboardServiceInput, type DashboardInput } from '@app/graphql/generated/client/graphql'; +import { API_VERSION } from '@app/environment'; +import { DynamicRemoteAccessType } from '@app/remoteAccess/types'; +import { DashboardInputSchema } from '@app/graphql/generate/validators'; +import { ZodError } from 'zod'; + +const getVmSummary = async (): Promise => { + try { + const hypervisor = await getHypervisor(); + if (!hypervisor) { + return { + installed: 0, + started: 0, + }; + } + + const activeDomains = await hypervisor.connectListAllDomains(ConnectListAllDomainsFlags.ACTIVE) as unknown[]; + const inactiveDomains = await hypervisor.connectListAllDomains(ConnectListAllDomainsFlags.INACTIVE) as unknown[]; + return { + installed: activeDomains.length + inactiveDomains.length, + started: activeDomains.length, + }; + } catch { + return { + installed: 0, + started: 0, + }; + } +}; + +/* +const twoFactor = (): Dashboard['twoFactor'] => { + const { isRemoteEnabled, isLocalEnabled } = checkTwoFactorEnabled(); + return { + remote: { + enabled: isRemoteEnabled, + }, + local: { + enabled: isLocalEnabled, + }, + }; +}; */ + +const getDynamicRemoteAccessService = (): DashboardServiceInput | null => { + const uptimeTimestamp = bootTimestamp.toISOString(); + + const { config, dynamicRemoteAccess } = store.getState(); + const enabledStatus = config.remote.dynamicRemoteAccessType; + + return { + name: 'dynamic-remote-access', + online: enabledStatus !== DynamicRemoteAccessType.DISABLED, + version: dynamicRemoteAccess.runningType, + uptime: { + timestamp: new Date(uptimeTimestamp), + }, + }; +}; + +const services = (): DashboardInput['services'] => { + const uptimeTimestamp = bootTimestamp.toISOString(); + const dynamicRemoteAccess = getDynamicRemoteAccessService(); + return [{ + name: 'unraid-api', + online: true, + uptime: { + timestamp: new Date(uptimeTimestamp), + }, + version: API_VERSION, + }, + ...(dynamicRemoteAccess ? [dynamicRemoteAccess] : [])]; +}; + +const getData = async (): Promise => { + const emhttp = getters.emhttp(); + const docker = getters.docker(); + + return { + vars: { + regState: emhttp.var.regState, + regTy: emhttp.var.regTy, + flashGuid: emhttp.var.flashGuid, + }, + apps: { + installed: docker.installed ?? 0, + started: docker.running ?? 0 + }, + versions: { + unraid: await getUnraidVersion(), + }, + os: { + hostname: emhttp.var.name, + uptime: new Date(bootTimestamp.toISOString()), + }, + vms: await getVmSummary(), + array: getArray(), + services: services(), + display: await display(), + config: emhttp.var.configValid ? { valid: true } : { + valid: false, + error: { + error: 'UNKNOWN_ERROR', + invalid: 'INVALID', + nokeyserver: 'NO_KEY_SERVER', + withdrawn: 'WITHDRAWN', + }[emhttp.var.configState] ?? 'UNKNOWN_ERROR', + }, + }; +}; + +export const generateData = async (): Promise => { + const data = await getData(); + + try { + // Validate generated data + // @TODO: Fix this runtype to use generated types from the Zod validators (as seen in mothership Codegen) + const result = DashboardInputSchema().parse(data) + + return result + + } catch (error: unknown) { + // Log error for user + if (error instanceof ZodError) { + dashboardLogger.error('Failed validation with issues: ' , error.issues.map(issue => ({ message: issue.message, path: issue.path.join(',') }))) + } else { + dashboardLogger.error('Failed validating dashboard object: ', error, data); + } + } + + return null; +}; + diff --git a/api/src/core/modules/docker/get-docker-containers.ts b/api/src/core/modules/docker/get-docker-containers.ts new file mode 100644 index 000000000..6c835b58a --- /dev/null +++ b/api/src/core/modules/docker/get-docker-containers.ts @@ -0,0 +1,92 @@ +/*! + * Copyright 2019-2022 Lime Technology Inc. All rights reserved. + * Written by: Alexis Tyler + */ + +import fs from 'fs'; +import camelCaseKeys from 'camelcase-keys'; +import { catchHandlers } from '@app/core/utils/misc/catch-handlers'; +import { getters, store } from '@app/store'; +import { updateDockerState } from '@app/store/modules/docker' + +import { + type ContainerPort, + ContainerPortType, + type DockerContainer, + ContainerState, +} from '@app/graphql/generated/api/types'; +import { dockerLogger } from '@app/core/log'; +import { docker } from '@app/core/utils/clients/docker'; + +/** + * Get all Docker containers. + * @returns All the in/active Docker containers on the system. + */ + +export const getDockerContainers = async ( + { useCache } = { useCache: true } +): Promise> => { + const dockerState = getters.docker() + if (useCache && dockerState.containers) { + dockerLogger.trace('Using docker container cache'); + return dockerState.containers; + } + + dockerLogger.trace('Skipping docker container cache'); + + /** + * Docker auto start file + * + * @note Doesn't exist if array is offline. + * @see https://github.com/limetech/webgui/issues/502#issue-480992547 + */ + const autoStartFile = await fs.promises + .readFile(getters.paths()['docker-autostart'], 'utf8') + .then((file) => file.toString()) + .catch(() => ''); + const autoStarts = autoStartFile.split('\n'); + const rawContainers = await docker + .listContainers({ + all: true, + size: true, + }) + .then((containers) => + containers.map((object) => camelCaseKeys(object, { deep: true })) + ) + // If docker throws an error return no containers + .catch(catchHandlers.docker); + + // Cleanup container object + const containers: Array = rawContainers.map( + (container) => { + const names = container.names[0]; + const containerData: DockerContainer = { + ...container, + labels: container.labels, + // @ts-expect-error sizeRootFs is not on the dockerode type, but is fetched when size: true is set + sizeRootFs: container.sizeRootFs ?? undefined, + imageId: container.imageID, + state: + typeof container?.state === 'string' + ? ContainerState[container.state.toUpperCase()] ?? + ContainerState.EXITED + : ContainerState.EXITED, + autoStart: autoStarts.includes(names.split('/')[1]), + ports: container.ports.map((port) => ({ + ...port, + type: ContainerPortType[port.type.toUpperCase()], + })), + }; + return containerData; + } + ); + + // Get all of the current containers + const installed = containers.length; + const running = containers.filter( + (container) => container.state === ContainerState.RUNNING + ).length; + + store.dispatch(updateDockerState({ containers, installed, running })) + return containers; +}; diff --git a/api/src/graphql/resolvers/query/docker.ts b/api/src/graphql/resolvers/query/docker.ts new file mode 100644 index 000000000..2f3f57139 --- /dev/null +++ b/api/src/graphql/resolvers/query/docker.ts @@ -0,0 +1,16 @@ +import { getDockerContainers } from "@app/core/modules/index"; +import { ensurePermission } from "@app/core/utils/permissions/ensure-permission"; +import { type QueryResolvers } from "@app/graphql/generated/api/types"; + +export const dockerContainersResolver: QueryResolvers['dockerContainers'] = async (_, __, context) => { + const { user } = context; + + // Check permissions + ensurePermission(user, { + resource: 'docker/container', + action: 'read', + possession: 'any', + }); + + return getDockerContainers(); +} \ No newline at end of file diff --git a/api/src/graphql/resolvers/query/index.ts b/api/src/graphql/resolvers/query/index.ts index 2d0bcbda6..9c2ce99f4 100644 --- a/api/src/graphql/resolvers/query/index.ts +++ b/api/src/graphql/resolvers/query/index.ts @@ -1,11 +1,11 @@ import { getArray } from '@app/core/modules/get-array'; -import { getDockerContainers } from '@app/core/modules/index'; import { type QueryResolvers } from '@app/graphql/generated/api/types'; import cloud from '@app/graphql/resolvers/query/cloud'; import { config } from '@app/graphql/resolvers/query/config'; import crashReportingEnabled from '@app/graphql/resolvers/query/crash-reporting-enabled'; import { disksResolver } from '@app/graphql/resolvers/query/disks'; import display from '@app/graphql/resolvers/query/display'; +import { dockerContainersResolver } from '@app/graphql/resolvers/query/docker'; import flash from '@app/graphql/resolvers/query/flash'; import online from '@app/graphql/resolvers/query/online'; import owner from '@app/graphql/resolvers/query/owner'; @@ -21,7 +21,7 @@ export const Query: QueryResolvers = { config, crashReportingEnabled, disks: disksResolver, - dockerContainers: getDockerContainers, + dockerContainers: dockerContainersResolver, display, flash, online, diff --git a/api/src/graphql/resolvers/subscription/remote-graphql/remote-query.ts b/api/src/graphql/resolvers/subscription/remote-graphql/remote-query.ts index b29775b22..fa1ae470e 100644 --- a/api/src/graphql/resolvers/subscription/remote-graphql/remote-query.ts +++ b/api/src/graphql/resolvers/subscription/remote-graphql/remote-query.ts @@ -25,7 +25,9 @@ export const executeRemoteGraphQLQuery = async ( upcApiKey: apiKey }); if (ENVIRONMENT === 'development') { - remoteQueryLogger.debug('Running query', parsedQuery.query); + remoteQueryLogger.addContext('query', parsedQuery.query); + remoteQueryLogger.debug('[DEVONLY] Running query'); + remoteQueryLogger.removeContext('query'); } const localResult = await localClient.query({ query: parsedQuery.query, diff --git a/api/src/index.ts b/api/src/index.ts index e356ded16..1c25aaadf 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -19,13 +19,13 @@ import { loadRegistrationKey } from '@app/store/modules/registration'; import { createApolloExpressServer } from '@app/server'; import { unlinkSync } from 'fs'; import { fileExistsSync } from '@app/core/utils/files/file-exists'; -import { setupDockerWatch } from '@app/store/watch/docker-watch'; import { PORT, environment } from '@app/environment'; import { shutdownApiEvent } from '@app/store/actions/shutdown-api-event'; import { PingTimeoutJobs } from '@app/mothership/jobs/ping-timeout-jobs'; import { type BaseContext, type ApolloServer } from '@apollo/server'; import { loadDynamixConfigFile } from '@app/store/modules/dynamix'; import { setupDynamixConfigWatch } from '@app/store/watch/dynamix-config-watch'; +import { setupVarRunWatch } from '@app/store/watch/var-run-watch'; let server: ApolloServer; @@ -68,7 +68,7 @@ void am( setupRegistrationKeyWatch(); // Start listening to docker events - setupDockerWatch(); + await setupVarRunWatch(); // Start listening to dynamix config file changes setupDynamixConfigWatch(); diff --git a/api/src/store/modules/docker.ts b/api/src/store/modules/docker.ts new file mode 100644 index 000000000..32abc987c --- /dev/null +++ b/api/src/store/modules/docker.ts @@ -0,0 +1,33 @@ +import { createSlice, type PayloadAction } from '@reduxjs/toolkit'; +import merge from 'lodash/merge'; +import { DaemonConnectionStatus } from '@app/store/types'; +import { type DockerContainer } from '@app/graphql/generated/api/types'; + +type DockerState = { + status: DaemonConnectionStatus; + installed: number | null; + running: number | null; + containers: DockerContainer[]; +}; + +const initialState: DockerState = { + status: DaemonConnectionStatus.DISCONNECTED, + installed: null, + running: null, + containers: [], +}; + +export const docker = createSlice({ + name: 'docker', + initialState, + reducers: { + updateDockerState(state, action: PayloadAction>) { + state.status = action.payload.status ?? initialState.status; + state.installed = action.payload.installed ?? initialState.installed; + state.running = action.payload.running ?? initialState.running; + state.containers = action.payload.containers ?? initialState.containers; + }, + }, +}); + +export const { updateDockerState } = docker.actions; diff --git a/api/src/store/modules/dynamix.ts b/api/src/store/modules/dynamix.ts index 53b59babd..b62469a67 100644 --- a/api/src/store/modules/dynamix.ts +++ b/api/src/store/modules/dynamix.ts @@ -1,12 +1,12 @@ import { parseConfig } from '@app/core/utils/misc/parse-config'; -import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit'; +import { createAsyncThunk, createSlice, type PayloadAction } from '@reduxjs/toolkit'; import { access } from 'fs/promises'; import merge from 'lodash/merge'; import { FileLoadStatus } from '@app/store/types'; import { F_OK } from 'constants'; -import { RecursivePartial, RecursiveNullable } from '@app/types'; +import { type RecursivePartial, type RecursiveNullable } from '@app/types'; import { toBoolean } from '@app/core/utils/casting'; -import { DynamixConfig } from '@app/core/types/ini'; +import { type DynamixConfig } from '@app/core/types/ini'; export type SliceState = { status: FileLoadStatus; diff --git a/api/src/store/modules/paths.ts b/api/src/store/modules/paths.ts index 19229553a..ca61d3bf8 100644 --- a/api/src/store/modules/paths.ts +++ b/api/src/store/modules/paths.ts @@ -20,7 +20,8 @@ const initialState = { 'myservers-env': '/boot/config/plugins/dynamix.my.servers/env' as const, 'keyfile-base': resolvePath(process.env.PATHS_KEYFILE_BASE ?? '/boot/config' as const), 'machine-id': resolvePath(process.env.PATHS_MACHINE_ID ?? '/var/lib/dbus/machine-id' as const), - 'log-base': resolvePath('/var/log/unraid-api/' as const) + 'log-base': resolvePath('/var/log/unraid-api/' as const), + 'var-run': '/var/run' as const, }; export const paths = createSlice({ diff --git a/api/src/store/watch/docker-watch.ts b/api/src/store/watch/docker-watch.ts new file mode 100644 index 000000000..0bd94b2a0 --- /dev/null +++ b/api/src/store/watch/docker-watch.ts @@ -0,0 +1,61 @@ +import { store } from '@app/store'; +import { dockerLogger } from '@app/core/log'; +import { updateDockerState } from '@app/store/modules/docker'; +import { getDockerContainers } from '@app/core/modules/index'; +import { ContainerState } from '@app/graphql/generated/api/types'; +import { docker } from '@app/core/utils/index'; +import DockerEE from 'docker-event-emitter'; +import { debounce } from 'lodash'; + +const updateContainerCache = async () => { + try { + await getDockerContainers({ useCache: false }); + } catch (err) { + dockerLogger.warn('Caught error getting containers %o', err) + store.dispatch(updateDockerState({ installed: null, running: null, containers: [] })) + } +}; + +const debouncedContainerCacheUpdate = debounce(updateContainerCache, 500); + +export const setupDockerWatch = async (): Promise => { + // Only watch container events equal to start/stop + const watchedActions = [ + 'die', + 'kill', + 'oom', + 'pause', + 'restart', + 'start', + 'stop', + 'unpause', + ]; + + // Create docker event emitter instance + dockerLogger.debug('Creating docker event emitter instance'); + + const dee = new DockerEE(docker); + // On Docker event update info with { apps: { installed, started } } + dee.on( + 'container', + async (data: { + Type: 'container'; + Action: 'start' | 'stop'; + from: string; + }) => { + // Only listen to container events + if (!watchedActions.includes(data.Action)) { + return; + } + dockerLogger.addContext('data', data); + dockerLogger.debug(`[${data.from}] ${data.Type}->${data.Action}`); + dockerLogger.removeContext('data'); + await debouncedContainerCacheUpdate() + } + ); + // Get docker container count on first start + await debouncedContainerCacheUpdate(); + await dee.start(); + dockerLogger.debug('Binding to docker events'); + return dee; +}; diff --git a/api/src/store/watch/var-run-watch.ts b/api/src/store/watch/var-run-watch.ts new file mode 100644 index 000000000..0b6a8ccbb --- /dev/null +++ b/api/src/store/watch/var-run-watch.ts @@ -0,0 +1,25 @@ +import { dockerLogger } from '@app/core/log'; +import { getters, store } from '@app/store/index'; +import { setupDockerWatch } from '@app/store/watch/docker-watch'; +import { watch } from 'chokidar'; +import type DockerEE from 'docker-event-emitter'; +import { updateDockerState } from '@app/store/modules/docker' + +export const setupVarRunWatch = () => { + const paths = getters.paths() + let dockerWatcher: null | typeof DockerEE = null; + watch(paths['var-run'], { ignoreInitial: false }).on('add', async (path) => { + if (path === paths['docker-socket']) { + dockerLogger.debug('Starting docker watch'); + dockerWatcher = await setupDockerWatch() + } + }).on('unlink', (path) => { + if (path === paths['docker-socket'] && dockerWatcher) { + dockerLogger.debug('Stopping docker watch') + dockerWatcher?.stop?.() + + store.dispatch(updateDockerState({ installed: null, running: null, containers: [] })) + } + }) + +} \ No newline at end of file