From 4e04548b79154a08cd3131e6cc806d9b5d8c8757 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 14 Oct 2025 11:26:55 -0400 Subject: [PATCH] map containers to their template files --- api/.env.development | 1 + api/.gitignore | 3 + api/generated-schema.graphql | 9 + api/package.json | 1 + api/src/environment.ts | 5 + .../resolvers/docker/docker-config.model.ts | 15 + .../resolvers/docker/docker-config.service.ts | 3 + .../docker/docker-template-scanner.model.ts | 17 + .../docker-template-scanner.service.spec.ts | 427 ++++++++++++++++++ .../docker/docker-template-scanner.service.ts | 206 +++++++++ .../graph/resolvers/docker/docker.model.ts | 3 + .../resolvers/docker/docker.module.spec.ts | 8 + .../graph/resolvers/docker/docker.module.ts | 3 +- .../resolvers/docker/docker.resolver.spec.ts | 13 + .../graph/resolvers/docker/docker.resolver.ts | 19 +- .../resolvers/docker/docker.service.spec.ts | 40 ++ .../graph/resolvers/docker/docker.service.ts | 43 +- pnpm-lock.yaml | 16 + 18 files changed, 823 insertions(+), 9 deletions(-) create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.model.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.spec.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.ts diff --git a/api/.env.development b/api/.env.development index 35a4b30d7..6a22de0ea 100644 --- a/api/.env.development +++ b/api/.env.development @@ -19,6 +19,7 @@ PATHS_LOGS_FILE=./dev/log/graphql-api.log PATHS_CONNECT_STATUS_FILE_PATH=./dev/connectStatus.json # Connect plugin status file PATHS_OIDC_JSON=./dev/configs/oidc.local.json PATHS_LOCAL_SESSION_FILE=./dev/local-session +PATHS_DOCKER_TEMPLATES=./dev/docker-templates ENVIRONMENT="development" NODE_ENV="development" PORT="3001" diff --git a/api/.gitignore b/api/.gitignore index 77fdfdbee..1bb009b34 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -96,3 +96,6 @@ dev/configs/oidc.local.json # local api keys dev/keys/* + +# dev docker templates +dev/docker-templates \ No newline at end of file diff --git a/api/generated-schema.graphql b/api/generated-schema.graphql index 190492b78..23d2da235 100644 --- a/api/generated-schema.graphql +++ b/api/generated-schema.graphql @@ -1108,6 +1108,7 @@ type DockerContainer implements Node { networkSettings: JSON mounts: [JSON!] autoStart: Boolean! + templatePath: String isUpdateAvailable: Boolean isRebuildReady: Boolean } @@ -1151,6 +1152,13 @@ type DockerContainerOverviewForm { data: JSON! } +type DockerTemplateSyncResult { + scanned: Int! + matched: Int! + skipped: Int! + errors: [String!]! +} + type ResolvedOrganizerView { id: String! name: String! @@ -2463,6 +2471,7 @@ type Mutation { moveDockerItemsToPosition(sourceEntryIds: [String!]!, destinationFolderId: String!, position: Float!): ResolvedOrganizerV1! renameDockerFolder(folderId: String!, newName: String!): ResolvedOrganizerV1! createDockerFolderWithItems(name: String!, parentId: String, sourceEntryIds: [String!], position: Float): ResolvedOrganizerV1! + syncDockerTemplatePaths: DockerTemplateSyncResult! refreshDockerDigests: Boolean! """Initiates a flash drive backup using a configured remote.""" diff --git a/api/package.json b/api/package.json index fd2ff9183..abb5a0f0c 100644 --- a/api/package.json +++ b/api/package.json @@ -104,6 +104,7 @@ "escape-html": "1.0.3", "execa": "9.6.0", "exit-hook": "4.0.0", + "fast-xml-parser": "^5.3.0", "fastify": "5.5.0", "filenamify": "7.0.0", "fs-extra": "11.3.1", diff --git a/api/src/environment.ts b/api/src/environment.ts index b1d3c2bad..2cd0b6c4a 100644 --- a/api/src/environment.ts +++ b/api/src/environment.ts @@ -111,5 +111,10 @@ export const PATHS_CONFIG_MODULES = export const PATHS_LOCAL_SESSION_FILE = process.env.PATHS_LOCAL_SESSION_FILE ?? '/var/run/unraid-api/local-session'; +export const PATHS_DOCKER_TEMPLATES = process.env.PATHS_DOCKER_TEMPLATES?.split(',') ?? [ + '/boot/config/plugins/dockerMan/templates-user', + '/boot/config/plugins/dockerMan/templates', +]; + /** feature flag for the upcoming docker release */ export const ENABLE_NEXT_DOCKER_RELEASE = process.env.ENABLE_NEXT_DOCKER_RELEASE === 'true'; diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts b/api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts index e7a47ae66..00fbbc97e 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts @@ -1,7 +1,22 @@ import { Field, ObjectType } from '@nestjs/graphql'; +import { IsObject, IsOptional, IsArray, IsString } from 'class-validator'; + +import { GraphQLJSON } from 'graphql-scalars'; @ObjectType() export class DockerConfig { @Field(() => String) + @IsString() updateCheckCronSchedule!: string; + + @Field(() => GraphQLJSON, { nullable: true }) + @IsOptional() + @IsObject() + templateMappings?: Record; + + @Field(() => [String], { nullable: true }) + @IsOptional() + @IsArray() + @IsString({ each: true }) + skipTemplatePaths?: string[]; } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts index 1ed27212f..74d89ab72 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts @@ -31,6 +31,8 @@ export class DockerConfigService extends ConfigFilePersister { defaultConfig(): DockerConfig { return { updateCheckCronSchedule: CronExpression.EVERY_DAY_AT_6AM, + templateMappings: {}, + skipTemplatePaths: [], }; } @@ -40,6 +42,7 @@ export class DockerConfigService extends ConfigFilePersister { if (!cronExpression.valid) { throw new AppError(`Cron expression not supported: ${dockerConfig.updateCheckCronSchedule}`); } + return dockerConfig; } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.model.ts b/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.model.ts new file mode 100644 index 000000000..021e416ab --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.model.ts @@ -0,0 +1,17 @@ +import { Field, Int, ObjectType } from '@nestjs/graphql'; + +@ObjectType() +export class DockerTemplateSyncResult { + @Field(() => Int) + scanned!: number; + + @Field(() => Int) + matched!: number; + + @Field(() => Int) + skipped!: number; + + @Field(() => [String]) + errors!: string[]; +} + diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.spec.ts new file mode 100644 index 000000000..2e07483fe --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.spec.ts @@ -0,0 +1,427 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { mkdir, rm, writeFile } from 'fs/promises'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; +import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js'; +import { DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; +import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; + +vi.mock('@app/environment.js', () => ({ + PATHS_DOCKER_TEMPLATES: ['/tmp/test-templates'], + ENABLE_NEXT_DOCKER_RELEASE: true, +})); + +describe('DockerTemplateScannerService', () => { + let service: DockerTemplateScannerService; + let dockerConfigService: DockerConfigService; + let dockerService: DockerService; + const testTemplateDir = '/tmp/test-templates'; + + beforeEach(async () => { + await mkdir(testTemplateDir, { recursive: true }); + + const mockDockerService = { + getContainers: vi.fn(), + }; + + const mockDockerConfigService = { + getConfig: vi.fn(), + replaceConfig: vi.fn(), + validate: vi.fn((config) => Promise.resolve(config)), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DockerTemplateScannerService, + { + provide: DockerConfigService, + useValue: mockDockerConfigService, + }, + { + provide: DockerService, + useValue: mockDockerService, + }, + ], + }).compile(); + + service = module.get(DockerTemplateScannerService); + dockerConfigService = module.get(DockerConfigService); + dockerService = module.get(DockerService); + }); + + afterEach(async () => { + await rm(testTemplateDir, { recursive: true, force: true }); + }); + + describe('parseTemplate', () => { + it('should parse valid XML template', async () => { + const templatePath = join(testTemplateDir, 'test.xml'); + const templateContent = ` + + test-container + test/image +`; + await writeFile(templatePath, templateContent); + + const result = await (service as any).parseTemplate(templatePath); + + expect(result).toEqual({ + filePath: templatePath, + name: 'test-container', + repository: 'test/image', + }); + }); + + it('should handle invalid XML gracefully by returning null', async () => { + const templatePath = join(testTemplateDir, 'invalid.xml'); + await writeFile(templatePath, 'not xml'); + + const result = await (service as any).parseTemplate(templatePath); + expect(result).toBeNull(); + }); + + it('should return null for XML without Container element', async () => { + const templatePath = join(testTemplateDir, 'no-container.xml'); + const templateContent = ``; + await writeFile(templatePath, templateContent); + + const result = await (service as any).parseTemplate(templatePath); + + expect(result).toBeNull(); + }); + }); + + describe('matchContainerToTemplate', () => { + it('should match by container name (exact match)', () => { + const container: DockerContainer = { + id: 'abc123', + names: ['/test-container'], + image: 'different/image:latest', + } as DockerContainer; + + const templates = [ + { filePath: '/path/1', name: 'test-container', repository: 'some/repo' }, + { filePath: '/path/2', name: 'other', repository: 'other/repo' }, + ]; + + const result = (service as any).matchContainerToTemplate(container, templates); + + expect(result).toEqual(templates[0]); + }); + + it('should match by repository when name does not match', () => { + const container: DockerContainer = { + id: 'abc123', + names: ['/my-container'], + image: 'test/image:v1.0', + } as DockerContainer; + + const templates = [ + { filePath: '/path/1', name: 'different', repository: 'other/repo' }, + { filePath: '/path/2', name: 'also-different', repository: 'test/image' }, + ]; + + const result = (service as any).matchContainerToTemplate(container, templates); + + expect(result).toEqual(templates[1]); + }); + + it('should strip tags when matching repository', () => { + const container: DockerContainer = { + id: 'abc123', + names: ['/my-container'], + image: 'test/image:latest', + } as DockerContainer; + + const templates = [ + { filePath: '/path/1', name: 'different', repository: 'test/image:v1.0' }, + ]; + + const result = (service as any).matchContainerToTemplate(container, templates); + + expect(result).toEqual(templates[0]); + }); + + it('should return null when no match found', () => { + const container: DockerContainer = { + id: 'abc123', + names: ['/my-container'], + image: 'test/image:latest', + } as DockerContainer; + + const templates = [ + { filePath: '/path/1', name: 'different', repository: 'other/image' }, + ]; + + const result = (service as any).matchContainerToTemplate(container, templates); + + expect(result).toBeNull(); + }); + + it('should be case-insensitive', () => { + const container: DockerContainer = { + id: 'abc123', + names: ['/Test-Container'], + image: 'Test/Image:latest', + } as DockerContainer; + + const templates = [ + { filePath: '/path/1', name: 'test-container', repository: 'test/image' }, + ]; + + const result = (service as any).matchContainerToTemplate(container, templates); + + expect(result).toEqual(templates[0]); + }); + }); + + describe('scanTemplates', () => { + it('should scan templates and create mappings', async () => { + const template1 = join(testTemplateDir, 'redis.xml'); + await writeFile( + template1, + ` + + redis + redis +` + ); + + const containers: DockerContainer[] = [ + { + id: 'container1', + names: ['/redis'], + image: 'redis:latest', + } as DockerContainer, + ]; + + vi.mocked(dockerService.getContainers).mockResolvedValue(containers); + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: [], + }); + + const result = await service.scanTemplates(); + + expect(result.scanned).toBe(1); + expect(result.matched).toBe(1); + expect(result.errors).toHaveLength(0); + expect(dockerConfigService.replaceConfig).toHaveBeenCalledWith( + expect.objectContaining({ + templateMappings: { + redis: template1, + }, + }) + ); + }); + + it('should skip containers in skipTemplatePaths', async () => { + const template1 = join(testTemplateDir, 'redis.xml'); + await writeFile( + template1, + ` + + redis + redis +` + ); + + const containers: DockerContainer[] = [ + { + id: 'container1', + names: ['/redis'], + image: 'redis:latest', + } as DockerContainer, + ]; + + vi.mocked(dockerService.getContainers).mockResolvedValue(containers); + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: ['redis'], + }); + + const result = await service.scanTemplates(); + + expect(result.skipped).toBe(1); + expect(result.matched).toBe(0); + }); + + it('should handle missing template directory gracefully', async () => { + await rm(testTemplateDir, { recursive: true, force: true }); + + const containers: DockerContainer[] = []; + + vi.mocked(dockerService.getContainers).mockResolvedValue(containers); + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: [], + }); + + const result = await service.scanTemplates(); + + expect(result.scanned).toBe(0); + expect(result.errors.length).toBeGreaterThan(0); + }); + + it('should handle docker service errors gracefully', async () => { + vi.mocked(dockerService.getContainers).mockRejectedValue(new Error('Docker error')); + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: [], + }); + + const result = await service.scanTemplates(); + + expect(result.errors.length).toBeGreaterThan(0); + expect(result.errors[0]).toContain('Failed to get containers'); + }); + + it('should set null mapping for unmatched containers', async () => { + const containers: DockerContainer[] = [ + { + id: 'container1', + names: ['/unknown'], + image: 'unknown:latest', + } as DockerContainer, + ]; + + vi.mocked(dockerService.getContainers).mockResolvedValue(containers); + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: [], + }); + + await service.scanTemplates(); + + expect(dockerConfigService.replaceConfig).toHaveBeenCalledWith( + expect.objectContaining({ + templateMappings: { + unknown: null, + }, + }) + ); + }); + }); + + describe('syncMissingContainers', () => { + it('should return true and trigger scan when containers are missing mappings', async () => { + const containers: DockerContainer[] = [ + { + id: 'container1', + names: ['/redis'], + image: 'redis:latest', + } as DockerContainer, + ]; + + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: [], + }); + + vi.mocked(dockerService.getContainers).mockResolvedValue(containers); + + const scanSpy = vi.spyOn(service, 'scanTemplates').mockResolvedValue({ + scanned: 0, + matched: 0, + skipped: 0, + errors: [], + }); + + const result = await service.syncMissingContainers(containers); + + expect(result).toBe(true); + expect(scanSpy).toHaveBeenCalled(); + }); + + it('should return false when all containers have mappings', async () => { + const containers: DockerContainer[] = [ + { + id: 'container1', + names: ['/redis'], + image: 'redis:latest', + } as DockerContainer, + ]; + + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: { + redis: '/path/to/template.xml', + }, + skipTemplatePaths: [], + }); + + const scanSpy = vi.spyOn(service, 'scanTemplates'); + + const result = await service.syncMissingContainers(containers); + + expect(result).toBe(false); + expect(scanSpy).not.toHaveBeenCalled(); + }); + + it('should not trigger scan for containers in skip list', async () => { + const containers: DockerContainer[] = [ + { + id: 'container1', + names: ['/redis'], + image: 'redis:latest', + } as DockerContainer, + ]; + + vi.mocked(dockerConfigService.getConfig).mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: ['redis'], + }); + + const scanSpy = vi.spyOn(service, 'scanTemplates'); + + const result = await service.syncMissingContainers(containers); + + expect(result).toBe(false); + expect(scanSpy).not.toHaveBeenCalled(); + }); + }); + + describe('normalizeContainerName', () => { + it('should remove leading slash', () => { + const result = (service as any).normalizeContainerName('/container-name'); + expect(result).toBe('container-name'); + }); + + it('should convert to lowercase', () => { + const result = (service as any).normalizeContainerName('/Container-Name'); + expect(result).toBe('container-name'); + }); + }); + + describe('normalizeRepository', () => { + it('should strip tag', () => { + const result = (service as any).normalizeRepository('redis:latest'); + expect(result).toBe('redis'); + }); + + it('should strip version tag', () => { + const result = (service as any).normalizeRepository('postgres:14.5'); + expect(result).toBe('postgres'); + }); + + it('should convert to lowercase', () => { + const result = (service as any).normalizeRepository('Redis:Latest'); + expect(result).toBe('redis'); + }); + + it('should handle repository without tag', () => { + const result = (service as any).normalizeRepository('nginx'); + expect(result).toBe('nginx'); + }); + }); +}); + diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.ts new file mode 100644 index 000000000..b05ebb668 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.ts @@ -0,0 +1,206 @@ +import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'; +import { Timeout } from '@nestjs/schedule'; +import { readdir, readFile } from 'fs/promises'; +import { join } from 'path'; + +import { XMLParser } from 'fast-xml-parser'; + +import { PATHS_DOCKER_TEMPLATES } from '@app/environment.js'; +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; +import { DockerTemplateSyncResult } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.model.js'; +import { DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; +import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; + +interface ParsedTemplate { + filePath: string; + name?: string; + repository?: string; +} + +@Injectable() +export class DockerTemplateScannerService { + private readonly logger = new Logger(DockerTemplateScannerService.name); + private readonly xmlParser = new XMLParser({ + ignoreAttributes: false, + parseAttributeValue: true, + trimValues: true, + }); + + constructor( + private readonly dockerConfigService: DockerConfigService, + @Inject(forwardRef(() => DockerService)) + private readonly dockerService: DockerService + ) {} + + @Timeout(5_000) + async bootstrapScan(attempt = 1, maxAttempts = 5): Promise { + try { + this.logger.log(`Starting template scan (attempt ${attempt}/${maxAttempts})`); + const result = await this.scanTemplates(); + this.logger.log( + `Template scan complete: ${result.matched} matched, ${result.scanned} scanned, ${result.skipped} skipped` + ); + } catch (error) { + if (attempt < maxAttempts) { + this.logger.warn( + `Template scan failed (attempt ${attempt}/${maxAttempts}), retrying in 60s: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + setTimeout(() => this.bootstrapScan(attempt + 1, maxAttempts), 60_000); + } else { + this.logger.error( + `Template scan failed after ${maxAttempts} attempts: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + } + } + } + + async syncMissingContainers(containers: DockerContainer[]): Promise { + const config = this.dockerConfigService.getConfig(); + const mappings = config.templateMappings || {}; + const skipSet = new Set(config.skipTemplatePaths || []); + + const needsSync = containers.filter((c) => { + const containerName = this.normalizeContainerName(c.names[0]); + return !mappings[containerName] && !skipSet.has(containerName); + }); + + if (needsSync.length > 0) { + this.logger.log(`Found ${needsSync.length} containers without template mappings, triggering sync`); + await this.scanTemplates(); + return true; + } + return false; + } + + async scanTemplates(): Promise { + const result: DockerTemplateSyncResult = { + scanned: 0, + matched: 0, + skipped: 0, + errors: [], + }; + + const templates = await this.loadAllTemplates(result); + + try { + const containers = await this.dockerService.getContainers({ skipCache: true }); + const config = this.dockerConfigService.getConfig(); + const currentMappings = config.templateMappings || {}; + const skipSet = new Set(config.skipTemplatePaths || []); + + const newMappings: Record = { ...currentMappings }; + + for (const container of containers) { + const containerName = this.normalizeContainerName(container.names[0]); + if (skipSet.has(containerName)) { + result.skipped++; + continue; + } + + const match = this.matchContainerToTemplate(container, templates); + if (match) { + newMappings[containerName] = match.filePath; + result.matched++; + } else { + newMappings[containerName] = null; + } + } + + await this.updateMappings(newMappings); + } catch (error) { + const errorMsg = `Failed to get containers: ${error instanceof Error ? error.message : 'Unknown error'}`; + this.logger.error(error, 'Failed to get containers'); + result.errors.push(errorMsg); + } + + return result; + } + + private async loadAllTemplates(result: DockerTemplateSyncResult): Promise { + const allTemplates: ParsedTemplate[] = []; + + for (const directory of PATHS_DOCKER_TEMPLATES) { + try { + const files = await readdir(directory); + const xmlFiles = files.filter((f) => f.endsWith('.xml')); + result.scanned += xmlFiles.length; + + for (const file of xmlFiles) { + const filePath = join(directory, file); + try { + const template = await this.parseTemplate(filePath); + if (template) { + allTemplates.push(template); + } + } catch (error) { + const errorMsg = `Failed to parse template ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`; + this.logger.warn(errorMsg); + result.errors.push(errorMsg); + } + } + } catch (error) { + const errorMsg = `Failed to read template directory ${directory}: ${error instanceof Error ? error.message : 'Unknown error'}`; + this.logger.warn(errorMsg); + result.errors.push(errorMsg); + } + } + + return allTemplates; + } + + private async parseTemplate(filePath: string): Promise { + const content = await readFile(filePath, 'utf-8'); + const parsed = this.xmlParser.parse(content); + + if (!parsed.Container) { + return null; + } + + const container = parsed.Container; + return { + filePath, + name: container.Name, + repository: container.Repository, + }; + } + + private matchContainerToTemplate( + container: DockerContainer, + templates: ParsedTemplate[] + ): ParsedTemplate | null { + const containerName = this.normalizeContainerName(container.names[0]); + const containerImage = this.normalizeRepository(container.image); + + for (const template of templates) { + if (template.name && this.normalizeContainerName(template.name) === containerName) { + return template; + } + } + + for (const template of templates) { + if (template.repository && this.normalizeRepository(template.repository) === containerImage) { + return template; + } + } + + return null; + } + + private normalizeContainerName(name: string): string { + return name.replace(/^\//, '').toLowerCase(); + } + + private normalizeRepository(repository: string): string { + return repository.split(':')[0].toLowerCase(); + } + + private async updateMappings(mappings: Record): Promise { + const config = this.dockerConfigService.getConfig(); + const updated = await this.dockerConfigService.validate({ + ...config, + templateMappings: mappings, + }); + this.dockerConfigService.replaceConfig(updated); + } +} + diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.model.ts b/api/src/unraid-api/graph/resolvers/docker/docker.model.ts index ff00c3211..a4b8b8f0e 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.model.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.model.ts @@ -119,6 +119,9 @@ export class DockerContainer extends Node { @Field(() => Boolean) autoStart!: boolean; + + @Field(() => String, { nullable: true }) + templatePath?: string; } @ObjectType({ implements: () => Node }) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts index 77edf3f4e..b31f63e10 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts @@ -6,6 +6,7 @@ import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/dock import { DockerEventService } from '@app/unraid-api/graph/resolvers/docker/docker-event.service.js'; import { DockerFormService } from '@app/unraid-api/graph/resolvers/docker/docker-form.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; +import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js'; import { DockerModule } from '@app/unraid-api/graph/resolvers/docker/docker.module.js'; import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; @@ -67,6 +68,13 @@ describe('DockerModule', () => { { provide: DockerFormService, useValue: { getContainerOverviewForm: vi.fn() } }, { provide: DockerOrganizerService, useValue: {} }, { provide: DockerPhpService, useValue: { getContainerUpdateStatuses: vi.fn() } }, + { + provide: DockerTemplateScannerService, + useValue: { + scanTemplates: vi.fn(), + syncMissingContainers: vi.fn(), + }, + }, ], }).compile(); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index f1c100126..0e760e0eb 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -7,6 +7,7 @@ import { DockerContainerResolver } from '@app/unraid-api/graph/resolvers/docker/ import { DockerFormService } from '@app/unraid-api/graph/resolvers/docker/docker-form.service.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; +import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js'; import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; @@ -24,7 +25,7 @@ import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/o DockerManifestService, DockerPhpService, DockerConfigService, - // DockerEventService, + DockerTemplateScannerService, // Jobs ContainerStatusJob, diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts index b0df40c5a..a10707413 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DockerFormService } from '@app/unraid-api/graph/resolvers/docker/docker-form.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; +import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js'; import { ContainerState, DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; @@ -50,6 +51,18 @@ describe('DockerResolver', () => { getContainerUpdateStatuses: vi.fn(), }, }, + { + provide: DockerTemplateScannerService, + useValue: { + scanTemplates: vi.fn().mockResolvedValue({ + scanned: 0, + matched: 0, + skipped: 0, + errors: [], + }), + syncMissingContainers: vi.fn().mockResolvedValue(false), + }, + }, ], }).compile(); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts index 6f16f9175..c203e839b 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts @@ -7,6 +7,8 @@ import { UsePermissions } from '@unraid/shared/use-permissions.directive.js'; import { UseFeatureFlag } from '@app/unraid-api/decorators/use-feature-flag.decorator.js'; import { DockerFormService } from '@app/unraid-api/graph/resolvers/docker/docker-form.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; +import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js'; +import { DockerTemplateSyncResult } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.model.js'; import { ExplicitStatusItem } from '@app/unraid-api/graph/resolvers/docker/docker-update-status.model.js'; import { Docker, @@ -26,7 +28,8 @@ export class DockerResolver { private readonly dockerService: DockerService, private readonly dockerFormService: DockerFormService, private readonly dockerOrganizerService: DockerOrganizerService, - private readonly dockerPhpService: DockerPhpService + private readonly dockerPhpService: DockerPhpService, + private readonly dockerTemplateScannerService: DockerTemplateScannerService ) {} @UsePermissions({ @@ -50,7 +53,9 @@ export class DockerResolver { @Info() info: GraphQLResolveInfo ) { const requestsSize = GraphQLFieldHelper.isFieldRequested(info, 'sizeRootFs'); - return this.dockerService.getContainers({ skipCache, size: requestsSize }); + const containers = await this.dockerService.getContainers({ skipCache, size: requestsSize }); + const wasSynced = await this.dockerTemplateScannerService.syncMissingContainers(containers); + return wasSynced ? await this.dockerService.getContainers({ skipCache: true }) : containers; } @UsePermissions({ @@ -219,4 +224,14 @@ export class DockerResolver { public async containerUpdateStatuses() { return this.dockerPhpService.getContainerUpdateStatuses(); } + + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @UsePermissions({ + action: AuthAction.UPDATE_ANY, + resource: Resource.DOCKER, + }) + @Mutation(() => DockerTemplateSyncResult) + public async syncDockerTemplatePaths() { + return this.dockerTemplateScannerService.scanTemplates(); + } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.service.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.service.spec.ts index ba7e974f2..020851bec 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.service.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.service.spec.ts @@ -7,6 +7,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; // Import the mocked pubsub parts import { pubsub, PUBSUB_CHANNEL } from '@app/core/pubsub.js'; +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; +import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js'; import { ContainerState, DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; @@ -79,6 +81,29 @@ const mockCacheManager = { del: vi.fn(), }; +// Mock DockerConfigService +const mockDockerConfigService = { + getConfig: vi.fn().mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: [], + }), + replaceConfig: vi.fn(), + validate: vi.fn((config) => Promise.resolve(config)), +}; + +// Mock DockerTemplateScannerService +const mockDockerTemplateScannerService = { + bootstrapScan: vi.fn().mockResolvedValue(undefined), + scanTemplates: vi.fn().mockResolvedValue({ + scanned: 0, + matched: 0, + skipped: 0, + errors: [], + }), + syncMissingContainers: vi.fn().mockResolvedValue(false), +}; + describe('DockerService', () => { let service: DockerService; @@ -91,6 +116,13 @@ describe('DockerService', () => { mockCacheManager.get.mockReset(); mockCacheManager.set.mockReset(); mockCacheManager.del.mockReset(); + mockDockerConfigService.getConfig.mockReturnValue({ + updateCheckCronSchedule: '0 6 * * *', + templateMappings: {}, + skipTemplatePaths: [], + }); + mockDockerTemplateScannerService.bootstrapScan.mockResolvedValue(undefined); + mockDockerTemplateScannerService.syncMissingContainers.mockResolvedValue(false); const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -99,6 +131,14 @@ describe('DockerService', () => { provide: CACHE_MANAGER, useValue: mockCacheManager, }, + { + provide: DockerConfigService, + useValue: mockDockerConfigService, + }, + { + provide: DockerTemplateScannerService, + useValue: mockDockerTemplateScannerService, + }, ], }).compile(); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker.service.ts index 4599b3e88..b5b1136d8 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.service.ts @@ -1,5 +1,12 @@ import { CACHE_MANAGER } from '@nestjs/cache-manager'; -import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { + forwardRef, + Inject, + Injectable, + Logger, + OnApplicationBootstrap, + OnModuleInit, +} from '@nestjs/common'; import { readFile } from 'fs/promises'; import { type Cache } from 'cache-manager'; @@ -9,6 +16,8 @@ import { pubsub, PUBSUB_CHANNEL } from '@app/core/pubsub.js'; import { catchHandlers } from '@app/core/utils/misc/catch-handlers.js'; import { sleep } from '@app/core/utils/misc/sleep.js'; import { getters } from '@app/store/index.js'; +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; +import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js'; import { ContainerPortType, ContainerState, @@ -25,7 +34,7 @@ interface NetworkListingOptions { } @Injectable() -export class DockerService { +export class DockerService implements OnApplicationBootstrap { private client: Docker; private autoStarts: string[] = []; private readonly logger = new Logger(DockerService.name); @@ -33,12 +42,21 @@ export class DockerService { public static readonly CONTAINER_CACHE_KEY = 'docker_containers'; public static readonly CONTAINER_WITH_SIZE_CACHE_KEY = 'docker_containers_with_size'; public static readonly NETWORK_CACHE_KEY = 'docker_networks'; - public static readonly CACHE_TTL_SECONDS = 60; // Cache for 60 seconds + public static readonly CACHE_TTL_SECONDS = 60; - constructor(@Inject(CACHE_MANAGER) private cacheManager: Cache) { + constructor( + @Inject(CACHE_MANAGER) private cacheManager: Cache, + private readonly dockerConfigService: DockerConfigService, + @Inject(forwardRef(() => DockerTemplateScannerService)) + private readonly templateScannerService: DockerTemplateScannerService + ) { this.client = this.getDockerClient(); } + async onApplicationBootstrap() { + await this.templateScannerService.bootstrapScan(); + } + public getDockerClient() { return new Docker({ socketPath: '/var/run/docker.sock', @@ -141,8 +159,21 @@ export class DockerService { this.autoStarts = await this.getAutoStarts(); const containers = rawContainers.map((container) => this.transformContainer(container)); - await this.cacheManager.set(cacheKey, containers, DockerService.CACHE_TTL_SECONDS * 1000); - return containers; + const config = this.dockerConfigService.getConfig(); + const containersWithTemplatePaths = containers.map((c) => { + const containerName = c.names[0]?.replace(/^\//, '').toLowerCase(); + return { + ...c, + templatePath: config.templateMappings?.[containerName] || undefined, + }; + }); + + await this.cacheManager.set( + cacheKey, + containersWithTemplatePaths, + DockerService.CACHE_TTL_SECONDS * 1000 + ); + return containersWithTemplatePaths; } /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a1dcb021b..edc1e0b2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -187,6 +187,9 @@ importers: exit-hook: specifier: 4.0.0 version: 4.0.0 + fast-xml-parser: + specifier: ^5.3.0 + version: 5.3.0 fastify: specifier: 5.5.0 version: 5.5.0 @@ -7735,6 +7738,10 @@ packages: resolution: {integrity: sha512-y655CeyUQ+jj7KBbYMc4FG01V8ZQqjN+gDYGJ50RtfsUB8iG9AmwmwoAgeKLJdmueKKMrH1RJ7yXHTSoczdv5w==} hasBin: true + fast-xml-parser@5.3.0: + resolution: {integrity: sha512-gkWGshjYcQCF+6qtlrqBqELqNqnt4CxruY6UVAWWnqb3DQ6qaNFEIKqzYep1XzHLM/QtrHVCxyPOtTk4LTQ7Aw==} + hasBin: true + fastify-plugin@4.5.1: resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==} @@ -11380,6 +11387,9 @@ packages: strnum@1.0.5: resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==} + strnum@2.1.1: + resolution: {integrity: sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==} + strtok3@10.3.1: resolution: {integrity: sha512-3JWEZM6mfix/GCJBBUrkA8p2Id2pBkyTkVCJKto55w080QBKZ+8R171fGrbiSp+yMO/u6F8/yUh7K4V9K+YCnw==} engines: {node: '>=18'} @@ -20091,6 +20101,10 @@ snapshots: dependencies: strnum: 1.0.5 + fast-xml-parser@5.3.0: + dependencies: + strnum: 2.1.1 + fastify-plugin@4.5.1: {} fastify-plugin@5.0.1: {} @@ -24252,6 +24266,8 @@ snapshots: strnum@1.0.5: {} + strnum@2.1.1: {} + strtok3@10.3.1: dependencies: '@tokenizer/token': 0.3.0