test: retrieveing remote container digest

This commit is contained in:
Pujit Mehrotra
2025-08-12 15:18:45 -04:00
parent 193be3df36
commit 81aff48821
5 changed files with 707 additions and 0 deletions

View File

@@ -0,0 +1,162 @@
import { beforeAll, describe, expect, it, vi } from 'vitest';
import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js';
import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js';
describe.skipIf(!!process.env.CI)('DockerManifestService - Real Integration Tests', () => {
let service: DockerManifestService;
let dockerAuthService: DockerAuthService;
beforeAll(() => {
dockerAuthService = new DockerAuthService();
service = new DockerManifestService(dockerAuthService);
}, 30000);
describe('headManifest - Real HTTP calls', () => {
it('should receive authentication challenge from Docker Hub', async () => {
const manifestURL = 'https://registry-1.docker.io/v2/library/alpine/manifests/latest';
const headers = {
Accept: 'application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json',
};
const result = await service.headManifest(manifestURL, headers);
expect(result.statusCode).toBe(401);
expect(result.headers['www-authenticate']).toContain('Bearer');
expect(result.headers['www-authenticate']).toContain('realm');
}, 15000);
it('should handle unauthorized requests gracefully', async () => {
const manifestURL = 'https://registry-1.docker.io/v2/library/hello-world/manifests/latest';
const headers = {
Accept: 'application/vnd.docker.distribution.manifest.v2+json',
};
const result = await service.headManifest(manifestURL, headers);
expect(result.statusCode).toBeGreaterThanOrEqual(200);
expect(result.statusCode).toBeLessThan(500);
}, 15000);
});
describe('getRemoteDigest - Real Docker Registry calls', () => {
it('should get digest for public Alpine image', async () => {
const digest = await service.getRemoteDigest('alpine:latest');
expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/);
}, 30000);
it('should get digest for public Nginx image', async () => {
const digest = await service.getRemoteDigest('nginx:latest');
expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/);
}, 30000);
it('should get digest for specific image tag', async () => {
const digest = await service.getRemoteDigest('alpine:3.18');
expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/);
}, 30000);
it('should handle non-existent image gracefully', async () => {
const digest = await service.getRemoteDigest('nonexistent/nonexistent:nonexistent');
expect(digest).toBeNull();
}, 30000);
it('should work with Docker Hub organization images', async () => {
const digest = await service.getRemoteDigest('library/hello-world:latest');
expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/);
}, 30000);
it('should handle anonymous access to public registries', async () => {
const digest = await service.getRemoteDigest('busybox:latest');
expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/);
}, 30000);
it('should work with images without explicit tags', async () => {
const digest = await service.getRemoteDigest('ubuntu');
expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/);
}, 30000);
it('should get digest with mocked empty docker auth', async () => {
const mockDockerAuthService = new DockerAuthService();
const mockManifestService = new DockerManifestService(mockDockerAuthService);
vi.spyOn(mockDockerAuthService, 'readDockerAuth').mockResolvedValue({});
const digest = await mockManifestService.getRemoteDigest('alpine:latest');
expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/);
}, 30000);
it('should investigate HEAD vs GET behavior with authentication', async () => {
const manifestURL = 'https://registry-1.docker.io/v2/library/alpine/manifests/latest';
const headers = { Accept: 'application/vnd.docker.distribution.manifest.v2+json' };
// Test initial HEAD request (should be 401)
const initialResult = await service.headManifest(manifestURL, headers);
// console.log('Initial HEAD request - Status:', initialResult.statusCode);
// Get Bearer token for anonymous access
const wwwAuth = (initialResult.headers?.['www-authenticate'] || '').toString();
const token = await dockerAuthService.getBearerToken(wwwAuth, {
username: '',
password: '',
});
// console.log('Got token:', !!token);
if (token) {
// Test authenticated HEAD request
const authResult = await service.headManifest(manifestURL, headers, {
Authorization: `Bearer ${token}`,
});
// console.log('Authenticated HEAD request - Status:', authResult.statusCode);
// console.log('Authenticated HEAD digest:', authResult.headers?.['docker-content-digest']);
expect(authResult.statusCode).toBeGreaterThanOrEqual(200);
expect(authResult.statusCode).toBeLessThan(300);
}
}, 30000);
it('should handle moderate concurrent requests', async () => {
// Start with fewer requests to identify the breaking point
const promises = Array.from({ length: 10 }, () => service.getRemoteDigest('alpine:latest'));
const results = await Promise.all(promises);
results.forEach((digest, index) => {
expect(digest, `Request ${index + 1} failed`).toMatch(/^sha256:[a-f0-9]{64}$/);
});
}, 60000);
it('should handle 100 requests with optimized batching', async () => {
const batchSize = 10;
const totalRequests = 100;
const results: (string | null)[] = [];
// Process in batches with minimal delays
for (let i = 0; i < totalRequests; i += batchSize) {
const batch = Array.from({ length: Math.min(batchSize, totalRequests - i) }, () =>
service.getRemoteDigest('alpine:latest')
);
const batchResults = await Promise.all(batch);
results.push(...batchResults);
// Minimal delay between batches
if (i + batchSize < totalRequests) {
await new Promise((resolve) => setTimeout(resolve, 50));
}
}
expect(results).toHaveLength(totalRequests);
results.forEach((digest, index) => {
expect(digest, `Request ${index + 1} failed`).toMatch(/^sha256:[a-f0-9]{64}$/);
});
}, 60000);
});
});

View File

@@ -0,0 +1,328 @@
import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js';
import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js';
vi.mock('got', () => ({
got: {
head: vi.fn(),
get: vi.fn(),
},
}));
vi.mock('@app/core/utils/index.js', () => ({
docker: {
getImage: vi.fn(() => ({
inspect: vi.fn(),
})),
getContainer: vi.fn(() => ({
inspect: vi.fn(),
})),
},
}));
describe('DockerManifestService', () => {
let service: DockerManifestService;
let mockDockerAuthService: any;
let mockGot: any;
beforeEach(async () => {
vi.clearAllMocks();
mockDockerAuthService = {
readDockerAuth: vi.fn(),
decodeAuth: vi.fn(),
getBearerToken: vi.fn(),
};
const { got } = await import('got');
mockGot = vi.mocked(got);
mockGot.head.mockReset();
mockGot.get.mockReset();
service = new DockerManifestService(mockDockerAuthService);
});
describe('parseImageRef - Unit Tests', () => {
it('should parse simple image name with default tag', () => {
const result = service.parseImageRef('nginx');
expect(result).toEqual({
registryBaseURL: 'https://registry-1.docker.io',
authConfigKey: 'https://index.docker.io/v1/',
repoPath: 'library/nginx',
tag: 'latest',
});
});
it('should parse image name with explicit tag', () => {
const result = service.parseImageRef('nginx:1.21');
expect(result).toEqual({
registryBaseURL: 'https://registry-1.docker.io',
authConfigKey: 'https://index.docker.io/v1/',
repoPath: 'library/nginx',
tag: '1.21',
});
});
it('should parse Docker Hub organization image', () => {
const result = service.parseImageRef('ubuntu/nginx:latest');
expect(result).toEqual({
registryBaseURL: 'https://registry-1.docker.io',
authConfigKey: 'https://index.docker.io/v1/',
repoPath: 'ubuntu/nginx',
tag: 'latest',
});
});
it('should parse custom registry with port', () => {
const result = service.parseImageRef('registry.example.com:5000/myorg/myapp:v1.0');
expect(result).toEqual({
registryBaseURL: 'https://registry.example.com:5000',
authConfigKey: 'registry.example.com:5000',
repoPath: 'myorg/myapp',
tag: 'v1.0',
});
});
it('should parse custom registry without port', () => {
const result = service.parseImageRef('registry.example.com/myorg/myapp:v1.0');
expect(result).toEqual({
registryBaseURL: 'https://registry.example.com',
authConfigKey: 'registry.example.com',
repoPath: 'myorg/myapp',
tag: 'v1.0',
});
});
it('should handle docker.io explicit registry', () => {
const result = service.parseImageRef('docker.io/nginx:latest');
expect(result).toEqual({
registryBaseURL: 'https://registry-1.docker.io',
authConfigKey: 'https://index.docker.io/v1/',
repoPath: 'docker.io/nginx',
tag: 'latest',
});
});
it('should handle complex tag with multiple colons', () => {
const result = service.parseImageRef('nginx:alpine-3.14.2');
expect(result).toEqual({
registryBaseURL: 'https://registry-1.docker.io',
authConfigKey: 'https://index.docker.io/v1/',
repoPath: 'library/nginx',
tag: 'alpine-3.14.2',
});
});
it('should handle localhost registry', () => {
const result = service.parseImageRef('localhost:5000/myapp:latest');
expect(result).toEqual({
registryBaseURL: 'https://localhost:5000',
authConfigKey: 'localhost:5000',
repoPath: 'myapp',
tag: 'latest',
});
});
it('should handle deep nested repository path', () => {
const result = service.parseImageRef('registry.example.com/team/project/service:v1.0.0');
expect(result).toEqual({
registryBaseURL: 'https://registry.example.com',
authConfigKey: 'registry.example.com',
repoPath: 'team/project/service',
tag: 'v1.0.0',
});
});
});
describe('headManifest - Unit Tests', () => {
it('should return response when request succeeds', async () => {
const mockResponse = {
statusCode: 200,
headers: { 'docker-content-digest': 'sha256:abc123' },
};
mockGot.head.mockResolvedValue(mockResponse);
const result = await service.headManifest(
'https://registry-1.docker.io/v2/library/nginx/manifests/latest',
{ Accept: 'application/vnd.docker.distribution.manifest.v2+json' },
{},
mockGot
);
expect(result).toEqual(mockResponse);
});
it('should fallback to GET when HEAD fails', async () => {
const mockGetResponse = {
statusCode: 200,
headers: { 'docker-content-digest': 'sha256:def456' },
};
mockGot.head.mockRejectedValue(new Error('HEAD not supported'));
mockGot.get.mockResolvedValue(mockGetResponse);
const result = await service.headManifest(
'https://registry-1.docker.io/v2/library/nginx/manifests/latest',
{ Accept: 'application/vnd.docker.distribution.manifest.v2+json' },
{},
mockGot
);
expect(result).toEqual(mockGetResponse);
});
});
describe('getRemoteDigest - Unit Tests', () => {
it('should return digest for public image (anonymous access)', async () => {
const mockResponse = {
statusCode: 200,
headers: { 'docker-content-digest': 'sha256:abc123def456' },
};
mockGot.head.mockResolvedValueOnce(mockResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({});
const result = await service.getRemoteDigest('nginx:latest');
expect(result).toBe('sha256:abc123def456');
});
it('should handle array digest header', async () => {
const mockResponse = {
statusCode: 200,
headers: { 'docker-content-digest': ['sha256:abc123def456', 'sha256:other'] },
};
mockGot.head.mockResolvedValueOnce(mockResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({});
const result = await service.getRemoteDigest('nginx:latest');
expect(result).toBe('sha256:abc123def456');
});
it('should authenticate with Bearer token when registry requires it', async () => {
const unauthorizedResponse = {
statusCode: 401,
headers: {
'www-authenticate':
'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/nginx:pull"',
},
};
const authorizedResponse = {
statusCode: 200,
headers: { 'docker-content-digest': 'sha256:authenticated' },
};
mockGot.head
.mockResolvedValueOnce(unauthorizedResponse)
.mockResolvedValueOnce(authorizedResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({
'https://index.docker.io/v1/': { auth: 'dXNlcjpwYXNz' },
});
mockDockerAuthService.decodeAuth.mockReturnValueOnce({ username: 'user', password: 'pass' });
mockDockerAuthService.getBearerToken.mockResolvedValueOnce('bearer-token-123');
const result = await service.getRemoteDigest('nginx:latest');
expect(result).toBe('sha256:authenticated');
});
it('should handle private registries requiring authentication', async () => {
const authorizedResponse = {
statusCode: 200,
headers: { 'docker-content-digest': 'sha256:private-registry' },
};
mockGot.head.mockResolvedValueOnce(authorizedResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({
'registry.example.com': { auth: 'dXNlcjpwYXNz' },
});
const result = await service.getRemoteDigest('registry.example.com/private/app:v1.0');
expect(result).toBe('sha256:private-registry');
});
it('should return null when authentication fails', async () => {
const unauthorizedResponse = {
statusCode: 401,
headers: {
'www-authenticate':
'Bearer realm="https://auth.docker.io/token",service="registry.docker.io"',
},
};
mockGot.head.mockResolvedValue(unauthorizedResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({});
mockDockerAuthService.decodeAuth.mockReturnValueOnce({ username: '', password: '' });
mockDockerAuthService.getBearerToken.mockResolvedValueOnce(null);
const result = await service.getRemoteDigest('nginx:latest');
expect(result).toBeNull();
});
it('should return null when no digest is available', async () => {
const mockResponse = {
statusCode: 200,
headers: {},
};
mockGot.head.mockResolvedValueOnce(mockResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({});
const result = await service.getRemoteDigest('nginx:latest');
expect(result).toBeNull();
});
it('should work with custom registries', async () => {
const authorizedResponse = {
statusCode: 200,
headers: { 'docker-content-digest': 'sha256:custom-registry' },
};
mockGot.head.mockResolvedValueOnce(authorizedResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({});
const result = await service.getRemoteDigest('registry.example.com/myapp:v1.0');
expect(result).toBe('sha256:custom-registry');
});
it('should handle images with no explicit tag (defaults to latest)', async () => {
const mockResponse = {
statusCode: 200,
headers: { 'docker-content-digest': 'sha256:latest-digest' },
};
mockGot.head.mockResolvedValueOnce(mockResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({});
const result = await service.getRemoteDigest('alpine');
expect(result).toBe('sha256:latest-digest');
});
it('should handle registry errors gracefully', async () => {
const errorResponse = {
statusCode: 500,
headers: {},
};
mockGot.head.mockResolvedValueOnce(errorResponse);
mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({});
const result = await service.getRemoteDigest('nginx:latest');
expect(result).toBeNull();
});
});
});

View File

@@ -0,0 +1,73 @@
import { Injectable, Logger } from '@nestjs/common';
import { readFile } from 'fs/promises';
import { join } from 'path';
import type { Got } from 'got';
import { ExtendOptions, got as gotClient } from 'got';
export type BasicDockerCreds = {
username: string;
password: string;
};
export type DockerWwwAuthParts = {
realm: string;
service: string;
scope: string;
};
@Injectable()
export class DockerAuthService {
private readonly logger = new Logger(DockerAuthService.name);
constructor() {}
async readDockerAuth(configPath?: string) {
try {
configPath ??= join(process.env.HOME || '/root', '.docker/config.json');
const cfg = JSON.parse(await readFile(configPath, 'utf8'));
return cfg.auths || {};
} catch (error) {
this.logger.debug(error, `Failed to read Docker auth from '${configPath}'`);
return {};
}
}
decodeAuth(auth: string): BasicDockerCreds {
try {
const [username, password] = Buffer.from(auth, 'base64').toString('utf8').split(':');
return { username, password };
} catch {
return { username: '', password: '' };
}
}
parseWWWAuth(wwwAuth: string): Partial<DockerWwwAuthParts> {
// www-authenticate: Bearer realm="...",service="...",scope="repository:repo/name:pull"
const parts: Partial<DockerWwwAuthParts> = {};
const rawParts = wwwAuth.replace(/^Bearer\s+/i, '').split(',') || [];
rawParts.forEach((pair) => {
const [k, v] = pair.split('=');
parts[k.trim()] = v?.replace(/^"|"$/g, '');
});
return parts;
}
async getBearerToken(
wwwAuth: string,
basicCreds: BasicDockerCreds,
got: Got<ExtendOptions> = gotClient
) {
const parts = this.parseWWWAuth(wwwAuth);
if (!parts.realm || !parts.service || !parts.scope) return null;
const { token } = await got
.get(parts.realm, {
searchParams: { service: parts.service, scope: parts.scope },
username: basicCreds.username,
password: basicCreds.password,
timeout: { request: 15_000 },
responseType: 'json',
})
.json<{ token?: string }>();
return token;
}
}

View File

@@ -0,0 +1,140 @@
import { Injectable } from '@nestjs/common';
import { ExtendOptions, Got, got as gotClient, OptionsOfTextResponseBody } from 'got';
import { docker } from '@app/core/utils/index.js';
import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js';
/** Accept header for Docker API manifest listing */
const ACCEPT_MANIFEST =
'application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json';
@Injectable()
export class DockerManifestService {
constructor(private readonly dockerAuthService: DockerAuthService) {}
parseImageRef(imageRef: string) {
// Normalize to repo:tag and extract registry/repo/name/tag
let ref = imageRef;
if (!ref.includes(':')) ref += ':latest';
// Registry present?
const firstSlash = ref.indexOf('/');
const maybeRegistry = firstSlash > -1 ? ref.slice(0, firstSlash) : '';
const hasDotOrColon = maybeRegistry.includes('.') || maybeRegistry.includes(':');
const isDockerHub = !hasDotOrColon || maybeRegistry === 'docker.io';
const registry = isDockerHub ? 'registry-1.docker.io' : maybeRegistry;
const rest = isDockerHub ? ref : ref.slice(maybeRegistry.length + 1);
const lastColon = rest.lastIndexOf(':');
const namePart = rest.slice(0, lastColon);
const tag = rest.slice(lastColon + 1);
// Ensure docker hub library namespace
const repoPath = isDockerHub && !namePart.includes('/') ? `library/${namePart}` : namePart;
return {
registryBaseURL: `https://${registry}`,
authConfigKey: isDockerHub ? 'https://index.docker.io/v1/' : maybeRegistry,
repoPath, // e.g. library/nginx or org/image
tag,
};
}
async headManifest(
url: string,
headers: Record<string, string>,
authHeader: Record<string, string> = {},
got: Got<ExtendOptions> = gotClient
) {
const opts: OptionsOfTextResponseBody = {
headers: { ...headers, ...authHeader },
timeout: { request: 15_000 },
throwHttpErrors: false,
};
try {
return await got.head(url, opts);
} catch {
// Some registries dont allow HEAD; try GET to read headers
return await got.get(url, opts);
}
}
async getRemoteDigest(imageRef) {
const { registryBaseURL, repoPath, tag, authConfigKey } = this.parseImageRef(imageRef);
const manifestURL = `${registryBaseURL}/v2/${repoPath}/manifests/${tag}`;
const dockerAuths = this.dockerAuthService.readDockerAuth();
const authEntry = dockerAuths[authConfigKey];
const basicCreds = authEntry?.auth
? this.dockerAuthService.decodeAuth(authEntry.auth)
: { username: '', password: '' };
// 1) Probe without auth to learn challenge
let resp = await this.headManifest(manifestURL, { Accept: ACCEPT_MANIFEST });
const digestHeaderRaw = resp.headers?.['docker-content-digest'];
const digestHeader = Array.isArray(digestHeaderRaw) ? digestHeaderRaw[0] : digestHeaderRaw;
if (resp.statusCode >= 200 && resp.statusCode < 300 && digestHeader) return digestHeader.trim();
const wwwAuth = (resp.headers?.['www-authenticate'] || '').toString();
if (/Bearer/i.test(wwwAuth)) {
const token = await this.dockerAuthService.getBearerToken(wwwAuth, basicCreds);
if (!token) return null;
// 2) Repeat with Bearer
resp = await this.headManifest(
manifestURL,
{ Accept: ACCEPT_MANIFEST },
{ Authorization: `Bearer ${token}` }
);
} else if (/Basic/i.test(wwwAuth) && basicCreds.username && basicCreds.password) {
// 2) Repeat with Basic
const basic =
'Basic ' +
Buffer.from(`${basicCreds.username}:${basicCreds.password}`).toString('base64');
resp = await this.headManifest(
manifestURL,
{ Accept: ACCEPT_MANIFEST },
{ Authorization: basic }
);
}
const digestRaw = resp.headers?.['docker-content-digest'];
const digest = Array.isArray(digestRaw) ? digestRaw[0] : digestRaw;
return digest ? digest.trim() : null;
}
async getLocalDigest(imageRef) {
try {
const data = await docker.getImage(imageRef).inspect();
const digests = data.RepoDigests || [];
if (digests.length === 0) return null;
// Prefer a digest matching this repo if present; else first
const pick = digests.find((d) => d.startsWith(imageRef.split(':')[0] + '@')) || digests[0];
const at = pick.indexOf('@');
return at >= 0 ? pick.slice(at + 1) : null;
} catch {
return null;
}
}
async isRebuildReady(networkMode) {
if (!networkMode || !networkMode.startsWith('container:')) return false;
const target = networkMode.slice('container:'.length);
try {
await docker.getContainer(target).inspect();
return false;
} catch {
return true; // unresolved target -> ':???' equivalent
}
}
async isUpdateAvailable(imageRef: string) {
const [local, remote] = await Promise.all([
this.getLocalDigest(imageRef),
this.getRemoteDigest(imageRef),
]);
if (local && remote) return local !== remote;
return null; // unknown
}
}

View File

@@ -1,6 +1,8 @@
import { Module } from '@nestjs/common';
import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js';
import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js';
import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js';
import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js';
import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js';
import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js';
@@ -12,6 +14,8 @@ import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.ser
DockerService,
DockerConfigService,
DockerOrganizerService,
DockerAuthService,
DockerManifestService,
// DockerEventService,
// Resolvers