mirror/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2025-12-21 12:59:52 -06:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Revert: single ai image entry point (#2131)" (#2143)

Browse Source 
This reverts commit 907d0db328.
This commit is contained in:
Daniel Salazar
2025-12-11 17:03:20 -08:00
committed by GitHub
parent 89fbcffc7a
commit 83eab0d6ac
20 changed files with 1455 additions and 943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
package-lock.json generated
View File

@@ -14597,7 +14597,9 @@
}
},
"node_modules/together-ai": {
"version": "0.32.0",
"version": "0.33.0",
"resolved": "https://registry.npmjs.org/together-ai/-/together-ai-0.33.0.tgz",
"integrity": "sha512-2JdxYwbw+Xw2bW2PHBGqbMTtYsQHoWO9UXvdwIfQkde/swoKp2x/hpxEjtTERzrMP4O5SdDPGxsjfcPXewDJ9A==",
"license": "Apache-2.0",
"bin": {
"together-ai": "bin/cli"
@@ -15830,7 +15832,7 @@
"svg-captcha": "^1.4.0",
"svgo": "^3.0.2",
"tiktoken": "^1.0.16",
"together-ai": "^0.32.0",
"together-ai": "^0.33.0",
"tweetnacl": "^1.0.3",
"ua-parser-js": "^1.0.38",
"uglify-js": "^3.17.4",

2
src/backend/package.json
View File

@@ -82,7 +82,7 @@
"svg-captcha": "^1.4.0",
"svgo": "^3.0.2",
"tiktoken": "^1.0.16",
"together-ai": "^0.32.0",
"together-ai": "^0.33.0",
"tweetnacl": "^1.0.3",
"ua-parser-js": "^1.0.38",
"uglify-js": "^3.17.4",

32
src/backend/src/modules/ai/PuterAIChatModule.js
View File

@@ -22,17 +22,14 @@ import { AdvancedBase } from '@heyputer/putility';
import config from '../../config.js';
import { AIInterfaceService } from '../../services/ai/AIInterfaceService.js';
import { AIChatService } from '../../services/ai/chat/AIChatService.js';
import { GeminiImageGenerationService } from '../../services/ai/image/GeminiImageGenerationService.js';
import { OpenAIImageGenerationService } from '../../services/ai/image/OpenAIImageGenerationService.js';
import { TogetherImageGenerationService } from '../../services/ai/image/TogetherImageGenerationService.js';
import { AIImageGenerationService } from '../../services/ai/image/AIImageGenerationService.js';
import { AWSTextractService } from '../../services/ai/ocr/AWSTextractService.js';
import { ElevenLabsVoiceChangerService } from '../../services/ai/sts/ElevenLabsVoiceChangerService.js';
import { OpenAISpeechToTextService } from '../../services/ai/stt/OpenAISpeechToTextService.js';
import { AWSPollyService } from '../../services/ai/tts/AWSPollyService.js';
import { ElevenLabsTTSService } from '../../services/ai/tts/ElevenLabsTTSService.js';
import { OpenAITTSService } from '../../services/ai/tts/OpenAITTSService.js';
import { OpenAIVideoGenerationService } from '../../services/ai/video/OpenAIVideoGenerationService.js';
import { TogetherVideoGenerationService } from '../../services/ai/video/TogetherVideoGenerationService.js';
// import { AIVideoGenerationService } from '../../services/ai/video/AIVideoGenerationService.js';
/**
* PuterAIModule class extends AdvancedBase to manage and register various AI services.
@@ -57,8 +54,13 @@ export class PuterAIModule extends AdvancedBase {
// completion ai service
services.registerService('ai-chat', AIChatService);
// TODO DS: centralize other service types too
// image generation ai service
services.registerService('ai-image', AIImageGenerationService);
// video generation ai service
// services.registerService('ai-video', AIVideoGenerationService);
// TODO DS: centralize other service types too
// TODO: services should govern their own availability instead of the module deciding what to register
if ( config?.services?.['aws-textract']?.aws ) {
@@ -78,25 +80,9 @@ export class PuterAIModule extends AdvancedBase {
if ( config?.services?.openai || config?.openai ) {
services.registerService('openai-image-generation', OpenAIImageGenerationService);
services.registerService('openai-video-generation', OpenAIVideoGenerationService);
services.registerService('openai-tts', OpenAITTSService);
services.registerService('openai-speech2txt', OpenAISpeechToTextService);
}
if ( config?.services?.['together-ai'] ) {
services.registerService('together-image-generation', TogetherImageGenerationService);
services.registerService('together-video-generation', TogetherVideoGenerationService);
}
if ( config?.services?.['gemini'] ) {
services.registerService('gemini-image-generation', GeminiImageGenerationService);
}
}
}
}

48
src/backend/src/services/ChatAPIService.js
View File

@@ -50,7 +50,7 @@ class ChatAPIService extends BaseService {
})();
// Register the router with the Express app
app.use('/puterai/chat', router);
app.use('/puterai', router);
// Install endpoints
this.install_chat_endpoints_({ router });
@@ -66,7 +66,7 @@ class ChatAPIService extends BaseService {
const Endpoint = this.require('Endpoint');
// Endpoint to list available AI chat models
Endpoint({
route: '/models',
route: '/chat/models',
methods: ['GET'],
handler: async (req, res) => {
try {
@@ -89,7 +89,7 @@ class ChatAPIService extends BaseService {
// Endpoint to get detailed information about available AI chat models
Endpoint({
route: '/models/details',
route: '/chat/models/details',
methods: ['GET'],
handler: async (req, res) => {
try {
@@ -109,6 +109,48 @@ class ChatAPIService extends BaseService {
}
},
}).attach(router);
Endpoint({
route: '/image/models',
methods: ['GET'],
handler: async (req, res) => {
try {
// Use SUService to access AIImageGenerationService as system user
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const svc_imageGen = this.services.get('ai-image');
// Return the simple model list which contains basic model information
return svc_imageGen.list();
});
// Return the list of models
res.json({ models });
} catch ( error ) {
this.log.error('Error fetching image models:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
Endpoint({
route: '/image/models/details',
methods: ['GET'],
handler: async (req, res) => {
try {
// Use SUService to access AIImageGenerationService as system user
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const svc_imageGen = this.services.get('ai-image');
// Return the detailed model list which includes cost and capability information
return svc_imageGen.models();
});
// Return the detailed list of models
res.json({ models });
} catch ( error ) {
this.log.error('Error fetching image model details:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
}
}

12
src/backend/src/services/ChatAPIService.test.js
View File

@@ -124,7 +124,11 @@ describe('ChatAPIService', () => {
// Verify
expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({
route: '/models',
route: '/chat/models',
methods: ['GET'],
}));
expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({
route: '/image/models',
methods: ['GET'],
}));
});
@@ -138,7 +142,11 @@ describe('ChatAPIService', () => {
// Verify
expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({
route: '/models/details',
route: '/chat/models/details',
methods: ['GET'],
}));
expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({
route: '/image/models/details',
methods: ['GET'],
}));
});

1
src/backend/src/services/PuterSiteService.js
View File

@@ -112,7 +112,6 @@ class PuterSiteService extends BaseService {
root_dir_id: this.config.devtest_directory,
};
}
console.log('???', subdomain, options);
const rows = await this.db.read(`SELECT * FROM subdomains WHERE ${
options.is_custom_domain ? 'domain' : 'subdomain'
} = ? LIMIT 1`,

4
src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts
View File

@@ -46,9 +46,9 @@ export class TogetherAIProvider implements IChatProvider {
let models: IChatModel[] | undefined = kv.get(this.#kvKey);
if ( models ) return models;
const api_models = await this.#together.models.list();
const apiModels = await this.#together.models.list();
models = [];
for ( const model of api_models ) {
for ( const model of apiModels ) {
if ( model.type === 'chat' || model.type === 'code' || model.type === 'language' || model.type === 'moderation' ) {
models.push({
id: `togetherai:${model.id}`,

2
src/backend/src/services/ai/image/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*.js
*.js.map

263
src/backend/src/services/ai/image/AIImageGenerationService.ts Normal file
View File

@@ -0,0 +1,263 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// METADATA // {"ai-commented":{"service":"claude"}}
import { APIError } from '../../../api/APIError.js';
import { ErrorService } from '../../../modules/core/ErrorService.js';
import { Context } from '../../../util/context.js';
import BaseService from '../../BaseService.js';
import { BaseDatabaseAccessService } from '../../database/BaseDatabaseAccessService.js';
import { DB_WRITE } from '../../database/consts.js';
import { DriverService } from '../../drivers/DriverService.js';
import { TypedValue } from '../../drivers/meta/Runtime.js';
import { EventService } from '../../EventService.js';
import { MeteringService } from '../../MeteringService/MeteringService.js';
import { GeminiImageGenerationProvider } from './providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.js';
import { OpenAiImageGenerationProvider } from './providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.js';
import { TogetherImageGenerationProvider } from './providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.js';
import { IGenerateParams, IImageModel, IImageProvider } from './providers/types.js';
export class AIImageGenerationService extends BaseService {
static SERVICE_NAME = 'ai-image';
static DEFAULT_PROVIDER = 'openai-image-generation';
get meteringService (): MeteringService {
return this.services.get('meteringService').meteringService;
}
get db (): BaseDatabaseAccessService {
return this.services.get('database').get(DB_WRITE, 'ai-service');
}
get errorService (): ErrorService {
return this.services.get('error-service');
}
get eventService (): EventService {
return this.services.get('event');
}
get driverService (): DriverService {
return this.services.get('driver');
}
getProvider (name: string): IImageProvider | undefined {
return this.#providers[name];
}
#providers: Record<string, IImageProvider> = {};
#modelIdMap: Record<string, IImageModel[]> = {};
/** Driver interfaces */
static IMPLEMENTS = {
['driver-capabilities']: {
supports_test_mode (iface: string, method_name: string) {
return iface === 'puter-image-generation' &&
method_name === 'generate';
},
},
['puter-image-generation']: {
async generate (...parameters: Parameters<AIImageGenerationService['generate']>) {
return (this as unknown as AIImageGenerationService).generate(...parameters);
},
},
};
getModel ({ modelId, provider}: { modelId: string, provider?: string }) {
const models = this.#modelIdMap[modelId];
if ( ! provider ) {
return models[0];
}
const model = models.find(m => m.provider === provider);
return model ?? models[0];
}
private async registerProviders () {
const openAiConfig = this.config.providers?.['openai-image-generation'] || this.global_config?.services?.['openai'] || this.global_config?.openai;
if ( openAiConfig && (openAiConfig.apiKey || openAiConfig.secret_key) ) {
this.#providers['openai-image-generation'] = new OpenAiImageGenerationProvider({ apiKey: openAiConfig.apiKey || openAiConfig.secret_key }, this.meteringService, this.errorService);
}
const geminiConfig = this.config.providers?.['gemini-image-generation'] || this.global_config?.services?.gemini;
if ( geminiConfig && (geminiConfig.apiKey || geminiConfig.secret_key) ) {
this.#providers['gemini-image-generation'] = new GeminiImageGenerationProvider({ apiKey: geminiConfig.apiKey || geminiConfig.secret_key }, this.meteringService, this.errorService);
}
const togetherConfig = this.config.providers?.['together-image-generation'] || this.global_config?.services?.['together-ai'];
if ( togetherConfig && (togetherConfig.apiKey || togetherConfig.secret_key) ) {
this.#providers['together-image-generation'] = new TogetherImageGenerationProvider({ apiKey: togetherConfig.apiKey || togetherConfig.secret_key }, this.meteringService, this.errorService, this.eventService);
}
// emit event for extensions to add providers
const extensionProviders = {} as Record<string, IImageProvider>;
await this.eventService.emit('ai.image.registerProviders', extensionProviders);
for ( const providerName in extensionProviders ) {
if ( this.#providers[providerName] ) {
console.warn('AIChatService: provider name conflict for ', providerName, ' registering with -extension suffix');
this.#providers[`${providerName}-extension`] = extensionProviders[providerName];
continue;
}
this.#providers[providerName] = extensionProviders[providerName];
}
}
protected async '__on_boot.consolidation' () {
// register chat providers here
await this.registerProviders();
// build model id map
for ( const providerName in this.#providers ) {
const provider = this.#providers[providerName];
// alias all driver requests to go here to support legacy routing
this.driverService.register_service_alias(AIImageGenerationService.SERVICE_NAME,
providerName,
{ iface: 'puter-image-generation' });
// build model id map
for ( const model of await provider.models() ) {
if ( ! this.#modelIdMap[model.id] ) {
this.#modelIdMap[model.id] = [];
}
this.#modelIdMap[model.id].push({ ...model, provider: providerName });
if ( model.aliases ) {
for ( const alias of model.aliases ) {
// join arrays which are aliased the same
if ( ! this.#modelIdMap[alias] ) {
this.#modelIdMap[alias] = this.#modelIdMap[model.id];
continue;
}
if ( this.#modelIdMap[alias] !== this.#modelIdMap[model.id] ) {
this.#modelIdMap[alias].push({ ...model, provider: providerName });
this.#modelIdMap[model.id] = this.#modelIdMap[alias];
continue;
}
}
}
this.#modelIdMap[model.id].sort((a, b) => a.costs[a.index_cost_key || Object.keys(a.costs)[0]] - b.costs[b.index_cost_key || Object.keys(b.costs)[0]]);
}
}
}
models () {
return Object.entries(this.#modelIdMap)
.map(([_, models]) => models)
.flat()
.sort((a, b) => {
if ( a.provider === b.provider ) {
return a.id.localeCompare(b.id);
}
return a.provider!.localeCompare(b.provider!);
});
}
list () {
return Object.keys(this.#modelIdMap).sort();
}
async generate (parameters: IGenerateParams) {
const clientDriverCall = Context.get('client_driver_call');
let { test_mode: testMode, intended_service: legacyProviderName } = clientDriverCall as { test_mode?: boolean; response_metadata: Record<string, unknown>; intended_service?: string };
const configuredProviders = Object.keys(this.#providers);
if ( configuredProviders.length === 0 ) {
throw new Error('no image generation providers configured');
}
let intendedProvider = (parameters.provider || (legacyProviderName === AIImageGenerationService.SERVICE_NAME ? '' : legacyProviderName)) ?? '';
if ( !parameters.model && !intendedProvider ) {
intendedProvider = configuredProviders.includes(AIImageGenerationService.DEFAULT_PROVIDER)
? AIImageGenerationService.DEFAULT_PROVIDER
: configuredProviders[0];
}
if ( intendedProvider && !this.#providers[intendedProvider] ) {
intendedProvider = configuredProviders[0];
}
if ( !parameters.model && intendedProvider ) {
parameters.model = this.#providers[intendedProvider].getDefaultModel();
}
const model = parameters.model ? this.getModel({ modelId: parameters.model, provider: intendedProvider }) : undefined;
if ( ! model ) {
const availableModelsUrl = `${this.global_config.origin }/puterai/image/models`;
throw APIError.create('field_invalid', undefined, {
key: 'model',
expected: `a valid model name from ${availableModelsUrl}`,
got: model,
});
}
// call model provider;
const provider = this.#providers[model.provider!];
if ( ! provider ) {
throw new Error(`no provider found for model ${model.id}`);
}
if ( model.allowedRatios?.length ) {
if ( parameters.ratio ) {
const isValidRatio = model.allowedRatios.some(r => r.w === parameters.ratio!.w && r.h === parameters.ratio!.h);
if ( ! isValidRatio ) {
parameters.ratio = model.allowedRatios[0];
}
} else {
parameters.ratio = model.allowedRatios[0];
}
}
if ( ! parameters.ratio ) {
parameters.ratio = { w: 1024, h: 1024 };
}
if ( model.allowedQualityLevels?.length ) {
if ( parameters.quality ) {
if ( ! model.allowedQualityLevels.includes(parameters.quality) ) {
parameters.quality = model.allowedQualityLevels[0];
}
} else {
parameters.quality = model.allowedQualityLevels[0];
}
}
const url = await provider.generate({
...parameters,
model: model.id,
provider: model.provider,
test_mode: testMode,
});
const isDataUrl = url.startsWith('data:');
const image = new TypedValue({
$: isDataUrl ? 'string:url:data' : 'string:url:web',
content_type: 'image',
}, url);
return image;
}
}

257
src/backend/src/services/ai/image/GeminiImageGenerationService.js
View File

@@ -1,257 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// METADATA // {"ai-commented":{"service":"claude"}}
const APIError = require('../../../api/APIError');
const BaseService = require('../../BaseService');
const { TypedValue } = require('../../drivers/meta/Runtime');
const { Context } = require('../../../util/context');
const { GoogleGenAI } = require('@google/genai');
/**
* Service class for generating images using Gemini's API
* Extends BaseService to provide image generation capabilities through
* the puter-image-generation interface.
*/
class GeminiImageGenerationService extends BaseService {
/** @type {import('../../MeteringService/MeteringService').MeteringService} */
get meteringService () {
return this.services.get('meteringService').meteringService;
}
static MODULES = {
};
_construct () {
this.models_ = {
'gemini-2.5-flash-image-preview': {
'1024x1024': 0.039,
},
'gemini-3-pro-image-preview': {
'1024x1024': 0.156,
},
};
}
/**
* Initializes the Gemini client with API credentials from config
* @private
* @async
* @returns {Promise<void>}
*/
async _init () {
this.genAI = new GoogleGenAI({ apiKey: this.global_config.services.gemini.apiKey });
}
static IMPLEMENTS = {
['driver-capabilities']: {
supports_test_mode (iface, method_name) {
return iface === 'puter-image-generation' &&
method_name === 'generate';
},
},
['puter-image-generation']: {
/**
* Generates an image using Gemini's gemini-2.5-flash-image-preview
* @param {string} prompt - The text description of the image to generate
* @param {Object} options - Generation options
* @param {Object} options.ratio - Image dimensions ratio object with w/h properties
* @param {string} [options.model='gemini-2.5-flash-image-preview'] - The model to use for generation
* @param {string} [options.input_image] - Base64 encoded input image for image-to-image generation
* @param {string} [options.input_image_mime_type] - MIME type of the input image
* @returns {Promise<string>} URL of the generated image
* @throws {Error} If prompt is not a string or ratio is invalid
*/
async generate (params) {
const { prompt, quality, test_mode, model, ratio, input_image, input_image_mime_type } = params;
if ( test_mode ) {
return new TypedValue({
$: 'string:url:web',
content_type: 'image',
}, 'https://puter-sample-data.puter.site/image_example.png');
}
const url = await this.generate(prompt, {
quality,
ratio: ratio || this.constructor.RATIO_SQUARE,
model,
input_image,
input_image_mime_type,
});
// Determine if this is a data URL or web URL
const isDataUrl = url.startsWith('data:');
const image = new TypedValue({
$: isDataUrl ? 'string:url:data' : 'string:url:web',
content_type: 'image',
}, url);
return image;
},
},
};
static RATIO_SQUARE = { w: 1024, h: 1024 };
async generate (prompt, {
ratio,
model,
input_image,
input_image_mime_type,
}) {
if ( typeof prompt !== 'string' ) {
throw new Error('`prompt` must be a string');
}
if ( !ratio || !this._validate_ratio(ratio, model) ) {
throw new Error(`\`ratio\` must be a valid ratio for model ${ model}`);
}
// Validate input image if provided
if ( input_image && !input_image_mime_type ) {
throw new Error('`input_image_mime_type` is required when `input_image` is provided');
}
if ( input_image_mime_type && !input_image ) {
throw new Error('`input_image` is required when `input_image_mime_type` is provided');
}
if ( input_image_mime_type && !this._validate_image_mime_type(input_image_mime_type) ) {
throw new Error('`input_image_mime_type` must be a valid image MIME type (image/png, image/jpeg, image/webp)');
}
// Somewhat sane defaults
model = model ?? 'gemini-2.5-flash-image-preview';
if ( ! this.models_[model] ) {
throw APIError.create('field_invalid', null, {
key: 'model',
expected: `one of: ${
Object.keys(this.models_).join(', ')}`,
got: model,
});
}
const price_key = `${ratio.w}x${ratio.h}`;
if ( ! this.models_[model][price_key] ) {
const availableSizes = Object.keys(this.models_[model]);
throw APIError.create('field_invalid', null, {
key: 'size/quality combination',
expected: `one of: ${ availableSizes.join(', ')}`,
got: price_key,
});
}
const actor = Context.get('actor');
const user_private_uid = actor?.private_uid ?? 'UNKNOWN';
if ( user_private_uid === 'UNKNOWN' ) {
this.errors.report('chat-completion-service:unknown-user', {
message: 'failed to get a user ID for a Gemini request',
alarm: true,
trace: true,
});
}
const usageType = `gemini:${model}:${price_key}`;
const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1);
if ( ! usageAllowed ) {
throw APIError.create('insufficient_funds');
}
// Construct the prompt based on whether we have an input image
let contents;
if ( input_image && input_image_mime_type ) {
// Image-to-image generation
contents = [
{ text: `Generate a picture of dimensions ${parseInt(ratio.w)}x${parseInt(ratio.h)} with the prompt: ${prompt}` },
{
inlineData: {
mimeType: input_image_mime_type,
data: input_image,
},
},
];
} else {
// Text-to-image generation
contents = `Generate a picture of dimensions ${parseInt(ratio.w)}x${parseInt(ratio.h)} with the prompt: ${prompt}`;
}
const response = await this.genAI.models.generateContent({
model,
contents,
});
// Metering usage tracking
// Gemini usage: always 1 image, resolution, cost, model
this.meteringService.incrementUsage(actor, usageType, 1);
let url = undefined;
for ( const part of response.candidates[0].content.parts ) {
if ( part.text ) {
// do nothing here
} else if ( part.inlineData ) {
const imageData = part.inlineData.data;
url = `data:image/png;base64,${ imageData}`;
}
}
if ( ! url ) {
throw new Error('Failed to extract image URL from Gemini response');
}
return url;
}
/**
* Get valid ratios for a specific model
* @param {string} model - The model name
* @returns {Array<Object>} Array of valid ratio objects
* @private
*/
_getValidRatios (model) {
if (
model === 'gemini-2.5-flash-image-preview' ||
model === 'gemini-3-pro-image-preview'
) {
return [this.constructor.RATIO_SQUARE];
}
return [];
}
_validate_ratio (ratio, model) {
const validRatios = this._getValidRatios(model);
return validRatios.includes(ratio);
}
/**
* Validates if the provided MIME type is supported for input images
* @param {string} mimeType - The MIME type to validate
* @returns {boolean} True if the MIME type is supported
* @private
*/
_validate_image_mime_type (mimeType) {
const supportedTypes = ['image/png', 'image/jpeg', 'image/jpg', 'image/webp'];
return supportedTypes.includes(mimeType.toLowerCase());
}
}
module.exports = {
GeminiImageGenerationService,
};

359
src/backend/src/services/ai/image/OpenAIImageGenerationService.js
View File

@@ -1,359 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// METADATA // {"ai-commented":{"service":"claude"}}
const APIError = require('../../../api/APIError');
const BaseService = require('../../BaseService');
const { TypedValue } = require('../../drivers/meta/Runtime');
const { Context } = require('../../../util/context');
/**
* Service class for generating images using OpenAI's DALL-E API.
* Extends BaseService to provide image generation capabilities through
* the puter-image-generation interface. Supports different aspect ratios
* (square, portrait, landscape) and handles API authentication, request
* validation, and spending tracking.
*/
class OpenAIImageGenerationService extends BaseService {
/** @type {import('../../MeteringService/MeteringService').MeteringService} */
get meteringService () {
return this.services.get('meteringService').meteringService;
}
static MODULES = {
openai: require('openai'),
};
_construct () {
this.models_ = {
'gpt-image-1-mini': {
'low:1024x1024': 0.005,
'low:1024x1536': 0.006,
'low:1536x1024': 0.006,
'medium:1024x1024': 0.011,
'medium:1024x1536': 0.015,
'medium:1536x1024': 0.015,
'high:1024x1024': 0.036,
'high:1024x1536': 0.052,
'high:1536x1024': 0.052,
},
'gpt-image-1': {
'low:1024x1024': 0.011,
'low:1024x1536': 0.016,
'low:1536x1024': 0.016,
'medium:1024x1024': 0.042,
'medium:1024x1536': 0.063,
'medium:1536x1024': 0.063,
'high:1024x1024': 0.167,
'high:1024x1536': 0.25,
'high:1536x1024': 0.25,
},
'dall-e-3': {
'1024x1024': 0.04,
'1024x1792': 0.08,
'1792x1024': 0.08,
'hd:1024x1024': 0.08,
'hd:1024x1792': 0.12,
'hd:1792x1024': 0.12,
},
'dall-e-2': {
'1024x1024': 0.02,
'512x512': 0.018,
'256x256': 0.016,
},
};
}
/**
* Initializes the OpenAI client with API credentials from config
* @private
* @async
* @returns {Promise<void>}
*/
async _init () {
let apiKey =
this.config?.services?.openai?.apiKey ??
this.global_config?.services?.openai?.apiKey;
if ( ! apiKey ) {
apiKey =
this.config?.openai?.secret_key ??
this.global_config.openai?.secret_key;
// Log a warning to inform users about the deprecated format
console.warn('The `openai.secret_key` configuration format is deprecated. ' +
'Please use `services.openai.apiKey` instead.');
}
this.openai = new this.modules.openai.OpenAI({
apiKey,
});
}
static IMPLEMENTS = {
['driver-capabilities']: {
supports_test_mode (iface, method_name) {
return iface === 'puter-image-generation' &&
method_name === 'generate';
},
},
['puter-image-generation']: {
/**
* Generates an image using OpenAI's DALL-E API
* @param {string} prompt - The text description of the image to generate
* @param {Object} options - Generation options
* @param {Object} options.ratio - Image dimensions ratio object with w/h properties
* @param {string} [options.model='dall-e-3'] - The model to use for generation
* @returns {Promise<string>} URL of the generated image
* @throws {Error} If prompt is not a string or ratio is invalid
*/
async generate (params) {
const { prompt, quality, test_mode, model, ratio } = params;
if ( test_mode ) {
return new TypedValue({
$: 'string:url:web',
content_type: 'image',
}, 'https://puter-sample-data.puter.site/image_example.png');
}
const url = await this.generate(prompt, {
quality,
ratio: ratio || this.constructor.RATIO_SQUARE,
model,
});
const image = new TypedValue({
$: 'string:url:web',
content_type: 'image',
}, url);
return image;
},
},
};
static RATIO_SQUARE = { w: 1024, h: 1024 };
static RATIO_PORTRAIT = { w: 1024, h: 1792 };
static RATIO_LANDSCAPE = { w: 1792, h: 1024 };
// GPT-Image-1 specific ratios
static RATIO_GPT_PORTRAIT = { w: 1024, h: 1536 };
static RATIO_GPT_LANDSCAPE = { w: 1536, h: 1024 };
async generate (prompt, {
ratio,
model,
quality,
}) {
if ( typeof prompt !== 'string' ) {
throw new Error('`prompt` must be a string');
}
if ( !ratio || !this._validate_ratio(ratio, model) ) {
throw new Error(`\`ratio\` must be a valid ratio for model ${ model}`);
}
// Somewhat sane defaults
model = model ?? 'gpt-image-1-mini';
quality = quality ?? 'low';
if ( ! this.models_[model] ) {
throw APIError.create('field_invalid', null, {
key: 'model',
expected: `one of: ${
Object.keys(this.models_).join(', ')}`,
got: model,
});
}
// Validate quality based on the model
const validQualities = this._getValidQualities(model);
if ( quality !== undefined && !validQualities.includes(quality) ) {
throw APIError.create('field_invalid', null, {
key: 'quality',
expected: `one of: ${ validQualities.join(', ').replace(/^$/, 'none (no quality)')}`,
got: quality,
});
}
const size = `${ratio.w}x${ratio.h}`;
const price_key = this._buildPriceKey(model, quality, size);
if ( ! this.models_[model][price_key] ) {
const availableSizes = Object.keys(this.models_[model]);
throw APIError.create('field_invalid', null, {
key: 'size/quality combination',
expected: `one of: ${ availableSizes.join(', ')}`,
got: price_key,
});
}
const actor = Context.get('actor');
const user_private_uid = actor?.private_uid ?? 'UNKNOWN';
if ( user_private_uid === 'UNKNOWN' ) {
this.errors.report('chat-completion-service:unknown-user', {
message: 'failed to get a user ID for an OpenAI request',
alarm: true,
trace: true,
});
}
const usageType = `openai:${model}:${price_key}`;
const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1);
if ( ! usageAllowed ) {
throw APIError.create('insufficient_funds');
}
// Build API parameters based on model
const apiParams = this._buildApiParams(model, {
user: user_private_uid,
prompt,
size,
quality,
});
const result = await this.openai.images.generate(apiParams);
// For image generation, usage is typically image count and resolution
this.meteringService.incrementUsage(actor, usageType, 1);
const spending_meta = {
model,
size: `${ratio.w}x${ratio.h}`,
};
if ( quality ) {
spending_meta.size = `${quality }:${ spending_meta.size}`;
}
const url = result.data?.[0]?.url || (result.data?.[0]?.b64_json ? `data:image/png;base64,${ result.data[0].b64_json}` : null);
if ( ! url ) {
throw new Error('Failed to extract image URL from OpenAI response');
}
return url;
}
/**
* Get valid quality levels for a specific model
* @param {string} model - The model name
* @returns {Array<string>} Array of valid quality levels
* @private
*/
_getValidQualities (model) {
if ( model === 'gpt-image-1-mini' ) {
return ['low', 'medium', 'high'];
}
if ( model === 'gpt-image-1' ) {
return ['low', 'medium', 'high'];
}
if ( model === 'dall-e-2' ) {
return [''];
}
if ( model === 'dall-e-3' ) {
return ['', 'hd'];
}
// Fallback for unknown models - assume no quality tiers
return [''];
}
/**
* Build the price key for a model based on quality and size
* @param {string} model - The model name
* @param {string} quality - The quality level
* @param {string} size - The image size (e.g., "1024x1024")
* @returns {string} The price key
* @private
*/
_buildPriceKey (model, quality, size) {
if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) {
// gpt-image-1 and gpt-image-1-mini use format: "quality:size" - default to low if not specified
const qualityLevel = quality || 'low';
return `${qualityLevel}:${size}`;
} else {
// dall-e models use format: "hd:size" or just "size"
return (quality === 'hd' ? 'hd:' : '') + size;
}
}
/**
* Build API parameters based on the model
* @param {string} model - The model name
* @param {Object} baseParams - Base parameters for the API call
* @returns {Object} API parameters object
* @private
*/
_buildApiParams (model, baseParams) {
const apiParams = {
user: baseParams.user,
prompt: baseParams.prompt,
size: baseParams.size,
};
if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) {
// gpt-image-1 requires the model parameter and uses different quality mapping
apiParams.model = model;
// Default to low quality if not specified, consistent with _buildPriceKey
apiParams.quality = baseParams.quality || 'low';
} else {
// dall-e models
apiParams.model = model;
if ( baseParams.quality === 'hd' ) {
apiParams.quality = 'hd';
}
}
return apiParams;
}
/**
* Get valid ratios for a specific model
* @param {string} model - The model name
* @returns {Array<Object>} Array of valid ratio objects
* @private
*/
_getValidRatios (model) {
const commonRatios = [this.constructor.RATIO_SQUARE];
if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) {
return [
...commonRatios,
this.constructor.RATIO_GPT_PORTRAIT,
this.constructor.RATIO_GPT_LANDSCAPE,
];
} else {
// DALL-E models
return [
...commonRatios,
this.constructor.RATIO_PORTRAIT,
this.constructor.RATIO_LANDSCAPE,
];
}
}
_validate_ratio (ratio, model) {
const validRatios = this._getValidRatios(model);
return validRatios.includes(ratio);
}
}
module.exports = {
OpenAIImageGenerationService,
};

287
src/backend/src/services/ai/image/TogetherImageGenerationService.js
View File

@@ -1,287 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// METADATA // {"ai-commented":{"service":"claude"}}
import { APIError } from 'openai';
import { Together } from 'together-ai';
import { Context } from '../../../util/context.js';
import BaseService from '../../BaseService.js';
import { TypedValue } from '../../drivers/meta/Runtime.js';
/**
* Service class for generating images using Together AI models.
* Extends BaseService to provide image generation capabilities through the
* puter-image-generation interface. Handles authentication, request validation,
* and metering integration.
*/
export class TogetherImageGenerationService extends BaseService {
DEFAULT_MODEL = 'black-forest-labs/FLUX.1-schnell';
DEFAULT_RATIO = { w: 1024, h: 1024 };
CONDITION_IMAGE_MODELS = [
'black-forest-labs/flux.1-kontext-dev',
'black-forest-labs/flux.1-kontext-pro',
'black-forest-labs/flux.1-kontext-max',
];
/** @type {import('../../MeteringService/MeteringService.js').MeteringService} */
get meteringService () {
return this.services.get('meteringService').meteringService;
}
async _init () {
const apiKey =
this.config?.apiKey ??
this.global_config?.services?.['together-ai']?.apiKey;
if ( ! apiKey ) {
throw new Error('Together AI image generation requires an API key');
}
this.client = new Together({ apiKey });
}
static IMPLEMENTS = {
['driver-capabilities']: {
supports_test_mode (iface, method_name) {
return iface === 'puter-image-generation' &&
method_name === 'generate';
},
},
['puter-image-generation']: {
async generate (...args) {
return this.generate(...args);
},
},
['models']: {
},
};
/**
* Generates an image using Together AI image models
* @param {object} params - Generation parameters
* @param {string} params.prompt - Prompt describing the desired image
* @param {string} [params.model] - Together AI model identifier
* @param {object} [params.ratio] - Width/height ratio object (e.g., { w: 1024, h: 1024 })
* @param {number} [params.width] - Explicit width override
* @param {number} [params.height] - Explicit height override
* @param {string} [params.aspect_ratio] - Aspect ratio string (e.g., "16:9")
* @param {number} [params.steps] - Diffusion step count
* @param {number} [params.seed] - Seed for reproducibility
* @param {string} [params.negative_prompt] - Negative prompt text
* @param {number} [params.n] - Number of images to generate (default 1)
* @param {string} [params.image_url] - Reference image URL for image-to-image
* @param {string} [params.image_base64] - Base64 encoded reference image
* @param {boolean} [params.disable_safety_checker] - Disable Together AI safety checker
* @param {boolean} [params.test_mode] - Enable Puter test mode shortcut
* @returns {Promise<TypedValue>} TypedValue containing the generated image URL or data URI
*/
async generate (params) {
const {
prompt,
test_mode,
ratio,
model,
width,
height,
aspect_ratio,
steps,
seed,
negative_prompt,
n,
image_url,
image_base64,
mask_image_url,
mask_image_base64,
prompt_strength,
disable_safety_checker,
response_format,
} = params;
const svc_event = this.services.get('event');
svc_event.emit('ai.log.image', { actor: Context.get('actor'), parameters: params, completionId: '0', intended_service: params.model });
if ( test_mode ) {
return new TypedValue({
$: 'string:url:web',
content_type: 'image',
}, 'https://puter-sample-data.puter.site/image_example.png');
}
const url = await this.#generate(prompt, {
ratio,
model,
width,
height,
aspect_ratio,
steps,
seed,
negative_prompt,
n,
image_url,
image_base64,
mask_image_url,
mask_image_base64,
prompt_strength,
disable_safety_checker,
response_format,
});
const isDataUrl = url.startsWith('data:');
return new TypedValue({
$: isDataUrl ? 'string:url:data' : 'string:url:web',
content_type: 'image',
}, url);
}
async #generate (prompt, options) {
if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) {
throw new Error('`prompt` must be a non-empty string');
}
const request = this.#buildRequest(prompt, options);
const actor = Context.get('actor');
if ( ! actor ) {
throw new Error('actor not found in context');
}
const usageType = `together-image:${request.model}`;
const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1);
if ( ! usageAllowed ) {
throw APIError.create('insufficient_funds');
}
const response = await this.client.images.generate(request);
if ( ! response?.data?.length ) {
throw new Error('Together AI response did not include image data');
}
this.meteringService.incrementUsage(actor, usageType, 1);
const first = response.data[0];
if ( first.url ) {
return first.url;
}
if ( first.b64_json ) {
return `data:image/png;base64,${ first.b64_json}`;
}
throw new Error('Together AI response did not include an image URL');
}
/**
* Normalizes Together AI image generation request parameters
*/
#buildRequest (prompt, options = {}) {
const {
ratio,
model,
width,
height,
aspect_ratio,
steps,
seed,
negative_prompt,
n,
image_url,
image_base64,
mask_image_url,
mask_image_base64,
prompt_strength,
disable_safety_checker,
response_format,
input_image,
} = options;
const request = {
prompt,
model: model ?? this.constructor.DEFAULT_MODEL,
};
const requiresConditionImage =
this.#modelRequiresConditionImage(request.model);
const ratioWidth = (ratio && ratio.w !== undefined) ? Number(ratio.w) : undefined;
const ratioHeight = (ratio && ratio.h !== undefined) ? Number(ratio.h) : undefined;
const normalizedWidth = this.#normalizeDimension(width !== undefined ? Number(width) : (ratioWidth ?? this.DEFAULT_RATIO.w));
const normalizedHeight = this.#normalizeDimension(height !== undefined ? Number(height) : (ratioHeight ?? this.DEFAULT_RATIO.h));
if ( aspect_ratio ) {
request.aspect_ratio = aspect_ratio;
} else {
if ( normalizedWidth ) request.width = normalizedWidth;
if ( normalizedHeight ) request.height = normalizedHeight;
}
if ( typeof steps === 'number' && Number.isFinite(steps) ) {
request.steps = Math.max(1, Math.min(50, Math.round(steps)));
}
if ( typeof seed === 'number' && Number.isFinite(seed) ) request.seed = Math.round(seed);
if ( typeof negative_prompt === 'string' ) request.negative_prompt = negative_prompt;
if ( typeof n === 'number' && Number.isFinite(n) ) {
request.n = Math.max(1, Math.min(4, Math.round(n)));
}
if ( typeof disable_safety_checker === 'boolean' ) {
request.disable_safety_checker = disable_safety_checker;
}
if ( typeof response_format === 'string' ) request.response_format = response_format;
const resolvedImageBase64 = typeof image_base64 === 'string'
? image_base64
: (typeof input_image === 'string' ? input_image : undefined);
if ( typeof image_url === 'string' ) request.image_url = image_url;
if ( resolvedImageBase64 ) request.image_base64 = resolvedImageBase64;
if ( typeof mask_image_url === 'string' ) request.mask_image_url = mask_image_url;
if ( typeof mask_image_base64 === 'string' ) request.mask_image_base64 = mask_image_base64;
if ( typeof prompt_strength === 'number' && Number.isFinite(prompt_strength) ) {
request.prompt_strength = Math.max(0, Math.min(1, prompt_strength));
}
if ( requiresConditionImage ) {
const conditionSource = resolvedImageBase64
? resolvedImageBase64
: (typeof image_url === 'string' ? image_url : undefined);
if ( ! conditionSource ) {
throw new Error(`Model ${request.model} requires an image_url or image_base64 input`);
}
request.condition_image = conditionSource;
}
return request;
}
#normalizeDimension (value) {
if ( typeof value !== 'number' ) return undefined;
const rounded = Math.max(64, Math.round(value));
// Flux models expect multiples of 8. Snap to the nearest multiple without going below 64.
return Math.max(64, Math.round(rounded / 8) * 8);
}
#modelRequiresConditionImage (model) {
if ( typeof model !== 'string' || model.trim() === '' ) {
return false;
}
const normalized = model.toLowerCase();
return this.CONDITION_IMAGE_MODELS.some(required => normalized === required);
}
}

171
src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts Normal file
View File

@@ -0,0 +1,171 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// METADATA // {"ai-commented":{"service":"claude"}}
import { GenerateContentResponse, GoogleGenAI } from '@google/genai';
import APIError from '../../../../../api/APIError.js';
import { ErrorService } from '../../../../../modules/core/ErrorService.js';
import { Context } from '../../../../../util/context.js';
import { MeteringService } from '../../../../MeteringService/MeteringService.js';
import { GEMINI_DEFAULT_RATIO, GEMINI_IMAGE_GENERATION_MODELS } from './models.js';
import { IGenerateParams, IImageModel, IImageProvider } from '../types.js';
type GeminiGenerateParams = IGenerateParams & {
input_image?: string;
input_image_mime_type?: string;
};
export class GeminiImageGenerationProvider implements IImageProvider {
#meteringService: MeteringService;
#client: GoogleGenAI;
#errors: ErrorService;
constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService) {
if ( ! config.apiKey ) {
throw new Error('Gemini image generation requires an API key');
}
this.#meteringService = meteringService;
this.#client = new GoogleGenAI({ apiKey: config.apiKey });
this.#errors = errorService;
}
models (): IImageModel[] {
return GEMINI_IMAGE_GENERATION_MODELS;
}
getDefaultModel (): string {
return GEMINI_IMAGE_GENERATION_MODELS[0].id;
}
async generate (params: IGenerateParams): Promise<string> {
const { prompt, test_mode } = params;
let { model, ratio, quality } = params;
const { input_image, input_image_mime_type } = params as GeminiGenerateParams;
const selectedModel = this.models().find(m => m.id === model) || this.models().find(m => m.id === this.getDefaultModel())!;
if ( test_mode ) {
return 'https://puter-sample-data.puter.site/image_example.png';
}
if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) {
throw new Error('`prompt` must be a non-empty string');
}
const allowedRatios = selectedModel.allowedRatios ?? [GEMINI_DEFAULT_RATIO];
ratio = ratio && this.#isValidRatio(ratio, allowedRatios) ? ratio : allowedRatios[0];
if ( input_image && !input_image_mime_type ) {
throw new Error('`input_image_mime_type` is required when `input_image` is provided');
}
if ( input_image_mime_type && !input_image ) {
throw new Error('`input_image` is required when `input_image_mime_type` is provided');
}
if ( input_image_mime_type && !this.#isValidImageMimeType(input_image_mime_type) ) {
throw new Error('`input_image_mime_type` must be a valid image MIME type (image/png, image/jpeg, image/webp)');
}
const priceKey = `${quality ? `${quality}:` : ''}${ratio.w}x${ratio.h}`;
const priceInCents = selectedModel.costs[priceKey];
if ( priceInCents === undefined ) {
const availableSizes = Object.keys(selectedModel.costs);
throw APIError.create('field_invalid', undefined, {
key: 'size/quality combination',
expected: `one of: ${ availableSizes.join(', ')}`,
got: priceKey,
});
}
const actor = Context.get('actor');
const user_private_uid = actor?.private_uid ?? 'UNKNOWN';
if ( user_private_uid === 'UNKNOWN' ) {
this.#errors.report('gemini-image-generation:unknown-user', {
message: 'failed to get a user ID for a Gemini request',
alarm: true,
trace: true,
});
}
const costInMicroCents = priceInCents * 1_000_000;
const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents);
if ( ! usageAllowed ) {
throw APIError.create('insufficient_funds');
}
const contents = this.#buildContents(prompt, ratio, input_image, input_image_mime_type);
const response = await this.#client.models.generateContent({
model: selectedModel.id,
contents,
});
const usageType = `gemini:${selectedModel.id}:${priceKey}`;
this.#meteringService.incrementUsage(actor, usageType, 1, costInMicroCents);
const url = this.#extractImageUrl(response);
if ( ! url ) {
throw new Error('Failed to extract image URL from Gemini response');
}
return url;
}
#buildContents (prompt: string, ratio: { w: number; h: number }, input_image?: string, input_image_mime_type?: string) {
if ( input_image && input_image_mime_type ) {
return [
{ text: `Generate a picture of dimensions ${parseInt(`${ratio.w}`)}x${parseInt(`${ratio.h}`)} with the prompt: ${prompt}` },
{
inlineData: {
mimeType: input_image_mime_type,
data: input_image,
},
},
];
}
return `Generate a picture of dimensions ${parseInt(`${ratio.w}`)}x${parseInt(`${ratio.h}`)} with the prompt: ${prompt}`;
}
#extractImageUrl (response: GenerateContentResponse): string | undefined {
const parts = response?.candidates?.[0]?.content?.parts;
if ( ! Array.isArray(parts) ) {
return undefined;
}
for ( const part of parts ) {
if ( part?.inlineData?.data ) {
return `data:image/png;base64,${ part.inlineData.data}`;
}
}
return undefined;
}
#isValidRatio (ratio: { w: number; h: number }, allowedRatios: { w: number; h: number }[]) {
return allowedRatios.some(r => r.w === ratio.w && r.h === ratio.h);
}
#isValidImageMimeType (mimeType?: string) {
if ( ! mimeType ) return false;
const supportedTypes = ['image/png', 'image/jpeg', 'image/jpg', 'image/webp'];
return supportedTypes.includes(mimeType.toLowerCase());
}
}

111
src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts Normal file
View File

@@ -0,0 +1,111 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { IImageModel } from '../types';
export const GEMINI_DEFAULT_RATIO = { w: 1024, h: 1024 };
export const GEMINI_IMAGE_GENERATION_MODELS: IImageModel[] = [
{
id: 'gemini-2.5-flash-image-preview',
aliases: ['gemini-2.5-flash-image'],
name: 'Gemini 2.5 Flash Image Preview',
version: '1.0',
costs_currency: 'usd-cents',
index_cost_key: '1024x1024',
allowedQualityLevels: [''],
costs: {
'1x1': 3.9, // $0.039 per image
'2x3': 3.9, // $0.039 per image
'3x2': 3.9, // $0.039 per image
'3x4': 3.9, // $0.039 per image
'4x3': 3.9, // $0.039 per image
'4x5': 3.9, // $0.039 per image
'5x4': 3.9, // $0.039 per image
'9x16': 3.9, // $0.039 per image
'16x9': 3.9, // $0.039 per image
'21x9': 3.9, // $0.039 per image
},
allowedRatios: [
{ w: 1, h: 1 },
{ w: 2, h: 3 },
{ w: 3, h: 2 },
{ w: 3, h: 4 },
{ w: 4, h: 3 },
{ w: 4, h: 5 },
{ w: 5, h: 4 },
{ w: 9, h: 16 },
{ w: 16, h: 9 },
{ w: 21, h: 9 },
],
},
{
id: 'gemini-3-pro-image-preview',
name: 'Gemini 3 Pro Image Preview',
version: '1.0',
costs_currency: 'usd-cents',
index_cost_key: '1024x1024',
aliases: ['gemini-3-pro-image'],
allowedQualityLevels: ['1K', '2K', '4K'],
allowedRatios: [
{ w: 1, h: 1 },
{ w: 2, h: 3 },
{ w: 3, h: 2 },
{ w: 3, h: 4 },
{ w: 4, h: 3 },
{ w: 4, h: 5 },
{ w: 5, h: 4 },
{ w: 9, h: 16 },
{ w: 16, h: 9 },
{ w: 21, h: 9 },
],
costs: {
'1K:1x1': 13.51, // $0.1351 per image
'1K:2x3': 13.51, // $0.1351 per image
'1K:3x2': 13.51, // $0.1351 per image
'1K:3x4': 13.51, // $0.1351 per image
'1K:4x3': 13.51, // $0.1351 per image
'1K:4x5': 13.51, // $0.1351 per image
'1K:5x4': 13.51, // $0.1351 per image
'1K:9x16': 13.51, // $0.1351 per image
'1K:16x9': 13.51, // $0.1351 per image
'1K:21x9': 13.51, // $0.1351 per image
'2K:1x1': 13.51, // $0.1351 per image
'2K:2x3': 13.51, // $0.1351 per image
'2K:3x2': 13.51, // $0.1351 per image
'2K:3x4': 13.51, // $0.1351 per image
'2K:4x3': 13.51, // $0.1351 per image
'2K:4x5': 13.51, // $0.1351 per image
'2K:5x4': 13.51, // $0.1351 per image
'2K:9x16': 13.51, // $0.1351 per image
'2K:16x9': 13.51, // $0.1351 per image
'2K:21x9': 13.51, // $0.1351 per image
'4K:1x1': 24.1, // $0.24 per image
'4K:2x3': 24.1, // $0.24 per image
'4K:3x2': 24.1, // $0.24 per image
'4K:3x4': 24.1, // $0.24 per image
'4K:4x3': 24.1, // $0.24 per image
'4K:4x5': 24.1, // $0.24 per image
'4K:5x4': 24.1, // $0.24 per image
'4K:9x16': 24.1, // $0.24 per image
'4K:16x9': 24.1, // $0.24 per image
'4K:21x9': 24.1, // $0.24 per image
},
},
];

176
src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts Normal file
View File

@@ -0,0 +1,176 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// METADATA // {"ai-commented":{"service":"claude"}}
import openai, { OpenAI } from 'openai';
import { ImageGenerateParamsNonStreaming } from 'openai/resources/images.js';
import APIError from '../../../../../api/APIError.js';
import { ErrorService } from '../../../../../modules/core/ErrorService.js';
import { Context } from '../../../../../util/context.js';
import { MeteringService } from '../../../../MeteringService/MeteringService.js';
import { IGenerateParams, IImageProvider } from '../types.js';
import { OPEN_AI_IMAGE_GENERATION_MODELS } from './models.js';
/**
* Service class for generating images using OpenAI's DALL-E API.
* Extends BaseService to provide image generation capabilities through
* the puter-image-generation interface. Supports different aspect ratios
* (square, portrait, landscape) and handles API authentication, request
* validation, and spending tracking.
*/
export class OpenAiImageGenerationProvider implements IImageProvider {
#meteringService: MeteringService;
#openai: OpenAI;
#errors: ErrorService;
constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService) {
this.#meteringService = meteringService;
this.#openai = new openai.OpenAI({
apiKey: config.apiKey,
});
this.#errors = errorService;
}
models () {
return OPEN_AI_IMAGE_GENERATION_MODELS;
}
getDefaultModel (): string {
return 'dall-e-2';
}
async generate ({ prompt, quality, test_mode, model, ratio }: IGenerateParams) {
const selectedModel = this.models().find(m => m.id === model) || this.models().find(m => m.id === this.getDefaultModel())!;
if ( test_mode ) {
return 'https://puter-sample-data.puter.site/image_example.png';
}
if ( typeof prompt !== 'string' ) {
throw new Error('`prompt` must be a string');
}
const validRations = selectedModel?.allowedRatios;
if ( validRations && (!ratio || !validRations.some(r => r.w === ratio.w && r.h === ratio.h)) ) {
ratio = validRations[0]; // Default to the first allowed ratio
}
if ( ! ratio ) {
ratio = { w: 1024, h: 1024 }; // Fallback ratio
}
const validQualities = selectedModel?.allowedQualityLevels;
if ( validQualities && (!quality || !validQualities.includes(quality)) ) {
quality = validQualities[0]; // Default to the first allowed quality
}
const size = `${ratio.w}x${ratio.h}`;
const price_key = this.#buildPriceKey(selectedModel.id, quality!, size);
if ( ! selectedModel?.costs[price_key] ) {
const availableSizes = Object.keys(selectedModel?.costs);
throw APIError.create('field_invalid', undefined, {
key: 'size/quality combination',
expected: `one of: ${ availableSizes.join(', ')}`,
got: price_key,
});
}
const actor = Context.get('actor');
const user_private_uid = actor?.private_uid ?? 'UNKNOWN';
if ( user_private_uid === 'UNKNOWN' ) {
this.#errors.report('chat-completion-service:unknown-user', {
message: 'failed to get a user ID for an OpenAI request',
alarm: true,
trace: true,
});
}
const costInMicroCents = selectedModel.costs[price_key] * 1_000_000;
const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents);
if ( ! usageAllowed ) {
throw APIError.create('insufficient_funds');
}
// Build API parameters based on model
const apiParams = this.#buildApiParams(selectedModel.id, {
user: user_private_uid,
prompt,
size,
quality,
} as Partial<ImageGenerateParamsNonStreaming>);
const result = await this.#openai.images.generate(apiParams);
// For image generation, usage is typically image count and resolution
const usageType = `openai:${selectedModel.id}:${price_key}`;
this.#meteringService.incrementUsage(actor, usageType, 1, costInMicroCents);
const spending_meta = {
model,
size: `${ratio.w}x${ratio.h}`,
};
if ( quality ) {
spending_meta.size = `${quality}:${ spending_meta.size}`;
}
const url = result.data?.[0]?.url || (result.data?.[0]?.b64_json ? `data:image/png;base64,${ result.data[0].b64_json}` : null);
if ( ! url ) {
throw new Error('Failed to extract image URL from OpenAI response');
}
return url;
}
#buildPriceKey (model: string, quality: string, size: string) {
if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) {
// gpt-image-1 and gpt-image-1-mini use format: "quality:size" - default to low if not specified
const qualityLevel = quality || 'low';
return `${qualityLevel}:${size}`;
} else {
// dall-e models use format: "hd:size" or just "size"
return (quality === 'hd' ? 'hd:' : '') + size;
}
}
#buildApiParams (model: string, baseParams: Partial<ImageGenerateParamsNonStreaming>): ImageGenerateParamsNonStreaming {
const apiParams = {
user: baseParams.user,
prompt: baseParams.prompt,
size: baseParams.size,
} as ImageGenerateParamsNonStreaming;
if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) {
// gpt-image-1 requires the model parameter and uses different quality mapping
apiParams.model = model;
// Default to low quality if not specified, consistent with _buildPriceKey
apiParams.quality = baseParams.quality || 'low';
} else {
// dall-e models
apiParams.model = model;
if ( baseParams.quality === 'hd' ) {
apiParams.quality = 'hd';
}
}
return apiParams;
}
}

71
src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts Normal file
View File

@@ -0,0 +1,71 @@
import { IImageModel } from '../types';
export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [
{ id: 'gpt-image-1-mini',
name: 'GPT Image 1 Mini',
version: '1.0',
costs_currency: 'usd-cents',
index_cost_key: 'low:1024x1024',
costs: {
'low:1024x1024': 0.5,
'low:1024x1536': 0.6,
'low:1536x1024': 0.6,
'medium:1024x1024': 1.1,
'medium:1024x1536': 1.5,
'medium:1536x1024': 1.5,
'high:1024x1024': 3.6,
'high:1024x1536': 5.2,
'high:1536x1024': 5.2,
},
allowedQualityLevels: ['low', 'medium', 'high'],
allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1536 }, { w: 1536, h: 1024 }],
},
{ id: 'gpt-image-1',
name: 'GPT Image 1',
version: '1.0',
costs_currency: 'usd-cents',
index_cost_key: 'low:1024x1024',
costs: {
'low:1024x1024': 1.1,
'low:1024x1536': 1.6,
'low:1536x1024': 1.6,
'medium:1024x1024': 4.2,
'medium:1024x1536': 6.3,
'medium:1536x1024': 6.3,
'high:1024x1024': 16.7,
'high:1024x1536': 25,
'high:1536x1024': 25,
},
allowedQualityLevels: ['low', 'medium', 'high'],
allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1536 }, { w: 1536, h: 1024 }],
},
{ id: 'dall-e-3',
name: 'DALL·E 3',
version: '1.0',
costs_currency: 'usd-cents',
index_cost_key: '1024x1024',
costs: {
'1024x1024': 4,
'1024x1792': 8,
'1792x1024': 8,
'hd:1024x1024': 8,
'hd:1024x1792': 12,
'hd:1792x1024': 12,
},
allowedQualityLevels: ['', 'hd'],
allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1792 }, { w: 1792, h: 1024 }],
},
{ id: 'dall-e-2',
name: 'DALL·E 2',
version: '1.0',
costs_currency: 'usd-cents',
index_cost_key: '1024x1024',
costs: {
'256x256': 1.6,
'512x512': 1.8,
'1024x1024': 2,
},
allowedRatios: [ { w: 256, h: 256 }, { w: 512, h: 512 }, { w: 1024, h: 1024 }],
},
];

242
src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts Normal file
View File

@@ -0,0 +1,242 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// METADATA // {"ai-commented":{"service":"claude"}}
import { Together } from 'together-ai';
import APIError from '../../../../../api/APIError.js';
import { ErrorService } from '../../../../../modules/core/ErrorService.js';
import { Context } from '../../../../../util/context.js';
import { EventService } from '../../../../EventService.js';
import { MeteringService } from '../../../../MeteringService/MeteringService.js';
import { IGenerateParams, IImageModel, IImageProvider } from '../types.js';
import { TOGETHER_IMAGE_GENERATION_MODELS } from './models.js';
const TOGETHER_DEFAULT_RATIO = { w: 1024, h: 1024 };
type TogetherGenerateParams = IGenerateParams & {
steps?: number;
seed?: number;
negative_prompt?: string;
n?: number;
image_url?: string;
image_base64?: string;
mask_image_url?: string;
mask_image_base64?: string;
prompt_strength?: number;
disable_safety_checker?: boolean;
response_format?: string;
input_image?: string;
};
const DEFAULT_MODEL = 'togetherai:black-forest-labs/FLUX.1-schnell';
const CONDITION_IMAGE_MODELS = [
'togetherai:black-forest-labs/flux.1-kontext-dev',
'togetherai:black-forest-labs/flux.1-kontext-pro',
'togetherai:black-forest-labs/flux.1-kontext-max',
];
export class TogetherImageGenerationProvider implements IImageProvider {
#client: Together;
#meteringService: MeteringService;
#errors: ErrorService;
#eventService: EventService;
constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService, eventService: EventService) {
if ( ! config.apiKey ) {
throw new Error('Together AI image generation requires an API key');
}
this.#meteringService = meteringService;
this.#errors = errorService;
this.#eventService = eventService;
this.#client = new Together({ apiKey: config.apiKey });
}
models (): IImageModel[] {
return TOGETHER_IMAGE_GENERATION_MODELS;
}
getDefaultModel (): string {
return DEFAULT_MODEL;
}
async generate (params: IGenerateParams): Promise<string> {
const { prompt, test_mode } = params;
let { model, ratio, quality } = params;
const options = params as TogetherGenerateParams;
const selectedModel = this.#getModel(model);
await this.#eventService.emit('ai.log.image', { actor: Context.get('actor'), parameters: params, completionId: '0', intended_service: selectedModel.id });
if ( test_mode ) {
return 'https://puter-sample-data.puter.site/image_example.png';
}
if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) {
throw new Error('`prompt` must be a non-empty string');
}
ratio = ratio || TOGETHER_DEFAULT_RATIO;
const actor = Context.get('actor');
if ( ! actor ) {
this.#errors.report('together-image-generation:unknown-actor', {
message: 'failed to resolve actor for Together image generation',
trace: true,
});
throw new Error('actor not found in context');
}
const priceKey = '1MP';
const centsPerMP = selectedModel.costs[priceKey]; // hardcoded for now since all together ai models use this type of pricing
if ( centsPerMP === undefined ) {
throw new Error(`No pricing configured for model ${selectedModel.id}`);
}
const usageType = `together-image:${selectedModel.id}:${priceKey}`;
let MP = (ratio.h * ratio.w) / 1_000_000;
if ( quality ) {
// if quality its gemini 3 image, so price based on those K sizes as MP
MP = parseInt(quality[0]) ; // convert to microcents
}
const costInMicroCents = centsPerMP * MP * 1_000_000; // cost in microcents
const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents);
if ( ! usageAllowed ) {
throw APIError.create('insufficient_funds');
}
const request = this.#buildRequest(prompt, { ...options, ratio, model: selectedModel.id.replace('togetherai:', '') }) as unknown as Together.Images.ImageGenerateParams;
try {
const response = await this.#client.images.generate(request);
if ( ! response?.data?.length ) {
throw new Error('Together AI response did not include image data');
}
this.#meteringService.incrementUsage(actor, usageType, MP, costInMicroCents);
const first = response.data[0] as { url?: string; b64_json?: string };
const url = first.url || (first.b64_json ? `data:image/png;base64,${ first.b64_json}` : undefined);
if ( ! url ) {
throw new Error('Together AI response did not include an image URL');
}
return url;
} catch ( error ) {
throw new Error(`Together AI image generation error: ${(error as Error).message}`);
}
}
#getModel (model?: string) {
return this.models().find(m => m.id === model) || this.models().find(m => m.id === DEFAULT_MODEL)!;
}
#buildRequest (prompt: string, options: TogetherGenerateParams) {
const {
ratio,
model,
steps,
seed,
negative_prompt,
n,
image_url,
image_base64,
mask_image_url,
mask_image_base64,
prompt_strength,
disable_safety_checker,
response_format,
input_image,
} = options;
const request: Record<string, unknown> = {
prompt,
model: model ?? DEFAULT_MODEL,
};
const requiresConditionImage = this.#modelRequiresConditionImage(request.model as string);
const ratioWidth = ratio?.w !== undefined ? Number(ratio.w) : undefined;
const ratioHeight = ratio?.h !== undefined ? Number(ratio.h) : undefined;
const normalizedWidth = this.#normalizeDimension((ratioWidth ?? TOGETHER_DEFAULT_RATIO.w));
const normalizedHeight = this.#normalizeDimension((ratioHeight ?? TOGETHER_DEFAULT_RATIO.h));
if ( normalizedWidth ) request.width = normalizedWidth;
if ( normalizedHeight ) request.height = normalizedHeight;
if ( typeof steps === 'number' && Number.isFinite(steps) ) {
request.steps = Math.max(1, Math.min(50, Math.round(steps)));
}
if ( typeof seed === 'number' && Number.isFinite(seed) ) request.seed = Math.round(seed);
if ( typeof negative_prompt === 'string' ) request.negative_prompt = negative_prompt;
if ( typeof n === 'number' && Number.isFinite(n) ) {
request.n = Math.max(1, Math.min(4, Math.round(n)));
}
if ( disable_safety_checker ) {
request.disable_safety_checker = true;
}
if ( typeof response_format === 'string' ) request.response_format = response_format;
const resolvedImageBase64 = typeof image_base64 === 'string'
? image_base64
: (typeof input_image === 'string' ? input_image : undefined);
if ( typeof image_url === 'string' ) request.image_url = image_url;
if ( resolvedImageBase64 ) request.image_base64 = resolvedImageBase64;
if ( typeof mask_image_url === 'string' ) request.mask_image_url = mask_image_url;
if ( typeof mask_image_base64 === 'string' ) request.mask_image_base64 = mask_image_base64;
if ( typeof prompt_strength === 'number' && Number.isFinite(prompt_strength) ) {
request.prompt_strength = Math.max(0, Math.min(1, prompt_strength));
}
if ( requiresConditionImage ) {
const conditionSource = resolvedImageBase64
? resolvedImageBase64
: (typeof image_url === 'string' ? image_url : undefined);
if ( ! conditionSource ) {
throw new Error(`Model ${request.model} requires an image_url or image_base64 input`);
}
request.condition_image = conditionSource;
}
return request;
}
#normalizeDimension (value?: number) {
if ( typeof value !== 'number' || Number.isNaN(value) ) return undefined;
const rounded = Math.max(64, Math.round(value));
// Flux models expect multiples of 8. Snap to the nearest multiple without going below 64.
return Math.max(64, Math.round(rounded / 8) * 8);
}
#modelRequiresConditionImage (modelId?: string) {
if ( typeof modelId !== 'string' || modelId.trim() === '' ) {
return false;
}
const normalized = modelId.toLowerCase();
return CONDITION_IMAGE_MODELS.some(required => normalized === required);
}
}

319
src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts Normal file
View File

@@ -0,0 +1,319 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { IImageModel } from '../types';
export const TOGETHER_IMAGE_GENERATION_MODELS: IImageModel[] = [
{
id: 'togetherai:ByteDance-Seed/Seedream-3.0',
aliases: ['ByteDance-Seed/Seedream-3.0'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'ByteDance-Seed/Seedream-3.0',
allowedQualityLevels: [''],
costs: { '1MP': 1.8 },
},
{
id: 'togetherai:ByteDance-Seed/Seedream-4.0',
aliases: ['ByteDance-Seed/Seedream-4.0'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'ByteDance-Seed/Seedream-4.0',
allowedQualityLevels: [''],
costs: { '1MP': 3 },
},
{
id: 'togetherai:HiDream-ai/HiDream-I1-Dev',
aliases: ['HiDream-ai/HiDream-I1-Dev'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'HiDream-ai/HiDream-I1-Dev',
allowedQualityLevels: [''],
costs: { '1MP': 0.45 },
},
{
id: 'togetherai:HiDream-ai/HiDream-I1-Fast',
aliases: ['HiDream-ai/HiDream-I1-Fast'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'HiDream-ai/HiDream-I1-Fast',
allowedQualityLevels: [''],
costs: { '1MP': 0.32 },
},
{
id: 'togetherai:HiDream-ai/HiDream-I1-Full',
aliases: ['HiDream-ai/HiDream-I1-Full'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'HiDream-ai/HiDream-I1-Full',
allowedQualityLevels: [''],
costs: { '1MP': 0.9 },
},
{
id: 'togetherai:Lykon/DreamShaper',
aliases: ['Lykon/DreamShaper'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'Lykon/DreamShaper',
allowedQualityLevels: [''],
costs: { '1MP': 0.06 },
},
{
id: 'togetherai:Qwen/Qwen-Image',
aliases: ['Qwen/Qwen-Image'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'Qwen/Qwen-Image',
allowedQualityLevels: [''],
costs: { '1MP': 0.58 },
},
{
id: 'togetherai:RunDiffusion/Juggernaut-pro-flux',
aliases: ['RunDiffusion/Juggernaut-pro-flux'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'RunDiffusion/Juggernaut-pro-flux',
allowedQualityLevels: [''],
costs: { '1MP': 0.49 },
},
{
id: 'togetherai:Rundiffusion/Juggernaut-Lightning-Flux',
aliases: ['Rundiffusion/Juggernaut-Lightning-Flux'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'Rundiffusion/Juggernaut-Lightning-Flux',
allowedQualityLevels: [''],
costs: { '1MP': 0.17 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-dev',
aliases: ['black-forest-labs/FLUX.1-dev'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-dev',
allowedQualityLevels: [''],
costs: { '1MP': 2.5 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-dev-lora',
aliases: ['black-forest-labs/FLUX.1-dev-lora'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-dev-lora',
allowedQualityLevels: [''],
costs: { '1MP': 2.5 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-kontext-dev',
aliases: ['black-forest-labs/FLUX.1-kontext-dev'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-kontext-dev',
allowedQualityLevels: [''],
costs: { '1MP': 2.5 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-kontext-max',
aliases: ['black-forest-labs/FLUX.1-kontext-max'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-kontext-max',
allowedQualityLevels: [''],
costs: { '1MP': 8 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-kontext-pro',
aliases: ['black-forest-labs/FLUX.1-kontext-pro'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-kontext-pro',
allowedQualityLevels: [''],
costs: { '1MP': 4 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-krea-dev',
aliases: ['black-forest-labs/FLUX.1-krea-dev'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-krea-dev',
allowedQualityLevels: [''],
costs: { '1MP': 2.5 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-pro',
aliases: ['black-forest-labs/FLUX.1-pro'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-pro',
allowedQualityLevels: [''],
costs: { '1MP': 5 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-schnell',
aliases: ['black-forest-labs/FLUX.1-schnell'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-schnell',
allowedQualityLevels: [''],
costs: { '1MP': 0.27 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1.1-pro',
aliases: ['black-forest-labs/FLUX.1.1-pro'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1.1-pro',
allowedQualityLevels: [''],
costs: { '1MP': 4 },
},
{
id: 'togetherai:black-forest-labs/FLUX.2-pro',
aliases: ['black-forest-labs/FLUX.2-pro'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.2-pro',
allowedQualityLevels: [''],
costs: { '1MP': 3 },
},
{
id: 'togetherai:black-forest-labs/FLUX.2-flex',
aliases: ['black-forest-labs/FLUX.2-flex'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.2-flex',
allowedQualityLevels: [''],
costs: { '1MP': 3 },
},
{
id: 'togetherai:black-forest-labs/FLUX.2-dev',
aliases: ['black-forest-labs/FLUX.2-dev'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.2-dev',
allowedQualityLevels: [''],
costs: { '1MP': 3 },
},
{
id: 'togetherai:google/flash-image-2.5',
aliases: ['google/flash-image-2.5'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'google/flash-image-2.5',
allowedQualityLevels: ['1K'],
allowedRatios: [
{ w: 1024, h: 1024 },
{ w: 1248, h: 832 },
{ w: 832, h: 1248 },
{ w: 1184, h: 864 },
{ w: 864, h: 1184 },
{ w: 896, h: 1152 },
{ w: 1152, h: 896 },
{ w: 768, h: 1344 },
{ w: 1344, h: 768 },
{ w: 1536, h: 672 },
{ w: 672, h: 1536 },
],
costs: { '1MP': 3.91 },
},
{
id: 'togetherai:google/gemini-3-pro-image',
aliases: ['gemini-3-pro-image', 'google/gemini-3-pro-image'],
name: 'gemini-3-pro-image (Together AI)',
costs_currency: 'usd-cents',
index_cost_key: '1MP',
allowedQualityLevels: ['1K', '2K', '4K'],
allowedRatios: [
{ w: 1, h: 1 },
{ w: 2, h: 3 },
{ w: 3, h: 2 },
{ w: 3, h: 4 },
{ w: 4, h: 3 },
{ w: 4, h: 5 },
{ w: 5, h: 4 },
{ w: 9, h: 16 },
{ w: 16, h: 9 },
{ w: 21, h: 9 },
],
costs: { '1MP': 13.51 },
},
{
id: 'togetherai:google/imagen-4.0-fast',
aliases: ['google/imagen-4.0-fast'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'google/imagen-4.0-fast',
allowedQualityLevels: [''],
costs: { '1MP': 2 },
},
{
id: 'togetherai:google/imagen-4.0-preview',
aliases: ['google/imagen-4.0-preview'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'google/imagen-4.0-preview',
allowedQualityLevels: [''],
costs: { '1MP': 4 },
},
{
id: 'togetherai:google/imagen-4.0-ultra',
aliases: ['google/imagen-4.0-ultra'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'google/imagen-4.0-ultra',
allowedQualityLevels: [''],
costs: { '1MP': 6.02 },
},
{
id: 'togetherai:ideogram/ideogram-3.0',
aliases: ['ideogram/ideogram-3.0'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'ideogram/ideogram-3.0',
allowedQualityLevels: [''],
costs: { '1MP': 6.02 },
},
{
id: 'togetherai:stabilityai/stable-diffusion-3-medium',
aliases: ['stabilityai/stable-diffusion-3-medium'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'stabilityai/stable-diffusion-3-medium',
allowedQualityLevels: [''],
costs: { '1MP': 0.19 },
},
{
id: 'togetherai:stabilityai/stable-diffusion-xl-base-1.0',
aliases: ['stabilityai/stable-diffusion-xl-base-1.0'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'stabilityai/stable-diffusion-xl-base-1.0',
allowedQualityLevels: [''],
costs: { '1MP': 0.19 },
},
{
id: 'togetherai:black-forest-labs/FLUX.1-schnell-Free',
aliases: ['black-forest-labs/FLUX.1-schnell-Free', 'FLUX.1-schnell-Free'],
costs_currency: 'usd-cents',
index_cost_key: '1MP',
name: 'black-forest-labs/FLUX.1-schnell-Free',
allowedQualityLevels: [''],
costs: { '1MP': 0 },
},
];

29
src/backend/src/services/ai/image/providers/types.ts Normal file
View File

@@ -0,0 +1,29 @@
export interface IImageModel {
id: string;
name: string;
description?: string;
version?: string;
costs_currency: string;
index_cost_key?: string;
costs: Record<string, number>;
allowedQualityLevels?: string[];
allowedRatios?: { w: number, h: number }[];
provider?: string;
aliases?: string[];
}
export interface IGenerateParams {
prompt: string,
ratio: { w: number, h: number }
model: string,
provider?: string,
test_mode?: boolean
quality?: string,
};
export interface IImageProvider {
generate (params: IGenerateParams): Promise<string>;
models (): Promise<IImageModel[]> | IImageModel[];
getDefaultModel (): string;
}

6
src/backend/src/services/ai/ocr/MistralOCRService.js
View File

@@ -79,12 +79,6 @@ export class MistralOCRService extends BaseService {
apiKey: this.config.apiKey,
});
const svc_aiChat = this.services.get('ai-chat');
svc_aiChat.register_provider({
service_name: this.service_name,
alias: true,
});
this.meteringService = this.services.get('meteringService').meteringService;
}