From f823eddfa5f3f375da451d3c3d84694bc26ddfab Mon Sep 17 00:00:00 2001 From: Nariman Jelveh Date: Mon, 3 Nov 2025 19:13:39 -0800 Subject: [PATCH] make together.ai image and video models work! (#1881) * make together ai image models work! * add video models too! * add costmaps for together ai image and video --- package-lock.json | 223 +++++++++++++-- src/backend/package.json | 2 +- .../src/modules/puterai/AIInterfaceService.js | 12 + .../src/modules/puterai/PuterAIModule.js | 6 + .../puterai/TogetherImageGenerationService.js | 259 ++++++++++++++++++ .../puterai/TogetherVideoGenerationService.js | 247 +++++++++++++++++ .../costMaps/togetherCostMap.ts | 62 ++++- src/puter-js/index.d.ts | 26 +- src/puter-js/src/modules/AI.js | 100 ++++++- 9 files changed, 895 insertions(+), 42 deletions(-) create mode 100644 src/backend/src/modules/puterai/TogetherImageGenerationService.js create mode 100644 src/backend/src/modules/puterai/TogetherVideoGenerationService.js diff --git a/package-lock.json b/package-lock.json index 6b07030a..a6b10813 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8566,6 +8566,15 @@ "node": ">=8" } }, + "node_modules/brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.1.2" + } + }, "node_modules/brotli-dec-wasm": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/brotli-dec-wasm/-/brotli-dec-wasm-2.3.0.tgz", @@ -8625,6 +8634,15 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.6.19" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -9648,6 +9666,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", @@ -12470,6 +12494,12 @@ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "license": "ISC" }, + "node_modules/int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha512-a5jlKftS7HUOhkUyYD7j2sJ/ZnvWiNlZS1ldR+g1ifQ+/UuZXIE+YTc/lK1qGj/GwAU5F8Z0e1eVq2t1J5Ob2g==", + "license": "BSD-3-Clause" + }, "node_modules/interpret": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", @@ -12764,6 +12794,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, "node_modules/isbot": { "version": "3.8.0", "resolved": "https://registry.npmjs.org/isbot/-/isbot-3.8.0.tgz", @@ -13677,6 +13713,24 @@ "lru-cache": "6.0.0" } }, + "node_modules/lzo": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz", + "integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "bindings": "~1.2.1" + } + }, + "node_modules/lzo/node_modules/bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==", + "license": "MIT", + "optional": true + }, "node_modules/magic-string": { "version": "0.30.19", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.19.tgz", @@ -14448,6 +14502,12 @@ "node": ">= 6.13.0" } }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "license": "MIT" + }, "node_modules/node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -14804,6 +14864,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/object-stream": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz", + "integrity": "sha512-+NPJnRvX9RDMRY9mOWOo/NDppBjbZhXirNNSu2IBnuNboClC9h1ZGHXgHBLDbJMHsxeJDq922aVmG5xs24a/cA==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/omggif": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", @@ -15037,6 +15105,27 @@ "node": ">=6" } }, + "node_modules/parquetjs": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz", + "integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==", + "license": "MIT", + "dependencies": { + "brotli": "^1.3.0", + "bson": "^1.0.4", + "int53": "^0.2.4", + "object-stream": "0.0.1", + "snappyjs": "^0.6.0", + "thrift": "^0.11.0", + "varint": "^5.0.0" + }, + "engines": { + "node": ">=7.6" + }, + "optionalDependencies": { + "lzo": "^0.4.0" + } + }, "node_modules/parse-bmfont-ascii": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", @@ -15514,6 +15603,12 @@ "node": ">= 0.6.0" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/process-on-spawn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.1.0.tgz", @@ -15537,6 +15632,16 @@ "node": ">=0.4.0" } }, + "node_modules/progress-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/progress-stream/-/progress-stream-2.0.0.tgz", + "integrity": "sha512-xJwOWR46jcXUq6EH9yYyqp+I52skPySOeHfkxOZ2IY1AiBi/sFJhbhAKHoV3OTw/omQ45KTio9215dRJ2Yxd3Q==", + "license": "BSD-2-Clause", + "dependencies": { + "speedometer": "~1.0.0", + "through2": "~2.0.3" + } + }, "node_modules/prompt-sync": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/prompt-sync/-/prompt-sync-4.2.0.tgz", @@ -15660,6 +15765,17 @@ "node": ">=6" } }, + "node_modules/q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", + "deprecated": "You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.\n\n(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)", + "license": "MIT", + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } + }, "node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", @@ -16774,6 +16890,12 @@ "node": ">=8" } }, + "node_modules/snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==", + "license": "MIT" + }, "node_modules/socket.io": { "version": "4.8.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.1.tgz", @@ -16971,6 +17093,12 @@ "node": ">=8" } }, + "node_modules/speedometer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/speedometer/-/speedometer-1.0.0.tgz", + "integrity": "sha512-lgxErLl/7A5+vgIIXsh9MbeukOaCb2axgQ+bKCdIE+ibNT4XNYGNCR1qFEGq6F+YDASXK3Fh/c5FgtZchFolxw==", + "license": "MIT" + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -17591,6 +17719,60 @@ "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", "license": "MIT" }, + "node_modules/thrift": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz", + "integrity": "sha512-UpsBhOC45a45TpeHOXE4wwYwL8uD2apbHTbtBvkwtUU4dNwCjC7DpQTjw2Q6eIdfNtw+dKthdwq94uLXTJPfFw==", + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0", + "q": "^1.5.0", + "ws": ">= 2.2.3" + }, + "engines": { + "node": ">= 4.1.0" + } + }, + "node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "license": "MIT", + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/through2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/through2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/through2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/tiktoken": { "version": "1.0.22", "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz", @@ -17719,35 +17901,18 @@ } }, "node_modules/together-ai": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/together-ai/-/together-ai-0.6.0.tgz", - "integrity": "sha512-l5rT9lzpHXA0e6zEdBwlVKY9wb5XQaX5hpandKPvHI5n6Bap4UTynF8Q2RSsRSAz3auyeEGzFKLE5XI301hOtA==", + "version": "0.29.0", + "resolved": "https://registry.npmjs.org/together-ai/-/together-ai-0.29.0.tgz", + "integrity": "sha512-V4c6AVddCpFOlXCeP7J/lVWy5uyuk3bR1h8LAT2q9rJYx782h6dv8omATYMyCrDwWKjtROqmktsnWm7Imu2mvg==", "license": "Apache-2.0", "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" + "parquetjs": "^0.11.2", + "progress-stream": "^2.0.0" + }, + "bin": { + "together-ai": "bin/cli" } }, - "node_modules/together-ai/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/together-ai/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -18157,6 +18322,12 @@ "node": ">= 0.10" } }, + "node_modules/varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==", + "license": "MIT" + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -19352,7 +19523,7 @@ "svg-captcha": "^1.4.0", "svgo": "^3.0.2", "tiktoken": "^1.0.16", - "together-ai": "^0.6.0-alpha.4", + "together-ai": "^0.29.0", "tweetnacl": "^1.0.3", "ua-parser-js": "^1.0.38", "uglify-js": "^3.17.4", diff --git a/src/backend/package.json b/src/backend/package.json index 42a2eaad..4c2c9ace 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -81,7 +81,7 @@ "svg-captcha": "^1.4.0", "svgo": "^3.0.2", "tiktoken": "^1.0.16", - "together-ai": "^0.6.0-alpha.4", + "together-ai": "^0.29.0", "tweetnacl": "^1.0.3", "ua-parser-js": "^1.0.38", "uglify-js": "^3.17.4", diff --git a/src/backend/src/modules/puterai/AIInterfaceService.js b/src/backend/src/modules/puterai/AIInterfaceService.js index 41abc73d..7031d814 100644 --- a/src/backend/src/modules/puterai/AIInterfaceService.js +++ b/src/backend/src/modules/puterai/AIInterfaceService.js @@ -137,6 +137,18 @@ class AIInterfaceService extends BaseService { duration: { type: 'number', optional: true }, size: { type: 'string', optional: true }, resolution: { type: 'string', optional: true }, + width: { type: 'number', optional: true }, + height: { type: 'number', optional: true }, + fps: { type: 'number', optional: true }, + steps: { type: 'number', optional: true }, + guidance_scale: { type: 'number', optional: true }, + seed: { type: 'number', optional: true }, + output_format: { type: 'string', optional: true }, + output_quality: { type: 'number', optional: true }, + negative_prompt: { type: 'string', optional: true }, + reference_images: { type: 'json', optional: true }, + frame_images: { type: 'json', optional: true }, + metadata: { type: 'json', optional: true }, input_reference: { type: 'file', optional: true }, }, result_choices: [ diff --git a/src/backend/src/modules/puterai/PuterAIModule.js b/src/backend/src/modules/puterai/PuterAIModule.js index 1b1f18c2..b92bc3cf 100644 --- a/src/backend/src/modules/puterai/PuterAIModule.js +++ b/src/backend/src/modules/puterai/PuterAIModule.js @@ -80,6 +80,12 @@ class PuterAIModule extends AdvancedBase { if ( config?.services?.['together-ai'] ) { const { TogetherAIService } = require('./TogetherAIService'); services.registerService('together-ai', TogetherAIService); + + const { TogetherImageGenerationService } = require('./TogetherImageGenerationService'); + services.registerService('together-image-generation', TogetherImageGenerationService); + + const { TogetherVideoGenerationService } = require('./TogetherVideoGenerationService'); + services.registerService('together-video-generation', TogetherVideoGenerationService); } if ( config?.services?.['mistral'] ) { diff --git a/src/backend/src/modules/puterai/TogetherImageGenerationService.js b/src/backend/src/modules/puterai/TogetherImageGenerationService.js new file mode 100644 index 00000000..9f0a5470 --- /dev/null +++ b/src/backend/src/modules/puterai/TogetherImageGenerationService.js @@ -0,0 +1,259 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// METADATA // {"ai-commented":{"service":"claude"}} +const APIError = require('../../api/APIError'); +const BaseService = require('../../services/BaseService'); +const { TypedValue } = require('../../services/drivers/meta/Runtime'); +const { Context } = require('../../util/context'); +const { Together } = require('together-ai'); + +/** +* Service class for generating images using Together AI models. +* Extends BaseService to provide image generation capabilities through the +* puter-image-generation interface. Handles authentication, request validation, +* and metering integration. +*/ +class TogetherImageGenerationService extends BaseService { + /** @type {import('../../services/MeteringService/MeteringService').MeteringService} */ + get meteringService() { + return this.services.get('meteringService').meteringService; + } + + static MODULES = {}; + + async _init() { + const apiKey = + this.config?.apiKey ?? + this.global_config?.services?.['together-ai']?.apiKey; + + if ( !apiKey ) { + throw new Error('Together AI image generation requires an API key'); + } + + this.client = new Together({ apiKey }); + } + + static IMPLEMENTS = { + ['driver-capabilities']: { + supports_test_mode(iface, method_name) { + return iface === 'puter-image-generation' && + method_name === 'generate'; + }, + }, + ['puter-image-generation']: { + /** + * Generates an image using Together AI image models + * @param {object} params - Generation parameters + * @param {string} params.prompt - Prompt describing the desired image + * @param {string} [params.model] - Together AI model identifier + * @param {object} [params.ratio] - Width/height ratio object (e.g., { w: 1024, h: 1024 }) + * @param {number} [params.width] - Explicit width override + * @param {number} [params.height] - Explicit height override + * @param {string} [params.aspect_ratio] - Aspect ratio string (e.g., "16:9") + * @param {number} [params.steps] - Diffusion step count + * @param {number} [params.seed] - Seed for reproducibility + * @param {string} [params.negative_prompt] - Negative prompt text + * @param {number} [params.n] - Number of images to generate (default 1) + * @param {string} [params.image_url] - Reference image URL for image-to-image + * @param {string} [params.image_base64] - Base64 encoded reference image + * @param {boolean} [params.disable_safety_checker] - Disable Together AI safety checker + * @param {boolean} [params.test_mode] - Enable Puter test mode shortcut + * @returns {Promise} TypedValue containing the generated image URL or data URI + */ + async generate(params) { + const { + prompt, + test_mode, + ratio, + model, + width, + height, + aspect_ratio, + steps, + seed, + negative_prompt, + n, + image_url, + image_base64, + mask_image_url, + mask_image_base64, + prompt_strength, + disable_safety_checker, + response_format, + } = params; + + if ( test_mode ) { + return new TypedValue({ + $: 'string:url:web', + content_type: 'image', + }, 'https://puter-sample-data.puter.site/image_example.png'); + } + + const url = await this.generate(prompt, { + ratio, + model, + width, + height, + aspect_ratio, + steps, + seed, + negative_prompt, + n, + image_url, + image_base64, + mask_image_url, + mask_image_base64, + prompt_strength, + disable_safety_checker, + response_format, + }); + + const isDataUrl = url.startsWith('data:'); + return new TypedValue({ + $: isDataUrl ? 'string:url:data' : 'string:url:web', + content_type: 'image', + }, url); + }, + }, + }; + + static DEFAULT_MODEL = 'black-forest-labs/FLUX.1-schnell'; + static DEFAULT_RATIO = { w: 1024, h: 1024 }; + + /** + * Generates an image using Together AI client + * @private + */ + async generate(prompt, options) { + if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { + throw new Error('`prompt` must be a non-empty string'); + } + + const request = this._buildRequest(prompt, options); + + const actor = Context.get('actor'); + if ( !actor ) { + throw new Error('actor not found in context'); + } + + const usageType = `together-image:${request.model}`; + const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1); + if ( !usageAllowed ) { + throw APIError.create('insufficient_funds'); + } + + const response = await this.client.images.create(request); + if ( !response?.data?.length ) { + throw new Error('Together AI response did not include image data'); + } + + this.meteringService.incrementUsage(actor, usageType, 1); + + const first = response.data[0]; + if ( first.url ) { + return first.url; + } + if ( first.b64_json ) { + return 'data:image/png;base64,' + first.b64_json; + } + + throw new Error('Together AI response did not include an image URL'); + } + + /** + * Normalizes Together AI image generation request parameters + * @private + */ + _buildRequest(prompt, options = {}) { + const { + ratio, + model, + width, + height, + aspect_ratio, + steps, + seed, + negative_prompt, + n, + image_url, + image_base64, + mask_image_url, + mask_image_base64, + prompt_strength, + disable_safety_checker, + response_format, + } = options; + + const request = { + prompt, + model: model ?? this.constructor.DEFAULT_MODEL, + }; + + const ratioWidth = (ratio && ratio.w !== undefined) ? Number(ratio.w) : undefined; + const ratioHeight = (ratio && ratio.h !== undefined) ? Number(ratio.h) : undefined; + + const normalizedWidth = this._normalizeDimension( + width !== undefined ? Number(width) : (ratioWidth ?? this.constructor.DEFAULT_RATIO.w) + ); + const normalizedHeight = this._normalizeDimension( + height !== undefined ? Number(height) : (ratioHeight ?? this.constructor.DEFAULT_RATIO.h) + ); + + if ( aspect_ratio ) { + request.aspect_ratio = aspect_ratio; + } else { + if ( normalizedWidth ) request.width = normalizedWidth; + if ( normalizedHeight ) request.height = normalizedHeight; + } + + if ( typeof steps === 'number' && Number.isFinite(steps) ) { + request.steps = Math.max(1, Math.min(50, Math.round(steps))); + } + if ( typeof seed === 'number' && Number.isFinite(seed) ) request.seed = Math.round(seed); + if ( typeof negative_prompt === 'string' ) request.negative_prompt = negative_prompt; + if ( typeof n === 'number' && Number.isFinite(n) ) { + request.n = Math.max(1, Math.min(4, Math.round(n))); + } + if ( typeof disable_safety_checker === 'boolean' ) { + request.disable_safety_checker = disable_safety_checker; + } + if ( typeof response_format === 'string' ) request.response_format = response_format; + if ( typeof image_url === 'string' ) request.image_url = image_url; + if ( typeof image_base64 === 'string' ) request.image_base64 = image_base64; + if ( typeof mask_image_url === 'string' ) request.mask_image_url = mask_image_url; + if ( typeof mask_image_base64 === 'string' ) request.mask_image_base64 = mask_image_base64; + if ( typeof prompt_strength === 'number' && Number.isFinite(prompt_strength) ) { + request.prompt_strength = Math.max(0, Math.min(1, prompt_strength)); + } + + return request; + } + + _normalizeDimension(value) { + if ( typeof value !== 'number' ) return undefined; + const rounded = Math.max(64, Math.round(value)); + // Flux models expect multiples of 8. Snap to the nearest multiple without going below 64. + return Math.max(64, Math.round(rounded / 8) * 8); + } +} + +module.exports = { + TogetherImageGenerationService, +}; diff --git a/src/backend/src/modules/puterai/TogetherVideoGenerationService.js b/src/backend/src/modules/puterai/TogetherVideoGenerationService.js new file mode 100644 index 00000000..84c2886e --- /dev/null +++ b/src/backend/src/modules/puterai/TogetherVideoGenerationService.js @@ -0,0 +1,247 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// METADATA // {"ai-commented":{"service":"claude"}} +const APIError = require('../../api/APIError'); +const BaseService = require('../../services/BaseService'); +const { TypedValue } = require('../../services/drivers/meta/Runtime'); +const { Context } = require('../../util/context'); +const { Together } = require('together-ai'); + +const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; +const POLL_INTERVAL_MS = 5_000; +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes +const DEFAULT_MODEL = 'minimax/video-01-director'; +const DEFAULT_DURATION_SECONDS = 6; +const DEFAULT_USAGE_KEY = 'together-video:default'; + +class TogetherVideoGenerationService extends BaseService { + /** @type {import('../../services/MeteringService/MeteringService').MeteringService} */ + get meteringService() { + return this.services.get('meteringService').meteringService; + } + + static MODULES = {}; + + async _init() { + const apiKey = + this.config?.apiKey ?? + this.global_config?.services?.['together-ai']?.apiKey; + + if ( !apiKey ) { + throw new Error('Together AI video generation requires an API key'); + } + + this.client = new Together({ apiKey }); + } + + static IMPLEMENTS = { + ['driver-capabilities']: { + supports_test_mode(iface, method_name) { + return iface === 'puter-video-generation' && + method_name === 'generate'; + }, + }, + ['puter-video-generation']: { + async generate(params) { + return await this.generateVideo(params); + }, + }, + }; + + async generateVideo(params) { + const { + prompt, + model: requestedModel, + seconds, + duration, + width, + height, + fps, + steps, + guidance_scale: guidanceScale, + seed, + output_format: outputFormat, + output_quality: outputQuality, + negative_prompt: negativePrompt, + reference_images: referenceImages, + frame_images: frameImages, + metadata, + test_mode: testMode, + } = params ?? {}; + + if ( typeof prompt !== 'string' || !prompt.trim() ) { + throw APIError.create('field_invalid', null, { + key: 'prompt', + expected: 'a non-empty string', + got: prompt, + }); + } + + const model = requestedModel ?? DEFAULT_MODEL; + + if ( testMode ) { + return new TypedValue({ + $: 'string:url:web', + content_type: 'video', + }, DEFAULT_TEST_VIDEO_URL); + } + + const normalizedSeconds = this.#coercePositiveInteger(seconds ?? duration) ?? DEFAULT_DURATION_SECONDS; + + const actor = Context.get('actor'); + if ( !actor ) { + throw new Error('actor not found in context'); + } + + const estimatedUsageUnits = 1; // Together video billing is per generated video + const usageKey = this.#determineUsageKey(model); + + const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageKey, estimatedUsageUnits); + if ( !usageAllowed ) { + throw APIError.create('insufficient_funds'); + } + + const createPayload = { + prompt, + model, + }; + + if ( normalizedSeconds ) { + createPayload.seconds = normalizedSeconds; + } + if ( this.#isFiniteNumber(width) ) { + createPayload.width = Number(width); + } + if ( this.#isFiniteNumber(height) ) { + createPayload.height = Number(height); + } + if ( this.#isFiniteNumber(fps) ) { + createPayload.fps = Number(fps); + } + if ( this.#isFiniteNumber(steps) ) { + createPayload.steps = Number(steps); + } + if ( this.#isFiniteNumber(guidanceScale) ) { + createPayload.guidance_scale = Number(guidanceScale); + } + if ( this.#isFiniteNumber(seed) ) { + createPayload.seed = Number(seed); + } + if ( typeof outputFormat === 'string' && outputFormat.trim() ) { + createPayload.output_format = outputFormat.trim(); + } + if ( this.#isFiniteNumber(outputQuality) ) { + createPayload.output_quality = Number(outputQuality); + } + if ( typeof negativePrompt === 'string' && negativePrompt.trim() ) { + createPayload.negative_prompt = negativePrompt; + } + if ( Array.isArray(referenceImages) && referenceImages.length > 0 ) { + createPayload.reference_images = referenceImages.filter(item => typeof item === 'string' && item.trim().length > 0); + } + if ( Array.isArray(frameImages) && frameImages.length > 0 ) { + createPayload.frame_images = frameImages.filter(frame => frame && typeof frame === 'object'); + } + if ( metadata && typeof metadata === 'object' ) { + createPayload.metadata = metadata; + } + + const job = await this.client.videos.create(createPayload); + const finalJob = await this.#pollUntilComplete(job.id); + + if ( finalJob.status === 'failed' ) { + const errorMessage = finalJob?.info?.errors?.[0]?.message ?? + finalJob?.info?.errors?.message ?? + finalJob?.info?.errors ?? + 'Video generation failed'; + throw new Error(errorMessage); + } + + if ( finalJob.status === 'cancelled' ) { + throw new Error('Video generation was cancelled'); + } + + this.meteringService.incrementUsage(actor, usageKey, 1); + + const videoUrl = finalJob?.outputs?.video_url; + if ( typeof videoUrl === 'string' && videoUrl.trim() ) { + return new TypedValue({ + $: 'string:url:web', + content_type: 'video', + }, videoUrl); + } + + throw new Error('Together AI response did not include a video URL'); + } + + async #pollUntilComplete(jobId) { + let job = await this.client.videos.retrieve(jobId); + const start = Date.now(); + + while ( job.status === 'queued' || job.status === 'in_progress' ) { + if ( Date.now() - start > DEFAULT_TIMEOUT_MS ) { + throw new Error('Timed out waiting for Together AI video generation to complete'); + } + + await this.#delay(POLL_INTERVAL_MS); + job = await this.client.videos.retrieve(jobId); + } + + return job; + } + + async #delay(ms) { + return await new Promise(resolve => setTimeout(resolve, ms)); + } + + #determineUsageKey(model) { + if ( typeof model === 'string' && model.trim() ) { + return `together-video:${model}`; + } + return DEFAULT_USAGE_KEY; + } + + #coercePositiveInteger(value) { + if ( typeof value === 'number' && Number.isFinite(value) ) { + const rounded = Math.round(value); + return rounded > 0 ? rounded : undefined; + } + if ( typeof value === 'string' ) { + const numeric = Number.parseInt(value, 10); + return Number.isFinite(numeric) && numeric > 0 ? numeric : undefined; + } + return undefined; + } + + #isFiniteNumber(value) { + if ( typeof value === 'number' ) { + return Number.isFinite(value); + } + if ( typeof value === 'string' ) { + const numeric = Number(value); + return Number.isFinite(numeric); + } + return false; + } +} + +module.exports = { + TogetherVideoGenerationService, +}; diff --git a/src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts b/src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts index a6a9bf6b..d5ddfb10 100644 --- a/src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts +++ b/src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts @@ -1,12 +1,62 @@ // TogetherAI Cost Map -// Note: TogetherAI uses dynamic pricing fetched from their API. This map only includes static/hardcoded test models. -// For production, model costs should be fetched dynamically via the Together API. -// -// Most TogetherAI models are not listed here due to dynamic pricing. -// Only hardcoded test models are included. export const TOGETHER_COST_MAP = { // Test model (hardcoded) 'together:model-fallback-test-1:input': 10, 'together:model-fallback-test-1:output': 10, -}; \ No newline at end of file + + // Image generation placeholder (actual pricing is fetched dynamically via Together API) + 'together-image:default': 0, + 'together-image:ByteDance-Seed/Seedream-3.0': 0.018 * 100_000_000, + 'together-image:ByteDance-Seed/Seedream-4.0': 0.03 * 100_000_000, + 'together-image:HiDream-ai/HiDream-I1-Dev': 0.0045 * 100_000_000, + 'together-image:HiDream-ai/HiDream-I1-Fast': 0.0032 * 100_000_000, + 'together-image:HiDream-ai/HiDream-I1-Full': 0.009 * 100_000_000, + 'together-image:Lykon/DreamShaper': 0.0006 * 100_000_000, + 'together-image:Qwen/Qwen-Image': 0.0058 * 100_000_000, + 'together-image:RunDiffusion/Juggernaut-pro-flux': 0.0049 * 100_000_000, + 'together-image:Rundiffusion/Juggernaut-Lightning-Flux': 0.0017 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-Canny-pro': 0.05 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-dev': 0.025 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-dev-lora': 0.025 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-kontext-dev': 0.025 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-kontext-max': 0.08 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-kontext-pro': 0.04 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-krea-dev': 0.025 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-pro': 0.05 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-schnell': 0.0027 * 100_000_000, + 'together-image:black-forest-labs/FLUX.1-schnell-Free': 0, + 'together-image:black-forest-labs/FLUX.1.1-pro': 0.05 * 100_000_000, + 'together-image:google/flash-image-2.5': 0.039 * 100_000_000, + 'together-image:google/imagen-4.0-fast': 0.02 * 100_000_000, + 'together-image:google/imagen-4.0-preview': 0.04 * 100_000_000, + 'together-image:google/imagen-4.0-ultra': 0.06 * 100_000_000, + 'together-image:ideogram/ideogram-3.0': 0.06 * 100_000_000, + 'together-image:stabilityai/stable-diffusion-3-medium': 0.0019 * 100_000_000, + 'together-image:stabilityai/stable-diffusion-xl-base-1.0': 0.0045 * 100_000_000, + + // Video generation placeholder (per-video pricing). Update with real pricing when available. + 'together-video:default': 0, + 'together-video:ByteDance/Seedance-1.0-lite': 0.14 * 100_000_000, + 'together-video:ByteDance/Seedance-1.0-pro': 0.57 * 100_000_000, + 'together-video:Wan-AI/Wan2.2-I2V-A14B': 0.31 * 100_000_000, + 'together-video:Wan-AI/Wan2.2-T2V-A14B': 0.66 * 100_000_000, + 'together-video:google/veo-2.0': 2.50 * 100_000_000, + 'together-video:google/veo-3.0': 1.60 * 100_000_000, + 'together-video:google/veo-3.0-audio': 3.20 * 100_000_000, + 'together-video:google/veo-3.0-fast': 0.80 * 100_000_000, + 'together-video:google/veo-3.0-fast-audio': 1.20 * 100_000_000, + 'together-video:kwaivgI/kling-1.6-pro': 0.32 * 100_000_000, + 'together-video:kwaivgI/kling-1.6-standard': 0.19 * 100_000_000, + 'together-video:kwaivgI/kling-2.0-master': 0.92 * 100_000_000, + 'together-video:kwaivgI/kling-2.1-master': 0.92 * 100_000_000, + 'together-video:kwaivgI/kling-2.1-pro': 0.32 * 100_000_000, + 'together-video:kwaivgI/kling-2.1-standard': 0.18 * 100_000_000, + 'together-video:minimax/hailuo-02': 0.56 * 100_000_000, + 'together-video:minimax/video-01-director': 0.28 * 100_000_000, + 'together-video:openai/sora-2': 0.80 * 100_000_000, + 'together-video:openai/sora-2-pro': 4.00 * 100_000_000, + 'together-video:pixverse/pixverse-v5': 0.30 * 100_000_000, + 'together-video:vidu/vidu-2.0': 0.28 * 100_000_000, + 'together-video:vidu/vidu-q1': 0.22 * 100_000_000, +}; diff --git a/src/puter-js/index.d.ts b/src/puter-js/index.d.ts index e9ddd540..c3486a55 100644 --- a/src/puter-js/index.d.ts +++ b/src/puter-js/index.d.ts @@ -125,11 +125,27 @@ interface Txt2ImgOptions { interface Txt2VidOptions { prompt?: string; - model?: 'sora-2' | 'sora-2-pro'; - duration?: 4 | 8 | 12; - seconds?: 4 | 8 | 12; - size?: '720x1280' | '1280x720' | '1024x1792' | '1792x1024'; - resolution?: '720x1280' | '1280x720' | '1024x1792' | '1792x1024'; + model?: string; + duration?: number; + seconds?: number; + size?: string; + resolution?: string; + width?: number; + height?: number; + fps?: number; + steps?: number; + guidance_scale?: number; + seed?: number; + output_format?: string; + output_quality?: number; + negative_prompt?: string; + reference_images?: string[]; + frame_images?: Array>; + metadata?: Record; + provider?: string; + service?: string; + driver?: string; + test_mode?: boolean; } interface Txt2SpeechOptions { diff --git a/src/puter-js/src/modules/AI.js b/src/puter-js/src/modules/AI.js index c708ea89..55a7aa34 100644 --- a/src/puter-js/src/modules/AI.js +++ b/src/puter-js/src/modules/AI.js @@ -10,6 +10,36 @@ const normalizeTTSProvider = (value) => { return value; }; +const TOGETHER_IMAGE_MODEL_PREFIXES = [ + 'black-forest-labs/', + 'stabilityai/', + 'togethercomputer/', + 'playgroundai/', + 'runwayml/', + 'lightricks/', + 'sg161222/', + 'wavymulder/', + 'prompthero/', +]; + +const TOGETHER_IMAGE_MODEL_KEYWORDS = [ + 'flux', + 'kling', + 'sd3', + 'stable-diffusion', + 'kolors', +]; + +const TOGETHER_VIDEO_MODEL_PREFIXES = [ + 'minimax/', + 'google/', + 'bytedance/', + 'pixverse/', + 'kwaivgi/', + 'vidu/', + 'wan-ai/', +]; + class AI{ /** * Creates a new instance with the given authentication token, API origin, and app ID, @@ -808,15 +838,51 @@ class AI{ if (options.model === "nano-banana") options.model = "gemini-2.5-flash-image-preview"; - if (options.model === "gemini-2.5-flash-image-preview") + const driverHint = typeof options.driver === 'string' ? options.driver : undefined; + const providerRaw = typeof options.provider === 'string' + ? options.provider + : (typeof options.service === 'string' ? options.service : undefined); + const providerHint = typeof providerRaw === 'string' ? providerRaw.toLowerCase() : undefined; + const modelLower = typeof options.model === 'string' ? options.model.toLowerCase() : ''; + + const looksLikeTogetherModel = + typeof options.model === 'string' && + (TOGETHER_IMAGE_MODEL_PREFIXES.some(prefix => modelLower.startsWith(prefix)) || + TOGETHER_IMAGE_MODEL_KEYWORDS.some(keyword => modelLower.includes(keyword))); + + if (driverHint) { + AIService = driverHint; + } else if (providerHint === 'gemini') { AIService = "gemini-image-generation"; + } else if (providerHint === 'together' || providerHint === 'together-ai') { + AIService = "together-image-generation"; + } else if (options.model === "gemini-2.5-flash-image-preview") { + AIService = "gemini-image-generation"; + } else if (looksLikeTogetherModel) { + AIService = "together-image-generation"; + } // Call the original chat.complete method return await utils.make_driver_method(['prompt'], 'puter-image-generation', AIService, 'generate', { responseType: 'blob', test_mode: testMode ?? false, - transform: async blob => { + transform: async result => { + let url; + if ( typeof result === 'string' ) { + url = result; + } else if ( result instanceof Blob ) { + url = await utils.blob_to_url(result); + } else if ( result instanceof ArrayBuffer ) { + const blob = new Blob([result]); + url = await utils.blob_to_url(blob); + } else if ( result && typeof result === 'object' && typeof result.arrayBuffer === 'function' ) { + const arrayBuffer = await result.arrayBuffer(); + const blob = new Blob([arrayBuffer], { type: result.type || undefined }); + url = await utils.blob_to_url(blob); + } else { + throw { message: 'Unexpected image response format', code: 'invalid_image_response' }; + } let img = new Image(); - img.src = await utils.blob_to_url(blob); + img.src = url; img.toString = () => img.src; img.valueOf = () => img.src; return img; @@ -861,7 +927,33 @@ class AI{ options.seconds = options.duration; } - return await utils.make_driver_method(['prompt'], 'puter-video-generation', 'openai-video-generation', 'generate', { + let videoService = 'openai-video-generation'; + const driverHint = typeof options.driver === 'string' ? options.driver : undefined; + const driverHintLower = driverHint ? driverHint.toLowerCase() : undefined; + const providerRaw = typeof options.provider === 'string' + ? options.provider + : (typeof options.service === 'string' ? options.service : undefined); + const providerHint = typeof providerRaw === 'string' ? providerRaw.toLowerCase() : undefined; + const modelLower = typeof options.model === 'string' ? options.model.toLowerCase() : ''; + + const looksLikeTogetherVideoModel = typeof options.model === 'string' && + TOGETHER_VIDEO_MODEL_PREFIXES.some(prefix => modelLower.startsWith(prefix)); + + if (driverHintLower === 'together' || driverHintLower === 'together-ai') { + videoService = 'together-video-generation'; + } else if (driverHintLower === 'together-video-generation') { + videoService = 'together-video-generation'; + } else if (driverHintLower === 'openai') { + videoService = 'openai-video-generation'; + } else if (driverHint) { + videoService = driverHint; + } else if (providerHint === 'together' || providerHint === 'together-ai') { + videoService = 'together-video-generation'; + } else if (looksLikeTogetherVideoModel) { + videoService = 'together-video-generation'; + } + + return await utils.make_driver_method(['prompt'], 'puter-video-generation', videoService, 'generate', { responseType: 'blob', test_mode: testMode ?? false, transform: async result => {