diff --git a/package-lock.json b/package-lock.json index f15dee87..97b1540c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15389,6 +15389,31 @@ "node": ">=8.0" } }, + "node_modules/together-ai": { + "version": "0.6.0-alpha.4", + "resolved": "https://registry.npmjs.org/together-ai/-/together-ai-0.6.0-alpha.4.tgz", + "integrity": "sha512-aeMAMkDJn6Pzdr93cDc9dQksshGMBr+6v1096NI82iAnozuT/11ngrW4h9iiQWChh/CoiD0vWDASendG/v84CQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + } + }, + "node_modules/together-ai/node_modules/@types/node": { + "version": "18.19.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.45.tgz", + "integrity": "sha512-VZxPKNNhjKmaC1SUYowuXSRSMGyQGmQjvvA1xE4QZ0xce2kLtEhPDS+kqpCPBZYgqblCLQ2DAjSzmgCM5auvhA==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -16650,6 +16675,7 @@ "string-length": "^6.0.0", "svgo": "^3.0.2", "tiktoken": "^1.0.15", + "together-ai": "^0.6.0-alpha.4", "tweetnacl": "^1.0.3", "ua-parser-js": "^1.0.38", "uglify-js": "^3.17.4", diff --git a/src/backend/package.json b/src/backend/package.json index c8a2d59d..5d09800e 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -69,6 +69,7 @@ "string-length": "^6.0.0", "svgo": "^3.0.2", "tiktoken": "^1.0.15", + "together-ai": "^0.6.0-alpha.4", "tweetnacl": "^1.0.3", "ua-parser-js": "^1.0.38", "uglify-js": "^3.17.4", diff --git a/src/backend/src/modules/puterai/PuterAIModule.js b/src/backend/src/modules/puterai/PuterAIModule.js index 977afec2..fdb67a41 100644 --- a/src/backend/src/modules/puterai/PuterAIModule.js +++ b/src/backend/src/modules/puterai/PuterAIModule.js @@ -33,6 +33,11 @@ class PuterAIModule extends AdvancedBase { const { ClaudeService } = require('./ClaudeService'); services.registerService('claude', ClaudeService); } + + if ( !! config?.services?.['together-ai'] ) { + const { TogetherAIService } = require('./TogetherAIService'); + services.registerService('together-ai', TogetherAIService); + } } } diff --git a/src/backend/src/modules/puterai/TogetherAIService.js b/src/backend/src/modules/puterai/TogetherAIService.js new file mode 100644 index 00000000..851987ac --- /dev/null +++ b/src/backend/src/modules/puterai/TogetherAIService.js @@ -0,0 +1,67 @@ +const { PassThrough } = require("stream"); +const BaseService = require("../../services/BaseService"); +const { TypedValue } = require("../../services/drivers/meta/Runtime"); +const { nou } = require("../../util/langutil"); + +class TogetherAIService extends BaseService { + static MODULES = { + ['together-ai']: require('together-ai'), + kv: globalThis.kv, + uuidv4: require('uuid').v4, + } + + async _init () { + const require = this.require; + const Together = require('together-ai'); + this.together = new Together({ + apiKey: this.config.apiKey + }); + this.kvkey = this.modules.uuidv4(); + debugger; + } + + static IMPLEMENTS = { + ['puter-chat-completion']: { + async complete ({ messages, stream, model }) { + console.log('model?', model); + const completion = await this.together.chat.completions.create({ + model: model ?? + 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo', + messages: messages, + stream, + }); + + if ( stream ) { + const stream = new PassThrough(); + const retval = new TypedValue({ + $: 'stream', + content_type: 'application/x-ndjson', + chunked: true, + }, stream); + (async () => { + for await ( const chunk of completion ) { + console.log('IT IS THIS STRING', chunk); + if ( chunk.choices.length < 1 ) continue; + if ( chunk.choices[0].finish_reason ) { + stream.end(); + break; + } + if ( nou(chunk.choices[0].delta.content) ) continue; + const str = JSON.stringify({ + text: chunk.choices[0].delta.content + }); + stream.write(str + '\n'); + } + })(); + return retval; + } + + return completion.choices[0]; + } + } + } +} + +module.exports = { + TogetherAIService, +};