mirror/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2026-04-29 19:50:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into warning_dialog

Browse Source
This commit is contained in:
Nariman Jelveh
2024-12-08 11:29:52 -08:00
committed by GitHub
parent 2ca1f71595 4c5f427ac2
commit c8894c7d4c
232 changed files with 6426 additions and 2938 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CONTRIBUTING.md
+6 -3
View File
@@ -45,15 +45,18 @@ If you'd like to contribute code to Puter, you need to fork the project and subm
We'll review your pull request and work with you to get your changes merged into the project.
## Repository Structure
![file structure](./doc/File%20Structure.drawio.png)
## Your first code contribution
We maintain a list of issues that are good for first-time contributors. You can find these issues by searching for the [`good first issue`](https://github.com/HeyPuter/puter/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) label in our [GitHub repository](https://github.com/HeyPuter/puter). These issues are designed to be relatively easy to fix, and we're happy to help you get started. Pick an issue that interests you, and leave a comment on the issue to let us know you're working on it.
<br>
## Documentation for Contributors
See [doc/contributors/index.md](./doc/contributors/index.md) for more information.
### Backend
See [src/backend/CONTRIBUTING.md](src/backend/CONTRIBUTING.md)
<br>
awesome/#DoesItRunPuter.md
+1
View File
@@ -13,3 +13,4 @@
- [Steam Deck](https://twitter.com/everythingSung/status/1782162352403828793)
- [Ladybird Browser](https://x.com/HeyPuter/status/1810783504503800035)
- [Garry's Mod](https://x.com/HeyPuter/status/1850587712786722862)
- [Samsung Q88BA](https://x.com/AmirIsAround/status/1862614583263076540)
doc/File Structure.drawio
+214
View File
File diff suppressed because one or more lines are too long
doc/File Structure.drawio.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 187 KiB

doc/contributors/extensions.md
+3 -2
View File
@@ -1,2 +1,3 @@
### `vscode`
- `es6-string-html`
## Puter Extensions
See the [Wiki Page](https://github.com/HeyPuter/puter/wiki/ex_extensions)
doc/contributors/vscode.md
+2
View File
@@ -0,0 +1,2 @@
### `vscode`
- `es6-string-html`
experiments/x86emu/www/main.js
-1
View File
@@ -17,7 +17,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#!/usr/bin/env node
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
mods/mods_available/kdmod/ShareTestService.js
+2 -2
View File
@@ -21,7 +21,6 @@
// we have these things registered in "useapi".
const {
get_user,
generate_system_fsentries,
invalidate_cached_user,
deleteUser,
} = require('../../../src/backend/src/helpers.js');
@@ -146,7 +145,8 @@ class ShareTestService extends use.Service {
],
);
const user = await get_user({ username });
await generate_system_fsentries(user);
const svc_user = this.services.get('user');
await svc_user.generate_default_fsentries({ user });
invalidate_cached_user(user);
return user;
}
package-lock.json
Generated
+49 -15
View File
@@ -2014,6 +2014,19 @@
"semver": "bin/semver.js"
}
},
"node_modules/@babel/helper-environment-visitor": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.24.7.tgz",
"integrity": "sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/types": "^7.24.7"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-module-imports": {
"version": "7.25.9",
"resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz",
@@ -2154,7 +2167,6 @@
"version": "11.12.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
"integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=4"
@@ -2514,8 +2526,8 @@
"form-data": "^4.0.0"
}
},
"node_modules/@heyputer/parsely": {
"resolved": "src/parsely",
"node_modules/@heyputer/parsers": {
"resolved": "src/parsers",
"link": true
},
"node_modules/@heyputer/phoenix": {
@@ -9010,10 +9022,6 @@
"node": ">= 0.6"
}
},
"node_modules/contextlink": {
"resolved": "src/contextlink",
"link": true
},
"node_modules/convert-source-map": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
@@ -9588,6 +9596,17 @@
"node": ">=8"
}
},
"node_modules/doctrine": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
"integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
"dependencies": {
"esutils": "^2.0.2"
},
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/dom-converter": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz",
@@ -10061,7 +10080,6 @@
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
"license": "BSD-2-Clause",
"bin": {
"esparse": "bin/esparse.js",
"esvalidate": "bin/esvalidate.js"
@@ -12224,9 +12242,7 @@
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true,
"license": "MIT"
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="
},
"node_modules/js-yaml": {
"version": "4.1.0",
@@ -13299,6 +13315,10 @@
"integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==",
"license": "MIT"
},
"node_modules/module-docgen": {
"resolved": "tools/module-docgen",
"link": true
},
"node_modules/moment": {
"version": "2.30.1",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
@@ -15839,10 +15859,6 @@
"node": ">= 0.8"
}
},
"node_modules/strataparse": {
"resolved": "src/strataparse",
"link": true
},
"node_modules/streamsearch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
@@ -17863,6 +17879,7 @@
},
"src/contextlink": {
"version": "0.0.0",
"extraneous": true,
"license": "AGPL-3.0-only",
"devDependencies": {
"mocha": "^10.2.0"
@@ -17940,6 +17957,12 @@
"src/parsely": {
"name": "@heyputer/parsely",
"version": "1.0.0",
"extraneous": true,
"license": "AGPL-3.0-only"
},
"src/parsers": {
"name": "@heyputer/parsers",
"version": "1.0.0",
"license": "AGPL-3.0-only"
},
"src/phoenix": {
@@ -18045,6 +18068,7 @@
},
"src/strataparse": {
"version": "0.0.0",
"extraneous": true,
"license": "AGPL-3.0-only"
},
"src/terminal": {
@@ -18203,6 +18227,16 @@
"node": ">=18"
}
},
"tools/module-docgen": {
"version": "1.0.0",
"license": "AGPL-3.0-only",
"dependencies": {
"@babel/parser": "^7.26.2",
"@babel/traverse": "^7.25.9",
"dedent": "^1.5.3",
"doctrine": "^3.0.0"
}
},
"tools/token-count-accuracy": {
"version": "1.0.0",
"license": "AGPL-3.0-only"
package.json
+1 -1
View File
@@ -30,7 +30,7 @@
"webpack-cli": "^5.1.1"
},
"scripts": {
"test": "npx mocha src/phoenix/test src/contextlink/test && node src/backend/tools/test",
"test": "npx mocha src/phoenix/test && node src/backend/tools/test",
"start=gui": "nodemon --exec \"node dev-server.js\" ",
"start": "node ./tools/run-selfhosted.js",
"build": "cd src/gui; node ./build.js",
src/backend/doc/contributors/index.md → src/backend/CONTRIBUTING.md
+19 -15
View File
@@ -1,18 +1,22 @@
# Contributing to Puter's Backend
## File Structure
## Architecture
- [boot sequence](./boot-sequence.md)
- [modules and services](./modules.md)
- [boot sequence](./doc/contributors/boot-sequence.md)
- [modules and services](./doc/contributors/modules.md)
## Features
- [protected apps](../features/protected-apps.md)
- [service scripts](../features/service-scripts.md)
- [protected apps](./doc/features/protected-apps.md)
- [service scripts](./doc/features/service-scripts.md)
## Lists of Things
- [list of permissions](../lists-of-things/list-of-permissions.md)
- [list of permissions](./doc/lists-of-things/list-of-permissions.md)
## Code-First Approach
@@ -20,21 +24,21 @@ If you prefer to understand a system by looking at the
first files which are invoked and starting from there,
here's a handy list!
- [Kernel](../../src/Kernel.js), despite its intimidating name, is a
- [Kernel](./src/Kernel.js), despite its intimidating name, is a
relatively simple (< 200 LOC) class which loads the modules
(modules register services), and then starts all the services.
- [RuntimeEnvironment](../../src/boot/RuntimeEnvironment.js)
- [RuntimeEnvironment](./src/boot/RuntimeEnvironment.js)
sets the configuration and runtime directories. It's invoked by Kernel.
- The default setup for running a self-hosted Puter loads these modules:
- [CoreModule](../../src/CoreModule.js)
- [DatabaseModule](../../src/DatabaseModule.js)
- [LocalDiskStorageModule](../../src/LocalDiskStorageModule.js)
- [CoreModule](./src/CoreModule.js)
- [DatabaseModule](./src/DatabaseModule.js)
- [LocalDiskStorageModule](./src/LocalDiskStorageModule.js)
- HTTP endpoints are registered with
[WebServerService](../../src/services/WebServerService.js)
[WebServerService](./src/services/WebServerService.js)
by these services:
- [ServeGUIService](../../src/services/ServeGUIService.js)
- [PuterAPIService](../../src/services/PuterAPIService.js)
- [FilesystemAPIService](../../src/services/FilesystemAPIService.js)
- [ServeGUIService](./src/services/ServeGUIService.js)
- [PuterAPIService](./src/services/PuterAPIService.js)
- [FilesystemAPIService](./src/services/FilesystemAPIService.js)
## Development Philosophies
@@ -71,7 +75,7 @@ doing the useless work that reveals what the useful work is.
## Underlying Constructs
- [putility's README.md](../../packages/putility/README.md)
- [putility's README.md](../putility/README.md)
- Whenever you see `AdvancedBase`, that's from here
- Many things in backend extend this. Anything that doesn't only doesn't
because it was written before `AdvancedBase` existed.
src/backend/doc/Kernel.md
+65
View File
@@ -0,0 +1,65 @@
# Puter Kernel Documentation
## Overview
The **Puter Kernel** is the core runtime component of the Puter system. It provides the foundational infrastructure for:
- Initializing the runtime environment
- Managing internal and external modules (extensions)
- Setting up and booting core services
- Configuring logging and debugging utilities
- Integrating with third-party modules and performing dependency installs at runtime
This kernel is responsible for orchestrating the startup sequence and ensuring that all necessary services, modules, and environmental configurations are properly loaded before the application enters its operational state.
---
## Features
1. **Modular Architecture**:
The Kernel supports both internal and external modules:
- **Internal Modules**: Provided to Kernel by an initializing script, such
as `tools/run-selfhosted.js`, via the `add_module()` method.
- **External Modules**: Discovered in configured module directories and installed
dynamically. This includes resolving and executing `package.json` entries and
running `npm install` as needed.
2. **Service Container & Registry**:
The Kernel initializes a service container that manages a wide range of services. Services can:
- Register modules
- Initialize dependencies
- Emit lifecycle events (`boot.consolidation`, `boot.activation`, `boot.ready`) to
orchestrate a stable and consistent environment.
3. **Runtime Environment Setup**:
The Kernel sets up a `RuntimeEnvironment` to determine configuration paths and environment parameters. It also provides global helpers like `kv` for key-value storage and `cl` for simplified console logging.
4. **Logging and Debugging**:
Uses a temporary `BootLogger` for the initialization phase until LogService is
initialized, at which point it will replace the boot logger. Debugging features
(`ll`, `xtra_log`) are enabled in development environments for convenience.
## Initialization & Boot Process
1. **Constructor**:
When a Kernel instance is created, it sets up basic parameters, initializes an empty
module list, and prepares `useapi()` integration.
2. **Booting**:
The `boot()` method:
- Parses CLI arguments using `yargs`.
- Calls `_runtime_init()` to set up the `RuntimeEnvironment` and boot logger.
- Initializes global debugging/logging utilities.
- Sets up the service container (usually called `services`c instance of **Container**).
- Invokes module installation and service bootstrapping processes.
3. **Module Installation**:
Internal modules are registered and installed first.
External modules are discovered, packaged, installed, and their code is executed.
External modules are given a special context with access to `useapi()`, a dynamic
import mechanism for Puter modules and extensions.
4. **Service Bootstrapping**:
After modules and extensions are installed, services are initialized and activated.
For more information about how this works, see [boot-sequence.md](./contributors/boot-sequence.md).
src/backend/exports.js
+11
View File
@@ -28,6 +28,9 @@ const { Context } = require("./src/util/context.js");
const { TestDriversModule } = require("./src/modules/test-drivers/TestDriversModule.js");
const { PuterAIModule } = require("./src/modules/puterai/PuterAIModule.js");
const { BroadcastModule } = require("./src/modules/broadcast/BroadcastModule.js");
const { WebModule } = require("./src/modules/web/WebModule.js");
const { Core2Module } = require("./src/modules/core/Core2Module.js");
const { TemplateModule } = require("./src/modules/template/TemplateModule.js");
module.exports = {
@@ -42,9 +45,17 @@ module.exports = {
Context,
Kernel,
EssentialModules: [
Core2Module,
CoreModule,
WebModule,
TemplateModule,
],
// Pre-built modules
CoreModule,
WebModule,
DatabaseModule,
PuterDriversModule,
LocalDiskStorageModule,
src/backend/src/CoreModule.js
+31 -26
View File
@@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@@ -23,6 +24,16 @@ const { ProtectedAppES } = require("./om/entitystorage/ProtectedAppES");
const { Context } = require('./util/context');
/**
* Core module for the Puter platform that includes essential services including
* authentication, filesystems, rate limiting, permissions, and various API endpoints.
*
* This is a monolithic module. Incrementally, services should be migrated to
* Core2Module and other modules instead. Core2Module has a smaller scope, and each
* new module will be a cohesive concern. Once CoreModule is empty, it will be removed
* and Core2Module will take on its name.
*/
class CoreModule extends AdvancedBase {
dirname () { return __dirname; }
async install (context) {
@@ -33,11 +44,16 @@ class CoreModule extends AdvancedBase {
await install({ services, app, useapi, modapi });
}
// Some services were created before the BaseService
// class existed. They don't listen to the init event
// and the order in which they're instantiated matters.
// They all need to be installed after the init event
// is dispatched, so they get a separate install method.
/**
* Installs legacy services that don't extend BaseService and require special handling.
* These services were created before the BaseService class existed and don't listen
* to the init event. They need to be installed after the init event is dispatched
* due to initialization order dependencies.
*
* @param {Object} context - The context object containing service references
* @param {Object} context.services - Service registry for registering legacy services
* @returns {Promise<void>} Resolves when legacy services are installed
*/
async install_legacy (context) {
const services = context.get('services');
await install_legacy({ services });
@@ -52,6 +68,9 @@ module.exports = CoreModule;
const install = async ({ services, app, useapi, modapi }) => {
const config = require('./config');
// === LIBRARIES ===
useapi.withuse(() => {
def('Service', require('./services/BaseService'));
def('Module', AdvancedBase);
@@ -68,7 +87,6 @@ const install = async ({ services, app, useapi, modapi }) => {
def('core.config', config);
});
// === LIBRARIES ===
useapi.withuse(() => {
const ArrayUtil = require('./libraries/ArrayUtil');
services.registerService('util-array', ArrayUtil);
@@ -82,16 +100,11 @@ const install = async ({ services, app, useapi, modapi }) => {
// === SERVICES ===
// /!\ IMPORTANT /!\
// For new services, put the import immediate above the
// For new services, put the import immediately above the
// call to services.registerService. We'll clean this up
// in a future PR.
const { LogService } = require('./services/runtime-analysis/LogService');
const { PagerService } = require('./services/runtime-analysis/PagerService');
const { AlarmService } = require('./services/runtime-analysis/AlarmService');
const { ErrorService } = require('./services/runtime-analysis/ErrorService');
const { CommandService } = require('./services/CommandService');
const { ExpectationService } = require('./services/runtime-analysis/ExpectationService');
const { HTTPThumbnailService } = require('./services/thumbnails/HTTPThumbnailService');
const { PureJSThumbnailService } = require('./services/thumbnails/PureJSThumbnailService');
const { NAPIThumbnailService } = require('./services/thumbnails/NAPIThumbnailService');
@@ -124,12 +137,10 @@ const install = async ({ services, app, useapi, modapi }) => {
const { ESBuilder } = require('./om/entitystorage/ESBuilder');
const { Eq, Or } = require('./om/query/query');
const { TrackSpendingService } = require('./services/TrackSpendingService');
const { ServerHealthService } = require('./services/runtime-analysis/ServerHealthService');
const { MakeProdDebuggingLessAwfulService } = require('./services/MakeProdDebuggingLessAwfulService');
const { ConfigurableCountingService } = require('./services/ConfigurableCountingService');
const { FSLockService } = require('./services/fs/FSLockService');
const { StrategizedService } = require('./services/StrategizedService');
const WebServerService = require('./services/WebServerService');
const FilesystemAPIService = require('./services/FilesystemAPIService');
const ServeGUIService = require('./services/ServeGUIService');
const PuterAPIService = require('./services/PuterAPIService');
@@ -140,17 +151,10 @@ const install = async ({ services, app, useapi, modapi }) => {
// === Services which extend BaseService ===
services.registerService('system-validation', SystemValidationService);
services.registerService('server-health', ServerHealthService);
services.registerService('log-service', LogService);
services.registerService('commands', CommandService);
services.registerService('web-server', WebServerService, { app });
services.registerService('__api-filesystem', FilesystemAPIService);
services.registerService('__api', PuterAPIService);
services.registerService('__gui', ServeGUIService);
services.registerService('expectations', ExpectationService);
services.registerService('pager', PagerService);
services.registerService('alarm', AlarmService);
services.registerService('error-service', ErrorService);
services.registerService('registry', RegistryService);
services.registerService('__registrant', RegistrantService);
services.registerService('fslock', FSLockService);
@@ -351,24 +355,25 @@ const install = async ({ services, app, useapi, modapi }) => {
const { ReferralCodeService } = require('./services/ReferralCodeService');
services.registerService('referral-code', ReferralCodeService);
const { UserService } = require('./services/UserService');
services.registerService('user', UserService);
const { WSPushService } = require('./services/WSPushService');
services.registerService('__event-push-ws', WSPushService);
}
const install_legacy = async ({ services }) => {
const { ProcessEventService } = require('./services/runtime-analysis/ProcessEventService');
// const { FilesystemService } = require('./filesystem/FilesystemService');
const PerformanceMonitor = require('./monitor/PerformanceMonitor');
const { OperationTraceService } = require('./services/OperationTraceService');
const { WSPushService } = require('./services/WSPushService');
const { ClientOperationService } = require('./services/ClientOperationService');
const { EngPortalService } = require('./services/EngPortalService');
const { AppInformationService } = require('./services/AppInformationService');
const { FileCacheService } = require('./services/file-cache/FileCacheService');
// === Services which do not yet extend BaseService ===
services.registerService('process-event', ProcessEventService);
// services.registerService('filesystem', FilesystemService);
services.registerService('operationTrace', OperationTraceService);
services.registerService('__event-push-ws', WSPushService);
services.registerService('file-cache', FileCacheService);
services.registerService('client-operation', ClientOperationService);
services.registerService('app-information', AppInformationService);
src/backend/src/Extension.js
+26
View File
@@ -3,6 +3,10 @@ const EmitterFeature = require("@heyputer/putility/src/features/EmitterFeature")
const { Context } = require("./util/context");
const { ExtensionServiceState } = require("./ExtensionService");
/**
* This class creates the `extension` global that is seem by Puter backend
* extensions.
*/
class Extension extends AdvancedBase {
static FEATURES = [
EmitterFeature({
@@ -24,6 +28,9 @@ class Extension extends AdvancedBase {
console.log('Example method called by an extension.');
}
/**
* This will get a database instance from the default service.
*/
get db () {
const db = this.service.values.get('db');
if ( ! db ) {
@@ -35,6 +42,12 @@ class Extension extends AdvancedBase {
return db;
}
/**
* This will create a GET endpoint on the default service.
* @param {*} path - route for the endpoint
* @param {*} handler - function to handle the endpoint
* @param {*} options - options like noauth (bool) and mw (array)
*/
get (path, handler, options) {
// this extension will have a default service
this.ensure_service_();
@@ -51,6 +64,12 @@ class Extension extends AdvancedBase {
});
}
/**
* This will create a POST endpoint on the default service.
* @param {*} path - route for the endpoint
* @param {*} handler - function to handle the endpoint
* @param {*} options - options like noauth (bool) and mw (array)
*/
post (path, handler, options) {
// this extension will have a default service
this.ensure_service_();
@@ -67,6 +86,13 @@ class Extension extends AdvancedBase {
});
}
/**
* This method will create the "default service" for an extension.
* This is specifically for Puter extensions that do not define their
* own service classes.
*
* @returns {void}
*/
ensure_service_ () {
if ( this.service ) {
return;
src/backend/src/ExtensionService.js
+5
View File
@@ -5,6 +5,11 @@ const configurable_auth = require("./middleware/configurable_auth");
const { Context } = require("./util/context");
const { DB_READ, DB_WRITE } = require("./services/database/consts");
/**
* State shared with the default service and the `extension` global so that
* methods on `extension` can register routes (and make other changes in the
* future) to the default service.
*/
class ExtensionServiceState extends AdvancedBase {
constructor (...a) {
super(...a);
src/backend/src/Kernel.js
+3 -6
View File
@@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const { AdvancedBase, libs } = require("@heyputer/putility");
const { Context } = require('./util/context');
const BaseService = require("./services/BaseService");
const useapi = require('useapi');
@@ -25,7 +25,8 @@ const { hideBin } = require('yargs/helpers');
const { Extension } = require("./Extension");
const { ExtensionModule } = require("./ExtensionModule");
const { spawn } = require("node:child_process");
const { quot } = require("./util/strutil");
const { quot } = libs.string;
class Kernel extends AdvancedBase {
constructor ({ entry_path } = {}) {
@@ -78,8 +79,6 @@ class Kernel extends AdvancedBase {
this._runtime_init({ args });
// const express = require('express')
// const app = express();
const config = require('./config');
globalThis.ll = o => o;
@@ -112,7 +111,6 @@ class Kernel extends AdvancedBase {
const services = new Container({ logger: this.bootLogger });
this.services = services;
// app.set('services', services);
const root_context = Context.create({
environment: this.environment,
@@ -130,7 +128,6 @@ class Kernel extends AdvancedBase {
});
// Error.stackTraceLimit = Infinity;
Error.stackTraceLimit = 200;
}
src/backend/src/api/APIError.js
+1 -1
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { URLSearchParams } = require("node:url");
const { quot } = require("../util/strutil");
const { quot } = require('@heyputer/putility').libs.string;
/**
* APIError represents an error that can be sent to the client.
src/backend/src/api/eggspress.js
+2 -198
View File
@@ -1,198 +1,2 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const express = require('express');
const multer = require('multer');
const multest = require('@heyputer/multest');
const api_error_handler = require('../api/api_error_handler.js');
const fsBeforeMW = require('../middleware/fs');
const APIError = require('./APIError.js');
const { Context } = require('../util/context.js');
/**
* eggspress() is a factory function for creating express routers.
*
* @param {*} route the route to the router
* @param {*} settings the settings for the router. The following
* properties are supported:
* - auth: whether or not to use the auth middleware
* - fs: whether or not to use the fs middleware
* - json: whether or not to use the json middleware
* - customArgs: custom arguments to pass to the router
* - allowedMethods: the allowed HTTP methods
* @param {*} handler the handler for the router
* @returns {express.Router} the router
*/
module.exports = function eggspress (route, settings, handler) {
const router = express.Router();
const mw = [];
const afterMW = [];
// These flags enable specific middleware.
if ( settings.abuse ) mw.push(require('../middleware/abuse')(settings.abuse));
if ( settings.auth ) mw.push(require('../middleware/auth'));
if ( settings.auth2 ) mw.push(require('../middleware/auth2'));
if ( settings.fs ) {
mw.push(fsBeforeMW);
}
if ( settings.verified ) mw.push(require('../middleware/verified'));
if ( settings.json ) mw.push(express.json());
// The `files` setting is an array of strings. Each string is the name
// of a multipart field that contains files. `multer` is used to parse
// the multipart request and store the files in `req.files`.
if ( settings.files ) {
for ( const key of settings.files ) {
mw.push(multer().array(key));
}
}
if ( settings.multest ) {
mw.push(multest());
}
// The `multipart_jsons` setting is an array of strings. Each string
// is the name of a multipart field that contains JSON. This middleware
// parses the JSON in each field and stores the result in `req.body`.
if ( settings.multipart_jsons ) {
for ( const key of settings.multipart_jsons ) {
mw.push((req, res, next) => {
try {
if ( ! Array.isArray(req.body[key]) ) {
req.body[key] = [JSON.parse(req.body[key])];
} else {
req.body[key] = req.body[key].map(JSON.parse);
}
} catch (e) {
return res.status(400).send({
error: {
message: `Invalid JSON in multipart field ${key}`
}
});
}
next();
});
}
}
// The `alias` setting is an object. Each key is the name of a
// parameter. Each value is the name of a parameter that should
// be aliased to the key.
if ( settings.alias ) {
for ( const alias in settings.alias ) {
const target = settings.alias[alias];
mw.push((req, res, next) => {
const values = req.method === 'GET' ? req.query : req.body;
if ( values[alias] ) {
values[target] = values[alias];
}
next();
});
}
}
// The `parameters` setting is an object. Each key is the name of a
// parameter. Each value is a `Param` object. The `Param` object
// specifies how to validate the parameter.
if ( settings.parameters ) {
for ( const key in settings.parameters ) {
const param = settings.parameters[key];
mw.push(async (req, res, next) => {
if ( ! req.values ) req.values = {};
const values = req.method === 'GET' ? req.query : req.body;
const getParam = (key) => values[key];
try {
const result = await param.consolidate({ req, getParam });
req.values[key] = result;
} catch (e) {
api_error_handler(e, req, res, next);
return;
}
next();
});
}
}
// what if I wanted to pass arguments to, for example, `json`?
if ( settings.customArgs ) mw.push(settings.customArgs);
if ( settings.alarm_timeout ) {
mw.push((req, res, next) => {
setTimeout(() => {
if ( ! res.headersSent ) {
const log = req.services.get('log-service').create('eggspress:timeout');
const errors = req.services.get('error-service').create(log);
let id = Array.isArray(route) ? route[0] : route;
id = id.replace(/\//g, '_');
errors.report(id, {
source: new Error('Response timed out.'),
message: 'Response timed out.',
trace: true,
alarm: true,
});
}
}, settings.alarm_timeout);
next();
});
}
if ( settings.response_timeout ) {
mw.push((req, res, next) => {
setTimeout(() => {
if ( ! res.headersSent ) {
api_error_handler(APIError.create('response_timeout'), req, res, next);
}
}, settings.response_timeout);
next();
});
}
if ( settings.mw ) mw.push(...settings.mw);
const errorHandledHandler = async function (req, res, next) {
if ( settings.subdomain ) {
if ( require('../helpers').subdomain(req) !== settings.subdomain ) {
return next();
}
}
try {
const expected_ctx = res.locals.ctx;
const received_ctx = Context.get(undefined, { allow_fallback: true });
if ( expected_ctx != received_ctx ) {
await expected_ctx.arun(async () => {
await handler(req, res, next);
});
} else await handler(req, res, next);
} catch (e) {
api_error_handler(e, req, res, next);
}
};
if ( settings.allowedMethods.includes('GET') ) {
router.get(route, ...mw, errorHandledHandler, ...afterMW);
}
if ( settings.allowedMethods.includes('POST') ) {
router.post(route, ...mw, errorHandledHandler, ...afterMW);
}
return router;
}
// This file is a legacy alias
module.exports = require('../modules/web/lib/eggspress.js');
src/backend/src/boot/RuntimeEnvironment.js
+3 -7
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const { quot } = require("../util/strutil");
const { quot } = require('@heyputer/putility').libs.string;
const { TechnicalError } = require("../errors/TechnicalError");
const { print_error_help } = require("../errors/error_help_details");
const default_config = require("./default_config");
@@ -233,18 +233,14 @@ class RuntimeEnvironment extends AdvancedBase {
]
);
// Note: there used to be a 'mods_path_entry' here too
// but it was never used
const pwd_path_entry = this.get_first_suitable_path_(
{ pathFor: 'working directory' },
this.runtime_paths,
[ this.path_checks.require_write_permission ]
);
const mods_path_entry = this.get_first_suitable_path_(
{ pathFor: 'mods', optional: true },
this.mod_paths,
[ this.path_checks.require_read_permission ],
);
process.chdir(pwd_path_entry.path);
// Check for a valid config file in the config path
src/backend/src/config/ConfigLoader.js
+1 -1
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const { quot } = require("../util/strutil");
const { quot } = require('@heyputer/putility').libs.string;
class ConfigLoader extends AdvancedBase {
static MODULES = {
src/backend/src/errors/error_help_details.js
+1 -1
View File
@@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { quot, osclink } = require("../util/strutil");
const { quot, osclink } = require('@heyputer/putility').libs.string;
const reused = {
runtime_env_references: [
src/backend/src/filesystem/FilesystemService.js
-1
View File
@@ -44,7 +44,6 @@ class FilesystemService extends BaseService {
static MODULES = {
_path: require('path'),
uuidv4: require('uuid').v4,
socketio: require('../socketio.js'),
config: require('../config.js'),
}
src/backend/src/filesystem/backends/Test.js
-24
View File
@@ -1,24 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Test = void 0;
class Test {
}
exports.Test = Test;
src/backend/src/filesystem/backends/Test.ts
-3
View File
@@ -1,3 +0,0 @@
export class Test {
//
}
src/backend/src/filesystem/batch/BatchExecutor.js
+2 -2
View File
@@ -19,11 +19,11 @@
const { AdvancedBase } = require('@heyputer/putility');
const PathResolver = require('../../routers/filesystem_api/batch/PathResolver');
const commands = require('./commands').commands;
const { WorkUnit } = require('../../services/runtime-analysis/ExpectationService');
const APIError = require('../../api/APIError');
const { Context } = require('../../util/context');
const config = require('../../config');
const { TeePromise } = require('../../util/promise');
const { TeePromise } = require('@heyputer/putility');
const { WorkUnit } = require('../../modules/core/lib/expect');
class BatchExecutor extends AdvancedBase {
constructor (x, { actor, log, errors }) {
src/backend/src/filesystem/core/.gitignore
-2
View File
@@ -1,2 +0,0 @@
# Typescript directory
*.js
src/backend/src/filesystem/core/BackendAPI.ts
-85
View File
@@ -1,85 +0,0 @@
import { ISelector } from "./Selector";
type PuterUserID = number;
export const enum FSBackendSupportFlags {
None = 0,
// Platform-related flags
PlatformCaseSensitive = 1 << 1,
// Puter support flags
// PuterStatOwner indicates the backend can store `user_id`
PuterStatOwner = 1 << 2,
// PuterStatApp indicates the backend can store `associated_app_id`
PuterStatApp = 1 << 3,
// DetailVerboseReaddir indicates the backend will provide a full
// stat() result for each entry in readdir().
DetailVerboseReaddir = 1 << 4,
}
export const enum FSNodeType {
File,
Directory,
PuterShortcut,
SymbolicLink,
KVStore,
Socket,
}
export interface IOverwriteOptions {
readonly overwrite: boolean;
UserID: PuterUserID,
}
export interface IWriteOptions extends IOverwriteOptions {
readonly create: boolean;
}
export interface IDeleteOptions {
readonly recursive: boolean;
}
export interface IStatOptions {
followSymlinks?: boolean;
}
export interface IStatResult {
uuid: string;
name: string;
type: FSNodeType;
size: number;
mtime: Date;
ctime: Date;
atime: Date;
immutable: boolean;
}
export interface IMiniStatResult {
uuid: string;
name: string;
type: FSNodeType;
}
type ReaddirResult = IMiniStatResult | IStatResult;
export interface IMkdirOptions {
// Not for permission checks by the storage backend.
// A supporting storage backend will simply store this and
// return it in the stat() call.
UserID: PuterUserID,
}
export interface BackendAPI {
stat (selector: ISelector, options: IStatOptions): Promise<IStatResult>;
readdir (selector: ISelector): Promise<[string, ReaddirResult][]>;
mkdir (selector: ISelector, name: string): Promise<void>;
copy (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise<void>;
rename (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise<void>;
delete (selector: ISelector, options: IDeleteOptions): Promise<void>;
read_file (selector: ISelector): Promise<Buffer>;
write_file (selector: ISelector, data: Buffer, options: IOverwriteOptions): Promise<void>;
}
src/backend/src/filesystem/core/FSEntry.ts
View File
src/backend/src/filesystem/core/Selector.ts
-65
View File
@@ -1,65 +0,0 @@
import * as _path from 'path';
import * as _util from 'util';
type TemporeryNodeType = any;
export interface ISelector {
describe (showDebug?: boolean): string;
setPropertiesKnownBySelector (node: object): void;
}
export class NodePathSelector {
public value: string;
constructor (path: string) {
this.value = path;
}
public describe (showDebug?: boolean): string {
return this.value;
}
public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
node.path = this.value;
node.name = _path.basename(this.value);
}
}
export class NodeInternalUIDSelector {
public value: string;
constructor (uid: string) {
this.value = uid;
}
public describe (showDebug?: boolean): string {
return `[uid:${this.value}]`;
}
public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
node.uid = this.value;
}
}
export class NodeInternalIDSelector {
constructor (
public service: string,
public id: number,
public debugInfo: any
) { }
public describe (showDebug?: boolean): string {
if ( showDebug ) {
return `[db:${this.id}] (${
_util.inspect(this.debugInfo)
})`;
}
return `[db:${this.id}]`;
}
public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
if ( this.service === 'mysql' ) {
node.id = this.id;
}
}
}
src/backend/src/filesystem/hl_operations/hl_mkdir.js
+3 -10
View File
@@ -73,8 +73,7 @@ class MkTree extends HLFilesystemOperation {
}
async create_branch_ ({ parent_node, tree, parent_exists }) {
const { context, values } = this;
const { _path } = this.modules;
const { context } = this;
const fs = context.get('services').get('filesystem');
const actor = context.get('actor');
@@ -82,7 +81,6 @@ class MkTree extends HLFilesystemOperation {
const branches = tree.slice(1);
let current = parent_node.selector;
let lastCreatedSelector = parent_node.selector;
// trunk = a/b/c
@@ -242,7 +240,6 @@ class HLMkdir extends HLFilesystemOperation {
static MODULES = {
_path: require('path'),
socketio: require('../../socketio.js'),
}
static PROPERTIES = {
@@ -258,7 +255,7 @@ class HLMkdir extends HLFilesystemOperation {
async _run () {
const { context, values } = this;
const { _path, socketio } = this.modules;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
if ( ! is_valid_path(values.path, {
@@ -385,12 +382,8 @@ class HLMkdir extends HLFilesystemOperation {
}
async _create_parents ({ parent_node }) {
const { context, values } = this;
const { values } = this;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
let current = parent_node.selector;
let lastCreatedSelector = null;
const tree_op = new MkTree();
await tree_op.run({
src/backend/src/filesystem/hl_operations/hl_mklink.js
+1 -2
View File
@@ -36,11 +36,10 @@ class HLMkLink extends HLFilesystemOperation {
async _run () {
const { context, values } = this;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
const { target, parent, user } = values;
let { name, dedupe_name } = values;
let { name } = values;
if ( ! name ) {
throw APIError.create('field_empty', null, { key: 'name' });
src/backend/src/filesystem/hl_operations/hl_mkshortcut.js
-1
View File
@@ -40,7 +40,6 @@ class HLMkShortcut extends HLFilesystemOperation {
async _run () {
console.log('HLMKSHORTCUT IS HAPPENING')
const { context, values } = this;
const { _path, socketio } = this.modules;
const fs = context.get('services').get('filesystem');
const { target, parent, user } = values;
src/backend/src/filesystem/hl_operations/hl_read.js
-1
View File
@@ -28,7 +28,6 @@ class HLRead extends HLFilesystemOperation {
}
async _run () {
const { context } = this;
const {
fsNode, actor,
line_count, byte_count,
src/backend/src/filesystem/hl_operations/hl_readdir.js
-7
View File
@@ -81,13 +81,6 @@ class HLReadDir extends HLFilesystemOperation {
await child.fetchSuggestedApps(user);
await child.fetchSubdomains(user);
}
const fs = require('fs');
// fs.appendFileSync('/tmp/children.log',
// JSON.stringify({
// no_thumbs,
// no_assocs,
// entry: child.entry,
// }) + '\n');
return await child.getSafeEntry({ thumbnail: ! no_thumbs });
}));
}
src/backend/src/filesystem/hl_operations/hl_write.js
+1 -2
View File
@@ -23,7 +23,7 @@ const StringParam = require("../../api/filesystem/StringParam");
const UserParam = require("../../api/filesystem/UserParam");
const config = require("../../config");
const { chkperm, validate_fsentry_name } = require("../../helpers");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require("@heyputer/putility").libs.promise;
const { pausing_tee, logging_stream, offset_write_stream, stream_to_the_void } = require("../../util/streamutil");
const { TYPE_DIRECTORY } = require("../FSNodeContext");
const { LLRead } = require("../ll_operations/ll_read");
@@ -116,7 +116,6 @@ class HLWrite extends HLFilesystemOperation {
static MODULES = {
_path: require('path'),
socketio: require('../../socketio.js'),
mime: require('mime-types'),
}
src/backend/src/filesystem/ll_operations/ll_readdir.js
+1 -1
View File
@@ -25,7 +25,7 @@ const { LLFilesystemOperation } = require("./definitions");
class LLReadDir extends LLFilesystemOperation {
async _run () {
const { context } = this;
const { subject: subject_let, user, actor, no_acl } = this.values;
const { subject: subject_let, actor, no_acl } = this.values;
let subject = subject_let;
if ( ! await subject.exists() ) {
src/backend/src/filesystem/ll_operations/ll_readshares.js
+1 -1
View File
@@ -36,7 +36,7 @@ class LLReadShares extends LLFilesystemOperation {
`;
async _run () {
const { subject, user, actor, depth = 0 } = this.values;
const { subject, user, actor } = this.values;
const svc = this.context.get('services');
src/backend/src/filesystem/storage/ResourceService.js
-1
View File
@@ -103,7 +103,6 @@ class ResourceService {
}
async waitForResource (selector) {
const i = waiti++;
if ( selector instanceof NodePathSelector ) {
await this.waitForResourceByPath(selector.value);
}
src/backend/src/filesystem/storage/SystemFSEntryService.js
-1
View File
@@ -95,7 +95,6 @@ class SystemFSEntryService {
async get_uuid_from_path (path) {
path = PuterPath.adapt(path);
let current = path.reference;
let pathOfReference = path.reference === PuterPath.NULL_UUID
? '/' : this.get_path_from_uuid(path.reference);
src/backend/src/fun/dev-console-ui-utils.js
+1 -1
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const config = require('../config');
const { TeePromise } = require('../util/promise');
const { TeePromise } = require('@heyputer/putility').libs.promise;
const es_import_promise = new TeePromise();
let stringLength;
src/backend/src/helpers.js
+87 -394
View File
@@ -22,7 +22,6 @@ const micromatch = require('micromatch');
const config = require('./config')
const mime = require('mime-types');
const PerformanceMonitor = require('./monitor/PerformanceMonitor.js');
const { generate_identifier } = require('./util/identifier.js');
const { ManagedError } = require('./util/errorutil.js');
const { spanify } = require('./util/otelutil.js');
const APIError = require('./api/APIError.js');
@@ -1012,7 +1011,6 @@ async function gen_public_token(file_uuid, ttl = 24 * 60 * 60){
}
const uid = fsentry.uuid;
const expires = Math.ceil(Date.now() / 1000) + ttl;
const token = uuidv4();
const contentType = mime.contentType(fsentry.name);
@@ -1155,201 +1153,6 @@ async function jwt_auth(req){
return ancestors;
}
// THIS LEGACY FUNCTION IS STILL IN USE
// by: generate_system_fsentries
// TODO: migrate generate_system_fsentries to use QuickMkdir
async function mkdir(options){
const fs = systemfs;
debugger;
const resolved_path = PathBuilder.resolve(options.path, { puterfs: true });
const dirpath = _path.dirname(resolved_path);
let target_name = _path.basename(resolved_path);
const overwrite = options.overwrite ?? false;
const dedupe_name = options.dedupe_name ?? false;
const immutable = options.immutable ?? false;
const return_id = options.return_id ?? false;
const no_perm_check = options.no_perm_check ?? false;
// make parent directories as needed
const create_missing_parents = options.create_missing_parents ?? false;
// hold a list of all parent directories created in the process
let parent_dirs_created = [];
let overwritten_uid;
// target_name validation
try{
validate_fsentry_name(target_name)
}catch(e){
throw e.message;
}
// resolve dirpath to its fsentry
let parent = await convert_path_to_fsentry(dirpath);
// dirpath not found
if(parent === false && !create_missing_parents)
throw new Error("Target path not found");
// create missing parent directories
else if(parent === false && create_missing_parents){
const dirs = _path.resolve('/', dirpath).split('/');
let cur_path = '';
for(let j=0; j < dirs.length; j++){
if(dirs[j] === '')
continue;
cur_path += '/'+dirs[j];
// skip creating '/[username]'
if(j === 1)
continue;
try{
let d = await mkdir(fs, {path: cur_path, user: options.user});
d.path = cur_path;
parent_dirs_created.push(d);
}catch(e){
console.log(`Skipped mkdir ${cur_path}`);
}
}
// try setting parent again
parent = await convert_path_to_fsentry(dirpath);
if(parent === false)
throw new Error("Target path not found");
}
// check permission
if(!no_perm_check && !await chkperm(parent, options.user.id, 'write'))
throw { code:`forbidden`, message: `permission denied.`};
// check if a fsentry with the same name exists under this path
const existing_fsentry = await convert_path_to_fsentry(_path.resolve('/', dirpath + '/' + target_name ));
/** @type BaseDatabaseAccessService */
const db = services.get('database').get(DB_WRITE, 'filesystem');
// if trying to create a directory with an existing path and overwrite==false, throw an error
if(!overwrite && !dedupe_name && existing_fsentry !== false){
throw {
code: 'path_exists',
message:"A file/directory with the same path already exists.",
entry_name: existing_fsentry.name,
existing_fsentry: {
name: existing_fsentry.name,
uid: existing_fsentry.uuid,
}
};
}
else if(overwrite && existing_fsentry){
overwritten_uid = existing_fsentry.uuid;
// check permission
if(!await chkperm(existing_fsentry, options.user.id, 'write'))
throw {code:`forbidden`, message: `permission denied.`};
// delete existing dir
await db.write(
`DELETE FROM fsentries WHERE id = ? AND user_id = ?`,
[
//parent_uid
existing_fsentry.uuid,
//user_id
options.user.id,
]);
}
// dedupe name, generate a new name until its unique
else if(dedupe_name && existing_fsentry !== false){
for( let i = 1; ; i++){
let try_new_name = existing_fsentry.name + ' (' + i + ')';
let check_dupe = await db.read(
"SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1",
[existing_fsentry.parent_uid, try_new_name]
);
if(check_dupe[0] === undefined){
target_name = try_new_name;
break;
}
}
}
// shrotcut?
let shortcut_fsentry;
if(options.shortcut_to){
shortcut_fsentry = await uuid2fsentry(options.shortcut_to);
if(shortcut_fsentry === false){
throw ({ code:`not_found`, message: `shortcut_to not found.`})
}else if(!parent.is_dir){
throw ({ code:`not_dir`, message: `parent of shortcut_to must be a directory`})
}else if(!await chkperm(shortcut_fsentry, options.user.id, 'read')){
throw ({ code:`forbidden`, message: `shortcut_to permission denied.`})
}
}
// current epoch
const ts = Math.round(Date.now() / 1000)
const uid = uuidv4();
// record in db
let user_id = (parent === null ? options.user.id : parent.user_id);
const { insertId: mkdir_db_id } = await db.write(
`INSERT INTO fsentries
(uuid, parent_uid, user_id, name, is_dir, created, modified, immutable, shortcut_to, is_shortcut) VALUES
( ?, ?, ?, ?, true, ?, ?, ?, ?, ?)`,
[
//uuid
uid,
//parent_uid
(parent === null) ? null : parent.uuid,
//user_id
user_id,
//name
target_name,
//created
ts,
//modified
ts,
//immutable
immutable,
//shortcut_to,
shortcut_fsentry ? shortcut_fsentry.id : null,
//is_shortcut,
shortcut_fsentry ? 1 : 0,
]
);
const ret_obj = {
uid : uid,
name: target_name,
immutable: immutable,
is_dir: true,
path: options.path ?? false,
dirpath: dirpath,
is_shared: await is_shared_with_anyone(mkdir_db_id),
overwritten_uid: overwritten_uid,
shortcut_to: shortcut_fsentry ? shortcut_fsentry.uuid : null,
shortcut_to_path: shortcut_fsentry ? await id2path(shortcut_fsentry.id) : null,
parent_dirs_created: parent_dirs_created,
original_client_socket_id: options.original_client_socket_id,
};
// add existing_fsentry if exists
if(existing_fsentry){
ret_obj.existing_fsentry ={
name: existing_fsentry.name,
uid: existing_fsentry.uuid,
}
}
if(return_id)
ret_obj.id = mkdir_db_id;
// send realtime success msg to client
let socketio = require('./socketio.js').getio();
if(socketio){
socketio.to(user_id).emit('item.added', ret_obj)
}
return ret_obj;
}
function is_valid_uuid ( uuid ) {
let s = "" + uuid;
s = s.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
@@ -1412,100 +1215,6 @@ async function app_name_exists(name){
return true;
}
// generates all the default files and directories a user needs,
// generally used for a brand new account
async function generate_system_fsentries(user){
/** @type BaseDatabaseAccessService */
const db = services.get('database').get(DB_WRITE, 'filesystem');
//-------------------------------------------------------------
// create root `/[username]/`
//-------------------------------------------------------------
const root_dir = await mkdir({
path: '/' + user.username,
user: user,
immutable: true,
no_perm_check: true,
return_id: true,
});
// Normally, it is recommended to use mkdir() to create new folders,
// but during signup this could result in multiple queries to the DB server
// and for servers in remote regions such as Asia this could result in a
// very long time for /signup to finish, sometimes up to 30-40 seconds!
// by combining as many queries as we can into one and avoiding multiple back-and-forth
// with the DB server, we can speed this process up significantly.
const ts = Date.now()/1000;
// Generate UUIDs for all the default folders and files
let trash_uuid = uuidv4();
let appdata_uuid = uuidv4();
let desktop_uuid = uuidv4();
let documents_uuid = uuidv4();
let pictures_uuid = uuidv4();
let videos_uuid = uuidv4();
let public_uuid = uuidv4();
const insert_res = await db.write(
`INSERT INTO fsentries
(uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true)
`,
[
// Trash
trash_uuid, root_dir.uid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts,
// AppData
appdata_uuid, root_dir.uid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts,
// Desktop
desktop_uuid, root_dir.uid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts,
// Documents
documents_uuid, root_dir.uid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts,
// Pictures
pictures_uuid, root_dir.uid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts,
// Videos
videos_uuid, root_dir.uid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts,
// Public
public_uuid, root_dir.uid, user.id, 'Public', `/${user.username}/Public`, ts, ts,
]
);
// https://stackoverflow.com/a/50103616
let trash_id = insert_res.insertId;
let appdata_id = insert_res.insertId + 1;
let desktop_id = insert_res.insertId + 2;
let documents_id = insert_res.insertId + 3;
let pictures_id = insert_res.insertId + 4;
let videos_id = insert_res.insertId + 5;
let public_id = insert_res.insertId + 6;
// Asynchronously set the user's system folders uuids in database
// This is for caching purposes, so we don't have to query the DB every time we need to access these folders
// This is also possible because we know the user's system folders uuids will never change
// TODO: pass to IIAFE manager to avoid unhandled promise rejection
// (IIAFE manager doesn't exist yet, hence this is a TODO)
db.write(
`UPDATE user SET
trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?,
trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=?
WHERE id=?`,
[
trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid,
trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id,
user.id
]
);
invalidate_cached_user(user);
}
function send_email_verification_code(email_confirm_code, email){
const svc_email = Context.get('services').get('email');
svc_email.send_email({ email }, 'email_verification_code', {
@@ -1519,19 +1228,11 @@ function send_email_verification_token(email_confirm_token, email, user_uuid){
svc_email.send_email({ email }, 'email_verification_link', { link });
}
async function generate_random_username(){
let username;
do {
username = generate_identifier();
} while (await username_exists(username));
return username;
}
function generate_random_str(length) {
var result = '';
var characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
var charactersLength = characters.length;
for ( var i = 0; i < length; i++ ) {
let result = '';
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
const charactersLength = characters.length;
for ( let i = 0; i < length; i++ ) {
result += characters.charAt(Math.floor(Math.random() *
charactersLength));
}
@@ -1546,11 +1247,11 @@ function generate_random_str(length) {
* @throws {TypeError} If the `seconds` parameter is not a number.
*/
function seconds_to_string(seconds) {
var numyears = Math.floor(seconds / 31536000);
var numdays = Math.floor((seconds % 31536000) / 86400);
var numhours = Math.floor(((seconds % 31536000) % 86400) / 3600);
var numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60);
var numseconds = (((seconds % 31536000) % 86400) % 3600) % 60;
const numyears = Math.floor(seconds / 31536000);
const numdays = Math.floor((seconds % 31536000) / 86400);
const numhours = Math.floor(((seconds % 31536000) % 86400) / 3600);
const numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60);
const numseconds = (((seconds % 31536000) % 86400) % 3600) % 60;
return numyears + " years " + numdays + " days " + numhours + " hours " + numminutes + " minutes " + numseconds + " seconds";
}
@@ -1565,14 +1266,11 @@ async function suggest_app_for_fsentry(fsentry, options){
const suggested_apps = [];
let content_type = mime.contentType(fsentry.name);
if(content_type === null || content_type === undefined || content_type === false)
content_type = '';
if( ! content_type ) content_type = '';
// IIFE just so fsname can stay `const`
const fsname = (() => {
if ( ! fsentry.name ) {
const fs = require('fs');
fs.writeFileSync('/tmp/missing-fsentry-name.txt', JSON.stringify(fsentry, null, 2));
return 'missing-fsentry-name';
}
let fsname = fsentry.name.toLowerCase();
@@ -1581,74 +1279,79 @@ async function suggest_app_for_fsentry(fsentry, options){
return fsname;
})();
const file_extension = _path.extname(fsname).toLowerCase();
const any_of = (list, name) => {
return list.some(v => name.endsWith(v));
}
//---------------------------------------------
// Code
//---------------------------------------------
if(
fsname.endsWith('.asm') ||
fsname.endsWith('.asp') ||
fsname.endsWith('.aspx') ||
fsname.endsWith('.bash') ||
fsname.endsWith('.c') ||
fsname.endsWith('.cpp') ||
fsname.endsWith('.css') ||
fsname.endsWith('.csv') ||
fsname.endsWith('.dhtml') ||
fsname.endsWith('.f') ||
fsname.endsWith('.go') ||
fsname.endsWith('.h') ||
fsname.endsWith('.htm') ||
fsname.endsWith('.html') ||
fsname.endsWith('.html5') ||
fsname.endsWith('.java') ||
fsname.endsWith('.jl') ||
fsname.endsWith('.js') ||
fsname.endsWith('.jsa') ||
fsname.endsWith('.json') ||
fsname.endsWith('.jsonld') ||
fsname.endsWith('.jsf') ||
fsname.endsWith('.jsp') ||
fsname.endsWith('.kt') ||
fsname.endsWith('.log') ||
fsname.endsWith('.lock') ||
fsname.endsWith('.lua') ||
fsname.endsWith('.md') ||
fsname.endsWith('.perl') ||
fsname.endsWith('.phar') ||
fsname.endsWith('.php') ||
fsname.endsWith('.pl') ||
fsname.endsWith('.py') ||
fsname.endsWith('.r') ||
fsname.endsWith('.rb') ||
fsname.endsWith('.rdata') ||
fsname.endsWith('.rda') ||
fsname.endsWith('.rdf') ||
fsname.endsWith('.rds') ||
fsname.endsWith('.rs') ||
fsname.endsWith('.rlib') ||
fsname.endsWith('.rpy') ||
fsname.endsWith('.scala') ||
fsname.endsWith('.sc') ||
fsname.endsWith('.scm') ||
fsname.endsWith('.sh') ||
fsname.endsWith('.sol') ||
fsname.endsWith('.sql') ||
fsname.endsWith('.ss') ||
fsname.endsWith('.svg') ||
fsname.endsWith('.swift') ||
fsname.endsWith('.toml') ||
fsname.endsWith('.ts') ||
fsname.endsWith('.wasm') ||
fsname.endsWith('.xhtml') ||
fsname.endsWith('.xml') ||
fsname.endsWith('.yaml') ||
// files with no extension
!fsname.includes('.')
){
const exts_code = [
'.asm',
'.asp',
'.aspx',
'.bash',
'.c',
'.cpp',
'.css',
'.csv',
'.dhtml',
'.f',
'.go',
'.h',
'.htm',
'.html',
'.html5',
'.java',
'.jl',
'.js',
'.jsa',
'.json',
'.jsonld',
'.jsf',
'.jsp',
'.kt',
'.log',
'.lock',
'.lua',
'.md',
'.perl',
'.phar',
'.php',
'.pl',
'.py',
'.r',
'.rb',
'.rdata',
'.rda',
'.rdf',
'.rds',
'.rs',
'.rlib',
'.rpy',
'.scala',
'.sc',
'.scm',
'.sh',
'.sol',
'.sql',
'.ss',
'.svg',
'.swift',
'.toml',
'.ts',
'.wasm',
'.xhtml',
'.xml',
'.yaml',
];
if ( any_of(exts_code, fsname) || !fsname.includes('.') ) {
suggested_apps.push(await get_app({name: 'code'}))
suggested_apps.push(await get_app({name: 'editor'}))
}
//---------------------------------------------
// Editor
//---------------------------------------------
@@ -1712,19 +1415,17 @@ async function suggest_app_for_fsentry(fsentry, options){
//---------------------------------------------
// 3rd-party apps
//---------------------------------------------
const apps = kv.get(`assocs:${file_extension.slice(1)}:apps`)
const apps = kv.get(`assocs:${file_extension.slice(1)}:apps`) ?? [];
monitor.label("third party associations");
if(apps && apps.length > 0){
for (let index = 0; index < apps.length; index++) {
// retrieve app from DB
const third_party_app = await get_app({id: apps[index]})
if ( ! third_party_app ) continue;
// only add if the app is approved for opening items or the app is owned by this user
if( third_party_app.approved_for_opening_items ||
(options !== undefined && options.user !== undefined && options.user.id === third_party_app.owner_user_id))
suggested_apps.push(third_party_app)
}
for ( const app_id of apps ) {
// retrieve app from DB
const third_party_app = await get_app({id: app_id})
if ( ! third_party_app ) continue;
// only add if the app is approved for opening items or the app is owned by this user
if( third_party_app.approved_for_opening_items ||
(options !== undefined && options.user !== undefined && options.user.id === third_party_app.owner_user_id))
suggested_apps.push(third_party_app)
}
monitor.stamp();
monitor.end();
@@ -1741,10 +1442,6 @@ async function suggest_app_for_fsentry(fsentry, options){
});
}
function build_item_object(item){
}
async function get_taskbar_items(user) {
/** @type BaseDatabaseAccessService */
const db = services.get('database').get(DB_WRITE, 'filesystem');
@@ -1867,13 +1564,13 @@ async function mv(options){
function number_format (number, decimals, dec_point, thousands_sep) {
// Strip all characters but numerical ones.
number = (number + '').replace(/[^0-9+\-Ee.]/g, '');
var n = !isFinite(+number) ? 0 : +number,
let n = !isFinite(+number) ? 0 : +number,
prec = !isFinite(+decimals) ? 0 : Math.abs(decimals),
sep = (typeof thousands_sep === 'undefined') ? ',' : thousands_sep,
dec = (typeof dec_point === 'undefined') ? '.' : dec_point,
s = '',
toFixedFix = function (n, prec) {
var k = Math.pow(10, prec);
const k = Math.pow(10, prec);
return '' + Math.round(n * k) / k;
};
// Fix for IE parseFloat(0.55).toFixed(0) = 0;
@@ -1893,7 +1590,6 @@ module.exports = {
app_name_exists,
app_exists,
body_parser_error_handler,
build_item_object,
byte_format,
change_username,
chkperm,
@@ -1905,9 +1601,7 @@ module.exports = {
gen_public_token,
get_taskbar_items,
get_url_from_req,
generate_system_fsentries,
generate_random_str,
generate_random_username,
get_app,
get_user,
invalidate_cached_user,
@@ -1926,7 +1620,6 @@ module.exports = {
is_specifically_uuidv4,
is_valid_url,
jwt_auth,
mkdir,
mv,
number_format,
refresh_apps_cache,
src/backend/src/services/runtime-analysis/AlarmService.js → src/backend/src/modules/core/AlarmService.js
+42 -29
View File
@@ -24,18 +24,20 @@ const util = require('util');
const _path = require('path');
const fs = require('fs');
const { fallbackRead } = require('../../util/files.js');
const { generate_identifier } = require('../../util/identifier.js');
const { stringify_log_entry } = require('./LogService.js');
const BaseService = require('../BaseService.js');
const { split_lines } = require('../../util/stdioutil.js');
const { Context } = require('../../util/context.js');
const BaseService = require('../../services/BaseService.js');
/**
* @classdesc AlarmService class is responsible for managing alarms. It provides methods for creating, clearing, and handling alarms.
*/
* AlarmService class is responsible for managing alarms.
* It provides methods for creating, clearing, and handling alarms.
*/
class AlarmService extends BaseService {
static USE = {
logutil: 'core.util.logutil',
identutil: 'core.util.identutil',
stdioutil: 'core.util.stdioutil',
Context: 'core.context',
}
/**
* This method initializes the AlarmService by setting up its internal data structures and initializing any required dependencies.
*
@@ -59,23 +61,6 @@ class AlarmService extends BaseService {
// TODO:[self-hosted] fix this properly
this.known_errors = [];
// (async () => {
// try {
// this.known_errors = JSON5.parse(
// await fallbackRead(
// 'data/known_errors.json5',
// '/var/puter/data/known_errors.json5',
// ),
// );
// } catch (e) {
// this.create(
// 'missing-known-errors',
// e.message,
// )
// }
// })();
this._register_commands(services.get('commands'));
if ( this.global_config.env === 'dev' ) {
/**
@@ -94,7 +79,7 @@ class AlarmService extends BaseService {
const line =
`\x1B[31;1m [alarm]\x1B[0m ` +
`${alarm.id_string}: ${alarm.message} (${alarm.count})`;
const line_lines = split_lines(line);
const line_lines = this.stdioutil.split_lines(line);
lines.push(...line_lines);
}
@@ -102,6 +87,14 @@ class AlarmService extends BaseService {
}
}
}
/**
* AlarmService registers its commands at the consolidation phase because
* the '_init' method of CommandService may not have been called yet.
*/
['__on_boot.consolidation'] () {
this._register_commands(this.services.get('commands'));
}
adapt_id_ (id) {
// let shorten = false;
@@ -114,12 +107,21 @@ class AlarmService extends BaseService {
if ( shorten ) {
const rng = seedrandom(id);
id = generate_identifier('-', rng);
id = this.identutil.generate_identifier('-', rng);
}
return id;
}
/**
* Method to create an alarm with the given ID, message, and fields.
* If the ID already exists, it will be updated with the new fields
* and the occurrence count will be incremented.
*
* @param {string} id - Unique identifier for the alarm.
* @param {string} message - Message associated with the alarm.
* @param {object} fields - Additional information about the alarm.
*/
create (id, message, fields) {
this.log.error(`upcoming alarm: ${id}: ${message}`);
let existing = false;
@@ -223,6 +225,11 @@ class AlarmService extends BaseService {
}
}
/**
* Method to clear an alarm with the given ID.
* @param {*} id - The ID of the alarm to clear.
* @returns {void}
*/
clear (id) {
const alarm = this.alarms[id];
if ( !alarm ) {
@@ -305,7 +312,7 @@ class AlarmService extends BaseService {
svc_devConsole.add_widget(this.alarm_widget);
}
const args = Context.get('args') ?? {};
const args = this.Context.get('args') ?? {};
if ( args['quit-on-alarm'] ) {
const svc_shutdown = this.services.get('shutdown');
svc_shutdown.shutdown({
@@ -366,6 +373,12 @@ class AlarmService extends BaseService {
);
}
/**
* Method to get an alarm by its ID.
*
* @param {*} id - The ID of the alarm to get.
* @returns
*/
get_alarm (id) {
return this.alarms[id] ?? this.alarm_aliases[id];
}
@@ -492,7 +505,7 @@ class AlarmService extends BaseService {
}
log.log(`┏━━ Logs before: ${alarm.id_string} ━━━━`);
for ( const lg of occurance.logs ) {
log.log("┃ " + stringify_log_entry(lg));
log.log("┃ " + this.logutil.stringify_log_entry(lg));
}
log.log(`┗━━ Logs before: ${alarm.id_string} ━━━━`);
},
src/backend/src/modules/core/Core2Module.js
+55
View File
@@ -0,0 +1,55 @@
const { AdvancedBase } = require("@heyputer/putility");
/**
* A replacement for CoreModule with as few external relative requires as possible.
* This will eventually be the successor to CoreModule, the main module for Puter's backend.
*
* The scope of this module is:
* - logging and error handling
* - alarm handling
* - services that are tightly coupled with alarm handling are allowed
* - any essential information about server stats or health
* - any very generic service which other services can register
* behavior to.
*/
class Core2Module extends AdvancedBase {
async install (context) {
// === LIBS === //
const useapi = context.get('useapi');
const lib = require('./lib/__lib__.js');
for ( const k in lib ) {
useapi.def(`core.${k}`, lib[k], { assign: true });
}
useapi.def('core.context', require('../../util/context.js').Context);
// === SERVICES === //
const services = context.get('services');
const { LogService } = require('./LogService.js');
services.registerService('log-service', LogService);
const { AlarmService } = require("./AlarmService.js");
services.registerService('alarm', AlarmService);
const { ErrorService } = require("./ErrorService.js");
services.registerService('error-service', ErrorService);
const { PagerService } = require("./PagerService.js");
services.registerService('pager', PagerService);
const { ExpectationService } = require("./ExpectationService.js");
services.registerService('expectations', ExpectationService);
const { ProcessEventService } = require("./ProcessEventService.js");
services.registerService('process-event', ProcessEventService);
const { ServerHealthService } = require("./ServerHealthService.js");
services.registerService('server-health', ServerHealthService);
}
}
module.exports = {
Core2Module,
};
src/backend/src/services/runtime-analysis/ErrorService.js → src/backend/src/modules/core/ErrorService.js
+22 -3
View File
@@ -17,7 +17,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const BaseService = require("../BaseService");
const BaseService = require("../../services/BaseService");
/**
@@ -47,10 +47,11 @@ class ErrorContext {
/**
* The ErrorService class is responsible for handling and reporting errors within the system.
* It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms.
* @class ErrorService
* @extends BaseService
* @description The ErrorService class is responsible for handling and reporting errors within the system.
* It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms.
*/
class ErrorService extends BaseService {
/**
@@ -66,9 +67,27 @@ class ErrorService extends BaseService {
this.alarm = services.get('alarm');
this.backupLogger = services.get('log-service').create('error-service');
}
/**
* Creates an ErrorContext instance with the provided logging context.
*
* @param {*} log_context The logging context to associate with the error reports.
* @returns {ErrorContext} An ErrorContext instance.
*/
create (log_context) {
return new ErrorContext(this, log_context);
}
/**
* Reports an error with the specified location and details.
* The "location" is a string up to the callers discretion to identify
* the source of the error.
*
* @param {*} location The location where the error occurred.
* @param {*} fields The error details to report.
* @param {boolean} [alarm=true] Whether to raise an alarm for the error.
* @returns {void}
*/
report (location, { source, logger, trace, extra, message }, alarm = true) {
message = message ?? source?.message;
logger = logger ?? this.backupLogger;
src/backend/src/services/runtime-analysis/ExpectationService.js → src/backend/src/modules/core/ExpectationService.js
+43 -104
View File
@@ -18,82 +18,8 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { v4: uuidv4 } = require('uuid');
const { quot } = require('../../util/strutil');
const BaseService = require('../BaseService');
const BaseService = require('../../services/BaseService');
/**
* @class WorkUnit
* @description The WorkUnit class represents a unit of work that can be tracked and monitored for checkpoints.
* It includes methods to create instances, set checkpoints, and manage the state of the work unit.
*/
class WorkUnit {
/**
* Represents a unit of work with checkpointing capabilities.
*
* @class
*/
/**
* Creates and returns a new instance of WorkUnit.
*
* @static
* @returns {WorkUnit} A new instance of WorkUnit.
*/
static create () {
return new WorkUnit();
}
/**
* Creates a new instance of the WorkUnit class.
* @static
* @returns {WorkUnit} A new WorkUnit instance.
*/
constructor () {
this.id = uuidv4();
this.checkpoint_ = null;
}
checkpoint (label) {
console.log('CHECKPOINT', label);
this.checkpoint_ = label;
}
}
/**
* @class CheckpointExpectation
* @classdesc The CheckpointExpectation class is used to represent an expectation that a specific checkpoint
* will be reached during the execution of a work unit. It includes methods to check if the checkpoint has
* been reached and to report the results of this check.
*/
class CheckpointExpectation {
constructor (workUnit, checkpoint) {
this.workUnit = workUnit;
this.checkpoint = checkpoint;
}
/**
* Constructor for CheckpointExpectation class.
* Initializes the instance with a WorkUnit and a checkpoint label.
* @param {WorkUnit} workUnit - The work unit associated with the checkpoint.
* @param {string} checkpoint - The checkpoint label to be checked.
*/
check () {
// TODO: should be true if checkpoint was ever reached
return this.workUnit.checkpoint_ == this.checkpoint;
}
report (log) {
if ( this.check() ) return;
log.log(
`operation(${this.workUnit.id}): ` +
`expected ${quot(this.checkpoint)} ` +
`and got ${quot(this.workUnit.checkpoint_)}.`
);
}
}
/**
* This service helps diagnose errors involving the potentially
* complex relationships between asynchronous operations.
*/
/**
* @class ExpectationService
* @extends BaseService
@@ -108,6 +34,10 @@ class CheckpointExpectation {
* runtime behaviors in a system.
*/
class ExpectationService extends BaseService {
static USE = {
expect: 'core.expect'
};
/**
* Constructs the ExpectationService and initializes its internal state.
* This method is intended to be called asynchronously.
@@ -119,34 +49,12 @@ class ExpectationService extends BaseService {
this.expectations_ = [];
}
/**
* Initializes the ExpectationService, setting up interval functions and registering commands.
*
* This method sets up a periodic interval to purge expectations and registers a command
* to list pending expectations. The interval invokes `purgeExpectations_` every second.
* The command 'pending' allows users to list and log all pending expectations.
*
* @returns {Promise<void>} A promise that resolves when initialization is complete.
*/
async _init () {
const services = this.services;
// TODO: service to track all interval functions?
/**
* Initializes the service by setting up interval functions and registering commands.
* This method sets up a periodic interval function to purge expectations and registers
* a command to list pending expectations.
*
* @returns {void}
*/
// The comment should be placed above the method at line 68
setInterval(() => {
this.purgeExpectations_();
}, 1000);
const commands = services.get('commands');
* ExpectationService registers its commands at the consolidation phase because
* the '_init' method of CommandService may not have been called yet.
*/
['__on_boot.consolidation'] () {
const commands = this.services.get('commands');
commands.registerCommands('expectations', [
{
id: 'pending',
@@ -165,6 +73,31 @@ class ExpectationService extends BaseService {
]);
}
/**
* Initializes the ExpectationService, setting up interval functions and registering commands.
*
* This method sets up a periodic interval to purge expectations and registers a command
* to list pending expectations. The interval invokes `purgeExpectations_` every second.
* The command 'pending' allows users to list and log all pending expectations.
*
* @returns {Promise<void>} A promise that resolves when initialization is complete.
*/
async _init () {
// TODO: service to track all interval functions?
/**
* Initializes the service by setting up interval functions and registering commands.
* This method sets up a periodic interval function to purge expectations and registers
* a command to list pending expectations.
*
* @returns {void}
*/
// The comment should be placed above the method at line 68
setInterval(() => {
this.purgeExpectations_();
}, 1000);
}
/**
* Purges expectations that have been met.
@@ -186,14 +119,20 @@ class ExpectationService extends BaseService {
// this.expectations_ = this.expectations_.filter(v => v !== null);
}
/**
* Registers an expectation to be tracked by the service.
*
* @param {Object} workUnit - The work unit to track
* @param {string} checkpoint - The checkpoint to expect
* @returns {void}
*/
expect_eventually ({ workUnit, checkpoint }) {
this.expectations_.push(new CheckpointExpectation(workUnit, checkpoint));
this.expectations_.push(new this.expect.CheckpointExpectation(workUnit, checkpoint));
}
}
module.exports = {
WorkUnit,
ExpectationService
};
src/backend/src/services/runtime-analysis/LogService.js → src/backend/src/modules/core/LogService.js
+94 -242
View File
@@ -1,4 +1,4 @@
// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}}
// METADATA // {"ai-commented":{"service":"xai"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@@ -17,12 +17,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// Defines a function to create log severity objects used for logging
// The function is used to define various log levels and their properties
const logSeverity = (ordinal, label, esc, winst) => ({ ordinal, label, esc, winst });
// TODO: support the following annotation in tools/comment-writer/main.js
// AI-COMMENT-WRITER // SKIP 7 LINES
const LOG_LEVEL_ERRO = logSeverity(0, 'ERRO', '31;1', 'error');
const LOG_LEVEL_WARN = logSeverity(1, 'WARN', '33;1', 'warn');
const LOG_LEVEL_INFO = logSeverity(2, 'INFO', '36;1', 'info');
@@ -33,7 +28,8 @@ const LOG_LEVEL_SYSTEM = logSeverity(4, 'SYSTEM', '33;1', 'system');
const winston = require('winston');
const { Context } = require('../../util/context');
const BaseService = require('../BaseService');
const BaseService = require('../../services/BaseService');
const { stringify_log_entry } = require('./lib/log');
require('winston-daily-rotate-file');
const WINSTON_LEVELS = {
@@ -49,11 +45,10 @@ const WINSTON_LEVELS = {
/**
* Represents a logging context within the LogService.
* This class is used to manage logging operations with specific context information,
* allowing for hierarchical logging structures and dynamic field additions.
* @class LogContext
* @classdesc The LogContext class provides a structured way to handle logging within the application.
* It encapsulates the logging service, breadcrumbs for context, and fields for additional information.
* This class includes methods for different logging levels such as info, warn, debug, error, tick,
* and system logs. It also provides utility methods for sub-contexts, caching, and trace identification.
*/
class LogContext {
constructor (logService, { crumbs, fields }) {
@@ -110,14 +105,11 @@ class LogContext {
);
}
// convenience method to get a trace id that isn't as difficult
// for a human to read as a uuid.
/**
* Generates a human-readable trace ID.
* This method creates a trace ID that is easier for humans to read compared to a UUID.
* The trace ID is composed of two random alphanumeric strings joined by a hyphen.
*
* @returns {string} A human-readable trace ID.
* Generates a human-readable trace ID for logging purposes.
*
* @returns {string} A trace ID in the format 'xxxxxx-xxxxxx' where each segment is a
* random string of six lowercase letters and digits.
*/
mkid () {
// generate trace id
@@ -128,13 +120,9 @@ class LogContext {
return trace_id.join('-');
}
// add a trace id to this logging context
/**
* Adds a trace ID to the logging context.
* This method generates a new trace ID and assigns it to the logging context's fields.
* It then returns the modified logging context.
*
* @returns {LogContext} The modified logging context with the trace ID added.
* Adds a trace id to this logging context for tracking purposes.
* @returns {LogContext} The current logging context with the trace id added.
*/
traceOn () {
this.fields.trace_id = this.mkid();
@@ -143,81 +131,28 @@ class LogContext {
/**
* Retrieves the current log buffer.
*
* @returns {Array} The current log buffer containing log entries.
*/
* Gets the log buffer maintained by the LogService. This shows the most
* recent log entries.
* @returns {Array} An array of log entries stored in the buffer.
*/
get_log_buffer () {
return this.logService.get_log_buffer();
}
}
let log_epoch = Date.now();
/**
* Function to initialize the log epoch timestamp.
* This timestamp is used to calculate the time difference for log entries.
* Timestamp in milliseconds since the epoch, used for calculating log entry duration.
*/
const stringify_log_entry = ({ prefix, log_lvl, crumbs, message, fields, objects }) => {
const { colorize } = require('json-colorizer');
let lines = [], m;
/**
* Converts a log entry into a formatted string for display.
*
* This method formats log entries by combining the prefix, log level, crumbs (breadcrumbs),
* message, fields, and objects into a readable string. It includes color coding for log levels
* and timestamp information if available. The method processes each log entry into a multi-line
* string for enhanced readability.
*
* @param {Object} entry - The log entry object to be stringified.
* @param {string} entry.prefix - The optional prefix to prepend to the log message.
* @param {Object} entry.log_lvl - The log level object containing label and escape sequences.
* @param {Array} entry.crumbs - An array of breadcrumbs for context.
* @param {string} entry.message - The main log message.
* @param {Object} entry.fields - Additional fields to include in the log entry.
* @param {Object} entry.objects - Additional objects to include in the log entry.
* @returns {string} - The formatted log entry string.
*/
const lf = () => {
if ( ! m ) return;
lines.push(m);
m = '';
}
m = prefix ? `${prefix} ` : '';
m += `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`;
for ( const crumb of crumbs ) {
m += `::${crumb}`;
}
m += `\x1B[${log_lvl.esc}m]\x1B[0m`;
if ( fields.timestamp ) {
// display seconds since logger epoch
const n = (fields.timestamp - log_epoch) / 1000;
m += ` (${n.toFixed(3)}s)`;
}
m += ` ${message} `;
lf();
for ( const k in fields ) {
if ( k === 'timestamp' ) continue;
let v; try {
v = colorize(JSON.stringify(fields[k]));
} catch (e) {
v = '' + fields[k];
}
m += ` \x1B[1m${k}:\x1B[0m ${v}`;
lf();
}
return lines.join('\n');
};
/**
* @class DevLogger
* @description The DevLogger class is responsible for handling logging operations in a development environment.
* It can delegate logging to another logger and manage log output to a file. This class provides methods for
* logging messages at different levels and managing the state of logging, such as turning logging on or off
* and recording log output to a file. It is particularly useful for debugging and development purposes.
* @classdesc
* A development logger class designed for logging messages during development.
* This logger can either log directly to console or delegate logging to another logger.
* It provides functionality to turn logging on/off, and can optionally write logs to a file.
*
* @param {function} log - The logging function, typically `console.log` or similar.
* @param {object} [opt_delegate] - An optional logger to which log messages can be delegated.
*/
class DevLogger {
// TODO: this should eventually delegate to winston logger
@@ -260,11 +195,10 @@ class DevLogger {
/**
* @class
* @classdesc The `NullLogger` class is a logging utility that does not perform any actual logging.
* It is designed to be used as a placeholder or for environments where logging is not desired.
* This class can be extended or used as a base for other logging implementations that need to
* delegate logging responsibilities to another logger.
* @class NullLogger
* @description A logger that does nothing, effectively disabling logging.
* This class is used when logging is not desired or during development
* to avoid performance overhead or for testing purposes.
*/
class NullLogger {
// TODO: this should eventually delegate to winston logger
@@ -275,26 +209,15 @@ class NullLogger {
this.delegate = opt_delegate;
}
}
/**
* Constructor for the NullLogger class.
* This method initializes a new instance of the NullLogger class.
* It optionally accepts a delegate logger to which it can pass log messages.
*
* @param {function} log - The logging function to use (e.g., console.log).
* @param {Object} opt_delegate - An optional delegate logger to pass log messages to.
*/
onLogMessage () {
}
}
/**
* @class WinstonLogger
* @classdesc The WinstonLogger class is responsible for integrating the Winston logging library
* into the logging system. It handles forwarding log messages to Winston transports, which can
* include various logging destinations such as files, consoles, and remote logging services.
* This class is a key component in ensuring that log messages are appropriately recorded and
* managed, providing a structured and configurable logging mechanism.
* WinstonLogger Class
*
* A logger that delegates log messages to a Winston logger instance.
*/
class WinstonLogger {
constructor (winst) {
@@ -311,99 +234,13 @@ class WinstonLogger {
}
/**
* @class LogContext
* @description The `LogContext` class provides a context for logging messages within the application.
* It encapsulates the log service, a list of breadcrumbs (contextual information for the logs),
* and fields that can be attached to log messages.
*
* This class includes methods for various log levels (info, warn, debug, error, etc.),
* allowing for structured logging with contextual information.
*
* It also provides methods for creating sub-contexts, generating trace IDs, and managing log buffers.
*/
/**
* @class DevLogger
* @description The `DevLogger` class is a simple logger that outputs log messages to the console.
* It is primarily used in development environments. This logger can also delegate log messages to another logger.
*
* The class includes methods for toggling log output, recording log messages to a file,
* and adding timestamps to log entries.
*/
/**
* @class NullLogger
* @description The `NullLogger` class is a logger that does not output any log messages.
* It is used when logging is disabled or when logging is not required.
*/
/**
* @class WinstonLogger
* @description The `WinstonLogger` class is a logger that integrates with the Winston logging library.
* It provides a structured way to log messages with different log levels and transports.
*
* This logger can be used to log messages to files, with options for daily rotation and compression.
*/
/**
* @class TimestampLogger
* @description The `TimestampLogger` class is a decorator logger that adds timestamps to log messages.
* It delegates the actual logging to another logger.
*
* This class ensures that each log message includes a timestamp, which can be useful for debugging and performance monitoring.
*/
/**
* @class BufferLogger
* @description The `BufferLogger` class is a logger that maintains a buffer of log messages.
* It delegates the actual logging to another logger.
*
* This logger can be used to keep a limited number of recent log messages in memory,
* which can be useful for debugging and troubleshooting.
*/
/**
* @class CustomLogger
* @description The `CustomLogger` class is a flexible logger that allows for custom log message processing.
* It delegates the actual logging to another logger.
*
* This logger can be used to modify log messages before they are logged,
* allowing for custom log message formatting and processing.
*/
/**
* @class LogService
* @description The `LogService` class is a service that provides logging functionality for the application.
* It manages a collection of loggers, a buffer of recent log messages, and log directories.
*
* This class includes methods for registering log middleware, creating log contexts,
* and logging messages at various log levels. It also ensures that log messages are only output if they are at or above the configured output level.
*
* The `LogService` class is responsible for initializing loggers based on the application's configuration,
* ensuring that log directories exist, and providing methods for retrieving log files and buffers.
*/
/**
* @class
* @description The `TimestampLogger` class is a logger that adds timestamps to log messages. It delegates the actual logging to another logger.
* This class ensures that each log message includes a timestamp, which can be useful for debugging and performance monitoring.
*/
/**
* @class BufferLogger
* @description The `BufferLogger` class is a logger that maintains a buffer of log messages. It delegates the actual logging to another logger.
* This logger can be used to keep a limited number of recent log messages in memory, which can be useful for debugging and troubleshooting.
*/
/**
* @class CustomLogger
* @description The `CustomLogger` class is a flexible logger that allows for custom log message processing. It delegates the actual logging to another logger.
* This logger can be used to modify log messages before they are logged, allowing for custom log message formatting and processing.
*/
/**
* @class
* @description The `LogService` class is a service that provides logging functionality for the application.
* It manages a collection of loggers, a buffer of recent log messages, and log directories.
* This class includes methods for registering log middleware, creating log contexts,
* and logging messages at various log levels. It also ensures that log messages are only output if they are at or above the configured output level.
* The `LogService` class is responsible for initializing loggers based on the application's configuration,
* ensuring that log directories exist, and providing methods for retrieving log files and buffers.
*/
/**
* @class
* @description The `TimestampLogger` class is a logger that adds timestamps to log messages. It delegates the actual logging to another logger.
* This class ensures that each log message includes a timestamp, which can be useful for debugging and performance monitoring.
* @classdesc A logger that adds timestamps to log messages before delegating them to another logger.
* This class wraps another logger instance to ensure that all log messages include a timestamp,
* which can be useful for tracking the sequence of events in a system.
*
* @param {Object} delegate - The logger instance to which the timestamped log messages are forwarded.
*/
class TimestampLogger {
constructor (delegate) {
@@ -417,17 +254,12 @@ class TimestampLogger {
/**
* The LogService class is a core service that manages logging across the application.
* It facilitates the creation and management of various logging middleware, such as
* DevLogger, NullLogger, WinstonLogger, and more. This class extends BaseService and
* includes methods for initializing and configuring loggers, ensuring log directories,
* and handling log messages. It also allows for the registration of custom log middleware
* via the register_log_middleware method.
*
* The LogService class supports multiple logging levels, each with its own file and
* transport mechanisms. It includes utility methods for creating new log contexts,
* logging messages, and getting the log buffer. This class is essential for tracking
* and monitoring application behavior, errors, and system events.
* The `BufferLogger` class extends the logging functionality by maintaining a buffer of log entries.
* This class is designed to:
* - Store a specified number of recent log messages.
* - Allow for retrieval of these logs for debugging or monitoring purposes.
* - Ensure that the log buffer does not exceed the defined size by removing older entries when necessary.
* - Delegate logging messages to another logger while managing its own buffer.
*/
class BufferLogger {
constructor (size, delegate) {
@@ -446,11 +278,11 @@ class BufferLogger {
/**
* The `CustomLogger` class is a specialized logger that allows for custom
* logging behavior by applying a callback function to modify log entries
* before they are passed to the delegate logger. This class is part of the
* logging infrastructure, providing flexibility to alter log messages, fields,
* or other parameters dynamically based on the context in which the logging occurs.
* Represents a custom logger that can modify log messages before they are passed to another logger.
* @class CustomLogger
* @extends {Object}
* @param {Object} delegate - The delegate logger to which modified log messages will be passed.
* @param {Function} callback - A callback function that modifies log parameters before delegation.
*/
class CustomLogger {
constructor (delegate, callback) {
@@ -486,29 +318,37 @@ class CustomLogger {
/**
* The `LogService` class extends the `BaseService` and is responsible for managing logging operations.
* It handles the registration of log middleware, initializes various logging mechanisms, and provides
* methods to log messages at different severity levels. The class ensures that log directories are
* properly set up and manages the logging output levels based on configuration.
* The `LogService` class extends `BaseService` and is responsible for managing and
* orchestrating various logging functionalities within the application. It handles
* log initialization, middleware registration, log directory management, and
* provides methods for creating log contexts and managing log output levels.
*/
class LogService extends BaseService {
static MODULES = {
path: require('path'),
}
/**
* Initializes the log service by setting up the logging directory, configuring loggers,
* and registering commands for log management.
*
* @async
* @returns {Promise<void>} A promise that resolves when the initialization is complete.
* Defines the modules required by the LogService class.
* This static property contains modules that are used for file path operations.
* @property {Object} MODULES - An object containing required modules.
* @property {Object} MODULES.path - The Node.js path module for handling and resolving file paths.
*/
async _construct () {
this.loggers = [];
this.bufferLogger = null;
}
/**
* Registers a custom logging middleware with the LogService.
* @param {*} callback - The callback function that modifies log parameters before delegation.
*/
register_log_middleware (callback) {
this.loggers[0] = new CustomLogger(this.loggers[0], callback);
}
/**
* Registers logging commands with the command service.
*/
['__on_boot.consolidation'] () {
const commands = this.services.get('commands');
commands.registerCommands('logs', [
@@ -552,13 +392,13 @@ class LogService extends BaseService {
]);
}
/**
* Registers log-related commands for the service.
*
* This method defines a set of commands for managing log output,
* such as toggling log visibility, starting/stopping log recording to a file,
* and toggling log indentation.
*
* @param {Object} commands - The commands object to register commands to.
* Registers logging commands with the command service.
*
* This method sets up various logging commands that can be used to
* interact with the log output, such as toggling log display,
* starting/stopping log recording, and toggling log indentation.
*
* @memberof LogService
*/
async _init () {
const config = this.global_config;
@@ -651,6 +491,13 @@ class LogService extends BaseService {
globalThis.root_context.set('logger', this.create('root-context'));
}
/**
* Create a new log context with the specified prefix
*
* @param {1} prefix - The prefix for the log context
* @param {*} fields - Optional fields to include in the log context
* @returns {LogContext} A new log context with the specified prefix and fields
*/
create (prefix, fields = {}) {
const logContext = new LogContext(
this,
@@ -687,10 +534,12 @@ class LogService extends BaseService {
/**
* Ensures that the log directory exists by attempting to create it in several
* predefined locations. If none of the locations are available, an error is thrown.
*
* @throws {Error} If the log directory cannot be created or found.
* Ensures that a log directory exists for logging purposes.
* This method attempts to create or locate a directory for log files,
* falling back through several predefined paths if the preferred
* directory does not exist or cannot be created.
*
* @throws {Error} If no suitable log directory can be found or created.
*/
ensure_log_directory_ () {
// STEP 1: Try /var/puter/logs/heyputer
@@ -741,6 +590,12 @@ class LogService extends BaseService {
throw new Error('Unable to create or find log directory');
}
/**
* Generates a sanitized file path for log files.
*
* @param {string} name - The name of the log file, which will be sanitized to remove any path characters.
* @returns {string} A sanitized file path within the log directory.
*/
get_log_file (name) {
// sanitize name: cannot contain path characters
name = name.replace(/[^a-zA-Z0-9-_]/g, '_');
@@ -749,13 +604,10 @@ class LogService extends BaseService {
/**
* Retrieves the log buffer.
*
* This method returns the current log buffer, which is an array of log entries.
* Each log entry contains details such as the log level, crumbs, message, and fields.
*
* @returns {Array} The log buffer containing log entries.
*/
* Get the most recent log entries from the buffer maintained by the LogService.
* By default, the buffer contains the last 20 log entries.
* @returns
*/
get_log_buffer () {
return this.bufferLogger.buffer;
}
src/backend/src/services/runtime-analysis/PagerService.js → src/backend/src/modules/core/PagerService.js
+15 -8
View File
@@ -18,9 +18,8 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const pdjs = require('@pagerduty/pdjs');
const BaseService = require('../BaseService');
const BaseService = require('../../services/BaseService');
const util = require('util');
const { Context } = require('../../util/context');
/**
@@ -33,11 +32,24 @@ const { Context } = require('../../util/context');
* command registration.
*/
class PagerService extends BaseService {
static USE = {
Context: 'core.context',
}
async _construct () {
this.config = this.global_config.pager;
this.alertHandlers_ = [];
}
/**
* PagerService registers its commands at the consolidation phase because
* the '_init' method of CommandService may not have been called yet.
*/
['__on_boot.consolidation'] () {
this._register_commands(this.services.get('commands'));
}
/**
* Initializes the PagerService instance by setting the configuration and
* initializing an empty alert handler array.
@@ -47,8 +59,6 @@ class PagerService extends BaseService {
* @returns {Promise<void>}
*/
async _init () {
const services = this.services;
this.alertHandlers_ = [];
if ( ! this.config ) {
@@ -56,11 +66,8 @@ class PagerService extends BaseService {
}
this.onInit();
this._register_commands(services.get('commands'));
}
/**
* Initializes PagerDuty configuration and registers alert handlers.
* If PagerDuty is enabled in the configuration, it sets up an alert handler
@@ -83,7 +90,7 @@ class PagerService extends BaseService {
server_id: this.global_config.server_id,
};
const ctx = Context.get(undefined, { allow_fallback: true });
const ctx = this.Context.get(undefined, { allow_fallback: true });
// Add request payload if any exists
const req = ctx.get('req');
src/backend/src/services/runtime-analysis/ProcessEventService.js → src/backend/src/modules/core/ProcessEventService.js
+10 -5
View File
@@ -17,8 +17,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { Context } = require("../../util/context");
const BaseService = require("../../services/BaseService");
/**
* Service class that handles process-wide events and errors.
@@ -28,8 +28,13 @@ const { Context } = require("../../util/context");
*
* @class ProcessEventService
*/
class ProcessEventService {
constructor ({ services }) {
class ProcessEventService extends BaseService {
static USE = {
Context: 'core.context',
};
_init () {
const services = this.services;
const log = services.get('log-service').create('process-event-service');
const errors = services.get('error-service').create(log);
@@ -44,7 +49,7 @@ class ProcessEventService {
* @param {string} origin - The origin of the uncaught exception
* @returns {Promise<void>}
*/
await Context.allow_fallback(async () => {
await this.Context.allow_fallback(async () => {
errors.report('process:uncaughtException', {
source: err,
origin,
@@ -62,7 +67,7 @@ class ProcessEventService {
* @param {Promise} promise - The rejected promise
* @returns {Promise<void>} Resolves when error is reported
*/
await Context.allow_fallback(async () => {
await this.Context.allow_fallback(async () => {
errors.report('process:unhandledRejection', {
source: reason,
promise,
src/backend/src/modules/core/README.md
+269
View File
@@ -0,0 +1,269 @@
# Core2Module
A replacement for CoreModule with as few external relative requires as possible.
This will eventually be the successor to CoreModule, the main module for Puter's backend.
## Services
### AlarmService
AlarmService class is responsible for managing alarms.
It provides methods for creating, clearing, and handling alarms.
#### Listeners
##### `boot.consolidation`
AlarmService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `create`
Method to create an alarm with the given ID, message, and fields.
If the ID already exists, it will be updated with the new fields
and the occurrence count will be incremented.
###### Parameters
- **id:** Unique identifier for the alarm.
- **message:** Message associated with the alarm.
- **fields:** Additional information about the alarm.
##### `clear`
Method to clear an alarm with the given ID.
###### Parameters
- **id:** The ID of the alarm to clear.
##### `get_alarm`
Method to get an alarm by its ID.
###### Parameters
- **id:** The ID of the alarm to get.
### ErrorService
The ErrorService class is responsible for handling and reporting errors within the system.
It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms.
#### Methods
##### `init`
Initializes the ErrorService, setting up the alarm and backup logger services.
##### `create`
Creates an ErrorContext instance with the provided logging context.
###### Parameters
- **log_context:** The logging context to associate with the error reports.
##### `report`
Reports an error with the specified location and details.
The "location" is a string up to the callers discretion to identify
the source of the error.
###### Parameters
- **location:** The location where the error occurred.
- **fields:** The error details to report.
### ExpectationService
#### Listeners
##### `boot.consolidation`
ExpectationService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `expect_eventually`
Registers an expectation to be tracked by the service.
###### Parameters
- **workUnit:** The work unit to track
- **checkpoint:** The checkpoint to expect
### LogService
The `LogService` class extends `BaseService` and is responsible for managing and
orchestrating various logging functionalities within the application. It handles
log initialization, middleware registration, log directory management, and
provides methods for creating log contexts and managing log output levels.
#### Listeners
##### `boot.consolidation`
Registers logging commands with the command service.
#### Methods
##### `register_log_middleware`
Registers a custom logging middleware with the LogService.
###### Parameters
- **callback:** The callback function that modifies log parameters before delegation.
##### `create`
Create a new log context with the specified prefix
###### Parameters
- **prefix:** The prefix for the log context
- **fields:** Optional fields to include in the log context
##### `get_log_file`
Generates a sanitized file path for log files.
###### Parameters
- **name:** The name of the log file, which will be sanitized to remove any path characters.
##### `get_log_buffer`
Get the most recent log entries from the buffer maintained by the LogService.
By default, the buffer contains the last 20 log entries.
### PagerService
#### Listeners
##### `boot.consolidation`
PagerService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `onInit`
Initializes PagerDuty configuration and registers alert handlers.
If PagerDuty is enabled in the configuration, it sets up an alert handler
to send alerts to PagerDuty.
##### `alert`
Sends an alert to all registered alert handlers.
This method iterates through all alert handlers and attempts to send the alert.
If any handler fails to send the alert, an error message is logged.
###### Parameters
- **alert:** The alert object containing details about the alert.
### ProcessEventService
Service class that handles process-wide events and errors.
Provides centralized error handling for uncaught exceptions and unhandled promise rejections.
Sets up event listeners on the process object to capture and report critical errors
through the logging and error reporting services.
## Libraries
### core.expect
### core.util.identutil
#### Functions
##### `randomItem`
Select a random item from an array using a random number generator function.
###### Parameters
- **arr:** The array to select an item from
### core.util.logutil
#### Functions
##### `stringify_log_entry`
Stringifies a log entry into a formatted string for console output.
###### Parameters
- **logEntry:** The log entry object containing:
### stdio
#### Functions
##### `visible_length`
METADATA // {"ai-commented":{"service":"claude"}}
##### `split_lines`
Split a string into lines according to the terminal width,
preserving ANSI escape sequences, and return an array of lines.
###### Parameters
- **str:** The string to split into lines
### core.util.strutil
#### Functions
##### `quot`
METADATA // {"def":"core.util.strutil","ai-params":{"service":"claude"},"ai-commented":{"service":"claude"}}
##### `osclink`
Creates an OSC 8 hyperlink sequence for terminal output
###### Parameters
- **url:** The URL to link to
##### `format_as_usd`
Formats a number as a USD currency string with appropriate decimal places
###### Parameters
- **amount:** The amount to format
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../services/BaseService.js`
- `../../util/context.js`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../util/context`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
src/backend/src/services/runtime-analysis/ServerHealthService.js → src/backend/src/modules/core/ServerHealthService.js
+11 -10
View File
@@ -17,10 +17,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const BaseService = require("../BaseService");
const { SECOND } = require("../../util/time");
const { parse_meminfo } = require("../../util/linux");
const { asyncSafeSetInterval, TeePromise } = require("../../util/promise");
const BaseService = require("../../services/BaseService");
const { time, promise } = require("@heyputer/putility").libs;
/**
@@ -35,6 +33,10 @@ const { asyncSafeSetInterval, TeePromise } = require("../../util/promise");
* from `/proc/meminfo` and handling alarms via an external 'alarm' service.
*/
class ServerHealthService extends BaseService {
static USE = {
linuxutil: 'core.util.linuxutil'
};
static MODULES = {
fs: require('fs'),
}
@@ -70,7 +72,6 @@ class ServerHealthService extends BaseService {
*/
const min_free_KiB = 1024 * 1024; // 1 GiB
const min_available_KiB = 1024 * 1024 * 2; // 2 GiB
const svc_alarm = this.services.get('alarm');
@@ -96,7 +97,7 @@ class ServerHealthService extends BaseService {
const meminfo_text = await this.modules.fs.promises.readFile(
'/proc/meminfo', 'utf8'
);
const meminfo = parse_meminfo(meminfo_text);
const meminfo = this.linuxutil.parse_meminfo(meminfo_text);
const alarm_fields = {
mem_free: meminfo.MemFree,
mem_available: meminfo.MemAvailable,
@@ -135,11 +136,11 @@ class ServerHealthService extends BaseService {
* @param {none} - No parameters are passed to this method.
* @returns {void}
*/
asyncSafeSetInterval(async () => {
promise.asyncSafeSetInterval(async () => {
this.log.tick('service checks');
const check_failures = [];
for ( const { name, fn, chainable } of this.checks_ ) {
const p_timeout = new TeePromise();
const p_timeout = new promise.TeePromise();
/**
* Creates a TeePromise to handle potential timeouts during health checks.
*
@@ -147,7 +148,7 @@ class ServerHealthService extends BaseService {
*/
const timeout = setTimeout(() => {
p_timeout.reject(new Error('Health check timed out'));
}, 5 * SECOND);
}, 5 * time.SECOND);
try {
await Promise.race([
fn(),
@@ -180,7 +181,7 @@ class ServerHealthService extends BaseService {
}
this.failures_ = check_failures;
}, 10 * SECOND, null, {
}, 10 * time.SECOND, null, {
onBehindSchedule: (drift) => {
svc_alarm.create(
'health-checks-behind-schedule',
src/backend/src/modules/core/lib/__lib__.js
+9
View File
@@ -0,0 +1,9 @@
module.exports = {
util: {
logutil: require('./log.js'),
identutil: require('./identifier.js'),
stdioutil: require('./stdio.js'),
linuxutil: require('./linux.js'),
},
expect: require('./expect.js'),
};
src/backend/src/modules/core/lib/expect.js
+74
View File
@@ -0,0 +1,74 @@
// METADATA // {"def":"core.expect"}
const { v4: uuidv4 } = require('uuid');
/**
* @class WorkUnit
* @description The WorkUnit class represents a unit of work that can be tracked and monitored for checkpoints.
* It includes methods to create instances, set checkpoints, and manage the state of the work unit.
*/
class WorkUnit {
/**
* Represents a unit of work with checkpointing capabilities.
*
* @class
*/
/**
* Creates and returns a new instance of WorkUnit.
*
* @static
* @returns {WorkUnit} A new instance of WorkUnit.
*/
static create () {
return new WorkUnit();
}
/**
* Creates a new instance of the WorkUnit class.
* @static
* @returns {WorkUnit} A new WorkUnit instance.
*/
constructor () {
this.id = uuidv4();
this.checkpoint_ = null;
}
checkpoint (label) {
console.log('CHECKPOINT', label);
this.checkpoint_ = label;
}
}
/**
* @class CheckpointExpectation
* @classdesc The CheckpointExpectation class is used to represent an expectation that a specific checkpoint
* will be reached during the execution of a work unit. It includes methods to check if the checkpoint has
* been reached and to report the results of this check.
*/
class CheckpointExpectation {
constructor (workUnit, checkpoint) {
this.workUnit = workUnit;
this.checkpoint = checkpoint;
}
/**
* Constructor for CheckpointExpectation class.
* Initializes the instance with a WorkUnit and a checkpoint label.
* @param {WorkUnit} workUnit - The work unit associated with the checkpoint.
* @param {string} checkpoint - The checkpoint label to be checked.
*/
check () {
// TODO: should be true if checkpoint was ever reached
return this.workUnit.checkpoint_ == this.checkpoint;
}
report (log) {
if ( this.check() ) return;
log.log(
`operation(${this.workUnit.id}): ` +
`expected ${JSON.stringify(this.checkpoint)} ` +
`and got ${JSON.stringify(this.workUnit.checkpoint_)}.`
);
}
}
module.exports = {
WorkUnit,
CheckpointExpectation,
};
src/backend/src/modules/core/lib/identifier.js
+128
View File
@@ -0,0 +1,128 @@
// METADATA // {"def":"core.util.identutil","ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const adjectives = [
'amazing', 'ambitious', 'articulate', 'cool', 'bubbly', 'mindful', 'noble', 'savvy', 'serene',
'sincere', 'sleek', 'sparkling', 'spectacular', 'splendid', 'spotless', 'stunning',
'awesome', 'beaming', 'bold', 'brilliant', 'cheerful', 'modest', 'motivated',
'friendly', 'fun', 'funny', 'generous', 'gifted', 'graceful', 'grateful',
'passionate', 'patient', 'peaceful', 'perceptive', 'persistent',
'helpful', 'sensible', 'loyal', 'honest', 'clever', 'capable',
'calm', 'smart', 'genius', 'bright', 'charming', 'creative', 'diligent', 'elegant', 'fancy',
'colorful', 'avid', 'active', 'gentle', 'happy', 'intelligent',
'jolly', 'kind', 'lively', 'merry', 'nice', 'optimistic', 'polite',
'quiet', 'relaxed', 'silly', 'witty', 'young',
'strong', 'brave', 'agile', 'bold', 'confident', 'daring',
'fearless', 'heroic', 'mighty', 'powerful', 'valiant', 'wise', 'wonderful', 'zealous',
'warm', 'swift', 'neat', 'tidy', 'nifty', 'lucky', 'keen',
'blue', 'red', 'aqua', 'green', 'orange', 'pink', 'purple', 'cyan', 'magenta', 'lime',
'teal', 'lavender', 'beige', 'maroon', 'navy', 'olive', 'silver', 'gold', 'ivory',
];
const nouns = [
'street', 'roof', 'floor', 'tv', 'idea', 'morning', 'game', 'wheel', 'bag', 'clock', 'pencil', 'pen',
'magnet', 'chair', 'table', 'house', 'room', 'book', 'car', 'tree', 'candle', 'light', 'planet',
'flower', 'bird', 'fish', 'sun', 'moon', 'star', 'cloud', 'rain', 'snow', 'wind', 'mountain',
'river', 'lake', 'sea', 'ocean', 'island', 'bridge', 'road', 'train', 'plane', 'ship', 'bicycle',
'circle', 'square', 'garden', 'harp', 'grass', 'forest', 'rock', 'cake', 'pie', 'cookie', 'candy',
'butterfly', 'computer', 'phone', 'keyboard', 'mouse', 'cup', 'plate', 'glass', 'door',
'window', 'key', 'wallet', 'pillow', 'bed', 'blanket', 'soap', 'towel', 'lamp', 'mirror',
'camera', 'hat', 'shirt', 'pants', 'shoes', 'watch', 'ring',
'necklace', 'ball', 'toy', 'doll', 'kite', 'balloon', 'guitar', 'violin', 'piano', 'drum',
'trumpet', 'flute', 'viola', 'cello', 'harp', 'banjo', 'tuba',
]
const words = {
adjectives,
nouns,
};
/**
* Select a random item from an array using a random number generator function.
*
* @param {Array<T>} arr - The array to select an item from
* @param {function} [random=Math.random] - Random number generator function
* @returns {T} A random item from the array
*/
const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)];
/**
* A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999).
* The result is returned as a string with components separated by the specified separator.
* It is useful when you need to create unique identifiers that are also human-friendly.
*
* @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided.
* @param {function} [rng=Math.random] - Random number generator function
* @returns {string} A unique, human-friendly identifier.
*
* @example
*
* let identifier = window.generate_identifier();
* // identifier would be something like 'clever-idea-123'
*
*/
function generate_identifier(separator = '_', rng = Math.random){
// return a random combination of first_adj + noun + number (between 0 and 9999)
// e.g. clever-idea-123
return [
randomItem(adjectives, rng),
randomItem(nouns, rng),
Math.floor(rng() * 10000),
].join(separator);
}
// Character set used for generating human-readable, case-insensitive random codes
const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
function generate_random_code(n, {
rng = Math.random,
chars = HUMAN_READABLE_CASE_INSENSITIVE
} = {}) {
let code = '';
for ( let i = 0 ; i < n ; i++ ) {
code += randomItem(chars, rng);
}
return code;
}
/**
* Composes a code by combining a mask string with a base-36 converted number
* @param {string} mask - Initial string template to use as base
* @param {number} value - Number to convert to base-36 and append to the right
* @returns {string} Combined uppercase code
*/
function compose_code(mask, value) {
const right_str = value.toString(36);
let out_str = mask;
console.log('right_str', right_str);
console.log('out_str', out_str);
for ( let i = 0 ; i < right_str.length ; i++ ) {
out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i];
}
out_str = out_str.toUpperCase();
return out_str;
}
module.exports = {
randomItem,
generate_identifier,
generate_random_code,
};
src/backend/src/util/linux.js → src/backend/src/modules/core/lib/linux.js
+4 -3
View File
@@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const SmolUtil = require("./smolutil");
const smol = require('@heyputer/putility').libs.smol;
const parse_meminfo = text => {
const lines = text.split('\n');
@@ -26,8 +26,8 @@ const parse_meminfo = text => {
for ( const line of lines ) {
if ( line.trim().length == 0 ) continue;
const [key, value_and_unit] = SmolUtil.split(line, ':', { trim: true });
const [value, _] = SmolUtil.split(value_and_unit, ' ', { trim: true });
const [key, value_and_unit] = smol.split(line, ':', { trim: true });
const [value, _] = smol.split(value_and_unit, ' ', { trim: true });
// note: unit is always 'kB' so we discard it
meminfo[key] = Number.parseInt(value);
}
@@ -38,3 +38,4 @@ const parse_meminfo = text => {
module.exports = {
parse_meminfo,
};
src/backend/src/modules/core/lib/log.js
+73
View File
@@ -0,0 +1,73 @@
// METADATA // {"def":"core.util.logutil","ai-commented":{"service":"openai-completion","model":"gpt-4o"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const log_epoch = Date.now();
/**
* Stringifies a log entry into a formatted string for console output.
* @param {Object} logEntry - The log entry object containing:
* @param {string} [prefix] - Optional prefix for the log message.
* @param {Object} log_lvl - Log level object with properties for label, escape code, etc.
* @param {string[]} crumbs - Array of context crumbs.
* @param {string} message - The log message.
* @param {Object} fields - Additional fields to be included in the log.
* @param {Object} objects - Objects to be logged.
* @returns {string} A formatted string representation of the log entry.
*/
const stringify_log_entry = ({ prefix, log_lvl, crumbs, message, fields, objects }) => {
const { colorize } = require('json-colorizer');
let lines = [], m;
const lf = () => {
if ( ! m ) return;
lines.push(m);
m = '';
}
m = prefix ? `${prefix} ` : '';
m += `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`;
for ( const crumb of crumbs ) {
m += `::${crumb}`;
}
m += `\x1B[${log_lvl.esc}m]\x1B[0m`;
if ( fields.timestamp ) {
// display seconds since logger epoch
const n = (fields.timestamp - log_epoch) / 1000;
m += ` (${n.toFixed(3)}s)`;
}
m += ` ${message} `;
lf();
for ( const k in fields ) {
if ( k === 'timestamp' ) continue;
let v; try {
v = colorize(JSON.stringify(fields[k]));
} catch (e) {
v = '' + fields[k];
}
m += ` \x1B[1m${k}:\x1B[0m ${v}`;
lf();
}
return lines.join('\n');
};
module.exports = {
stringify_log_entry,
log_epoch,
};
src/backend/src/modules/core/lib/stdio.js
+70
View File
@@ -0,0 +1,70 @@
// METADATA // {"ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* Strip ANSI escape sequences from a string (e.g. color codes)
* and then return the length of the resulting string.
*
* @param {string} str - The string to calculate visible length for
* @returns {number} The length of the string without ANSI escape sequences
*/
const visible_length = (str) => {
// eslint-disable-next-line no-control-regex
return str.replace(/\x1b\[[0-9;]*m/g, '').length;
};
/**
* Split a string into lines according to the terminal width,
* preserving ANSI escape sequences, and return an array of lines.
*
* @param {string} str The string to split into lines
* @returns {string[]} Array of lines split according to terminal width
*/
const split_lines = (str) => {
const lines = [];
let line = '';
let line_length = 0;
for (const c of str) {
line += c;
if (c === '\n') {
lines.push(line);
line = '';
line_length = 0;
} else {
line_length++;
if (line_length >= process.stdout.columns) {
lines.push(line);
line = '';
line_length = 0;
}
}
}
if (line.length) {
lines.push(line);
}
return lines;
};
module.exports = {
visible_length,
split_lines,
};
src/backend/src/modules/puterai/AIChatService.js
+145 -2
View File
@@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const APIError = require("../../api/APIError");
const { PermissionUtil } = require("../../services/auth/PermissionService");
const BaseService = require("../../services/BaseService");
@@ -6,14 +7,31 @@ const { TypeSpec } = require("../../services/drivers/meta/Construct");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { Context } = require("../../util/context");
// Maximum number of fallback attempts when a model fails, including the first attempt
const MAX_FALLBACKS = 3 + 1; // includes first attempt
/**
* AIChatService class extends BaseService to provide AI chat completion functionality.
* Manages multiple AI providers, models, and fallback mechanisms for chat interactions.
* Handles model registration, usage tracking, cost calculation, content moderation,
* and implements the puter-chat-completion driver interface. Supports streaming responses
* and maintains detailed model information including pricing and capabilities.
*/
class AIChatService extends BaseService {
static MODULES = {
kv: globalThis.kv,
uuidv4: require('uuid').v4,
}
/**
* Initializes the service by setting up core properties.
* Creates empty arrays for providers and model lists,
* and initializes an empty object for the model map.
* Called during service instantiation.
* @private
*/
_construct () {
this.providers = [];
@@ -21,6 +39,13 @@ class AIChatService extends BaseService {
this.detail_model_list = [];
this.detail_model_map = {};
}
/**
* Initializes the service by setting up empty arrays and maps for providers and models.
* This method is called during service construction to establish the initial state.
* Creates empty arrays for providers, simple model list, and detailed model list,
* as well as an empty object for the detailed model map.
* @private
*/
_init () {
this.kvkey = this.modules.uuidv4();
@@ -28,6 +53,8 @@ class AIChatService extends BaseService {
const svc_event = this.services.get('event');
svc_event.on('ai.prompt.report-usage', async (_, details) => {
if ( details.service_used === 'fake-chat' ) return;
const values = {
user_id: details.actor?.type?.user?.id,
app_id: details.actor?.type?.app?.id ?? null,
@@ -67,6 +94,19 @@ class AIChatService extends BaseService {
});
}
/**
* Handles consolidation during service boot by registering service aliases
* and populating model lists/maps from providers.
*
* Registers each provider as an 'ai-chat' service alias and fetches their
* available models and pricing information. Populates:
* - simple_model_list: Basic list of supported models
* - detail_model_list: Detailed model info including costs
* - detail_model_map: Maps model IDs/aliases to their details
*
* @returns {Promise<void>}
*/
async ['__on_boot.consolidation'] () {
{
const svc_driver = this.services.get('driver')
@@ -83,6 +123,15 @@ class AIChatService extends BaseService {
// Populate simple model list
{
/**
* Populates the simple model list by fetching available models from the delegate service.
* Wraps the delegate.list() call in a try-catch block to handle potential errors gracefully.
* If the call fails, logs the error and returns an empty array to avoid breaking the service.
* The fetched models are added to this.simple_model_list.
*
* @private
* @returns {Promise<void>}
*/
const models = await (async () => {
try {
return await delegate.list() ?? [];
@@ -96,6 +145,14 @@ class AIChatService extends BaseService {
// Populate detail model list and map
{
/**
* Populates the detail model list and map with model information from the provider.
* Fetches detailed model data including pricing and capabilities.
* Handles model aliases and potential conflicts by storing multiple models in arrays.
* Annotates models with their provider service name.
* Catches and logs any errors during model fetching.
* @private
*/
const models = await (async () => {
try {
return await delegate.models() ?? [];
@@ -112,6 +169,13 @@ class AIChatService extends BaseService {
});
}
this.detail_model_list.push(...annotated_models);
/**
* Helper function to set or push a model into the detail_model_map.
* If there's no existing entry for the key, sets it directly.
* If there's a conflict, converts the entry to an array and pushes the new model.
* @param {string} key - The model ID or alias
* @param {Object} model - The model details to add
*/
const set_or_push = (key, model) => {
// Typical case: no conflict
if ( ! this.detail_model_map[key] ) {
@@ -153,16 +217,46 @@ class AIChatService extends BaseService {
}
},
['puter-chat-completion']: {
/**
* Implements the 'puter-chat-completion' interface methods for AI chat functionality.
* Handles model selection, fallbacks, usage tracking, and moderation.
* Contains methods for listing available models, completing chat prompts,
* and managing provider interactions.
*
* @property {Object} models - Available AI models with details like costs
* @property {Object} list - Simplified list of available models
* @property {Object} complete - Main method for chat completion requests
* @param {Object} parameters - Chat completion parameters including model and messages
* @returns {Promise<Object>} Chat completion response with usage stats
* @throws {Error} If service is called directly or no fallback models available
*/
async models () {
const delegate = this.get_delegate();
if ( ! delegate ) return await this.models_();
return await delegate.models();
},
/**
* Returns list of available AI models with detailed information
*
* Delegates to the intended service's models() method if a delegate exists,
* otherwise returns the internal detail_model_list containing all available models
* across providers with their capabilities and pricing information.
*
* @returns {Promise<Array>} Array of model objects with details like id, provider, cost, etc.
*/
async list () {
const delegate = this.get_delegate();
if ( ! delegate ) return await this.list_();
return await delegate.list();
},
/**
* Lists available AI models in a simplified format
*
* Returns a list of basic model information from all registered providers.
* This is a simpler version compared to models() that returns less detailed info.
*
* @returns {Promise<Array>} Array of simplified model objects
*/
async complete (parameters) {
const client_driver_call = Context.get('client_driver_call');
let { test_mode, intended_service, response_metadata } = client_driver_call;
@@ -197,7 +291,9 @@ class AIChatService extends BaseService {
const svc_driver = this.services.get('driver');
let ret, error, errors = [];
let service_used = intended_service;
let model_used = this.get_model_from_request(parameters);
let model_used = this.get_model_from_request(parameters, {
intended_service
});
await this.check_usage_({
actor: Context.get('actor'),
service: service_used,
@@ -330,6 +426,17 @@ class AIChatService extends BaseService {
}
}
/**
* Checks if the user has permission to use AI services and verifies usage limits
*
* @param {Object} params - The check parameters
* @param {Object} params.actor - The user/actor making the request
* @param {string} params.service - The AI service being used
* @param {string} params.model - The model being accessed
* @throws {APIError} If usage is not allowed or limits are exceeded
* @private
*/
async check_usage_ ({ actor, service, model }) {
const svc_permission = this.services.get('permission');
const svc_event = this.services.get('event');
@@ -359,6 +466,20 @@ class AIChatService extends BaseService {
}
}
/**
* Moderates chat messages for inappropriate content using OpenAI's moderation service
*
* @param {Object} params - The parameters object
* @param {Array} params.messages - Array of chat messages to moderate
* @returns {Promise<boolean>} Returns true if content is appropriate, false if flagged
*
* @description
* Extracts text content from messages and checks each against OpenAI's moderation.
* Handles both string content and structured message objects.
* Returns false immediately if any message is flagged as inappropriate.
* Returns true if OpenAI service is unavailable or all messages pass moderation.
*/
async moderate ({ messages }) {
const svc_openai = this.services.get('openai-completion');
@@ -385,14 +506,28 @@ class AIChatService extends BaseService {
return true;
}
async models_ () {
return this.detail_model_list;
}
/**
* Returns a list of available AI models with basic details
* @returns {Promise<Array>} Array of simple model objects containing basic model information
*/
async list_ () {
return this.simple_model_list;
}
/**
* Gets the appropriate delegate service for handling chat completion requests.
* If the intended service is this service (ai-chat), returns undefined.
* Otherwise returns the intended service wrapped as a puter-chat-completion interface.
*
* @returns {Object|undefined} The delegate service or undefined if intended service is ai-chat
*/
get_delegate () {
const client_driver_call = Context.get('client_driver_call');
if ( client_driver_call.intended_service === this.service_name ) {
@@ -463,13 +598,21 @@ class AIChatService extends BaseService {
});
}
get_model_from_request (parameters) {
get_model_from_request (parameters, modified_context = {}) {
const client_driver_call = Context.get('client_driver_call');
let { intended_service } = client_driver_call;
if ( modified_context.intended_service ) {
intended_service = modified_context.intended_service;
}
let model = parameters.model;
if ( ! model ) {
const service = this.services.get(intended_service);
console.log({
what: intended_service,
w: service.get_default_model
});
if ( ! service.get_default_model ) {
throw new Error('could not infer model from service');
}
src/backend/src/modules/puterai/AIInterfaceService.js
+14
View File
@@ -1,6 +1,20 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const BaseService = require("../../services/BaseService");
/**
* Service class that manages AI interface registrations and configurations.
* Handles registration of various AI services including OCR, chat completion,
* image generation, and text-to-speech interfaces. Each interface defines
* its available methods, parameters, and expected results.
* @extends BaseService
*/
class AIInterfaceService extends BaseService {
/**
* Service class for managing AI interface registrations and configurations.
* Extends the base service to provide AI-related interface management.
* Handles registration of OCR, chat completion, image generation, and TTS interfaces.
*/
async ['__on_driver.register.interfaces'] () {
const svc_registry = this.services.get('registry');
const col_interfaces = svc_registry.get('interfaces');
src/backend/src/modules/puterai/AITestModeService.js
+12
View File
@@ -1,6 +1,18 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const BaseService = require("../../services/BaseService");
/**
* Service class that handles AI test mode functionality.
* Extends BaseService to register test services for AI chat completions.
* Used for testing and development of AI-related features by providing
* a mock implementation of the chat completion service.
*/
class AITestModeService extends BaseService {
/**
* Service for managing AI test mode functionality
* @extends BaseService
*/
async _init () {
const svc_driver = this.services.get('driver');
svc_driver.register_test_service('puter-chat-completion', 'ai-chat');
src/backend/src/modules/puterai/AWSPollyService.js
+55
View File
@@ -1,12 +1,28 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PollyClient, SynthesizeSpeechCommand, DescribeVoicesCommand } = require("@aws-sdk/client-polly");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
/**
* AWSPollyService class provides text-to-speech functionality using Amazon Polly.
* Extends BaseService to integrate with AWS Polly for voice synthesis operations.
* Implements voice listing, speech synthesis, and voice selection based on language.
* Includes caching for voice descriptions and supports both text and SSML inputs.
* @extends BaseService
*/
class AWSPollyService extends BaseService {
static MODULES = {
kv: globalThis.kv,
}
/**
* Initializes the service by creating an empty clients object.
* This method is called during service construction to set up
* the internal state needed for AWS Polly client management.
* @returns {Promise<void>}
*/
async _construct () {
this.clients_ = {};
}
@@ -18,6 +34,14 @@ class AWSPollyService extends BaseService {
}
},
['puter-tts']: {
/**
* Implements the driver interface methods for text-to-speech functionality
* Contains methods for listing available voices and synthesizing speech
* @interface
* @property {Object} list_voices - Lists available Polly voices with language info
* @property {Object} synthesize - Converts text to speech using specified voice/language
* @property {Function} supports_test_mode - Indicates test mode support for methods
*/
async list_voices () {
const polly_voices = await this.describe_voices();
@@ -64,6 +88,12 @@ class AWSPollyService extends BaseService {
}
}
/**
* Creates AWS credentials object for authentication
* @private
* @returns {Object} Object containing AWS access key ID and secret access key
*/
_create_aws_credentials () {
return {
accessKeyId: this.config.aws.access_key,
@@ -86,6 +116,13 @@ class AWSPollyService extends BaseService {
return this.clients_[region];
}
/**
* Describes available AWS Polly voices and caches the results
* @returns {Promise<Object>} Response containing array of voice details in Voices property
* @description Fetches voice information from AWS Polly API and caches it for 10 minutes
* Uses KV store for caching to avoid repeated API calls
*/
async describe_voices () {
let voices = this.modules.kv.get('svc:polly:voices');
if ( voices ) {
@@ -109,6 +146,17 @@ class AWSPollyService extends BaseService {
return response;
}
/**
* Synthesizes speech from text using AWS Polly
* @param {string} text - The text to synthesize
* @param {Object} options - Synthesis options
* @param {string} options.format - Output audio format (e.g. 'mp3')
* @param {string} [options.voice_id] - AWS Polly voice ID to use
* @param {string} [options.language] - Language code (e.g. 'en-US')
* @param {string} [options.text_type] - Type of input text ('text' or 'ssml')
* @returns {Promise<AWS.Polly.SynthesizeSpeechOutput>} The synthesized speech response
*/
async synthesize_speech (text, { format, voice_id, language, text_type }) {
const client = this._get_client(this.config.aws.region);
@@ -140,6 +188,13 @@ class AWSPollyService extends BaseService {
return response;
}
/**
* Attempts to find an appropriate voice for the given language code
* @param {string} language - The language code to find a voice for (e.g. 'en-US')
* @returns {Promise<?string>} The voice ID if found, null if no matching voice exists
* @private
*/
async maybe_get_language_appropriate_voice_ (language) {
const voices = await this.describe_voices();
src/backend/src/modules/puterai/AWSTextractService.js
+48 -1
View File
@@ -1,9 +1,23 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { TextractClient, AnalyzeDocumentCommand, InvalidS3ObjectException } = require("@aws-sdk/client-textract");
const BaseService = require("../../services/BaseService");
const APIError = require("../../api/APIError");
/**
* AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract
* Extends BaseService to integrate with AWS Textract for document analysis and text extraction.
* Implements driver capabilities and puter-ocr interface for document recognition.
* Handles both S3-stored and buffer-based document processing with automatic region management.
*/
class AWSTextractService extends BaseService {
/**
* AWS Textract service for OCR functionality
* Provides document analysis capabilities using AWS Textract API
* Implements interfaces for OCR recognition and driver capabilities
* @extends BaseService
*/
_construct () {
this.clients_ = {};
}
@@ -15,6 +29,13 @@ class AWSTextractService extends BaseService {
}
},
['puter-ocr']: {
/**
* Performs OCR recognition on a document using AWS Textract
* @param {Object} params - Recognition parameters
* @param {Object} params.source - The document source to analyze
* @param {boolean} params.test_mode - If true, returns sample test output instead of processing
* @returns {Promise<Object>} Recognition results containing blocks of text with confidence scores
*/
async recognize ({ source, test_mode }) {
if ( test_mode ) {
return {
@@ -61,6 +82,12 @@ class AWSTextractService extends BaseService {
},
};
/**
* Creates AWS credentials object for authentication
* @private
* @returns {Object} Object containing AWS access key ID and secret access key
*/
_create_aws_credentials () {
return {
accessKeyId: this.config.aws.access_key,
@@ -83,6 +110,15 @@ class AWSTextractService extends BaseService {
return this.clients_[region];
}
/**
* Analyzes a document using AWS Textract to extract text and layout information
* @param {FileFacade} file_facade - Interface to access the document file
* @returns {Promise<Object>} The raw Textract API response containing extracted text blocks
* @throws {Error} If document analysis fails or no suitable input format is available
* @description Processes document through Textract's AnalyzeDocument API with LAYOUT feature.
* Will attempt to use S3 direct access first, falling back to buffer upload if needed.
*/
async analyze_document (file_facade) {
const {
client, document, using_s3
@@ -119,6 +155,18 @@ class AWSTextractService extends BaseService {
throw new Error('expected to be unreachable');
}
/**
* Gets AWS client and document configuration for Textract processing
* @param {Object} file_facade - File facade object containing document source info
* @param {boolean} [force_buffer] - If true, forces using buffer instead of S3
* @returns {Promise<Object>} Object containing:
* - client: Configured AWS Textract client
* - document: Document configuration for Textract
* - using_s3: Boolean indicating if using S3 source
* @throws {APIError} If file does not exist
* @throws {Error} If no suitable input format is available
*/
async _get_client_and_document (file_facade, force_buffer) {
const try_s3info = await file_facade.get('s3-info');
if ( try_s3info && ! force_buffer ) {
@@ -137,7 +185,6 @@ class AWSTextractService extends BaseService {
const try_buffer = await file_facade.get('buffer');
if ( try_buffer ) {
const base64 = try_buffer.toString('base64');
return {
client: this._get_client(),
document: {
src/backend/src/modules/puterai/ClaudeEnoughService.js
+14
View File
@@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { XAIService } = require("./XAIService");
const CLAUDE_ENOUGH_PROMPT = `
@@ -19,7 +20,20 @@ const CLAUDE_ENOUGH_PROMPT = `
user of the driver interface (typically an app on Puter):
`.replace('\n', ' ').trim();
/**
* ClaudeEnoughService - A service class that implements a Claude-like AI interface
* Extends XAIService to provide Claude-compatible responses while using alternative AI models.
* Includes custom system prompts and model adaptation to simulate Claude's behavior
* in the Puter platform's chat completion interface.
*/
class ClaudeEnoughService extends XAIService {
/**
* Service that emulates Claude's behavior using alternative AI models
* @extends XAIService
* @description Provides a Claude-like interface while using other AI models as the backend.
* Includes custom system prompts and model adaptations to approximate Claude's behavior.
*/
get_system_prompt () {
return CLAUDE_ENOUGH_PROMPT;
}
src/backend/src/modules/puterai/ClaudeService.js
+68 -1
View File
@@ -1,10 +1,11 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { default: Anthropic } = require("@anthropic-ai/sdk");
const BaseService = require("../../services/BaseService");
const { whatis } = require("../../util/langutil");
const { PassThrough } = require("stream");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const APIError = require("../../api/APIError");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
const PUTER_PROMPT = `
You are running on an open-source platform called Puter,
@@ -15,13 +16,29 @@ const PUTER_PROMPT = `
user of the driver interface (typically an app on Puter):
`.replace('\n', ' ').trim();
// Maximum number of input tokens allowed for Claude API requests
const MAX_CLAUDE_INPUT_TOKENS = 10000;
/**
* ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models.
* Implements the puter-chat-completion interface for handling AI chat interactions.
* Manages message streaming, token limits, model selection, and API communication with Claude.
* Supports system prompts, message adaptation, and usage tracking.
* @extends BaseService
*/
class ClaudeService extends BaseService {
static MODULES = {
Anthropic: require('@anthropic-ai/sdk'),
}
/**
* Initializes the Claude service by creating an Anthropic client instance
* and registering this service as a provider with the AI chat service.
* @private
* @returns {Promise<void>}
*/
async _init () {
this.anthropic = new Anthropic({
apiKey: this.config.apiKey
@@ -34,15 +51,34 @@ class ClaudeService extends BaseService {
});
}
/**
* Returns the default model identifier for Claude API interactions
* @returns {string} The default model ID 'claude-3-5-sonnet-latest'
*/
get_default_model () {
return 'claude-3-5-sonnet-latest';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the puter-chat-completion interface for Claude AI models
* @param {Object} options - Configuration options for the chat completion
* @param {Array} options.messages - Array of message objects containing the conversation history
* @param {boolean} options.stream - Whether to stream the response
* @param {string} [options.model] - The Claude model to use, defaults to claude-3-5-sonnet-latest
* @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object
*/
async models () {
return await this.models_();
},
/**
* Returns a list of available model names including their aliases
* @returns {Promise<string[]>} Array of model identifiers and their aliases
* @description Retrieves all available Claude model IDs and their aliases,
* flattening them into a single array of strings that can be used for model selection
*/
async list () {
const models = await this.models_();
const model_names = [];
@@ -54,6 +90,15 @@ class ClaudeService extends BaseService {
}
return model_names;
},
/**
* Completes a chat interaction with the Claude AI model
* @param {Object} options - The completion options
* @param {Array} options.messages - Array of chat messages to process
* @param {boolean} options.stream - Whether to stream the response
* @param {string} [options.model] - The Claude model to use, defaults to service default
* @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object
* @throws {APIError} If input token count exceeds maximum allowed
*/
async complete ({ messages, stream, model }) {
const adapted_messages = [];
@@ -87,6 +132,15 @@ class ClaudeService extends BaseService {
}
}
/**
* Calculates the approximate token count for the input messages
* @private
* @returns {number} Estimated token count based on character length divided by 4
* @description Uses a simple character length based heuristic to estimate tokens.
* While not perfectly accurate, this provides a reasonable approximation for
* checking against max token limits before sending to Claude API.
*/
const token_count = (() => {
const text = JSON.stringify(adapted_messages) +
JSON.stringify(system_prompts);
@@ -165,6 +219,19 @@ class ClaudeService extends BaseService {
}
}
/**
* Retrieves available Claude AI models and their specifications
* @returns {Promise<Array>} Array of model objects containing:
* - id: Model identifier
* - name: Display name
* - aliases: Alternative names for the model
* - context: Maximum context window size
* - cost: Pricing details (currency, token counts, input/output costs)
* - qualitative_speed: Relative speed rating
* - max_output: Maximum output tokens
* - training_cutoff: Training data cutoff date
*/
async models_ () {
return [
{
src/backend/src/modules/puterai/FakeChatService.js
+25
View File
@@ -1,11 +1,36 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const BaseService = require("../../services/BaseService");
/**
* FakeChatService - A mock implementation of a chat service that extends BaseService.
* Provides fake chat completion responses using Lorem Ipsum text generation.
* Used for testing and development purposes when a real chat service is not needed.
* Implements the 'puter-chat-completion' interface with list() and complete() methods.
*/
class FakeChatService extends BaseService {
get_default_model () {
return 'fake';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implementation interface for the puter-chat-completion service.
* Provides fake chat completion functionality for testing purposes.
* Contains methods for listing available models and generating mock responses.
* @interface
*/
async list () {
return ['fake'];
},
/**
* Simulates a chat completion request by generating random Lorem Ipsum text
* @param {Object} params - The completion parameters
* @param {Array} params.messages - Array of chat messages (unused in fake implementation)
* @param {boolean} params.stream - Whether to stream the response (unused in fake implementation)
* @param {string} params.model - The model to use (unused in fake implementation)
* @returns {Object} A simulated chat completion response with Lorem Ipsum content
*/
async complete ({ messages, stream, model }) {
const { LoremIpsum } = require('lorem-ipsum');
const li = new LoremIpsum({
src/backend/src/modules/puterai/GroqAIService.js
+57 -1
View File
@@ -1,14 +1,31 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require("stream");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { nou } = require("../../util/langutil");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* Service class for integrating with Groq AI's language models.
* Extends BaseService to provide chat completion capabilities through the Groq API.
* Implements the puter-chat-completion interface for model management and text generation.
* Supports both streaming and non-streaming responses, handles multiple models including
* various versions of Llama, Mixtral, and Gemma, and manages usage tracking.
* @class GroqAIService
* @extends BaseService
*/
class GroqAIService extends BaseService {
static MODULES = {
Groq: require('groq-sdk'),
}
/**
* Initializes the GroqAI service by setting up the Groq client and registering with the AI chat provider
* @returns {Promise<void>}
* @private
*/
async _init () {
const Groq = require('groq-sdk');
this.client = new Groq({
@@ -22,20 +39,47 @@ class GroqAIService extends BaseService {
});
}
/**
* Returns the default model ID for the Groq AI service
* @returns {string} The default model ID 'llama-3.1-8b-instant'
*/
get_default_model () {
return 'llama-3.1-8b-instant';
}
static IMPLEMENTS = {
'puter-chat-completion': {
/**
* Defines the interface implementations for the puter-chat-completion service
* Contains methods for listing models and handling chat completions
* @property {Object} models - Returns available AI models
* @property {Object} list - Lists raw model data from the Groq API
* @property {Object} complete - Handles chat completion requests with optional streaming
* @returns {Object} Interface implementation object
*/
async models () {
return await this.models_();
},
/**
* Lists available AI models from the Groq API
* @returns {Promise<Array>} Array of model objects from the API's data field
* @description Unwraps and returns the model list from the Groq API response,
* which comes wrapped in an object with {object: "list", data: [...]}
*/
async list () {
// They send: { "object": "list", data }
const funny_wrapper = await this.client.models.list();
return funny_wrapper.data;
},
/**
* Completes a chat interaction using the Groq API
* @param {Object} options - The completion options
* @param {Array<Object>} options.messages - Array of message objects containing the conversation history
* @param {string} [options.model] - The model ID to use for completion. Defaults to service's default model
* @param {boolean} [options.stream] - Whether to stream the response
* @returns {TypedValue|Object} Returns either a TypedValue with streaming response or completion object with usage stats
*/
async complete ({ messages, model, stream }) {
for ( let i = 0; i < messages.length; i++ ) {
const message = messages[i];
@@ -101,6 +145,18 @@ class GroqAIService extends BaseService {
}
};
/**
* Returns an array of available AI models with their specifications
*
* Each model object contains:
* - id: Unique identifier for the model
* - name: Human-readable name
* - context: Maximum context window size in tokens
* - cost: Pricing details including currency and token rates
*
* @returns {Array<Object>} Array of model specification objects
*/
models_ () {
return [
{
src/backend/src/modules/puterai/MistralAIService.js
+53 -1
View File
@@ -1,15 +1,30 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require("stream");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { nou } = require("../../util/langutil");
const axios = require('axios');
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* MistralAIService class extends BaseService to provide integration with the Mistral AI API.
* Implements chat completion functionality with support for various Mistral models including
* mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and
* non-streaming responses, token usage tracking, and model management. Provides cost information
* for different models and implements the puter-chat-completion interface.
*/
class MistralAIService extends BaseService {
static MODULES = {
'@mistralai/mistralai': require('@mistralai/mistralai'),
}
/**
* Initializes the service's cost structure for different Mistral AI models.
* Sets up pricing information for various models including token costs for input/output.
* Each model entry specifies currency (usd-cents) and costs per million tokens.
* @private
*/
_construct () {
this.costs_ = {
'mistral-large-latest': {
@@ -80,6 +95,12 @@ class MistralAIService extends BaseService {
},
};
}
/**
* Initializes the service's cost structure for different Mistral AI models.
* Sets up pricing information for various models including token costs for input/output.
* Each model entry specifies currency (USD cents) and costs per million tokens.
* @private
*/
async _init () {
const require = this.require;
const { Mistral } = require('@mistralai/mistralai');
@@ -97,6 +118,13 @@ class MistralAIService extends BaseService {
// TODO: make this event-driven so it doesn't hold up boot
await this.populate_models_();
}
/**
* Populates the internal models array with available Mistral AI models and their configurations.
* Makes an API call to fetch model data, then processes and filters models based on cost information.
* Each model entry includes id, name, aliases, context window size, capabilities, and pricing.
* @private
* @returns {Promise<void>}
*/
async populate_models_ () {
const resp = await axios({
method: 'get',
@@ -131,17 +159,41 @@ class MistralAIService extends BaseService {
}
// return resp.data;
}
/**
* Populates the internal models array with available Mistral AI models and their metadata
* Fetches model data from the API, filters based on cost configuration, and stores
* model objects containing ID, name, aliases, context length, capabilities, and pricing
* @private
* @async
* @returns {void}
*/
get_default_model () {
return 'mistral-large-latest';
}
static IMPLEMENTS = {
'puter-chat-completion': {
/**
* Implements the puter-chat-completion interface for MistralAI service
* Provides methods for listing models and generating chat completions
* @interface
* @property {Function} models - Returns array of available model details
* @property {Function} list - Returns array of model IDs
* @property {Function} complete - Generates chat completion with optional streaming
*/
async models () {
return this.models_array_;
},
/**
* Returns an array of available AI models with their details
* @returns {Promise<Array>} Array of model objects containing id, name, aliases, context window size, capabilities, and cost information
*/
async list () {
return this.models_array_.map(m => m.id);
},
/**
* Returns an array of model IDs supported by the MistralAI service
* @returns {Promise<string[]>} Array of model identifier strings
*/
async complete ({ messages, stream, model }) {
for ( let i = 0; i < messages.length; i++ ) {
src/backend/src/modules/puterai/OpenAICompletionService.js
+77 -4
View File
@@ -1,17 +1,33 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require('stream');
const APIError = require('../../api/APIError');
const BaseService = require('../../services/BaseService');
const { TypedValue } = require('../../services/drivers/meta/Runtime');
const { Context } = require('../../util/context');
const SmolUtil = require('../../util/smolutil');
const smol = require('@heyputer/putility').libs.smol;
const { nou } = require('../../util/langutil');
const { TeePromise } = require('../../util/promise');
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* OpenAICompletionService class provides an interface to OpenAI's chat completion API.
* Extends BaseService to handle chat completions, message moderation, token counting,
* and streaming responses. Implements the puter-chat-completion interface and manages
* OpenAI API interactions with support for multiple models including GPT-4 variants.
* Handles usage tracking, spending records, and content moderation.
*/
class OpenAICompletionService extends BaseService {
static MODULES = {
openai: require('openai'),
tiktoken: require('tiktoken'),
}
/**
* Initializes the OpenAI service by setting up the API client with credentials
* and registering this service as a chat provider.
*
* @returns {Promise<void>} Resolves when initialization is complete
* @private
*/
async _init () {
const sk_key =
this.config?.openai?.secret_key ??
@@ -28,10 +44,21 @@ class OpenAICompletionService extends BaseService {
});
}
/**
* Gets the default model identifier for OpenAI completions
* @returns {string} The default model ID 'gpt-4o-mini'
*/
get_default_model () {
return 'gpt-4o-mini';
}
/**
* Returns an array of available AI models with their pricing information.
* Each model object includes an ID and cost details (currency, tokens, input/output rates).
* @returns {Promise<Array<{id: string, cost: {currency: string, tokens: number, input: number, output: number}}>}
*/
async models_ () {
return [
{
@@ -75,9 +102,25 @@ class OpenAICompletionService extends BaseService {
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the puter-chat-completion interface methods for model listing and chat completion
* @property {Object} models - Returns available AI models and their pricing
* @property {Function} list - Returns list of available model names/aliases
* @property {Function} complete - Handles chat completion requests with optional streaming
* @param {Object} params - Parameters for completion
* @param {Array} params.messages - Array of chat messages
* @param {boolean} params.test_mode - Whether to use test mode
* @param {boolean} params.stream - Whether to stream responses
* @param {string} params.model - Model ID to use
*/
async models () {
return await this.models_();
},
/**
* Retrieves a list of available AI models with their cost information
* @returns {Promise<Array>} Array of model objects containing id and cost details
* @private
*/
async list () {
const models = await this.models_();
const model_names = [];
@@ -89,6 +132,10 @@ class OpenAICompletionService extends BaseService {
}
return model_names;
},
/**
* Lists all available model names including aliases
* @returns {Promise<string[]>} Array of model IDs and their aliases
*/
async complete ({ messages, test_mode, stream, model }) {
// for now this code (also in AIChatService.js) needs to be
@@ -139,6 +186,14 @@ class OpenAICompletionService extends BaseService {
}
};
/**
* Checks text content against OpenAI's moderation API for inappropriate content
* @param {string} text - The text content to check for moderation
* @returns {Promise<Object>} Object containing flagged status and detailed results
* @property {boolean} flagged - Whether the content was flagged as inappropriate
* @property {Object} results - Raw moderation results from OpenAI API
*/
async check_moderation (text) {
// create moderation
const results = await this.openai.moderations.create({
@@ -160,6 +215,17 @@ class OpenAICompletionService extends BaseService {
};
}
/**
* Completes a chat conversation using OpenAI's API
* @param {Array} messages - Array of message objects or strings representing the conversation
* @param {Object} options - Configuration options
* @param {boolean} options.stream - Whether to stream the response
* @param {boolean} options.moderation - Whether to perform content moderation
* @param {string} options.model - The model to use for completion
* @returns {Promise<Object>} The completion response containing message and usage info
* @throws {Error} If messages are invalid or content is flagged by moderation
*/
async complete (messages, { stream, moderation, model }) {
// Validate messages
if ( ! Array.isArray(messages) ) {
@@ -234,7 +300,7 @@ class OpenAICompletionService extends BaseService {
if ( ! msg.content ) continue;
if ( typeof msg.content !== 'object' ) continue;
const content = SmolUtil.ensure_array(msg.content);
const content = smol.ensure_array(msg.content);
for ( const o of content ) {
if ( ! o.hasOwnProperty('image_url') ) continue;
@@ -260,7 +326,7 @@ class OpenAICompletionService extends BaseService {
if ( ! msg.content ) continue;
if ( typeof msg.content !== 'object' ) continue;
const content = SmolUtil.ensure_array(msg.content);
const content = smol.ensure_array(msg.content);
for ( const o of content ) {
// console.log('part of content', o);
@@ -338,6 +404,13 @@ class OpenAICompletionService extends BaseService {
const spending_meta = {};
spending_meta.timestamp = Date.now();
spending_meta.count_tokens_input = token_count;
/**
* Records spending metadata for the chat completion request and performs token counting.
* Initializes metadata object with timestamp and token counts for both input and output.
* Uses tiktoken to count output tokens from the completion response.
* Records spending data via spending service and increments usage counters.
* @private
*/
spending_meta.count_tokens_output = (() => {
// count output tokens (overestimate)
const enc = this.modules.tiktoken.encoding_for_model(model);
src/backend/src/modules/puterai/OpenAIImageGenerationService.js
+24
View File
@@ -1,11 +1,26 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { Context } = require("../../util/context");
/**
* Service class for generating images using OpenAI's DALL-E API.
* Extends BaseService to provide image generation capabilities through
* the puter-image-generation interface. Supports different aspect ratios
* (square, portrait, landscape) and handles API authentication, request
* validation, and spending tracking.
*/
class OpenAIImageGenerationService extends BaseService {
static MODULES = {
openai: require('openai'),
}
/**
* Initializes the OpenAI client with API credentials from config
* @private
* @async
* @returns {Promise<void>}
*/
async _init () {
const sk_key =
this.config?.openai?.secret_key ??
@@ -24,6 +39,15 @@ class OpenAIImageGenerationService extends BaseService {
}
},
['puter-image-generation']: {
/**
* Generates an image using OpenAI's DALL-E API
* @param {string} prompt - The text description of the image to generate
* @param {Object} options - Generation options
* @param {Object} options.ratio - Image dimensions ratio object with w/h properties
* @param {string} [options.model='dall-e-3'] - The model to use for generation
* @returns {Promise<string>} URL of the generated image
* @throws {Error} If prompt is not a string or ratio is invalid
*/
async generate ({ prompt, test_mode }) {
if ( test_mode ) {
return new TypedValue({
src/backend/src/modules/puterai/PuterAIModule.js
+16
View File
@@ -1,7 +1,23 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { AdvancedBase } = require("@heyputer/putility");
const config = require("../../config");
/**
* PuterAIModule class extends AdvancedBase to manage and register various AI services.
* This module handles the initialization and registration of multiple AI-related services
* including text processing, speech synthesis, chat completion, and image generation.
* Services are conditionally registered based on configuration settings, allowing for
* flexible deployment with different AI providers like AWS, OpenAI, Claude, Together AI,
* Mistral, Groq, and XAI.
* @extends AdvancedBase
*/
class PuterAIModule extends AdvancedBase {
/**
* Module for managing AI-related services in the Puter platform
* Extends AdvancedBase to provide core functionality
* Handles registration and configuration of various AI services like OpenAI, Claude, AWS services etc.
*/
async install (context) {
const services = context.get('services');
src/backend/src/modules/puterai/README.md
+333
View File
@@ -0,0 +1,333 @@
# PuterAIModule
PuterAIModule class extends AdvancedBase to manage and register various AI services.
This module handles the initialization and registration of multiple AI-related services
including text processing, speech synthesis, chat completion, and image generation.
Services are conditionally registered based on configuration settings, allowing for
flexible deployment with different AI providers like AWS, OpenAI, Claude, Together AI,
Mistral, Groq, and XAI.
## Services
### AIChatService
AIChatService class extends BaseService to provide AI chat completion functionality.
Manages multiple AI providers, models, and fallback mechanisms for chat interactions.
Handles model registration, usage tracking, cost calculation, content moderation,
and implements the puter-chat-completion driver interface. Supports streaming responses
and maintains detailed model information including pricing and capabilities.
#### Listeners
##### `boot.consolidation`
Handles consolidation during service boot by registering service aliases
and populating model lists/maps from providers.
Registers each provider as an 'ai-chat' service alias and fetches their
available models and pricing information. Populates:
- simple_model_list: Basic list of supported models
- detail_model_list: Detailed model info including costs
- detail_model_map: Maps model IDs/aliases to their details
#### Methods
##### `register_provider`
##### `moderate`
Moderates chat messages for inappropriate content using OpenAI's moderation service
###### Parameters
- **params:** The parameters object
- **params.messages:** Array of chat messages to moderate
##### `get_delegate`
Gets the appropriate delegate service for handling chat completion requests.
If the intended service is this service (ai-chat), returns undefined.
Otherwise returns the intended service wrapped as a puter-chat-completion interface.
##### `get_fallback_model`
Find an appropriate fallback model by sorting the list of models
by the euclidean distance of the input/output prices and selecting
the first one that is not in the tried list.
###### Parameters
- **param0:** null
##### `get_model_from_request`
### AIInterfaceService
Service class that manages AI interface registrations and configurations.
Handles registration of various AI services including OCR, chat completion,
image generation, and text-to-speech interfaces. Each interface defines
its available methods, parameters, and expected results.
#### Listeners
##### `driver.register.interfaces`
Service class for managing AI interface registrations and configurations.
Extends the base service to provide AI-related interface management.
Handles registration of OCR, chat completion, image generation, and TTS interfaces.
### AITestModeService
Service class that handles AI test mode functionality.
Extends BaseService to register test services for AI chat completions.
Used for testing and development of AI-related features by providing
a mock implementation of the chat completion service.
### AWSPollyService
AWSPollyService class provides text-to-speech functionality using Amazon Polly.
Extends BaseService to integrate with AWS Polly for voice synthesis operations.
Implements voice listing, speech synthesis, and voice selection based on language.
Includes caching for voice descriptions and supports both text and SSML inputs.
#### Methods
##### `describe_voices`
Describes available AWS Polly voices and caches the results
##### `synthesize_speech`
Synthesizes speech from text using AWS Polly
###### Parameters
- **text:** The text to synthesize
- **options:** Synthesis options
- **options.format:** Output audio format (e.g. 'mp3')
### AWSTextractService
AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract
Extends BaseService to integrate with AWS Textract for document analysis and text extraction.
Implements driver capabilities and puter-ocr interface for document recognition.
Handles both S3-stored and buffer-based document processing with automatic region management.
#### Methods
##### `analyze_document`
Analyzes a document using AWS Textract to extract text and layout information
###### Parameters
- **file_facade:** Interface to access the document file
### ClaudeEnoughService
ClaudeEnoughService - A service class that implements a Claude-like AI interface
Extends XAIService to provide Claude-compatible responses while using alternative AI models.
Includes custom system prompts and model adaptation to simulate Claude's behavior
in the Puter platform's chat completion interface.
#### Methods
##### `get_system_prompt`
Service that emulates Claude's behavior using alternative AI models
##### `adapt_model`
### ClaudeService
ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models.
Implements the puter-chat-completion interface for handling AI chat interactions.
Manages message streaming, token limits, model selection, and API communication with Claude.
Supports system prompts, message adaptation, and usage tracking.
#### Methods
##### `get_default_model`
Returns the default model identifier for Claude API interactions
### FakeChatService
FakeChatService - A mock implementation of a chat service that extends BaseService.
Provides fake chat completion responses using Lorem Ipsum text generation.
Used for testing and development purposes when a real chat service is not needed.
Implements the 'puter-chat-completion' interface with list() and complete() methods.
### GroqAIService
Service class for integrating with Groq AI's language models.
Extends BaseService to provide chat completion capabilities through the Groq API.
Implements the puter-chat-completion interface for model management and text generation.
Supports both streaming and non-streaming responses, handles multiple models including
various versions of Llama, Mixtral, and Gemma, and manages usage tracking.
#### Methods
##### `get_default_model`
Returns the default model ID for the Groq AI service
### MistralAIService
MistralAIService class extends BaseService to provide integration with the Mistral AI API.
Implements chat completion functionality with support for various Mistral models including
mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and
non-streaming responses, token usage tracking, and model management. Provides cost information
for different models and implements the puter-chat-completion interface.
#### Methods
##### `get_default_model`
Populates the internal models array with available Mistral AI models and their metadata
Fetches model data from the API, filters based on cost configuration, and stores
model objects containing ID, name, aliases, context length, capabilities, and pricing
### OpenAICompletionService
OpenAICompletionService class provides an interface to OpenAI's chat completion API.
Extends BaseService to handle chat completions, message moderation, token counting,
and streaming responses. Implements the puter-chat-completion interface and manages
OpenAI API interactions with support for multiple models including GPT-4 variants.
Handles usage tracking, spending records, and content moderation.
#### Methods
##### `get_default_model`
Gets the default model identifier for OpenAI completions
##### `check_moderation`
Checks text content against OpenAI's moderation API for inappropriate content
###### Parameters
- **text:** The text content to check for moderation
##### `complete`
Completes a chat conversation using OpenAI's API
###### Parameters
- **messages:** Array of message objects or strings representing the conversation
- **options:** Configuration options
- **options.stream:** Whether to stream the response
- **options.moderation:** Whether to perform content moderation
- **options.model:** The model to use for completion
### OpenAIImageGenerationService
Service class for generating images using OpenAI's DALL-E API.
Extends BaseService to provide image generation capabilities through
the puter-image-generation interface. Supports different aspect ratios
(square, portrait, landscape) and handles API authentication, request
validation, and spending tracking.
#### Methods
##### `generate`
### TogetherAIService
TogetherAIService class provides integration with Together AI's language models.
Extends BaseService to implement chat completion functionality through the
puter-chat-completion interface. Manages model listings, chat completions,
and streaming responses while handling usage tracking and model fallback testing.
#### Methods
##### `get_default_model`
Returns the default model ID for the Together AI service
### XAIService
XAIService class - Provides integration with X.AI's API for chat completions
Extends BaseService to implement the puter-chat-completion interface.
Handles model management, message adaptation, streaming responses,
and usage tracking for X.AI's language models like Grok.
#### Methods
##### `get_system_prompt`
Gets the system prompt used for AI interactions
##### `adapt_model`
##### `get_default_model`
Returns the default model identifier for the XAI service
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../api/APIError`
- `../../services/auth/PermissionService`
- `../../services/BaseService` (use.BaseService)
- `../../services/database/consts`
- `../../services/drivers/meta/Construct`
- `../../services/drivers/meta/Runtime`
- `../../util/context`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../services/BaseService` (use.BaseService)
- `../../api/APIError`
- `../../services/BaseService` (use.BaseService)
- `../../util/langutil`
- `../../services/drivers/meta/Runtime`
- `../../api/APIError`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/langutil`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/langutil`
- `../../util/promise`
- `../../api/APIError`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/context`
- `../../util/smolutil`
- `../../util/langutil`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/context`
- `../../config`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/langutil`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../util/langutil`
- `../../services/drivers/meta/Runtime`
- `../../util/promise`
src/backend/src/modules/puterai/TogetherAIService.js
+46 -1
View File
@@ -1,9 +1,18 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require("stream");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { nou } = require("../../util/langutil");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* TogetherAIService class provides integration with Together AI's language models.
* Extends BaseService to implement chat completion functionality through the
* puter-chat-completion interface. Manages model listings, chat completions,
* and streaming responses while handling usage tracking and model fallback testing.
* @extends BaseService
*/
class TogetherAIService extends BaseService {
static MODULES = {
['together-ai']: require('together-ai'),
@@ -11,6 +20,13 @@ class TogetherAIService extends BaseService {
uuidv4: require('uuid').v4,
}
/**
* Initializes the TogetherAI service by setting up the API client and registering as a chat provider
* @async
* @returns {Promise<void>}
* @private
*/
async _init () {
const require = this.require;
const Together = require('together-ai');
@@ -27,20 +43,41 @@ class TogetherAIService extends BaseService {
});
}
/**
* Returns the default model ID for the Together AI service
* @returns {string} The ID of the default model (meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo)
*/
get_default_model () {
return 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the puter-chat-completion interface for TogetherAI service
* Contains methods for listing models and generating chat completions
* @property {Object} models - Method to get available models
* @property {Object} list - Method to get list of model IDs
* @property {Object} complete - Method to generate chat completions
*/
async models () {
return await this.models_();
},
/**
* Retrieves available AI models from the Together API
* @returns {Promise<Array>} Array of model objects with their properties
* @implements {puter-chat-completion.models}
*/
async list () {
let models = this.modules.kv.get(`${this.kvkey}:models`);
if ( ! models ) models = await this.models_();
return models.map(model => model.id);
},
/**
* Lists available AI model IDs from the cache or fetches them if not cached
* @returns {Promise<string[]>} Array of model ID strings
*/
async complete ({ messages, stream, model }) {
if ( model === 'model-fallback-test-1' ) {
throw new Error('Model Fallback Test 1');
@@ -103,6 +140,14 @@ class TogetherAIService extends BaseService {
}
}
/**
* Fetches and caches available AI models from Together API
* @private
* @returns {Promise<Array>} Array of model objects containing id, name, context length,
* description and pricing information
* @remarks Models are cached for 5 minutes in KV store
*/
async models_ () {
let models = this.modules.kv.get(`${this.kvkey}:models`);
if ( models ) return models;
src/backend/src/modules/puterai/XAIService.js
+50 -1
View File
@@ -1,9 +1,10 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { default: Anthropic } = require("@anthropic-ai/sdk");
const BaseService = require("../../services/BaseService");
const { whatis, nou } = require("../../util/langutil");
const { PassThrough } = require("stream");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
const PUTER_PROMPT = `
You are running on an open-source platform called Puter,
@@ -14,11 +15,24 @@ const PUTER_PROMPT = `
user of the driver interface (typically an app on Puter):
`.replace('\n', ' ').trim();
/**
* XAIService class - Provides integration with X.AI's API for chat completions
* Extends BaseService to implement the puter-chat-completion interface.
* Handles model management, message adaptation, streaming responses,
* and usage tracking for X.AI's language models like Grok.
* @extends BaseService
*/
class XAIService extends BaseService {
static MODULES = {
openai: require('openai'),
}
/**
* Gets the system prompt used for AI interactions
* @returns {string} The base system prompt that identifies the AI as running on Puter
*/
get_system_prompt () {
return PUTER_PROMPT;
}
@@ -27,6 +41,12 @@ class XAIService extends BaseService {
return model;
}
/**
* Initializes the XAI service by setting up the OpenAI client and registering with the AI chat provider
* @private
* @returns {Promise<void>} Resolves when initialization is complete
*/
async _init () {
this.openai = new this.modules.openai.OpenAI({
apiKey: this.global_config.services.xai.apiKey,
@@ -40,15 +60,30 @@ class XAIService extends BaseService {
});
}
/**
* Returns the default model identifier for the XAI service
* @returns {string} The default model ID 'grok-beta'
*/
get_default_model () {
return 'grok-beta';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the interface for the puter-chat-completion driver
* Contains methods for listing models, getting model details,
* and handling chat completions with streaming support
* @type {Object}
*/
async models () {
return await this.models_();
},
/**
* Returns a list of available AI models with their capabilities and pricing details
* @returns {Promise<Array>} Array of model objects containing id, name, context window size, and cost information
*/
async list () {
const models = await this.models_();
const model_names = [];
@@ -60,6 +95,10 @@ class XAIService extends BaseService {
}
return model_names;
},
/**
* Returns a list of all available model names including their aliases
* @returns {Promise<string[]>} Array of model names and their aliases
*/
async complete ({ messages, stream, model }) {
model = this.adapt_model(model);
const adapted_messages = [];
@@ -162,6 +201,16 @@ class XAIService extends BaseService {
}
}
/**
* Retrieves available AI models and their specifications
* @returns {Promise<Array>} Array of model objects containing:
* - id: Model identifier string
* - name: Human readable model name
* - context: Maximum context window size
* - cost: Pricing information object with currency and rates
* @private
*/
async models_ () {
return [
{
src/backend/src/modules/selfhosted/ComplainAboutVersionsService.js
-1
View File
@@ -61,7 +61,6 @@ class ComplainAboutVersionsService extends BaseService {
let timeago = (() => {
let years = cur_date_obj.getFullYear() - eol_date.getFullYear();
let months = cur_date_obj.getMonth() - eol_date.getMonth();
let days = cur_date_obj.getDate() - eol_date.getDate();
let str = '';
while ( years > 0 ) {
src/backend/src/modules/selfhosted/DefaultUserService.js
+5 -4
View File
@@ -20,14 +20,14 @@ const { QuickMkdir } = require("../../filesystem/hl_operations/hl_mkdir");
const { HLWrite } = require("../../filesystem/hl_operations/hl_write");
const { NodePathSelector } = require("../../filesystem/node/selectors");
const { surrounding_box } = require("../../fun/dev-console-ui-utils");
const { get_user, generate_system_fsentries, invalidate_cached_user } = require("../../helpers");
const { get_user, invalidate_cached_user } = require("../../helpers");
const { Context } = require("../../util/context");
const { asyncSafeSetInterval } = require("../../util/promise");
const { asyncSafeSetInterval } = require('@heyputer/putility').libs.promise;
const { buffer_to_stream } = require("../../util/streamutil");
const BaseService = require("../../services/BaseService");
const { Actor, UserActorType } = require("../../services/auth/Actor");
const { DB_WRITE } = require("../../services/database/consts");
const { quot } = require("../../util/strutil");
const { quot } = require('@heyputer/putility').libs.string;
const USERNAME = 'admin';
@@ -165,7 +165,8 @@ class DefaultUserService extends BaseService {
],
);
user.password = password_hashed;
await generate_system_fsentries(user);
const svc_user = this.services.get('user');
await svc_user.generate_default_fsentries({ user });
// generate default files for admin user
const svc_fs = this.services.get('filesystem');
const make_tree_ = async ({ components, tree }) => {
src/backend/src/modules/template/README.md
+56
View File
@@ -0,0 +1,56 @@
# TemplateModule
This is a template module that you can copy and paste to create new modules.
This module is also included in `EssentialModules`, which means it will load
when Puter boots. If you're just testing something, you can add it here
temporarily.
## Services
### TemplateService
This is a template service that you can copy and paste to create new services.
You can also add to this service temporarily to test something.
#### Listeners
##### `install.routes`
TemplateService listens to this event to provide an example endpoint
##### `boot.consolidation`
TemplateService listens to this event to provide an example event
##### `boot.activation`
TemplateService listens to this event to show you that it's here
##### `start.webserver`
TemplateService listens to this event to show you that it's here
## Libraries
### hello_world
#### Functions
##### `hello_world`
This is a simple function that returns a string.
You can probably guess what string it returns.
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../util/context.js`
- `../../services/BaseService` (use.BaseService)
- `../../util/expressutil`
src/backend/src/modules/template/TemplateModule.js
+53
View File
@@ -0,0 +1,53 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
/**
* This is a template module that you can copy and paste to create new modules.
*
* This module is also included in `EssentialModules`, which means it will load
* when Puter boots. If you're just testing something, you can add it here
* temporarily.
*/
class TemplateModule extends AdvancedBase {
async install (context) {
// === LIBS === //
const useapi = context.get('useapi');
const lib = require('./lib/__lib__.js');
// In extensions: use('workinprogress').hello_world();
// In services classes: see TemplateService.js
useapi.def(`workinprogress`, lib, { assign: true });
useapi.def('core.context', require('../../util/context.js').Context);
// === SERVICES === //
const services = context.get('services');
const { TemplateService } = require('./TemplateService.js');
services.registerService('template-service', TemplateService);
}
}
module.exports = {
TemplateModule
};
src/backend/src/modules/template/TemplateService.js
+99
View File
@@ -0,0 +1,99 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// TODO: import via `USE` static member
const BaseService = require("../../services/BaseService");
const { Endpoint } = require("../../util/expressutil");
/**
* This is a template service that you can copy and paste to create new services.
* You can also add to this service temporarily to test something.
*/
class TemplateService extends BaseService {
static USE = {
// - Defined by lib/__lib__.js,
// - Exposed to `useapi` by TemplateModule.js
workinprogress: 'workinprogress'
}
_construct () {
// Use this override to initialize instance variables.
}
async _init () {
// This is where you initialize the service and prepare
// for the consolidation phase.
this.log.info("I am the template service.");
}
/**
* TemplateService listens to this event to provide an example endpoint
*/
['__on_install.routes'] (_, { app }) {
this.log.info("TemplateService get the event for installing endpoint.");
Endpoint({
route: '/example-endpoint',
methods: ['GET'],
handler: async (req, res) => {
res.send(this.workinprogress.hello_world());
}
}).attach(app);
// ^ Don't forget to attach the endpoint to the app!
// it's very easy to forget this step.
}
/**
* TemplateService listens to this event to provide an example event
*/
['__on_boot.consolidation'] () {
// At this stage, all services have been initialized and it is
// safe to start emitting events.
this.log.info("TemplateService sees consolidation boot phase.");
const svc_event = this.services.get('event');
svc_event.on('template-service.hello', (_eventid, event_data) => {
this.log.info('template-service said hello to itself; this is expected', {
event_data,
});
});
svc_event.emit('template-service.hello', {
message: 'Hello all you other services! I am the template service.'
});
}
/**
* TemplateService listens to this event to show you that it's here
*/
['__on_boot.activation'] () {
this.log.info("TemplateService sees activation boot phase.");
}
/**
* TemplateService listens to this event to show you that it's here
*/
['__on_start.webserver'] () {
this.log.info("TemplateService sees it's time to start web servers.");
}
}
module.exports = {
TemplateService
};
src/backend/src/modules/template/lib/__lib__.js
+3
View File
@@ -0,0 +1,3 @@
module.exports = {
hello_world: require('./hello_world.js'),
};
src/backend/src/modules/template/lib/hello_world.js
+10
View File
@@ -0,0 +1,10 @@
/**
* This is a simple function that returns a string.
* You can probably guess what string it returns.
*/
const hello_world = () => {
return "Hello, world!";
}
module.exports = hello_world;
src/backend/src/modules/web/README.md
+67
View File
@@ -0,0 +1,67 @@
# WebModule
This module initializes a pre-configured web server and socket.io server.
The main service, WebServerService, emits 'install.routes' and provides
the server instance to the callback.
## Services
### SocketioService
SocketioService provides a service for sending messages to clients.
socket.io is used behind the scenes. This service provides a simpler
interface for sending messages to rooms or socket ids.
#### Listeners
##### `install.socketio`
Initializes socket.io
###### Parameters
- **server:** The server to attach socket.io to.
### WebServerService
This class, WebServerService, is responsible for starting and managing the Puter web server.
It initializes the Express app, sets up middlewares, routes, and handles authentication and web sockets.
It also validates the host header and IP addresses to prevent security vulnerabilities.
#### Listeners
##### `boot.consolidation`
This method initializes the backend web server for Puter. It sets up the Express app, configures middleware, and starts the HTTP server.
##### `boot.activation`
Starts the web server and listens for incoming connections.
This method sets up the Express app, sets up middleware, and starts the server on the specified port.
It also sets up the Socket.io server for real-time communication.
##### `start.webserver`
This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use.
If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299.
Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events.
If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser.
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../services/BaseService` (use.BaseService)
- `../../util/context.js`
- `../../services/BaseService.js`
- `../../config.js`
- `../../middleware/auth.js`
- `../../util/strutil.js`
- `../../fun/dev-console-ui-utils.js`
- `../../helpers.js`
- `../../fun/logos.js`
src/backend/src/modules/web/SocketioService.js
+75
View File
@@ -0,0 +1,75 @@
// METADATA // {"ai-params":{"service":"claude"},"ai-commented":{"service":"claude"}}
const BaseService = require('../../services/BaseService');
/**
* SocketioService provides a service for sending messages to clients.
* socket.io is used behind the scenes. This service provides a simpler
* interface for sending messages to rooms or socket ids.
*/
class SocketioService extends BaseService {
static MODULES = {
socketio: require('socket.io'),
};
/**
* Initializes socket.io
*
* @evtparam server The server to attach socket.io to.
*/
['__on_install.socketio'] (_, { server }) {
const require = this.require;
const socketio = require('socket.io');
/**
* @type {import('socket.io').Server}
*/
this.io = socketio(server, {
cors: {
origin: '*',
}
});
}
/**
* Sends a message to specified socket(s) or room(s)
*
* @param {Array|Object} socket_specifiers - Single or array of objects specifying target sockets/rooms
* @param {string} key - The event key/name to emit
* @param {*} data - The data payload to send
* @returns {Promise<void>}
*/
async send (socket_specifiers, key, data) {
if ( ! Array.isArray(socket_specifiers) ) {
socket_specifiers = [socket_specifiers];
}
for ( const socket_specifier of socket_specifiers ) {
if ( socket_specifier.room ) {
this.io.to(socket_specifier.room).emit(key, data);
} else if ( socket_specifier.socket ) {
const io = this.io.sockets.sockets.get(socket_specifier.socket)
if ( ! io ) continue;
io.emit(key, data);
}
}
}
/**
* Checks if the specified socket or room exists
*
* @param {Object} socket_specifier - The socket specifier object
* @returns {boolean} True if the socket exists, false otherwise
*/
has (socket_specifier) {
if ( socket_specifier.room ) {
const room = this.io.sockets.adapter.rooms.get(socket_specifier.room);
return (!!room) && room.size > 0;
}
if ( socket_specifier.socket ) {
return this.io.sockets.sockets.has(socket_specifier.socket);
}
}
}
module.exports = SocketioService;
src/backend/src/modules/web/WebModule.js
+27
View File
@@ -0,0 +1,27 @@
const { AdvancedBase } = require("@heyputer/putility");
/**
* This module initializes a pre-configured web server and socket.io server.
* The main service, WebServerService, emits 'install.routes' and provides
* the server instance to the callback.
*/
class WebModule extends AdvancedBase {
async install (context) {
// === LIBS === //
const useapi = context.get('useapi');
useapi.def('web', require('./lib/__lib__.js'), { assign: true });
// === SERVICES === //
const services = context.get('services');
const SocketioService = require("./SocketioService");
services.registerService('socketio', SocketioService);
const WebServerService = require("./WebServerService");
services.registerService('web-server', WebServerService);
}
}
module.exports = {
WebModule,
};
src/backend/src/services/WebServerService.js → src/backend/src/modules/web/WebServerService.js
+21 -19
View File
@@ -18,18 +18,19 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const express = require('express');
const eggspress = require("../api/eggspress");
const { Context, ContextExpressMiddleware } = require("../util/context");
const BaseService = require("./BaseService");
const eggspress = require("./lib/eggspress.js");
const { Context, ContextExpressMiddleware } = require("../../util/context.js");
const BaseService = require("../../services/BaseService.js");
const config = require('../config');
const config = require('../../config.js');
const https = require('https')
var http = require('http');
const fs = require('fs');
const auth = require('../middleware/auth');
const { osclink } = require('../util/strutil');
const { surrounding_box, es_import_promise } = require('../fun/dev-console-ui-utils');
const auth = require('../../middleware/auth.js');
const { surrounding_box, es_import_promise } = require('../../fun/dev-console-ui-utils.js');
const relative_require = require;
const strutil = require('@heyputer/putility').libs.string;
/**
* This class, WebServerService, is responsible for starting and managing the Puter web server.
@@ -93,19 +94,15 @@ class WebServerService extends BaseService {
*
* @return {Promise} A promise that resolves when the server is up and running.
*/
// eslint-disable-next-line no-unused-vars
async ['__on_start.webserver'] () {
// ... rest of the method code
}
async ['__on_start.webserver'] () {
await es_import_promise;
// error handling middleware goes last, as per the
// expressjs documentation:
// https://expressjs.com/en/guide/error-handling.html
this.app.use(require('../api/api_error_handler'));
this.app.use(require('./lib/api_error_handler.js'));
const { jwt_auth } = require('../helpers');
const { jwt_auth } = require('../../helpers.js');
config.http_port = process.env.PORT ?? config.http_port;
@@ -201,7 +198,7 @@ class WebServerService extends BaseService {
*/
this.startup_widget = () => {
const link = `\x1B[34;1m${osclink(url)}\x1B[0m`;
const link = `\x1B[34;1m${strutil.osclink(url)}\x1B[0m`;
const lines = [
`Puter is now live at: ${link}`,
`Type web:dismiss to un-stick this message`,
@@ -224,7 +221,11 @@ class WebServerService extends BaseService {
// server.keepAliveTimeout = 1000 * 60 * 60 * 2; // 2 hours
// Socket.io server instance
const socketio = require('../socketio.js').init(server);
// const socketio = require('../../socketio.js').init(server);
// TODO: ^ Replace above line with the following code:
await this.services.emit('install.socketio', { server });
const socketio = this.services.get('socketio').io;
// Socket.io middleware for authentication
socketio.use(async (socket, next) => {
@@ -305,13 +306,13 @@ class WebServerService extends BaseService {
const require = this.require;
const config = this.global_config;
new ContextExpressMiddleware({
parent: globalThis.root_context.sub({
puter_environment: Context.create({
env: config.env,
version: require('../../package.json').version,
version: relative_require('../../../package.json').version,
}),
}, 'mw')
}).install(app);
@@ -580,6 +581,8 @@ class WebServerService extends BaseService {
app.options('/*', (_, res) => {
return res.sendStatus(200);
});
console.log('WEB SERVER INIT DONE');
}
_register_commands (commands) {
@@ -611,7 +614,7 @@ class WebServerService extends BaseService {
// comment above line 497
print_puter_logo_() {
if ( this.global_config.env !== 'dev' ) return;
const logos = require('../fun/logos.js');
const logos = require('../../fun/logos.js');
let last_logo = undefined;
for ( const logo of logos ) {
if ( logo.sz <= (process.stdout.columns ?? 0) ) {
@@ -622,7 +625,6 @@ class WebServerService extends BaseService {
const lines = last_logo.txt.split('\n');
const width = process.stdout.columns;
const pad = (width - last_logo.sz) / 2;
const asymmetrical = pad % 1 !== 0;
const pad_left = Math.floor(pad);
const pad_right = Math.ceil(pad);
for ( let i = 0 ; i < lines.length ; i++ ) {
src/backend/src/modules/web/lib/__lib__.js
+4
View File
@@ -0,0 +1,4 @@
module.exports = {
eggspress: require("./eggspress"),
api_error_handler: require("./api_error_handler"),
};
src/backend/src/modules/web/lib/api_error_handler.js
+78
View File
@@ -0,0 +1,78 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const APIError = require('../../../api/APIError.js');
/**
* api_error_handler() is an express error handler for API errors.
* It adheres to the express error handler signature and should be
* used as the last middleware in an express app.
*
* Since Express 5 is not yet released, this function is used by
* eggspress() to handle errors instead of as a middleware.
*
* @todo remove this function and use express error handling
* when Express 5 is released
*
* @param {*} err
* @param {*} req
* @param {*} res
* @param {*} next
* @returns
*/
module.exports = function api_error_handler (err, req, res, next) {
if (res.headersSent) {
console.error('error after headers were sent:', err);
return next(err)
}
// API errors might have a response to help the
// developer resolve the issue.
if ( err instanceof APIError ) {
return err.write(res);
}
if (
typeof err === 'object' &&
! (err instanceof Error) &&
err.hasOwnProperty('message')
) {
const apiError = APIError.create(400, err);
return apiError.write(res);
}
console.error('internal server error:', err);
const services = globalThis.services;
if ( services && services.has('alarm') ) {
const alarm = services.get('alarm');
alarm.create('api_error_handler', err.message, {
error: err,
url: req.url,
method: req.method,
body: req.body,
headers: req.headers,
});
}
req.__error_handled = true;
// Other errors should provide as little information
// to the client as possible for security reasons.
return res.send(500, 'Internal Server Error');
};
src/backend/src/modules/web/lib/eggspress.js
+199
View File
@@ -0,0 +1,199 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const express = require('express');
const multer = require('multer');
const multest = require('@heyputer/multest');
const api_error_handler = require('./api_error_handler.js');
const fsBeforeMW = require('../../../middleware/fs.js');
const APIError = require('../../../api/APIError.js');
const { Context } = require('../../../util/context.js');
const { subdomain } = require('../../../helpers.js');
/**
* eggspress() is a factory function for creating express routers.
*
* @param {*} route the route to the router
* @param {*} settings the settings for the router. The following
* properties are supported:
* - auth: whether or not to use the auth middleware
* - fs: whether or not to use the fs middleware
* - json: whether or not to use the json middleware
* - customArgs: custom arguments to pass to the router
* - allowedMethods: the allowed HTTP methods
* @param {*} handler the handler for the router
* @returns {express.Router} the router
*/
module.exports = function eggspress (route, settings, handler) {
const router = express.Router();
const mw = [];
const afterMW = [];
// These flags enable specific middleware.
if ( settings.abuse ) mw.push(require('../../../middleware/abuse')(settings.abuse));
if ( settings.auth ) mw.push(require('../../../middleware/auth'));
if ( settings.auth2 ) mw.push(require('../../../middleware/auth2'));
if ( settings.fs ) {
mw.push(fsBeforeMW);
}
if ( settings.verified ) mw.push(require('../../../middleware/verified'));
if ( settings.json ) mw.push(express.json());
// The `files` setting is an array of strings. Each string is the name
// of a multipart field that contains files. `multer` is used to parse
// the multipart request and store the files in `req.files`.
if ( settings.files ) {
for ( const key of settings.files ) {
mw.push(multer().array(key));
}
}
if ( settings.multest ) {
mw.push(multest());
}
// The `multipart_jsons` setting is an array of strings. Each string
// is the name of a multipart field that contains JSON. This middleware
// parses the JSON in each field and stores the result in `req.body`.
if ( settings.multipart_jsons ) {
for ( const key of settings.multipart_jsons ) {
mw.push((req, res, next) => {
try {
if ( ! Array.isArray(req.body[key]) ) {
req.body[key] = [JSON.parse(req.body[key])];
} else {
req.body[key] = req.body[key].map(JSON.parse);
}
} catch (e) {
return res.status(400).send({
error: {
message: `Invalid JSON in multipart field ${key}`
}
});
}
next();
});
}
}
// The `alias` setting is an object. Each key is the name of a
// parameter. Each value is the name of a parameter that should
// be aliased to the key.
if ( settings.alias ) {
for ( const alias in settings.alias ) {
const target = settings.alias[alias];
mw.push((req, res, next) => {
const values = req.method === 'GET' ? req.query : req.body;
if ( values[alias] ) {
values[target] = values[alias];
}
next();
});
}
}
// The `parameters` setting is an object. Each key is the name of a
// parameter. Each value is a `Param` object. The `Param` object
// specifies how to validate the parameter.
if ( settings.parameters ) {
for ( const key in settings.parameters ) {
const param = settings.parameters[key];
mw.push(async (req, res, next) => {
if ( ! req.values ) req.values = {};
const values = req.method === 'GET' ? req.query : req.body;
const getParam = (key) => values[key];
try {
const result = await param.consolidate({ req, getParam });
req.values[key] = result;
} catch (e) {
api_error_handler(e, req, res, next);
return;
}
next();
});
}
}
// what if I wanted to pass arguments to, for example, `json`?
if ( settings.customArgs ) mw.push(settings.customArgs);
if ( settings.alarm_timeout ) {
mw.push((req, res, next) => {
setTimeout(() => {
if ( ! res.headersSent ) {
const log = req.services.get('log-service').create('eggspress:timeout');
const errors = req.services.get('error-service').create(log);
let id = Array.isArray(route) ? route[0] : route;
id = id.replace(/\//g, '_');
errors.report(id, {
source: new Error('Response timed out.'),
message: 'Response timed out.',
trace: true,
alarm: true,
});
}
}, settings.alarm_timeout);
next();
});
}
if ( settings.response_timeout ) {
mw.push((req, res, next) => {
setTimeout(() => {
if ( ! res.headersSent ) {
api_error_handler(APIError.create('response_timeout'), req, res, next);
}
}, settings.response_timeout);
next();
});
}
if ( settings.mw ) mw.push(...settings.mw);
const errorHandledHandler = async function (req, res, next) {
if ( settings.subdomain ) {
if ( subdomain(req) !== settings.subdomain ) {
return next();
}
}
try {
const expected_ctx = res.locals.ctx;
const received_ctx = Context.get(undefined, { allow_fallback: true });
if ( expected_ctx != received_ctx ) {
await expected_ctx.arun(async () => {
await handler(req, res, next);
});
} else await handler(req, res, next);
} catch (e) {
api_error_handler(e, req, res, next);
}
};
if ( settings.allowedMethods.includes('GET') ) {
router.get(route, ...mw, errorHandledHandler, ...afterMW);
}
if ( settings.allowedMethods.includes('POST') ) {
router.post(route, ...mw, errorHandledHandler, ...afterMW);
}
return router;
}
src/backend/src/routers/_default.js
+2 -4
View File
@@ -289,10 +289,8 @@ router.all('*', async function(req, res, next) {
invalidate_cached_user(user);
// send realtime success msg to client
let socketio = require('../socketio.js').getio();
if(socketio){
socketio.to(user.id).emit('user.email_confirmed', {})
}
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: user.id }, 'user.email_confirmed', {});
// return results
h += `<p style="text-align:center; color:green;">Your email has been successfully confirmed.</p>`;
src/backend/src/routers/change_email.js
+2 -4
View File
@@ -83,10 +83,8 @@ const CHANGE_EMAIL_CONFIRM = eggspress('/change_email/confirm', {
});
invalidate_cached_user_by_id(user_id);
let socketio = require('../socketio.js').getio();
if(socketio){
socketio.to(user_id).emit('user.email_changed', {})
}
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: user_id }, 'user.email_changed', {});
const h = `<p style="text-align:center; color:green;">Your email has been successfully confirmed.</p>`;
return res.send(h);
src/backend/src/routers/confirm-email.js
+5 -5
View File
@@ -87,14 +87,14 @@ router.post('/confirm-email', auth, express.json(), async (req, res, next)=>{
// Send realtime success msg to client
if(req.body.code === req.user.email_confirm_code){
let socketio = require('../socketio.js').getio();
if(socketio){
socketio.to(req.user.id).emit('user.email_confirmed', {original_client_socket_id: req.body.original_client_socket_id})
}
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: req.user.id }, 'user.email_confirmed', {
original_client_socket_id: req.body.original_client_socket_id
});
}
// return results
return res.send(res_obj)
})
module.exports = router
module.exports = router
src/backend/src/routers/drivers/call.js
+1 -1
View File
@@ -23,7 +23,7 @@ const { TypeSpec } = require("../../services/drivers/meta/Construct");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { Context } = require("../../util/context");
const { whatis } = require("../../util/langutil");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
const { valid_file_size } = require("../../util/validutil");
let _handle_multipart;
src/backend/src/routers/filesystem_api/batch/all.js
+1 -2
View File
@@ -20,11 +20,10 @@ const APIError = require("../../../api/APIError");
const eggspress = require("../../../api/eggspress");
const config = require("../../../config");
const PathResolver = require("./PathResolver");
const { WorkUnit } = require("../../../services/runtime-analysis/ExpectationService");
const { Context } = require("../../../util/context");
const Busboy = require('busboy');
const { BatchExecutor } = require("../../../filesystem/batch/BatchExecutor");
const { TeePromise } = require("../../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
const { EWMA, MovingMode } = require("../../../util/opmath");
const { get_app } = require('../../../helpers');
const { valid_file_size } = require("../../../util/validutil");
src/backend/src/routers/filesystem_api/delete.js
+5 -5
View File
@@ -48,8 +48,6 @@ module.exports = eggspress('/delete', {
else if(paths.length === 0)
return res.status(400).send('paths cannot be empty')
const socketio = require('../../socketio.js').getio();
// try to delete each path in the array one by one (if glob, resolve first)
// TODO: remove this pseudo-batch
for(let j=0; j < paths.length; j++){
@@ -67,9 +65,11 @@ module.exports = eggspress('/delete', {
});
// send realtime success msg to client
if(socketio){
socketio.to(req.user.id).emit('item.removed', {path: item_path, descendants_only: descendants_only})
}
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: req.user.id }, 'item.removed', {
path: item_path,
descendants_only: descendants_only,
});
}
res.send({});
src/backend/src/routers/filesystem_api/rename.js
+2 -4
View File
@@ -174,10 +174,8 @@ module.exports = eggspress('/rename', {
};
// send realtime success msg to client
let socketio = require('../../socketio.js').getio();
if(socketio){
socketio.to(req.user.id).emit('item.renamed', return_obj)
}
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: req.user.id }, 'item.renamed', return_obj);
return res.send(return_obj);
});
src/backend/src/routers/filesystem_api/write.js
+3 -9
View File
@@ -23,9 +23,8 @@ const { HLWrite } = require('../../filesystem/hl_operations/hl_write.js');
const { boolify } = require('../../util/hl_types.js');
const { Context } = require('../../util/context.js');
const Busboy = require('busboy');
const { TeePromise } = require('../../util/promise.js');
const { TeePromise } = require('@heyputer/putility').libs.promise;
const APIError = require('../../api/APIError.js');
const api_error_handler = require('../../api/api_error_handler.js');
const { valid_file_size } = require('../../util/validutil.js');
// -----------------------------------------------------------------------//
@@ -172,13 +171,8 @@ module.exports = eggspress(['/up', '/write'], {
const values = req.method === 'GET' ? req.query : req.body;
const getParam = (key) => values[key];
try {
const result = await param.consolidate({ req, getParam });
req.values[key] = result;
} catch (e) {
api_error_handler(e, req, res, next);
return;
}
const result = await param.consolidate({ req, getParam });
req.values[key] = result;
}
if ( req.body.size === undefined ) {
src/backend/src/routers/get-dev-profile.js
+18 -6
View File
@@ -33,15 +33,27 @@ router.get('/get-dev-profile', auth, express.json(), async (req, response, next)
// check if user is verified
if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
return response.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
// TODO: we currently invalidate the cache on every request, this is because a developer may
// have been approved for the incentive program from one server, but the cache on another server
// may not have been updated yet. This is a temporary solution until we implement a better way to
// handle this. The better way would be for different servers to communicate with each other
// when a developer is approved for the incentive program (or any other change that affects the
// cache) and update the cache on all servers.
require('../helpers').invalidate_cached_user(req.user);
const { get_user } = require('../helpers');
let dev = await get_user(req.user);
dev = dev ?? {};
try{
// auth
response.send({
first_name: req.user.dev_first_name,
last_name: req.user.dev_last_name,
approved_for_incentive_program: req.user.dev_approved_for_incentive_program,
joined_incentive_program: req.user.dev_joined_incentive_program,
paypal: req.user.dev_paypal,
first_name: dev.dev_first_name,
last_name: dev.dev_last_name,
approved_for_incentive_program: dev.dev_approved_for_incentive_program,
joined_incentive_program: dev.dev_joined_incentive_program,
paypal: dev.dev_paypal,
});
}catch(e){
console.log(e)
src/backend/src/routers/hosting/puter-site.js
+1 -1
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const api_error_handler = require("../../api/api_error_handler");
const api_error_handler = require("../../modules/web/lib/api_error_handler");
const config = require("../../config");
const { get_user, get_app, id2path } = require("../../helpers");
const { Context } = require("../../util/context");
src/backend/src/routers/rao.js
+2 -2
View File
@@ -90,8 +90,8 @@ router.post('/rao', auth, express.json(), async (req, res, next)=>{
}
// Update clients
const socketio = require('../socketio.js').getio();
socketio.to(req.user.id).emit('app.opened', {
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: req.user.id }, 'app.opened', {
uuid: opened_app.uid,
uid: opened_app.uid,
name: opened_app.name,
src/backend/src/routers/signup.js
+11 -2
View File
@@ -17,12 +17,20 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
"use strict"
const {get_taskbar_items, generate_random_username, generate_system_fsentries, send_email_verification_code, send_email_verification_token, username_exists, invalidate_cached_user_by_id, get_user } = require('../helpers');
const {get_taskbar_items, generate_system_fsentries, send_email_verification_code, send_email_verification_token, username_exists, invalidate_cached_user_by_id, get_user } = require('../helpers');
const config = require('../config');
const eggspress = require('../api/eggspress');
const { Context } = require('../util/context');
const { DB_WRITE } = require('../services/database/consts');
async function generate_random_username () {
let username;
do {
username = generate_identifier();
} while (await username_exists(username));
return username;
}
// -----------------------------------------------------------------------//
// POST /signup
// -----------------------------------------------------------------------//
@@ -345,7 +353,8 @@ module.exports = eggspress(['/signup'], {
}
}
await generate_system_fsentries(user);
const svc_user = Context.get('services').get('user');
await svc_user.generate_default_fsentries({ user });
//set cookie
res.cookie(config.cookie_name, token, {
src/backend/src/routers/writeFile.js
+2 -4
View File
@@ -457,10 +457,8 @@ module.exports = eggspress('/writeFile', {
};
// send realtime success msg to client
let socketio = require('../socketio.js').getio();
if(socketio){
socketio.to(fsentry.user_id).emit('item.renamed', return_obj)
}
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: req.user.id }, 'item.renamed', return_obj);
return res.send(return_obj);
}

Some files were not shown because too many files have changed in this diff Show More