mirror/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2025-12-21 12:59:52 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat: metering service allowence checks and subscription integration 🚀 (#1749)

Browse Source 
* feat: metering allowence checks

* fix: bad math
This commit is contained in:
Daniel Salazar
2025-10-15 02:28:25 -07:00
committed by GitHub
parent 96a58ced29
commit e51d0c4600
28 changed files with 888 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@@ -45,5 +45,10 @@ jsconfig.json
# the exact tree installed in the node_modules folder
package-lock.json
# AI STUFF
AGENTS.md
.roo
# source maps
*.map

34
eslint.config.js
View File

@@ -1,10 +1,34 @@
import js from '@eslint/js';
import stylistic from '@stylistic/eslint-plugin';
import tseslintPlugin from '@typescript-eslint/eslint-plugin';
import tseslintParser from '@typescript-eslint/parser';
import { defineConfig } from 'eslint/config';
import globals from 'globals';
import controlStructureSpacing from './control-structure-spacing.js';
export default defineConfig([
// TypeScript support block
{
files: ['**/*.ts'],
languageOptions: {
parser: tseslintParser,
parserOptions: {
ecmaVersion: 'latest',
sourceType: 'module',
project: './tsconfig.json',
},
},
plugins: {
'@typescript-eslint': tseslintPlugin,
},
rules: {
// Recommended rules for TypeScript
'@typescript-eslint/no-explicit-any': 'warn',
'@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
'@typescript-eslint/ban-ts-comment': 'warn',
'@typescript-eslint/consistent-type-definitions': ['error', 'interface'],
},
},
{
plugins: {
js,
@@ -115,10 +139,10 @@ export default defineConfig([
},
},
{
files: ['**/*.{js,mjs,cjs}'],
files: ['**/*.{js,mjs,cjs,ts}'],
ignores: [
'src/backend/**/*.{js,mjs,cjs}',
'extensions/**/*.{js,mjs,cjs}',
'src/backend/**/*.{js,mjs,cjs,ts}',
'extensions/**/*.{js,mjs,cjs,ts}',
],
languageOptions: { globals: globals.browser },
rules: {
@@ -165,8 +189,8 @@ export default defineConfig([
},
},
{
files: ['**/*.{js,mjs,cjs}'],
ignores: ['src/backend/**/*.{js,mjs,cjs}'],
files: ['**/*.{js,mjs,cjs,ts}'],
ignores: ['src/backend/**/*.{js,mjs,cjs,ts}'],
languageOptions: { globals: globals.browser },
rules: {
'no-unused-vars': ['error', {

3
extensions/meteringAndBilling/config.json Normal file
View File

@@ -0,0 +1,3 @@
{
"unlimitedUsage": false
}

24
extensions/meteringAndBilling/eventListeners/subscriptionEvents.js Normal file
View File

@@ -0,0 +1,24 @@
extension.on('metering:overrideDefaultSubscription', async (/** @type {{actor: import('@heyputer/backend/src/services/auth/Actor').Actor, defaultSubscription: string}} */event) => {
// bit of a stub implementation for OSS, technically can be always free if you set this config true
if ( config.unlimitedUsage ) {
console.warn('WARNING!!! unlimitedUsage is enabled, this is not recommended for production use');
event.defaultSubscriptionId = 'unlimited';
}
});
extension.on('metering:registerAvailablePolicies', async (
/** @type {{actor: import('@heyputer/backend/src/services/auth/Actor').Actor, availablePolicies: unknown[]}} */event) => {
// bit of a stub implementation for OSS, technically can be always free if you set this config true
if ( config.unlimitedUsage ) {
console.warn('WARNING!!! unlimitedUsage is enabled, this is not recommended for production use');
event.availablePolicies.push({
id: 'unlimited',
monthUsageAllowence: 500_000_000 * 100_000_000, // unless you're like, jeff's, mark's and elon's illegitamate son, you probably won't hit $5m a month
monthlyStorageAllowence: 100_000 * 1024 * 1024, // 100MiB
});
}
});
extension.on('metering:getUserSubscription', async (/** @type {{actor: import('@heyputer/backend/src/services/auth/Actor').Actor, userSubscription: string}} */event) => {
event.userSubscriptionId = event.actor.type.user.subscription.tier;
});

2
extensions/meteringAndBilling/main.js Normal file
View File

@@ -0,0 +1,2 @@
import './eventListeners/subscriptionEvents.js';
import './routes/usage.js';

0
extensions/meteringService/package.json → extensions/meteringAndBilling/package.json
View File

13
extensions/meteringService/routes/usage.js → extensions/meteringAndBilling/routes/usage.js
View File

@@ -2,19 +2,22 @@
const meteringAndBillingServiceWrapper = extension.import('service:meteringService');
// TODO DS: move this to its own router and just use under this path
extension.get('/v2/usage', { subdomain: 'api' }, async (req, res) => {
extension.get('/meteringAndBilling/usage', { subdomain: 'api' }, async (req, res) => {
const meteringAndBillingService = meteringAndBillingServiceWrapper.meteringAndBillingService;
const actor = req.actor;
if ( !actor ) {
throw Error('actor not found in context');
}
const actorUsage = await meteringAndBillingService.getActorCurrentMonthUsageDetails(actor);
res.status(200).json(actorUsage);
const actorUsagePromise = meteringAndBillingService.getActorCurrentMonthUsageDetails(actor);
const actorAllowenceInfoPromise = meteringAndBillingService.getAllowedUsage(actor);
const [actorUsage, allowenceInfo] = await Promise.all([actorUsagePromise, actorAllowenceInfoPromise]);
res.status(200).json({ ...actorUsage, allowenceInfo });
return;
});
extension.get('/v2/usage/:appId', { subdomain: 'api' }, async (req, res) => {
extension.get('/meteringAndBilling/usage/:appId', { subdomain: 'api' }, async (req, res) => {
const meteringAndBillingService = meteringAndBillingServiceWrapper.meteringAndBillingService;
const actor = req.actor;
@@ -32,4 +35,4 @@ extension.get('/v2/usage/:appId', { subdomain: 'api' }, async (req, res) => {
return;
});
console.debug('Loaded /v2/usage route');
console.debug('Loaded /meteringAndBilling/usage route');

1
extensions/meteringService/main.js
View File

@@ -1 +0,0 @@
import './routes/usage.js';

12
extensions/tsconfig.json
View File

@@ -2,14 +2,18 @@
"compilerOptions": {
"target": "ES2020",
"allowJs": true,
"module": "esnext",
"moduleResolution": "node",
"module": "node16",
"moduleResolution": "node16",
"baseUrl": ".",
"outDir": "/dev/null",
"paths": {
"../src/*": ["../src/*"]
"../src/*": [
"../src/*"
]
},
"typeRoots": ["../node_modules/@types"],
"typeRoots": [
"../node_modules/@types"
],
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"strict": true

277
package-lock.json generated
View File

@@ -36,6 +36,8 @@
"devDependencies": {
"@eslint/js": "^9.35.0",
"@stylistic/eslint-plugin": "^5.3.1",
"@typescript-eslint/eslint-plugin": "^8.46.1",
"@typescript-eslint/parser": "^8.46.1",
"chalk": "^4.1.0",
"clean-css": "^5.3.2",
"dotenv": "^16.4.5",
@@ -7275,12 +7277,257 @@
"integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==",
"license": "MIT"
},
"node_modules/@typescript-eslint/types": {
"version": "8.46.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.46.0.tgz",
"integrity": "sha512-bHGGJyVjSE4dJJIO5yyEWt/cHyNwga/zXGJbJJ8TiO01aVREK6gCTu3L+5wrkb1FbDkQ+TKjMNe9R/QQQP9+rA==",
"node_modules/@typescript-eslint/eslint-plugin": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.46.1.tgz",
"integrity": "sha512-rUsLh8PXmBjdiPY+Emjz9NX2yHvhS11v0SR6xNJkm5GM1MO9ea/1GoDKlHHZGrOJclL/cZ2i/vRUYVtjRhrHVQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@eslint-community/regexpp": "^4.10.0",
"@typescript-eslint/scope-manager": "8.46.1",
"@typescript-eslint/type-utils": "8.46.1",
"@typescript-eslint/utils": "8.46.1",
"@typescript-eslint/visitor-keys": "8.46.1",
"graphemer": "^1.4.0",
"ignore": "^7.0.0",
"natural-compare": "^1.4.0",
"ts-api-utils": "^2.1.0"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"@typescript-eslint/parser": "^8.46.1",
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <6.0.0"
}
},
"node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": {
"version": "7.0.5",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz",
"integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/@typescript-eslint/parser": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.46.1.tgz",
"integrity": "sha512-6JSSaBZmsKvEkbRUkf7Zj7dru/8ZCrJxAqArcLaVMee5907JdtEbKGsZ7zNiIm/UAkpGUkaSMZEXShnN2D1HZA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "8.46.1",
"@typescript-eslint/types": "8.46.1",
"@typescript-eslint/typescript-estree": "8.46.1",
"@typescript-eslint/visitor-keys": "8.46.1",
"debug": "^4.3.4"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <6.0.0"
}
},
"node_modules/@typescript-eslint/project-service": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.46.1.tgz",
"integrity": "sha512-FOIaFVMHzRskXr5J4Jp8lFVV0gz5ngv3RHmn+E4HYxSJ3DgDzU7fVI1/M7Ijh1zf6S7HIoaIOtln1H5y8V+9Zg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/tsconfig-utils": "^8.46.1",
"@typescript-eslint/types": "^8.46.1",
"debug": "^4.3.4"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"typescript": ">=4.8.4 <6.0.0"
}
},
"node_modules/@typescript-eslint/scope-manager": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.46.1.tgz",
"integrity": "sha512-weL9Gg3/5F0pVQKiF8eOXFZp8emqWzZsOJuWRUNtHT+UNV2xSJegmpCNQHy37aEQIbToTq7RHKhWvOsmbM680A==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "8.46.1",
"@typescript-eslint/visitor-keys": "8.46.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
}
},
"node_modules/@typescript-eslint/tsconfig-utils": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.46.1.tgz",
"integrity": "sha512-X88+J/CwFvlJB+mK09VFqx5FE4H5cXD+H/Bdza2aEWkSb8hnWIQorNcscRl4IEo1Cz9VI/+/r/jnGWkbWPx54g==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"typescript": ">=4.8.4 <6.0.0"
}
},
"node_modules/@typescript-eslint/type-utils": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.46.1.tgz",
"integrity": "sha512-+BlmiHIiqufBxkVnOtFwjah/vrkF4MtKKvpXrKSPLCkCtAp8H01/VV43sfqA98Od7nJpDcFnkwgyfQbOG0AMvw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "8.46.1",
"@typescript-eslint/typescript-estree": "8.46.1",
"@typescript-eslint/utils": "8.46.1",
"debug": "^4.3.4",
"ts-api-utils": "^2.1.0"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <6.0.0"
}
},
"node_modules/@typescript-eslint/types": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.46.1.tgz",
"integrity": "sha512-C+soprGBHwWBdkDpbaRC4paGBrkIXxVlNohadL5o0kfhsXqOC6GYH2S/Obmig+I0HTDl8wMaRySwrfrXVP8/pQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
}
},
"node_modules/@typescript-eslint/typescript-estree": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.46.1.tgz",
"integrity": "sha512-uIifjT4s8cQKFQ8ZBXXyoUODtRoAd7F7+G8MKmtzj17+1UbdzFl52AzRyZRyKqPHhgzvXunnSckVu36flGy8cg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/project-service": "8.46.1",
"@typescript-eslint/tsconfig-utils": "8.46.1",
"@typescript-eslint/types": "8.46.1",
"@typescript-eslint/visitor-keys": "8.46.1",
"debug": "^4.3.4",
"fast-glob": "^3.3.2",
"is-glob": "^4.0.3",
"minimatch": "^9.0.4",
"semver": "^7.6.0",
"ts-api-utils": "^2.1.0"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"typescript": ">=4.8.4 <6.0.0"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
"integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
"integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
"dev": true,
"license": "ISC",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=16 || 14 >=14.17"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/@typescript-eslint/utils": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.46.1.tgz",
"integrity": "sha512-vkYUy6LdZS7q1v/Gxb2Zs7zziuXN0wxqsetJdeZdRe/f5dwJFglmuvZBfTUivCtjH725C1jWCDfpadadD95EDQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@eslint-community/eslint-utils": "^4.7.0",
"@typescript-eslint/scope-manager": "8.46.1",
"@typescript-eslint/types": "8.46.1",
"@typescript-eslint/typescript-estree": "8.46.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <6.0.0"
}
},
"node_modules/@typescript-eslint/visitor-keys": {
"version": "8.46.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.46.1.tgz",
"integrity": "sha512-ptkmIf2iDkNUjdeu2bQqhFPV1m6qTnFFjg7PPDjxKWaMaP0Z6I9l30Jr3g5QqbZGdw8YdYvLp+XnqnWWZOg/NA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "8.46.1",
"eslint-visitor-keys": "^4.2.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
@@ -11606,6 +11853,13 @@
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
"license": "ISC"
},
"node_modules/graphemer": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz",
"integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
"dev": true,
"license": "MIT"
},
"node_modules/groq-sdk": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.5.0.tgz",
@@ -17440,6 +17694,19 @@
"node": ">= 14.0.0"
}
},
"node_modules/ts-api-utils": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz",
"integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=18.12"
},
"peerDependencies": {
"typescript": ">=4.8.4"
}
},
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
@@ -19808,7 +20075,7 @@
},
"src/puter-js": {
"name": "@heyputer/puter.js",
"version": "2.0.14",
"version": "2.0.15",
"license": "Apache-2.0",
"dependencies": {
"@heyputer/kv.js": "^0.2.1",

2
package.json
View File

@@ -13,6 +13,8 @@
"devDependencies": {
"@eslint/js": "^9.35.0",
"@stylistic/eslint-plugin": "^5.3.1",
"@typescript-eslint/eslint-plugin": "^8.46.1",
"@typescript-eslint/parser": "^8.46.1",
"chalk": "^4.1.0",
"clean-css": "^5.3.2",
"dotenv": "^16.4.5",

6
src/backend/src/modules/core/AlarmService.d.ts vendored Normal file
View File

@@ -0,0 +1,6 @@
export class AlarmService {
create(id: string, message: string, fields?: object): void;
clear(id: string): void;
get_alarm(id: string): object | undefined;
// Add more methods/properties as needed for MeteringService usage
}

5
src/backend/src/routers/auth/check-permissions.js
View File

@@ -34,11 +34,6 @@ module.exports = eggspress('/auth/check-permissions', {
const actor = context.get('actor');
// Apps cannot (currently) check permissions on behalf of users
if ( ! ( actor.type instanceof UserActorType ) ) {
throw APIError.create('forbidden');
}
const permEntryPromises = [...new Set(permsToCheck)].map(async (perm) => {
try {
return [perm, permissionService.check(actor, perm)];

7
src/backend/src/services/EventService.d.ts vendored Normal file
View File

@@ -0,0 +1,7 @@
// Minimal EventService type declaration for MeteringService type safety
export interface EventService {
emit(key: string, data?: any, meta?: any): Promise<void>;
on(selector: string, callback: Function): { detach: () => void };
on_all(callback: Function): void;
get_scoped(scope: string): any;
}

216
src/backend/src/services/MeteringService/MeteringService.ts
View File

@@ -1,19 +1,11 @@
// @ts-ignore
import { SystemActorType, type Actor } from "../auth/Actor.js";
// @ts-ignore
import type { AlarmService } from "../../modules/core/AlarmService.js";
// @ts-ignore
import type { DBKVStore } from '../repositories/DBKVStore/DBKVStore.mjs';
// @ts-ignore
import type { SUService } from "../SUService.js";
import { COST_MAPS } from "./costMaps/index.js";
import { SUB_POLICIES } from "./subPolicies/index.js";
interface ActorWithType extends Actor {
type: {
app: { uid: string }
user: { uuid: string, username: string }
}
}
import type { AlarmService } from '../../modules/core/AlarmService.js';
import { SystemActorType, type Actor } from '../auth/Actor.js';
import type { EventService } from '../EventService'; // Type-only import for TS safety
import type { DBKVStore } from '../repositories/DBKVStore/DBKVStore';
import type { SUService } from '../SUService.js';
import { DEFAULT_FREE_SUBSCRIPTION, DEFAULT_TEMP_SUBSCRIPTION, GLOBAL_APP_KEY, METRICS_PREFIX, PERIOD_ESCAPE, POLICY_PREFIX } from './consts.js';
import { COST_MAPS } from './costMaps/index.js';
import { SUB_POLICIES } from './subPolicies/index.js';
interface PolicyAddOns {
purchasedCredits: number
@@ -27,26 +19,29 @@ interface UsageByType {
[serviceName: string]: number
}
const GLOBAL_APP_KEY = 'os-global'; // TODO DS: this should be loaded from config or db eventually
const METRICS_PREFIX = 'metering';
const POLICY_PREFIX = 'policy';
const PERIOD_ESCAPE = '_dot_'; // to replace dots in usage types for kvstore paths
interface MeteringAndBillingServiceDeps {
kvClientWrapper: DBKVStore,
superUserService: SUService,
alarmService: AlarmService
eventService: EventService
}
/**
* Handles usage metering and supports stubbs for billing methods for current scoped actor
*/
export class MeteringAndBillingService {
#kvClientWrapper: DBKVStore
#superUserService: SUService
#alarmService: AlarmService
constructor({ kvClientWrapper, superUserService, alarmService }: { kvClientWrapper: DBKVStore, superUserService: SUService, alarmService: AlarmService }) {
#kvClientWrapper: DBKVStore;
#superUserService: SUService;
#alarmService: AlarmService;
#eventService: EventService;
constructor({ kvClientWrapper, superUserService, alarmService, eventService }: MeteringAndBillingServiceDeps) {
this.#superUserService = superUserService;
this.#kvClientWrapper = kvClientWrapper;
this.#alarmService = alarmService;
this.#eventService = eventService;
}
utilRecordUsageObject(trackedUsageObject: Record<string, number>, actor: ActorWithType, modelPrefix: string) {
utilRecordUsageObject(trackedUsageObject: Record<string, number>, actor: Actor, modelPrefix: string) {
Object.entries(trackedUsageObject).forEach(([usageKind, amount]) => {
this.incrementUsage(actor, `${modelPrefix}:${usageKind}`, amount);
});
@@ -57,17 +52,16 @@ export class MeteringAndBillingService {
return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`;
}
// TODO DS: track daily and hourly usage as well
async incrementUsage(actor: ActorWithType, usageType: (keyof typeof COST_MAPS) | (string & {}), usageAmount: number, costOverride?: number) {
async incrementUsage(actor: Actor, usageType: (keyof typeof COST_MAPS) | (string & {}), usageAmount: number, costOverride?: number) {
try {
if (!usageAmount || !usageType || !actor) {
if ( !usageAmount || !usageType || !actor ) {
// silent fail for now;
console.warn("Invalid usage increment parameters", { actor, usageType, usageAmount, costOverride });
console.warn('Invalid usage increment parameters', { actor, usageType, usageAmount, costOverride });
return { total: 0 } as UsageByType;
}
if (actor.type instanceof SystemActorType || actor.type?.user?.username === 'system') {
if ( actor.type instanceof SystemActorType || actor.type?.user?.username === 'system' ) {
// Don't track for now since it will trigger infinite noise;
return { total: 0 } as UsageByType;
}
@@ -77,44 +71,50 @@ export class MeteringAndBillingService {
return this.#superUserService.sudo(async () => {
const totalCost = (costOverride ?? (COST_MAPS[usageType as keyof typeof COST_MAPS] || 0) * usageAmount) || 0; // TODO DS: apply our policy discounts here eventually
usageType = usageType.replace(/\./g, PERIOD_ESCAPE) as keyof typeof COST_MAPS; // replace dots with underscores for kvstore paths, TODO DS: map this back when reading
const appId = actor.type?.app?.uid || GLOBAL_APP_KEY
const actorId = actor.type?.user.uuid
const appId = actor.type?.app?.uid || GLOBAL_APP_KEY;
const actorId = actor.type?.user.uuid;
const pathAndAmountMap = {
'total': totalCost,
[`${usageType}.units`]: usageAmount,
[`${usageType}.cost`]: totalCost,
[`${usageType}.count`]: 1,
}
};
const lastUpdatedKey = `${METRICS_PREFIX}:actor:${actorId}:lastUpdated`;
const lastUpdatedPromise = this.#kvClientWrapper.set({
key: lastUpdatedKey,
value: Date.now(),
})
});
const actorUsageKey = `${METRICS_PREFIX}:actor:${actorId}:${currentMonth}`;
const actorUsagesPromise = this.#kvClientWrapper.incr({
key: actorUsageKey,
pathAndAmountMap,
})
});
const puterConsumptionKey = `${METRICS_PREFIX}:puter:${currentMonth}`; // global consumption across all users and apps
this.#kvClientWrapper.incr({
key: puterConsumptionKey,
pathAndAmountMap
}).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'puterConsumptionKey' with error: `, e) });
pathAndAmountMap,
}).catch((e: Error) => {
console.warn('Failed to increment aux usage data \'puterConsumptionKey\' with error: ', e);
});
const actorAppUsageKey = `${METRICS_PREFIX}:actor:${actorId}:app:${appId}:${currentMonth}`;
this.#kvClientWrapper.incr({
key: actorAppUsageKey,
pathAndAmountMap,
}).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'actorAppUsageKey' with error: `, e) });
}).catch((e: Error) => {
console.warn('Failed to increment aux usage data \'actorAppUsageKey\' with error: ', e);
});
const appUsageKey = `${METRICS_PREFIX}:app:${appId}:${currentMonth}`;
this.#kvClientWrapper.incr({
key: appUsageKey,
pathAndAmountMap,
}).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'appUsageKey' with error: `, e) });
}).catch((e: Error) => {
console.warn('Failed to increment aux usage data \'appUsageKey\' with error: ', e);
});
const actorAppTotalsKey = `${METRICS_PREFIX}:actor:${actorId}:apps:${currentMonth}`;
this.#kvClientWrapper.incr({
@@ -123,11 +123,12 @@ export class MeteringAndBillingService {
[`${appId}.total`]: totalCost,
[`${appId}.count`]: 1,
},
}).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'actorAppTotalsKey' with error: `, e) });
}).catch((e: Error) => {
console.warn('Failed to increment aux usage data \'actorAppTotalsKey\' with error: ', e);
});
return (await Promise.all([lastUpdatedPromise, actorUsagesPromise]))[1] as UsageByType;
})
});
} catch (e) {
console.error('Metering: Failed to increment usage for actor', actor, 'usageType', usageType, 'usageAmount', usageAmount, e);
this.#alarmService.create('metering-service-error', (e as Error).message, {
@@ -135,103 +136,149 @@ export class MeteringAndBillingService {
actor,
usageType,
usageAmount,
costOverride
costOverride,
});
return { total: 0 } as UsageByType;
}
}
async getActorCurrentMonthUsageDetails(actor: ActorWithType) {
if (!actor.type?.user?.uuid) {
async getActorCurrentMonthUsageDetails(actor: Actor) {
if ( !actor.type?.user?.uuid ) {
throw new Error('Actor must be a user to get usage details');
}
// batch get actor usage, per app usage, and actor app totals for the month
const currentMonth = this.#getMonthYearString();
const keys = [
`${METRICS_PREFIX}:actor:${actor.type.user.uuid}:${currentMonth}`,
`${METRICS_PREFIX}:actor:${actor.type.user.uuid}:apps:${currentMonth}`
]
`${METRICS_PREFIX}:actor:${actor.type.user.uuid}:apps:${currentMonth}`,
];
return await this.#superUserService.sudo(async () => {
const [usage, appTotals] = await this.#kvClientWrapper.get({ key: keys }) as [UsageByType | null, Record<string, UsageByType> | null];
// only show details of app based on actor, aggregate all as others, except if app is global one or null, then show all
const appId = actor.type?.app?.uid
if (appTotals && appId) {
const appId = actor.type?.app?.uid;
if ( appTotals && appId ) {
const filteredAppTotals: Record<string, UsageByType> = {};
let othersTotal: UsageByType | null = null;
Object.entries(appTotals).forEach(([appKey, appUsage]) => {
if (appKey === appId) {
if ( appKey === appId ) {
filteredAppTotals[appKey] = appUsage;
} else {
Object.entries(appUsage).forEach(([usageKind, amount]) => {
if (!othersTotal![usageKind]) {
if ( !othersTotal![usageKind] ) {
othersTotal![usageKind] = 0;
}
othersTotal![usageKind] += amount;
})
});
}
});
if (othersTotal) {
if ( othersTotal ) {
filteredAppTotals['others'] = othersTotal;
}
return {
usage: usage || { total: 0 },
appTotals: filteredAppTotals,
}
} else {
return {
usage: usage || { total: 0 },
appTotals: appTotals || {},
}
};
}
})
return {
usage: usage || { total: 0 },
appTotals: appTotals || {},
};
});
}
async getActorCurrentMonthAppUsageDetails(actor: ActorWithType, appId?: string) {
if (!actor.type?.user?.uuid) {
async getActorCurrentMonthAppUsageDetails(actor: Actor, appId?: string) {
if ( !actor.type?.user?.uuid ) {
throw new Error('Actor must be a user to get usage details');
}
appId = appId || actor.type?.app?.uid || GLOBAL_APP_KEY;
// batch get actor usage, per app usage, and actor app totals for the month
const currentMonth = this.#getMonthYearString();
const key = `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:app:${appId}:${currentMonth}`
const key = `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:app:${appId}:${currentMonth}`;
return await this.#superUserService.sudo(async () => {
const usage = await this.#kvClientWrapper.get({ key }) as UsageByType | null;
// only show usage if actor app is the same or if global app ( null appId )
const actorAppId = actor.type?.app?.uid
if (actorAppId && actorAppId !== appId && appId !== GLOBAL_APP_KEY) {
const actorAppId = actor.type?.app?.uid;
if ( actorAppId && actorAppId !== appId && appId !== GLOBAL_APP_KEY ) {
throw new Error('Actor can only get usage details for their own app or global app');
}
return usage || { total: 0 };
})
});
}
async getActorPolicy(actor: ActorWithType): Promise<(keyof typeof SUB_POLICIES) | null> {
if (!actor.type?.user.uuid) {
async getRemainingUsage(actor: Actor) {
const allowedUsage = await this.getAllowedUsage(actor);
return allowedUsage.remaining || 0;
}
async getAllowedUsage(actor: Actor) {
const userSubscriptionPromise = this.getActorSubscription(actor);
const userPolicyAddonsPromise = this.getActorPolicyAddons(actor);
const currentUsagePromise = this.getActorCurrentMonthUsageDetails(actor);
const [userSubscription, userPolicyAddons, currentMonthUsage] = await Promise.all([userSubscriptionPromise, userPolicyAddonsPromise, currentUsagePromise]);
return {
remaining: Math.max(0, userSubscription.monthUsageAllowence + (userPolicyAddons?.purchasedCredits || 0) - currentMonthUsage.usage.total),
monthUsageAllowence: userSubscription?.monthUsageAllowence,
userPolicyAddons,
};
}
async hasAnyUsage(actor: Actor) {
return (await this.getRemainingUsage(actor)) > 0;
}
async hasEnoughCreditsFor(actor: Actor, usageType: keyof typeof COST_MAPS, usageAmount: number) {
const remainingUsage = await this.getRemainingUsage(actor);
const cost = (COST_MAPS[usageType] || 0) * usageAmount;
return remainingUsage >= cost;
}
async hasEnoughCredits(actor: Actor, amount: number) {
const remainingUsage = await this.getRemainingUsage(actor);
return remainingUsage >= amount;
}
async getActorSubscription(actor: Actor): Promise<(typeof SUB_POLICIES)[number]> {
// TODO DS: maybe allow non-user actors to have subscriptions eventually
if ( !actor.type?.user.uuid ) {
throw new Error('Actor must be a user to get policy');
}
const key = `${POLICY_PREFIX}:actor:${actor.type.user.uuid}`;
return this.#superUserService.sudo(async () => {
const policy = await this.#kvClientWrapper.get({ key });
return policy as (keyof typeof SUB_POLICIES) || null;
})
const defaultUserSubscriptionId = (actor.type.user.email ? DEFAULT_FREE_SUBSCRIPTION : DEFAULT_TEMP_SUBSCRIPTION);
const defaultSubscriptionEvent = { actor, defaultSubscriptionId: '' };
const availablePoliciesEvent = { actor, availablePolicies: [] as (typeof SUB_POLICIES)[number][] };
const userSubscriptionEvent = { actor, userSubscriptionId: '' };
await Promise.allSettled([
this.#eventService.emit('metering:overrideDefaultSubscription', defaultSubscriptionEvent), // can override default subscription based on actor properties
this.#eventService.emit('metering:registerAvailablePolicies', availablePoliciesEvent), // will add or modify available policies
this.#eventService.emit('metering:getUserSubscription', userSubscriptionEvent), // will set userSubscription property on event
]);
const defaultSubscriptionId = defaultSubscriptionEvent.defaultSubscriptionId as unknown as (typeof SUB_POLICIES)[number]['id'] || defaultUserSubscriptionId;
const availablePolicies = [ ...availablePoliciesEvent.availablePolicies, ...SUB_POLICIES ];
const userSubscriptionId = userSubscriptionEvent.userSubscriptionId as unknown as typeof SUB_POLICIES[number]['id'] || defaultSubscriptionId;
return availablePolicies.find(({ id }) => id === userSubscriptionId || id === defaultSubscriptionId)!;
}
async getActorPolicyAddons(actor: ActorWithType) {
if (!actor.type?.user?.uuid) {
async getActorPolicyAddons(actor: Actor) {
if ( !actor.type?.user?.uuid ) {
throw new Error('Actor must be a user to get policy addons');
}
const key = `${POLICY_PREFIX}:actor:${actor.type.user?.uuid}:addons`;
return this.#superUserService.sudo(async () => {
const policyAddOns = await this.#kvClientWrapper.get({ key });
return (policyAddOns ?? {}) as PolicyAddOns;
})
});
}
async #updateAddonCredit(actor: ActorWithType, tokenAmount: number) {
if (!actor.type?.user?.uuid) {
// eslint-disable-next-line
async #updateAddonCredit(actor: Actor, tokenAmount: number) {
if ( !actor.type?.user?.uuid ) {
throw new Error('Actor must be a user to update extra credits');
}
const key = `${POLICY_PREFIX}:actor:${actor.type.user?.uuid}:addons`;
@@ -241,16 +288,15 @@ export class MeteringAndBillingService {
pathAndAmountMap: {
purchasedCredits: tokenAmount,
},
})
})
});
});
}
handlePolicyPurchase(actor: ActorWithType, policyType: keyof typeof SUB_POLICIES) {
handlePolicyPurchase(_actor: Actor, _policyType: keyof typeof SUB_POLICIES) {
// TODO DS: this should leverage extensions to call billing implementations
}
handleTokenPurchase(actor: ActorWithType, tokenAmount: number) {
handleTokenPurchase(_actor: Actor, _tokenAmount: number) {
// TODO DS: this should leverage extensions to call billing implementations
}

1
src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs
View File

@@ -10,6 +10,7 @@ export class MeteringAndBillingServiceWrapper extends BaseService {
kvClientWrapper: this.services.get('puter-kvstore').as('puter-kvstore'),
superUserService: this.services.get('su'),
alarmService: this.services.get('alarm'),
eventService: this.services.get('event'),
});
}
}

7
src/backend/src/services/MeteringService/consts.ts Normal file
View File

@@ -0,0 +1,7 @@
export const GLOBAL_APP_KEY = 'os-global'; // TODO DS: this should be loaded from config or db eventually
export const METRICS_PREFIX = 'metering';
export const POLICY_PREFIX = 'policy';
export const PERIOD_ESCAPE = '_dot_'; // to replace dots in usage types for kvstore paths
export const DEFAULT_FREE_SUBSCRIPTION = 'user_free'; // TODO DS: this should be loaded from config or db eventually
export const DEFAULT_TEMP_SUBSCRIPTION = 'temp_free'; // TODO DS: this should be loaded from config or db eventually

4
src/backend/src/services/MeteringService/subPolicies/index.ts
View File

@@ -1,7 +1,7 @@
import { REGISTERED_USER_FREE } from "./registeredUserFreePolicy";
import { TEMP_USER_FREE } from "./tempUserFreePolicy";
export const SUB_POLICIES = {
export const SUB_POLICIES = [
TEMP_USER_FREE,
REGISTERED_USER_FREE,
}
] as const;

1
src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts
View File

@@ -1,6 +1,7 @@
import { toMicroCents } from "../utils";
export const REGISTERED_USER_FREE = {
id: 'user_free',
monthUsageAllowence: toMicroCents(0.50),
monthlyStorageAllowence: 100 * 1024 * 1024, // 100MiB
};

1
src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts
View File

@@ -1,6 +1,7 @@
import { toMicroCents } from "../utils";
export const TEMP_USER_FREE = {
id: 'temp_free',
monthUsageAllowence: toMicroCents(0.25),
monthlyStorageAllowence: 100 * 1024 * 1024, // 100MiB
};

8
src/backend/src/services/SUService.d.ts vendored Normal file
View File

@@ -0,0 +1,8 @@
import type { Actor } from "./auth/Actor";
export class SUService {
_construct(): void;
get_system_actor(): Promise<any>;
sudo<T>(callback: () => Promise<T>): Promise<T>;
sudo<T>(actorOrCallback: Actor, callback: () => Promise<T>): Promise<T>;
}

15
src/backend/src/services/auth/Actor.d.ts vendored Normal file
View File

@@ -0,0 +1,15 @@
export class SystemActorType {
get uid(): string;
get_related_type(type_class: any): SystemActorType;
}
export class Actor {
type: {
app: { uid: string }
user: { uuid: string, username: string, email: string, subscription?: (typeof SUB_POLICIES)[keyof typeof SUB_POLICIES]['id'] }
}
get uid(): string;
clone(): Actor;
static get_system_actor(): Actor;
static adapt(actor?: any): Actor;
}

1
src/backend/src/services/repositories/DBKVStore/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*.js

354
src/backend/src/services/repositories/DBKVStore/DBKVStore.mjs
View File

@@ -1,354 +0,0 @@
import murmurhash from "murmurhash";
import APIError from '../../../api/APIError.js';
import { Context } from "../../../util/context.js";
const GLOBAL_APP_KEY = 'global';
export class DBKVStore {
#db;
/** @type {import('../../MeteringService/MeteringService.js').MeteringAndBillingService} */
#meteringService;
#global_config = {};
// TODO DS: make table name configurable
constructor({ sqlClient, meteringAndBillingService, globalConfig }) {
this.#db = sqlClient;
this.#meteringService = meteringAndBillingService;
this.#global_config = globalConfig;
}
async get({ key }) {
const actor = Context.get('actor');
// If the actor is an app then it gets its own KV store.
// The way this is implemented isn't ideal for future behaviour;
// a KV implementation specified by the user would have parameters
// that are scoped to the app, so this should eventually be
// changed to get the app ID from the same interface that would
// be used to obtain per-app user-specified implementation params.
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( !user ) {
throw new Error('User not found');
}
const deleteExpired = async (rows) => {
const query = `DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (${rows.map(() => '?').join(',')})`;
const params = [user.id, app?.uid ?? GLOBAL_APP_KEY, ...rows.map(r => r.kkey_hash)];
return await this.#db.write(query, params);
};
if ( Array.isArray(key) ) {
const keys = key;
const key_hashes = keys.map(key => murmurhash.v3(key));
const rows = app ? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (?)',
[user.id, app.uid, key_hashes]) : await this.#db.read(`SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') ` +
`AND kkey_hash IN (${key_hashes.map(() => '?').join(',')})`,
[user.id, key_hashes]);
const kv = {};
rows.forEach(row => {
row.value = this.#db.case({
mysql: () => row.value,
otherwise: () => JSON.parse(row.value ?? 'null'),
})();
kv[row.kkey] = row.value;
});
const expiredKeys = [];
rows.forEach(row => {
if ( row?.expireAt && row.expireAt < (Date.now() / 1000) ) {
expiredKeys.push(row);
kv[row.kkey] = null;
} else {
kv[row.kkey] = row.value ?? null;
}
});
// clean up expired keys asynchronously
if ( expiredKeys.length ) {
deleteExpired(expiredKeys);
}
return keys.map(key => kv[key]);
}
const key_hash = murmurhash.v3(key);
const kv = app ? await this.#db.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1',
[user.id, app.uid, key_hash]) : await this.#db.read(`SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') AND kkey_hash=? LIMIT 1`,
[user.id, key_hash]);
if ( kv[0] ) {
kv[0].value = this.#db.case({
mysql: () => kv[0].value,
otherwise: () => JSON.parse(kv[0].value ?? 'null'),
})();
}
if ( kv[0]?.expireAt && kv[0].expireAt < (Date.now() / 1000) ) {
// key has expired
// clean up asynchronously
deleteExpired([kv[0]]);
return null;
}
// TODO DS: we await because of the batching done for our sql db, we need to make OSS increment usage atomic
await this.#meteringService.incrementUsage(actor, 'kv:read', Array.isArray(key) ? key.length : 1);
return kv[0]?.value ?? null;
}
async set({ key, value, expireAt }) {
const actor = Context.get('actor');
const config = this.#global_config;
// Validate the key
// get() doesn't String() the key but it only passes it to
// murmurhash.v3() so it doesn't need to ¯\_(ツ)_/¯
key = String(key);
if ( Buffer.byteLength(key, 'utf8') > config.kv_max_key_size ) {
throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`);
}
// Validate the value
value = value === undefined ? null : value;
if (
value !== null &&
Buffer.byteLength(JSON.stringify(value), 'utf8') >
config.kv_max_value_size
) {
throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`);
}
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( !user ) {
throw new Error('User not found');
}
const key_hash = murmurhash.v3(key);
try {
await this.#db.write(`INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt)
VALUES (?, ?, ?, ?, ?, ?) ${
this.#db.case({
mysql: 'ON DUPLICATE KEY UPDATE value = ?',
sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET value = excluded.value',
})}`,
[
user.id, app?.uid ?? GLOBAL_APP_KEY, key_hash, key,
JSON.stringify(value), expireAt ?? null,
...this.#db.case({ mysql: [value], otherwise: [] }),
]);
} catch ( e ) {
// I discovered that my .sqlite file was corrupted and the update
// above didn't work. The current database initialization does not
// cause this issue so I'm adding this log as a safeguard.
// - KernelDeimos / ED
const svc_error = this.services.get('error-service');
svc_error.report('kvstore:sqlite_error', {
message: 'Broken database version - please contact maintainers',
source: e,
});
}
// TODO DS: we await because of the batching done for our sql db
await this.#meteringService.incrementUsage(actor, 'kv:write', 1);
return true;
}
async del({ key }) {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( !user ) {
throw new Error('User not found');
}
const key_hash = murmurhash.v3(key);
await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?',
[user.id, app?.uid ?? GLOBAL_APP_KEY, key_hash]);
await this.#meteringService.incrementUsage(actor, 'kv:write', 1);
return true;
}
async list({ as }) {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( !user ) {
throw new Error('User not found');
}
let rows = app ? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=?',
[user.id, app.uid]) : await this.#db.read(`SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}')`,
[user.id]);
rows = rows.filter (row => {
return !row?.expireAt || row?.expireAt > Date.now() / 1000;
});
rows = rows.map(row => ({
key: row.kkey,
value: this.#db.case({
mysql: () => row.value,
otherwise: () => JSON.parse(row.value ?? 'null'),
})(),
}));
as = as || 'entries';
if ( !['keys', 'values', 'entries'].includes(as) ) {
throw APIError.create('field_invalid', null, {
key: 'as',
expected: '"keys", "values", or "entries"',
});
}
if ( as === 'keys' ) {
rows = rows.map(row => row.key);
}
else if ( as === 'values' ) {
rows = rows.map(row => row.value);
}
await this.#meteringService.incrementUsage(actor, 'kv:read', rows.length);
return rows;
}
async flush() {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( !user ) {
throw new Error('User not found');
}
await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=?',
[user.id, app?.uid ?? GLOBAL_APP_KEY]);
// TODO DS: should handle actual number of deleted items
await this.#meteringService.incrementUsage(actor, 'kv:write', 1);
return true;
}
async expireAt({ key, timestamp }) {
if ( key === '' ) {
throw APIError.create('field_empty', null, {
key: 'key',
});
}
timestamp = Number(timestamp);
return await this.#expireat(key, timestamp);
}
async expire({ key, ttl }) {
if ( key === '' ) {
throw APIError.create('field_empty', null, {
key: 'key',
});
}
ttl = Number(ttl);
// timestamp in seconds
let timestamp = Math.floor(Date.now() / 1000) + ttl;
return await this.#expireat(key, timestamp);
}
/** @type {(params: {key:string, pathAndAmountMap: Record<string, number>}) => Promise<unknown>} */
async incr({ key, pathAndAmountMap }) {
let currVal = await this.get({ key });
const pathEntries = Object.entries(pathAndAmountMap);
if ( typeof currVal !== 'object' && pathEntries.length <= 1 && !pathEntries[0]?.[0] ){
const amount = pathEntries[0]?.[1] ?? 1;
this.set({ key, value: (Number(currVal) || 0) + amount });
return (Number(currVal) || 0) + amount;
}
// TODO DS: support arrays this also needs dynamodb implementation
if ( Array.isArray(currVal) ){
throw new Error('Current value is an array');
}
if ( !currVal ){
currVal = {};
}
if ( typeof currVal !== 'object' ){
throw new Error('Current value is not an object');
}
// create or change values as needed
for ( const [path, amount] of Object.entries(pathAndAmountMap) ){
const pathParts = path.split('.');
let obj = currVal;
if ( obj === null ) continue;
for ( let i = 0; i < pathParts.length - 1; i++ ){
const part = pathParts[i];
if ( !obj[part] ){
obj[part] = {};
}
if ( typeof obj[part] !== 'object' || Array.isArray(currVal) ){
throw new Error(`Path ${pathParts.slice(0, i + 1).join('.')} is not an object`);
}
obj = obj[part];
}
if ( obj === null ) continue;
const lastPart = pathParts[pathParts.length - 1];
if ( !obj[lastPart] ){
obj[lastPart] = 0;
}
if ( typeof obj[lastPart] !== 'number' ){
throw new Error(`Value at path ${path} is not a number`);
}
obj[lastPart] += amount;
}
this.set({ key, value: currVal });
return currVal;
}
async decr({ key, path = '', amount = 1 }) {
return await this.incr({ key, path, amount: -amount });
}
async #expireat(key, timestamp) {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( !user ) {
throw new Error('User not found');
}
const key_hash = murmurhash.v3(key);
try {
await this.#db.write(`INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt)
VALUES (?, ?, ?, ?, ?, ?) ${
this.#db.case({
mysql: 'ON DUPLICATE KEY UPDATE expireAt = ?',
sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET expireAt = excluded.expireAt',
})}`,
[
user.id,
app?.uid ?? GLOBAL_APP_KEY,
key_hash,
key,
null, // empty value
timestamp,
...this.#db.case({ mysql: [timestamp], otherwise: [] }),
]);
} catch ( e ) {
// I discovered that my .sqlite file was corrupted and the update
// above didn't work. The current database initialization does not
// cause this issue so I'm adding this log as a safeguard.
// - KernelDeimos / ED
const svc_error = this.services.get('error-service');
svc_error.report('kvstore:sqlite_error', {
message: 'Broken database version - please contact maintainers',
source: e,
});
}
}
}

350
src/backend/src/services/repositories/DBKVStore/DBKVStore.ts Normal file
View File

@@ -0,0 +1,350 @@
// TypeScript conversion of DBKVStore.mjs
import murmurhash from "murmurhash";
// @ts-ignore
import APIError from '../../../api/APIError.js';
// @ts-ignore
import { Context } from "../../../util/context.js";
const GLOBAL_APP_KEY = 'global';
export class DBKVStore {
#db: any;
#meteringService: any;
#global_config: any = {};
// TODO DS: make table name configurable
constructor({ sqlClient, meteringAndBillingService, globalConfig }: { sqlClient: any, meteringAndBillingService: any, globalConfig: any }) {
this.#db = sqlClient;
this.#meteringService = meteringAndBillingService;
this.#global_config = globalConfig;
}
async get({ key }: { key: any }): Promise<any> {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if (!user) {
throw new Error('User not found');
}
const deleteExpired = async (rows: any[]) => {
const query = `DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (${rows.map(() => '?').join(',')})`;
const params = [user.id, app?.uid ?? GLOBAL_APP_KEY, ...rows.map((r: any) => r.kkey_hash)];
return await this.#db.write(query, params);
};
if (Array.isArray(key)) {
const keys = key;
const key_hashes = keys.map((key: any) => murmurhash.v3(key));
const rows = app
? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (?)', [user.id, app.uid, key_hashes])
: await this.#db.read(
`SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') AND kkey_hash IN (${key_hashes.map(() => '?').join(',')})`,
[user.id, key_hashes]
);
const kv: any = {};
rows.forEach((row: any) => {
row.value = this.#db.case({
mysql: () => row.value,
otherwise: () => JSON.parse(row.value ?? 'null'),
})();
kv[row.kkey] = row.value;
});
const expiredKeys: any[] = [];
rows.forEach((row: any) => {
if (row?.expireAt && row.expireAt < Date.now() / 1000) {
expiredKeys.push(row);
kv[row.kkey] = null;
} else {
kv[row.kkey] = row.value ?? null;
}
});
// clean up expired keys asynchronously
if (expiredKeys.length) {
deleteExpired(expiredKeys);
}
return keys.map((key: any) => kv[key]);
}
const key_hash = murmurhash.v3(key);
const kv = app
? await this.#db.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1', [user.id, app.uid, key_hash])
: await this.#db.read(
`SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') AND kkey_hash=? LIMIT 1`,
[user.id, key_hash]
);
if (kv[0]) {
kv[0].value = this.#db.case({
mysql: () => kv[0].value,
otherwise: () => JSON.parse(kv[0].value ?? 'null'),
})();
}
if (kv[0]?.expireAt && kv[0].expireAt < Date.now() / 1000) {
// key has expired
// clean up asynchronously
deleteExpired([kv[0]]);
return null;
}
await this.#meteringService.incrementUsage(actor, 'kv:read', Array.isArray(key) ? key.length : 1);
return kv[0]?.value ?? null;
}
async set({ key, value, expireAt }: { key: any, value: any, expireAt?: any }): Promise<boolean> {
const actor = Context.get('actor');
const config = this.#global_config;
key = String(key);
if (Buffer.byteLength(key, 'utf8') > config.kv_max_key_size) {
throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`);
}
value = value === undefined ? null : value;
if (
value !== null &&
Buffer.byteLength(JSON.stringify(value), 'utf8') > config.kv_max_value_size
) {
throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`);
}
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if (!user) {
throw new Error('User not found');
}
const key_hash = murmurhash.v3(key);
try {
await this.#db.write(
`INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt)
VALUES (?, ?, ?, ?, ?, ?) ${this.#db.case({
mysql: 'ON DUPLICATE KEY UPDATE value = ?',
sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET value = excluded.value',
})
}`,
[
user.id,
app?.uid ?? GLOBAL_APP_KEY,
key_hash,
key,
JSON.stringify(value),
expireAt ?? null,
...this.#db.case({ mysql: [value], otherwise: [] }),
]
);
} catch (e: any) {
console.error(e);
}
await this.#meteringService.incrementUsage(actor, 'kv:write', 1);
return true;
}
async del({ key }: { key: any }): Promise<boolean> {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if (!user) {
throw new Error('User not found');
}
const key_hash = murmurhash.v3(key);
await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', [
user.id,
app?.uid ?? GLOBAL_APP_KEY,
key_hash,
]);
await this.#meteringService.incrementUsage(actor, 'kv:write', 1);
return true;
}
async list({ as }: { as?: any }): Promise<any> {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if (!user) {
throw new Error('User not found');
}
let rows = app
? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=?', [user.id, app.uid])
: await this.#db.read(
`SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}')`,
[user.id]
);
rows = rows.filter((row: any) => {
return !row?.expireAt || row?.expireAt > Date.now() / 1000;
});
rows = rows.map((row: any) => ({
key: row.kkey,
value: this.#db.case({
mysql: () => row.value,
otherwise: () => JSON.parse(row.value ?? 'null'),
})(),
}));
as = as || 'entries';
if (!['keys', 'values', 'entries'].includes(as)) {
throw APIError.create('field_invalid', null, {
key: 'as',
expected: '"keys", "values", or "entries"',
});
}
if (as === 'keys') {
rows = rows.map((row: any) => row.key);
} else if (as === 'values') {
rows = rows.map((row: any) => row.value);
}
await this.#meteringService.incrementUsage(actor, 'kv:read', rows.length);
return rows;
}
async flush(): Promise<boolean> {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if (!user) {
throw new Error('User not found');
}
await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=?', [
user.id,
app?.uid ?? GLOBAL_APP_KEY,
]);
await this.#meteringService.incrementUsage(actor, 'kv:write', 1);
return true;
}
async expireAt({ key, timestamp }: { key: any, timestamp: any }): Promise<any> {
if (key === '') {
throw APIError.create('field_empty', null, {
key: 'key',
});
}
timestamp = Number(timestamp);
return await this.#expireat(key, timestamp);
}
async expire({ key, ttl }: { key: any, ttl: any }): Promise<any> {
if (key === '') {
throw APIError.create('field_empty', null, {
key: 'key',
});
}
ttl = Number(ttl);
// timestamp in seconds
let timestamp = Math.floor(Date.now() / 1000) + ttl;
return await this.#expireat(key, timestamp);
}
async incr({ key, pathAndAmountMap }: { key: string, pathAndAmountMap: Record<string, number> }): Promise<any> {
let currVal = await this.get({ key });
const pathEntries = Object.entries(pathAndAmountMap);
if (typeof currVal !== 'object' && pathEntries.length <= 1 && !pathEntries[0]?.[0]) {
const amount = pathEntries[0]?.[1] ?? 1;
this.set({ key, value: (Number(currVal) || 0) + amount });
return (Number(currVal) || 0) + amount;
}
// TODO DS: support arrays this also needs dynamodb implementation
if (Array.isArray(currVal)) {
throw new Error('Current value is an array');
}
if (!currVal) {
currVal = {};
}
if (typeof currVal !== 'object') {
throw new Error('Current value is not an object');
}
// create or change values as needed
for (const [path, amount] of Object.entries(pathAndAmountMap)) {
const pathParts = path.split('.');
let obj = currVal;
if (obj === null) continue;
for (let i = 0; i < pathParts.length - 1; i++) {
const part = pathParts[i];
if (!obj[part]) {
obj[part] = {};
}
if (typeof obj[part] !== 'object' || Array.isArray(currVal)) {
throw new Error(`Path ${pathParts.slice(0, i + 1).join('.')} is not an object`);
}
obj = obj[part];
}
if (obj === null) continue;
const lastPart = pathParts[pathParts.length - 1];
if (!obj[lastPart]) {
obj[lastPart] = 0;
}
if (typeof obj[lastPart] !== 'number') {
throw new Error(`Value at path ${path} is not a number`);
}
obj[lastPart] += amount;
}
this.set({ key, value: currVal });
return currVal;
}
async decr(...params: Parameters<typeof DBKVStore.prototype.incr>): Promise<any> {
return await this.incr(...params);
}
async #expireat(key: any, timestamp: any): Promise<any> {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if (!user) {
throw new Error('User not found');
}
const key_hash = murmurhash.v3(key);
try {
await this.#db.write(
`INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt)
VALUES (?, ?, ?, ?, ?, ?) ${this.#db.case({
mysql: 'ON DUPLICATE KEY UPDATE expireAt = ?',
sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET expireAt = excluded.expireAt',
})
}`,
[
user.id,
app?.uid ?? GLOBAL_APP_KEY,
key_hash,
key,
null, // empty value
timestamp,
...this.#db.case({ mysql: [timestamp], otherwise: [] }),
]
);
} catch (e: any) {
console.error(e);
}
}
}

2
src/backend/src/services/repositories/DBKVStore/index.mjs
View File

@@ -1,6 +1,6 @@
import BaseService from '../../BaseService.js';
import { DB_READ } from '../../database/consts.js';
import { DBKVStore } from './DBKVStore.mjs';
import { DBKVStore } from './DBKVStore.js';
export class DBKVServiceWrapper extends BaseService {
kvStore = undefined;

4
src/puter-js/src/modules/Auth.js
View File

@@ -174,7 +174,7 @@ class Auth{
async getMonthlyUsage() {
try {
const resp = await fetch(`${this.APIOrigin}/v2/usage`, {
const resp = await fetch(`${this.APIOrigin}/meteringAndBilling/usage`, {
headers: {
Authorization: `Bearer ${this.authToken}`,
},
@@ -216,7 +216,7 @@ class Auth{
}
try {
const resp = await fetch(`${this.APIOrigin}/v2/usage/${appId}`, {
const resp = await fetch(`${this.APIOrigin}/meteringAndBilling/usage/${appId}`, {
headers: {
Authorization: `Bearer ${this.authToken}`,
},

3
tsconfig.json
View File

@@ -7,7 +7,8 @@
"strict": true,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"skipLibCheck": true
"skipLibCheck": true,
"sourceMap": true
},
"exclude": [
"**/*.test.ts",