diff --git a/.gitignore b/.gitignore index a40c72ef..03b96d05 100644 --- a/.gitignore +++ b/.gitignore @@ -45,5 +45,10 @@ jsconfig.json # the exact tree installed in the node_modules folder package-lock.json +# AI STUFF AGENTS.md .roo + + +# source maps +*.map \ No newline at end of file diff --git a/eslint.config.js b/eslint.config.js index e3b0bba1..2ee7cabb 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,10 +1,34 @@ import js from '@eslint/js'; import stylistic from '@stylistic/eslint-plugin'; +import tseslintPlugin from '@typescript-eslint/eslint-plugin'; +import tseslintParser from '@typescript-eslint/parser'; import { defineConfig } from 'eslint/config'; import globals from 'globals'; import controlStructureSpacing from './control-structure-spacing.js'; export default defineConfig([ + // TypeScript support block + { + files: ['**/*.ts'], + languageOptions: { + parser: tseslintParser, + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + project: './tsconfig.json', + }, + }, + plugins: { + '@typescript-eslint': tseslintPlugin, + }, + rules: { + // Recommended rules for TypeScript + '@typescript-eslint/no-explicit-any': 'warn', + '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + '@typescript-eslint/ban-ts-comment': 'warn', + '@typescript-eslint/consistent-type-definitions': ['error', 'interface'], + }, + }, { plugins: { js, @@ -115,10 +139,10 @@ export default defineConfig([ }, }, { - files: ['**/*.{js,mjs,cjs}'], + files: ['**/*.{js,mjs,cjs,ts}'], ignores: [ - 'src/backend/**/*.{js,mjs,cjs}', - 'extensions/**/*.{js,mjs,cjs}', + 'src/backend/**/*.{js,mjs,cjs,ts}', + 'extensions/**/*.{js,mjs,cjs,ts}', ], languageOptions: { globals: globals.browser }, rules: { @@ -165,8 +189,8 @@ export default defineConfig([ }, }, { - files: ['**/*.{js,mjs,cjs}'], - ignores: ['src/backend/**/*.{js,mjs,cjs}'], + files: ['**/*.{js,mjs,cjs,ts}'], + ignores: ['src/backend/**/*.{js,mjs,cjs,ts}'], languageOptions: { globals: globals.browser }, rules: { 'no-unused-vars': ['error', { diff --git a/extensions/meteringAndBilling/config.json b/extensions/meteringAndBilling/config.json new file mode 100644 index 00000000..29052280 --- /dev/null +++ b/extensions/meteringAndBilling/config.json @@ -0,0 +1,3 @@ +{ + "unlimitedUsage": false +} \ No newline at end of file diff --git a/extensions/meteringAndBilling/eventListeners/subscriptionEvents.js b/extensions/meteringAndBilling/eventListeners/subscriptionEvents.js new file mode 100644 index 00000000..f3a70a81 --- /dev/null +++ b/extensions/meteringAndBilling/eventListeners/subscriptionEvents.js @@ -0,0 +1,24 @@ +extension.on('metering:overrideDefaultSubscription', async (/** @type {{actor: import('@heyputer/backend/src/services/auth/Actor').Actor, defaultSubscription: string}} */event) => { + // bit of a stub implementation for OSS, technically can be always free if you set this config true + if ( config.unlimitedUsage ) { + console.warn('WARNING!!! unlimitedUsage is enabled, this is not recommended for production use'); + event.defaultSubscriptionId = 'unlimited'; + } +}); + +extension.on('metering:registerAvailablePolicies', async ( + /** @type {{actor: import('@heyputer/backend/src/services/auth/Actor').Actor, availablePolicies: unknown[]}} */event) => { + // bit of a stub implementation for OSS, technically can be always free if you set this config true + if ( config.unlimitedUsage ) { + console.warn('WARNING!!! unlimitedUsage is enabled, this is not recommended for production use'); + event.availablePolicies.push({ + id: 'unlimited', + monthUsageAllowence: 500_000_000 * 100_000_000, // unless you're like, jeff's, mark's and elon's illegitamate son, you probably won't hit $5m a month + monthlyStorageAllowence: 100_000 * 1024 * 1024, // 100MiB + }); + } +}); + +extension.on('metering:getUserSubscription', async (/** @type {{actor: import('@heyputer/backend/src/services/auth/Actor').Actor, userSubscription: string}} */event) => { + event.userSubscriptionId = event.actor.type.user.subscription.tier; +}); diff --git a/extensions/meteringAndBilling/main.js b/extensions/meteringAndBilling/main.js new file mode 100644 index 00000000..b376a793 --- /dev/null +++ b/extensions/meteringAndBilling/main.js @@ -0,0 +1,2 @@ +import './eventListeners/subscriptionEvents.js'; +import './routes/usage.js'; diff --git a/extensions/meteringService/package.json b/extensions/meteringAndBilling/package.json similarity index 100% rename from extensions/meteringService/package.json rename to extensions/meteringAndBilling/package.json diff --git a/extensions/meteringService/routes/usage.js b/extensions/meteringAndBilling/routes/usage.js similarity index 63% rename from extensions/meteringService/routes/usage.js rename to extensions/meteringAndBilling/routes/usage.js index cb30b9fa..5ef84cbc 100644 --- a/extensions/meteringService/routes/usage.js +++ b/extensions/meteringAndBilling/routes/usage.js @@ -2,19 +2,22 @@ const meteringAndBillingServiceWrapper = extension.import('service:meteringService'); // TODO DS: move this to its own router and just use under this path -extension.get('/v2/usage', { subdomain: 'api' }, async (req, res) => { +extension.get('/meteringAndBilling/usage', { subdomain: 'api' }, async (req, res) => { const meteringAndBillingService = meteringAndBillingServiceWrapper.meteringAndBillingService; const actor = req.actor; if ( !actor ) { throw Error('actor not found in context'); } - const actorUsage = await meteringAndBillingService.getActorCurrentMonthUsageDetails(actor); - res.status(200).json(actorUsage); + const actorUsagePromise = meteringAndBillingService.getActorCurrentMonthUsageDetails(actor); + const actorAllowenceInfoPromise = meteringAndBillingService.getAllowedUsage(actor); + + const [actorUsage, allowenceInfo] = await Promise.all([actorUsagePromise, actorAllowenceInfoPromise]); + res.status(200).json({ ...actorUsage, allowenceInfo }); return; }); -extension.get('/v2/usage/:appId', { subdomain: 'api' }, async (req, res) => { +extension.get('/meteringAndBilling/usage/:appId', { subdomain: 'api' }, async (req, res) => { const meteringAndBillingService = meteringAndBillingServiceWrapper.meteringAndBillingService; const actor = req.actor; @@ -32,4 +35,4 @@ extension.get('/v2/usage/:appId', { subdomain: 'api' }, async (req, res) => { return; }); -console.debug('Loaded /v2/usage route'); \ No newline at end of file +console.debug('Loaded /meteringAndBilling/usage route'); \ No newline at end of file diff --git a/extensions/meteringService/main.js b/extensions/meteringService/main.js deleted file mode 100644 index d0a519f7..00000000 --- a/extensions/meteringService/main.js +++ /dev/null @@ -1 +0,0 @@ -import './routes/usage.js'; diff --git a/extensions/tsconfig.json b/extensions/tsconfig.json index f531272a..fad607cc 100644 --- a/extensions/tsconfig.json +++ b/extensions/tsconfig.json @@ -2,14 +2,18 @@ "compilerOptions": { "target": "ES2020", "allowJs": true, - "module": "esnext", - "moduleResolution": "node", + "module": "node16", + "moduleResolution": "node16", "baseUrl": ".", "outDir": "/dev/null", "paths": { - "../src/*": ["../src/*"] + "../src/*": [ + "../src/*" + ] }, - "typeRoots": ["../node_modules/@types"], + "typeRoots": [ + "../node_modules/@types" + ], "esModuleInterop": true, "allowSyntheticDefaultImports": true, "strict": true diff --git a/package-lock.json b/package-lock.json index ecaaafa0..54cf3937 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,8 @@ "devDependencies": { "@eslint/js": "^9.35.0", "@stylistic/eslint-plugin": "^5.3.1", + "@typescript-eslint/eslint-plugin": "^8.46.1", + "@typescript-eslint/parser": "^8.46.1", "chalk": "^4.1.0", "clean-css": "^5.3.2", "dotenv": "^16.4.5", @@ -7275,12 +7277,257 @@ "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", "license": "MIT" }, - "node_modules/@typescript-eslint/types": { - "version": "8.46.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.46.0.tgz", - "integrity": "sha512-bHGGJyVjSE4dJJIO5yyEWt/cHyNwga/zXGJbJJ8TiO01aVREK6gCTu3L+5wrkb1FbDkQ+TKjMNe9R/QQQP9+rA==", + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.46.1.tgz", + "integrity": "sha512-rUsLh8PXmBjdiPY+Emjz9NX2yHvhS11v0SR6xNJkm5GM1MO9ea/1GoDKlHHZGrOJclL/cZ2i/vRUYVtjRhrHVQ==", "dev": true, "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "8.46.1", + "@typescript-eslint/type-utils": "8.46.1", + "@typescript-eslint/utils": "8.46.1", + "@typescript-eslint/visitor-keys": "8.46.1", + "graphemer": "^1.4.0", + "ignore": "^7.0.0", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.46.1", + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.46.1.tgz", + "integrity": "sha512-6JSSaBZmsKvEkbRUkf7Zj7dru/8ZCrJxAqArcLaVMee5907JdtEbKGsZ7zNiIm/UAkpGUkaSMZEXShnN2D1HZA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/scope-manager": "8.46.1", + "@typescript-eslint/types": "8.46.1", + "@typescript-eslint/typescript-estree": "8.46.1", + "@typescript-eslint/visitor-keys": "8.46.1", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.46.1.tgz", + "integrity": "sha512-FOIaFVMHzRskXr5J4Jp8lFVV0gz5ngv3RHmn+E4HYxSJ3DgDzU7fVI1/M7Ijh1zf6S7HIoaIOtln1H5y8V+9Zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.46.1", + "@typescript-eslint/types": "^8.46.1", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.46.1.tgz", + "integrity": "sha512-weL9Gg3/5F0pVQKiF8eOXFZp8emqWzZsOJuWRUNtHT+UNV2xSJegmpCNQHy37aEQIbToTq7RHKhWvOsmbM680A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.46.1", + "@typescript-eslint/visitor-keys": "8.46.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.46.1.tgz", + "integrity": "sha512-X88+J/CwFvlJB+mK09VFqx5FE4H5cXD+H/Bdza2aEWkSb8hnWIQorNcscRl4IEo1Cz9VI/+/r/jnGWkbWPx54g==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.46.1.tgz", + "integrity": "sha512-+BlmiHIiqufBxkVnOtFwjah/vrkF4MtKKvpXrKSPLCkCtAp8H01/VV43sfqA98Od7nJpDcFnkwgyfQbOG0AMvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.46.1", + "@typescript-eslint/typescript-estree": "8.46.1", + "@typescript-eslint/utils": "8.46.1", + "debug": "^4.3.4", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.46.1.tgz", + "integrity": "sha512-C+soprGBHwWBdkDpbaRC4paGBrkIXxVlNohadL5o0kfhsXqOC6GYH2S/Obmig+I0HTDl8wMaRySwrfrXVP8/pQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.46.1.tgz", + "integrity": "sha512-uIifjT4s8cQKFQ8ZBXXyoUODtRoAd7F7+G8MKmtzj17+1UbdzFl52AzRyZRyKqPHhgzvXunnSckVu36flGy8cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.46.1", + "@typescript-eslint/tsconfig-utils": "8.46.1", + "@typescript-eslint/types": "8.46.1", + "@typescript-eslint/visitor-keys": "8.46.1", + "debug": "^4.3.4", + "fast-glob": "^3.3.2", + "is-glob": "^4.0.3", + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.46.1.tgz", + "integrity": "sha512-vkYUy6LdZS7q1v/Gxb2Zs7zziuXN0wxqsetJdeZdRe/f5dwJFglmuvZBfTUivCtjH725C1jWCDfpadadD95EDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.7.0", + "@typescript-eslint/scope-manager": "8.46.1", + "@typescript-eslint/types": "8.46.1", + "@typescript-eslint/typescript-estree": "8.46.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.46.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.46.1.tgz", + "integrity": "sha512-ptkmIf2iDkNUjdeu2bQqhFPV1m6qTnFFjg7PPDjxKWaMaP0Z6I9l30Jr3g5QqbZGdw8YdYvLp+XnqnWWZOg/NA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.46.1", + "eslint-visitor-keys": "^4.2.1" + }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, @@ -11606,6 +11853,13 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "license": "ISC" }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true, + "license": "MIT" + }, "node_modules/groq-sdk": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.5.0.tgz", @@ -17440,6 +17694,19 @@ "node": ">= 14.0.0" } }, + "node_modules/ts-api-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", + "integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -19808,7 +20075,7 @@ }, "src/puter-js": { "name": "@heyputer/puter.js", - "version": "2.0.14", + "version": "2.0.15", "license": "Apache-2.0", "dependencies": { "@heyputer/kv.js": "^0.2.1", diff --git a/package.json b/package.json index dbb87b5d..be2e07da 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "devDependencies": { "@eslint/js": "^9.35.0", "@stylistic/eslint-plugin": "^5.3.1", + "@typescript-eslint/eslint-plugin": "^8.46.1", + "@typescript-eslint/parser": "^8.46.1", "chalk": "^4.1.0", "clean-css": "^5.3.2", "dotenv": "^16.4.5", diff --git a/src/backend/src/modules/core/AlarmService.d.ts b/src/backend/src/modules/core/AlarmService.d.ts new file mode 100644 index 00000000..1659cd7c --- /dev/null +++ b/src/backend/src/modules/core/AlarmService.d.ts @@ -0,0 +1,6 @@ +export class AlarmService { + create(id: string, message: string, fields?: object): void; + clear(id: string): void; + get_alarm(id: string): object | undefined; + // Add more methods/properties as needed for MeteringService usage +} \ No newline at end of file diff --git a/src/backend/src/routers/auth/check-permissions.js b/src/backend/src/routers/auth/check-permissions.js index 70f883e5..74b6937e 100644 --- a/src/backend/src/routers/auth/check-permissions.js +++ b/src/backend/src/routers/auth/check-permissions.js @@ -34,11 +34,6 @@ module.exports = eggspress('/auth/check-permissions', { const actor = context.get('actor'); - // Apps cannot (currently) check permissions on behalf of users - if ( ! ( actor.type instanceof UserActorType ) ) { - throw APIError.create('forbidden'); - } - const permEntryPromises = [...new Set(permsToCheck)].map(async (perm) => { try { return [perm, permissionService.check(actor, perm)]; diff --git a/src/backend/src/services/EventService.d.ts b/src/backend/src/services/EventService.d.ts new file mode 100644 index 00000000..8899eaae --- /dev/null +++ b/src/backend/src/services/EventService.d.ts @@ -0,0 +1,7 @@ +// Minimal EventService type declaration for MeteringService type safety +export interface EventService { + emit(key: string, data?: any, meta?: any): Promise; + on(selector: string, callback: Function): { detach: () => void }; + on_all(callback: Function): void; + get_scoped(scope: string): any; +} \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/MeteringService.ts b/src/backend/src/services/MeteringService/MeteringService.ts index 591a8d4e..a069c475 100644 --- a/src/backend/src/services/MeteringService/MeteringService.ts +++ b/src/backend/src/services/MeteringService/MeteringService.ts @@ -1,19 +1,11 @@ -// @ts-ignore -import { SystemActorType, type Actor } from "../auth/Actor.js"; -// @ts-ignore -import type { AlarmService } from "../../modules/core/AlarmService.js"; -// @ts-ignore -import type { DBKVStore } from '../repositories/DBKVStore/DBKVStore.mjs'; -// @ts-ignore -import type { SUService } from "../SUService.js"; -import { COST_MAPS } from "./costMaps/index.js"; -import { SUB_POLICIES } from "./subPolicies/index.js"; -interface ActorWithType extends Actor { - type: { - app: { uid: string } - user: { uuid: string, username: string } - } -} +import type { AlarmService } from '../../modules/core/AlarmService.js'; +import { SystemActorType, type Actor } from '../auth/Actor.js'; +import type { EventService } from '../EventService'; // Type-only import for TS safety +import type { DBKVStore } from '../repositories/DBKVStore/DBKVStore'; +import type { SUService } from '../SUService.js'; +import { DEFAULT_FREE_SUBSCRIPTION, DEFAULT_TEMP_SUBSCRIPTION, GLOBAL_APP_KEY, METRICS_PREFIX, PERIOD_ESCAPE, POLICY_PREFIX } from './consts.js'; +import { COST_MAPS } from './costMaps/index.js'; +import { SUB_POLICIES } from './subPolicies/index.js'; interface PolicyAddOns { purchasedCredits: number @@ -27,26 +19,29 @@ interface UsageByType { [serviceName: string]: number } -const GLOBAL_APP_KEY = 'os-global'; // TODO DS: this should be loaded from config or db eventually -const METRICS_PREFIX = 'metering'; -const POLICY_PREFIX = 'policy'; -const PERIOD_ESCAPE = '_dot_'; // to replace dots in usage types for kvstore paths - +interface MeteringAndBillingServiceDeps { + kvClientWrapper: DBKVStore, + superUserService: SUService, + alarmService: AlarmService + eventService: EventService +} /** * Handles usage metering and supports stubbs for billing methods for current scoped actor */ export class MeteringAndBillingService { - #kvClientWrapper: DBKVStore - #superUserService: SUService - #alarmService: AlarmService - constructor({ kvClientWrapper, superUserService, alarmService }: { kvClientWrapper: DBKVStore, superUserService: SUService, alarmService: AlarmService }) { + #kvClientWrapper: DBKVStore; + #superUserService: SUService; + #alarmService: AlarmService; + #eventService: EventService; + constructor({ kvClientWrapper, superUserService, alarmService, eventService }: MeteringAndBillingServiceDeps) { this.#superUserService = superUserService; this.#kvClientWrapper = kvClientWrapper; this.#alarmService = alarmService; + this.#eventService = eventService; } - utilRecordUsageObject(trackedUsageObject: Record, actor: ActorWithType, modelPrefix: string) { + utilRecordUsageObject(trackedUsageObject: Record, actor: Actor, modelPrefix: string) { Object.entries(trackedUsageObject).forEach(([usageKind, amount]) => { this.incrementUsage(actor, `${modelPrefix}:${usageKind}`, amount); }); @@ -57,17 +52,16 @@ export class MeteringAndBillingService { return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`; } - // TODO DS: track daily and hourly usage as well - async incrementUsage(actor: ActorWithType, usageType: (keyof typeof COST_MAPS) | (string & {}), usageAmount: number, costOverride?: number) { + async incrementUsage(actor: Actor, usageType: (keyof typeof COST_MAPS) | (string & {}), usageAmount: number, costOverride?: number) { try { - if (!usageAmount || !usageType || !actor) { + if ( !usageAmount || !usageType || !actor ) { // silent fail for now; - console.warn("Invalid usage increment parameters", { actor, usageType, usageAmount, costOverride }); + console.warn('Invalid usage increment parameters', { actor, usageType, usageAmount, costOverride }); return { total: 0 } as UsageByType; } - if (actor.type instanceof SystemActorType || actor.type?.user?.username === 'system') { + if ( actor.type instanceof SystemActorType || actor.type?.user?.username === 'system' ) { // Don't track for now since it will trigger infinite noise; return { total: 0 } as UsageByType; } @@ -77,44 +71,50 @@ export class MeteringAndBillingService { return this.#superUserService.sudo(async () => { const totalCost = (costOverride ?? (COST_MAPS[usageType as keyof typeof COST_MAPS] || 0) * usageAmount) || 0; // TODO DS: apply our policy discounts here eventually usageType = usageType.replace(/\./g, PERIOD_ESCAPE) as keyof typeof COST_MAPS; // replace dots with underscores for kvstore paths, TODO DS: map this back when reading - const appId = actor.type?.app?.uid || GLOBAL_APP_KEY - const actorId = actor.type?.user.uuid + const appId = actor.type?.app?.uid || GLOBAL_APP_KEY; + const actorId = actor.type?.user.uuid; const pathAndAmountMap = { 'total': totalCost, [`${usageType}.units`]: usageAmount, [`${usageType}.cost`]: totalCost, [`${usageType}.count`]: 1, - } + }; const lastUpdatedKey = `${METRICS_PREFIX}:actor:${actorId}:lastUpdated`; const lastUpdatedPromise = this.#kvClientWrapper.set({ key: lastUpdatedKey, value: Date.now(), - }) + }); const actorUsageKey = `${METRICS_PREFIX}:actor:${actorId}:${currentMonth}`; const actorUsagesPromise = this.#kvClientWrapper.incr({ key: actorUsageKey, pathAndAmountMap, - }) + }); const puterConsumptionKey = `${METRICS_PREFIX}:puter:${currentMonth}`; // global consumption across all users and apps this.#kvClientWrapper.incr({ key: puterConsumptionKey, - pathAndAmountMap - }).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'puterConsumptionKey' with error: `, e) }); + pathAndAmountMap, + }).catch((e: Error) => { + console.warn('Failed to increment aux usage data \'puterConsumptionKey\' with error: ', e); + }); const actorAppUsageKey = `${METRICS_PREFIX}:actor:${actorId}:app:${appId}:${currentMonth}`; this.#kvClientWrapper.incr({ key: actorAppUsageKey, pathAndAmountMap, - }).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'actorAppUsageKey' with error: `, e) }); + }).catch((e: Error) => { + console.warn('Failed to increment aux usage data \'actorAppUsageKey\' with error: ', e); + }); const appUsageKey = `${METRICS_PREFIX}:app:${appId}:${currentMonth}`; this.#kvClientWrapper.incr({ key: appUsageKey, pathAndAmountMap, - }).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'appUsageKey' with error: `, e) }); + }).catch((e: Error) => { + console.warn('Failed to increment aux usage data \'appUsageKey\' with error: ', e); + }); const actorAppTotalsKey = `${METRICS_PREFIX}:actor:${actorId}:apps:${currentMonth}`; this.#kvClientWrapper.incr({ @@ -123,11 +123,12 @@ export class MeteringAndBillingService { [`${appId}.total`]: totalCost, [`${appId}.count`]: 1, }, - }).catch((e: Error) => { console.warn(`Failed to increment aux usage data 'actorAppTotalsKey' with error: `, e) }); - + }).catch((e: Error) => { + console.warn('Failed to increment aux usage data \'actorAppTotalsKey\' with error: ', e); + }); return (await Promise.all([lastUpdatedPromise, actorUsagesPromise]))[1] as UsageByType; - }) + }); } catch (e) { console.error('Metering: Failed to increment usage for actor', actor, 'usageType', usageType, 'usageAmount', usageAmount, e); this.#alarmService.create('metering-service-error', (e as Error).message, { @@ -135,103 +136,149 @@ export class MeteringAndBillingService { actor, usageType, usageAmount, - costOverride + costOverride, }); return { total: 0 } as UsageByType; } } - async getActorCurrentMonthUsageDetails(actor: ActorWithType) { - if (!actor.type?.user?.uuid) { + async getActorCurrentMonthUsageDetails(actor: Actor) { + if ( !actor.type?.user?.uuid ) { throw new Error('Actor must be a user to get usage details'); } // batch get actor usage, per app usage, and actor app totals for the month const currentMonth = this.#getMonthYearString(); const keys = [ `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:${currentMonth}`, - `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:apps:${currentMonth}` - ] + `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:apps:${currentMonth}`, + ]; return await this.#superUserService.sudo(async () => { const [usage, appTotals] = await this.#kvClientWrapper.get({ key: keys }) as [UsageByType | null, Record | null]; // only show details of app based on actor, aggregate all as others, except if app is global one or null, then show all - const appId = actor.type?.app?.uid - if (appTotals && appId) { + const appId = actor.type?.app?.uid; + if ( appTotals && appId ) { const filteredAppTotals: Record = {}; let othersTotal: UsageByType | null = null; Object.entries(appTotals).forEach(([appKey, appUsage]) => { - if (appKey === appId) { + if ( appKey === appId ) { filteredAppTotals[appKey] = appUsage; } else { Object.entries(appUsage).forEach(([usageKind, amount]) => { - if (!othersTotal![usageKind]) { + if ( !othersTotal![usageKind] ) { othersTotal![usageKind] = 0; } othersTotal![usageKind] += amount; - }) + }); } }); - if (othersTotal) { + if ( othersTotal ) { filteredAppTotals['others'] = othersTotal; } return { usage: usage || { total: 0 }, appTotals: filteredAppTotals, - } - } else { - return { - usage: usage || { total: 0 }, - appTotals: appTotals || {}, - } + }; } - }) + return { + usage: usage || { total: 0 }, + appTotals: appTotals || {}, + }; + }); } - async getActorCurrentMonthAppUsageDetails(actor: ActorWithType, appId?: string) { - if (!actor.type?.user?.uuid) { + async getActorCurrentMonthAppUsageDetails(actor: Actor, appId?: string) { + if ( !actor.type?.user?.uuid ) { throw new Error('Actor must be a user to get usage details'); } - appId = appId || actor.type?.app?.uid || GLOBAL_APP_KEY; // batch get actor usage, per app usage, and actor app totals for the month const currentMonth = this.#getMonthYearString(); - const key = `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:app:${appId}:${currentMonth}` + const key = `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:app:${appId}:${currentMonth}`; return await this.#superUserService.sudo(async () => { const usage = await this.#kvClientWrapper.get({ key }) as UsageByType | null; // only show usage if actor app is the same or if global app ( null appId ) - const actorAppId = actor.type?.app?.uid - if (actorAppId && actorAppId !== appId && appId !== GLOBAL_APP_KEY) { + const actorAppId = actor.type?.app?.uid; + if ( actorAppId && actorAppId !== appId && appId !== GLOBAL_APP_KEY ) { throw new Error('Actor can only get usage details for their own app or global app'); } return usage || { total: 0 }; - }) + }); } - async getActorPolicy(actor: ActorWithType): Promise<(keyof typeof SUB_POLICIES) | null> { - if (!actor.type?.user.uuid) { + async getRemainingUsage(actor: Actor) { + const allowedUsage = await this.getAllowedUsage(actor); + return allowedUsage.remaining || 0; + + } + + async getAllowedUsage(actor: Actor) { + const userSubscriptionPromise = this.getActorSubscription(actor); + const userPolicyAddonsPromise = this.getActorPolicyAddons(actor); + const currentUsagePromise = this.getActorCurrentMonthUsageDetails(actor); + + const [userSubscription, userPolicyAddons, currentMonthUsage] = await Promise.all([userSubscriptionPromise, userPolicyAddonsPromise, currentUsagePromise]); + return { + remaining: Math.max(0, userSubscription.monthUsageAllowence + (userPolicyAddons?.purchasedCredits || 0) - currentMonthUsage.usage.total), + monthUsageAllowence: userSubscription?.monthUsageAllowence, + userPolicyAddons, + }; + } + + async hasAnyUsage(actor: Actor) { + return (await this.getRemainingUsage(actor)) > 0; + } + + async hasEnoughCreditsFor(actor: Actor, usageType: keyof typeof COST_MAPS, usageAmount: number) { + const remainingUsage = await this.getRemainingUsage(actor); + const cost = (COST_MAPS[usageType] || 0) * usageAmount; + return remainingUsage >= cost; + } + + async hasEnoughCredits(actor: Actor, amount: number) { + const remainingUsage = await this.getRemainingUsage(actor); + return remainingUsage >= amount; + } + + async getActorSubscription(actor: Actor): Promise<(typeof SUB_POLICIES)[number]> { + // TODO DS: maybe allow non-user actors to have subscriptions eventually + if ( !actor.type?.user.uuid ) { throw new Error('Actor must be a user to get policy'); } - const key = `${POLICY_PREFIX}:actor:${actor.type.user.uuid}`; - return this.#superUserService.sudo(async () => { - const policy = await this.#kvClientWrapper.get({ key }); - return policy as (keyof typeof SUB_POLICIES) || null; - }) + + const defaultUserSubscriptionId = (actor.type.user.email ? DEFAULT_FREE_SUBSCRIPTION : DEFAULT_TEMP_SUBSCRIPTION); + const defaultSubscriptionEvent = { actor, defaultSubscriptionId: '' }; + const availablePoliciesEvent = { actor, availablePolicies: [] as (typeof SUB_POLICIES)[number][] }; + const userSubscriptionEvent = { actor, userSubscriptionId: '' }; + + await Promise.allSettled([ + this.#eventService.emit('metering:overrideDefaultSubscription', defaultSubscriptionEvent), // can override default subscription based on actor properties + this.#eventService.emit('metering:registerAvailablePolicies', availablePoliciesEvent), // will add or modify available policies + this.#eventService.emit('metering:getUserSubscription', userSubscriptionEvent), // will set userSubscription property on event + ]); + + const defaultSubscriptionId = defaultSubscriptionEvent.defaultSubscriptionId as unknown as (typeof SUB_POLICIES)[number]['id'] || defaultUserSubscriptionId; + const availablePolicies = [ ...availablePoliciesEvent.availablePolicies, ...SUB_POLICIES ]; + const userSubscriptionId = userSubscriptionEvent.userSubscriptionId as unknown as typeof SUB_POLICIES[number]['id'] || defaultSubscriptionId; + + return availablePolicies.find(({ id }) => id === userSubscriptionId || id === defaultSubscriptionId)!; } - async getActorPolicyAddons(actor: ActorWithType) { - if (!actor.type?.user?.uuid) { + async getActorPolicyAddons(actor: Actor) { + if ( !actor.type?.user?.uuid ) { throw new Error('Actor must be a user to get policy addons'); } const key = `${POLICY_PREFIX}:actor:${actor.type.user?.uuid}:addons`; return this.#superUserService.sudo(async () => { const policyAddOns = await this.#kvClientWrapper.get({ key }); return (policyAddOns ?? {}) as PolicyAddOns; - }) + }); } - async #updateAddonCredit(actor: ActorWithType, tokenAmount: number) { - if (!actor.type?.user?.uuid) { + // eslint-disable-next-line + async #updateAddonCredit(actor: Actor, tokenAmount: number) { + if ( !actor.type?.user?.uuid ) { throw new Error('Actor must be a user to update extra credits'); } const key = `${POLICY_PREFIX}:actor:${actor.type.user?.uuid}:addons`; @@ -241,16 +288,15 @@ export class MeteringAndBillingService { pathAndAmountMap: { purchasedCredits: tokenAmount, }, - }) - }) - + }); + }); } - handlePolicyPurchase(actor: ActorWithType, policyType: keyof typeof SUB_POLICIES) { + handlePolicyPurchase(_actor: Actor, _policyType: keyof typeof SUB_POLICIES) { // TODO DS: this should leverage extensions to call billing implementations } - handleTokenPurchase(actor: ActorWithType, tokenAmount: number) { + handleTokenPurchase(_actor: Actor, _tokenAmount: number) { // TODO DS: this should leverage extensions to call billing implementations } diff --git a/src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs b/src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs index 77a11ef6..cf6149e5 100644 --- a/src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs +++ b/src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs @@ -10,6 +10,7 @@ export class MeteringAndBillingServiceWrapper extends BaseService { kvClientWrapper: this.services.get('puter-kvstore').as('puter-kvstore'), superUserService: this.services.get('su'), alarmService: this.services.get('alarm'), + eventService: this.services.get('event'), }); } } diff --git a/src/backend/src/services/MeteringService/consts.ts b/src/backend/src/services/MeteringService/consts.ts new file mode 100644 index 00000000..14b8a4e4 --- /dev/null +++ b/src/backend/src/services/MeteringService/consts.ts @@ -0,0 +1,7 @@ + +export const GLOBAL_APP_KEY = 'os-global'; // TODO DS: this should be loaded from config or db eventually +export const METRICS_PREFIX = 'metering'; +export const POLICY_PREFIX = 'policy'; +export const PERIOD_ESCAPE = '_dot_'; // to replace dots in usage types for kvstore paths +export const DEFAULT_FREE_SUBSCRIPTION = 'user_free'; // TODO DS: this should be loaded from config or db eventually +export const DEFAULT_TEMP_SUBSCRIPTION = 'temp_free'; // TODO DS: this should be loaded from config or db eventually \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/subPolicies/index.ts b/src/backend/src/services/MeteringService/subPolicies/index.ts index b439ee0f..44a6f0b5 100644 --- a/src/backend/src/services/MeteringService/subPolicies/index.ts +++ b/src/backend/src/services/MeteringService/subPolicies/index.ts @@ -1,7 +1,7 @@ import { REGISTERED_USER_FREE } from "./registeredUserFreePolicy"; import { TEMP_USER_FREE } from "./tempUserFreePolicy"; -export const SUB_POLICIES = { +export const SUB_POLICIES = [ TEMP_USER_FREE, REGISTERED_USER_FREE, -} \ No newline at end of file +] as const; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts b/src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts index b9c49a4a..5a04f536 100644 --- a/src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts +++ b/src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts @@ -1,6 +1,7 @@ import { toMicroCents } from "../utils"; export const REGISTERED_USER_FREE = { + id: 'user_free', monthUsageAllowence: toMicroCents(0.50), monthlyStorageAllowence: 100 * 1024 * 1024, // 100MiB }; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts b/src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts index 69957c5e..ba237784 100644 --- a/src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts +++ b/src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts @@ -1,6 +1,7 @@ import { toMicroCents } from "../utils"; export const TEMP_USER_FREE = { + id: 'temp_free', monthUsageAllowence: toMicroCents(0.25), monthlyStorageAllowence: 100 * 1024 * 1024, // 100MiB }; \ No newline at end of file diff --git a/src/backend/src/services/SUService.d.ts b/src/backend/src/services/SUService.d.ts new file mode 100644 index 00000000..9e24dbfc --- /dev/null +++ b/src/backend/src/services/SUService.d.ts @@ -0,0 +1,8 @@ +import type { Actor } from "./auth/Actor"; + +export class SUService { + _construct(): void; + get_system_actor(): Promise; + sudo(callback: () => Promise): Promise; + sudo(actorOrCallback: Actor, callback: () => Promise): Promise; +} \ No newline at end of file diff --git a/src/backend/src/services/auth/Actor.d.ts b/src/backend/src/services/auth/Actor.d.ts new file mode 100644 index 00000000..fa4730f3 --- /dev/null +++ b/src/backend/src/services/auth/Actor.d.ts @@ -0,0 +1,15 @@ +export class SystemActorType { + get uid(): string; + get_related_type(type_class: any): SystemActorType; +} + +export class Actor { + type: { + app: { uid: string } + user: { uuid: string, username: string, email: string, subscription?: (typeof SUB_POLICIES)[keyof typeof SUB_POLICIES]['id'] } + } + get uid(): string; + clone(): Actor; + static get_system_actor(): Actor; + static adapt(actor?: any): Actor; +} \ No newline at end of file diff --git a/src/backend/src/services/repositories/DBKVStore/.gitignore b/src/backend/src/services/repositories/DBKVStore/.gitignore new file mode 100644 index 00000000..4c43fe68 --- /dev/null +++ b/src/backend/src/services/repositories/DBKVStore/.gitignore @@ -0,0 +1 @@ +*.js \ No newline at end of file diff --git a/src/backend/src/services/repositories/DBKVStore/DBKVStore.mjs b/src/backend/src/services/repositories/DBKVStore/DBKVStore.mjs deleted file mode 100644 index 43b8aabc..00000000 --- a/src/backend/src/services/repositories/DBKVStore/DBKVStore.mjs +++ /dev/null @@ -1,354 +0,0 @@ -import murmurhash from "murmurhash"; -import APIError from '../../../api/APIError.js'; -import { Context } from "../../../util/context.js"; - -const GLOBAL_APP_KEY = 'global'; -export class DBKVStore { - #db; - /** @type {import('../../MeteringService/MeteringService.js').MeteringAndBillingService} */ - #meteringService; - - #global_config = {}; - // TODO DS: make table name configurable - constructor({ sqlClient, meteringAndBillingService, globalConfig }) { - this.#db = sqlClient; - this.#meteringService = meteringAndBillingService; - this.#global_config = globalConfig; - } - async get({ key }) { - const actor = Context.get('actor'); - - // If the actor is an app then it gets its own KV store. - // The way this is implemented isn't ideal for future behaviour; - // a KV implementation specified by the user would have parameters - // that are scoped to the app, so this should eventually be - // changed to get the app ID from the same interface that would - // be used to obtain per-app user-specified implementation params. - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - - if ( !user ) { - throw new Error('User not found'); - } - - const deleteExpired = async (rows) => { - const query = `DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (${rows.map(() => '?').join(',')})`; - const params = [user.id, app?.uid ?? GLOBAL_APP_KEY, ...rows.map(r => r.kkey_hash)]; - return await this.#db.write(query, params); - }; - - if ( Array.isArray(key) ) { - const keys = key; - const key_hashes = keys.map(key => murmurhash.v3(key)); - const rows = app ? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (?)', - [user.id, app.uid, key_hashes]) : await this.#db.read(`SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') ` + - `AND kkey_hash IN (${key_hashes.map(() => '?').join(',')})`, - [user.id, key_hashes]); - - const kv = {}; - rows.forEach(row => { - row.value = this.#db.case({ - mysql: () => row.value, - otherwise: () => JSON.parse(row.value ?? 'null'), - })(); - kv[row.kkey] = row.value; - }); - - const expiredKeys = []; - rows.forEach(row => { - if ( row?.expireAt && row.expireAt < (Date.now() / 1000) ) { - expiredKeys.push(row); - kv[row.kkey] = null; - } else { - kv[row.kkey] = row.value ?? null; - } - }); - - // clean up expired keys asynchronously - if ( expiredKeys.length ) { - deleteExpired(expiredKeys); - } - - return keys.map(key => kv[key]); - - } - - const key_hash = murmurhash.v3(key); - const kv = app ? await this.#db.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1', - [user.id, app.uid, key_hash]) : await this.#db.read(`SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') AND kkey_hash=? LIMIT 1`, - [user.id, key_hash]); - - if ( kv[0] ) { - kv[0].value = this.#db.case({ - mysql: () => kv[0].value, - otherwise: () => JSON.parse(kv[0].value ?? 'null'), - })(); - } - - if ( kv[0]?.expireAt && kv[0].expireAt < (Date.now() / 1000) ) { - // key has expired - // clean up asynchronously - deleteExpired([kv[0]]); - return null; - } - - // TODO DS: we await because of the batching done for our sql db, we need to make OSS increment usage atomic - await this.#meteringService.incrementUsage(actor, 'kv:read', Array.isArray(key) ? key.length : 1); - - return kv[0]?.value ?? null; - } - async set({ key, value, expireAt }) { - const actor = Context.get('actor'); - const config = this.#global_config; - - // Validate the key - // get() doesn't String() the key but it only passes it to - // murmurhash.v3() so it doesn't need to ¯\_(ツ)_/¯ - key = String(key); - if ( Buffer.byteLength(key, 'utf8') > config.kv_max_key_size ) { - throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`); - } - - // Validate the value - value = value === undefined ? null : value; - if ( - value !== null && - Buffer.byteLength(JSON.stringify(value), 'utf8') > - config.kv_max_value_size - ) { - throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`); - } - - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( !user ) { - throw new Error('User not found'); - } - - const key_hash = murmurhash.v3(key); - - try { - await this.#db.write(`INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt) - VALUES (?, ?, ?, ?, ?, ?) ${ - this.#db.case({ - mysql: 'ON DUPLICATE KEY UPDATE value = ?', - sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET value = excluded.value', - })}`, - [ - user.id, app?.uid ?? GLOBAL_APP_KEY, key_hash, key, - JSON.stringify(value), expireAt ?? null, - ...this.#db.case({ mysql: [value], otherwise: [] }), - ]); - } catch ( e ) { - // I discovered that my .sqlite file was corrupted and the update - // above didn't work. The current database initialization does not - // cause this issue so I'm adding this log as a safeguard. - // - KernelDeimos / ED - const svc_error = this.services.get('error-service'); - svc_error.report('kvstore:sqlite_error', { - message: 'Broken database version - please contact maintainers', - source: e, - }); - } - - // TODO DS: we await because of the batching done for our sql db - await this.#meteringService.incrementUsage(actor, 'kv:write', 1); - - return true; - } - async del({ key }) { - const actor = Context.get('actor'); - - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( !user ) { - throw new Error('User not found'); - } - - const key_hash = murmurhash.v3(key); - - await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', - [user.id, app?.uid ?? GLOBAL_APP_KEY, key_hash]); - - await this.#meteringService.incrementUsage(actor, 'kv:write', 1); - - return true; - } - async list({ as }) { - const actor = Context.get('actor'); - - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - - if ( !user ) { - throw new Error('User not found'); - } - - let rows = app ? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=?', - [user.id, app.uid]) : await this.#db.read(`SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}')`, - [user.id]); - - rows = rows.filter (row => { - return !row?.expireAt || row?.expireAt > Date.now() / 1000; - }); - - rows = rows.map(row => ({ - key: row.kkey, - value: this.#db.case({ - mysql: () => row.value, - otherwise: () => JSON.parse(row.value ?? 'null'), - })(), - })); - - as = as || 'entries'; - - if ( !['keys', 'values', 'entries'].includes(as) ) { - throw APIError.create('field_invalid', null, { - key: 'as', - expected: '"keys", "values", or "entries"', - }); - } - - if ( as === 'keys' ) { - rows = rows.map(row => row.key); - } - else if ( as === 'values' ) { - rows = rows.map(row => row.value); - } - - await this.#meteringService.incrementUsage(actor, 'kv:read', rows.length); - - return rows; - } - async flush() { - const actor = Context.get('actor'); - - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( !user ) { - throw new Error('User not found'); - } - - await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=?', - [user.id, app?.uid ?? GLOBAL_APP_KEY]); - - // TODO DS: should handle actual number of deleted items - await this.#meteringService.incrementUsage(actor, 'kv:write', 1); - - return true; - } - async expireAt({ key, timestamp }) { - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - timestamp = Number(timestamp); - - return await this.#expireat(key, timestamp); - } - async expire({ key, ttl }) { - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - ttl = Number(ttl); - - // timestamp in seconds - let timestamp = Math.floor(Date.now() / 1000) + ttl; - - return await this.#expireat(key, timestamp); - } - /** @type {(params: {key:string, pathAndAmountMap: Record}) => Promise} */ - async incr({ key, pathAndAmountMap }) { - let currVal = await this.get({ key }); - const pathEntries = Object.entries(pathAndAmountMap); - if ( typeof currVal !== 'object' && pathEntries.length <= 1 && !pathEntries[0]?.[0] ){ - const amount = pathEntries[0]?.[1] ?? 1; - this.set({ key, value: (Number(currVal) || 0) + amount }); - return (Number(currVal) || 0) + amount; - } - // TODO DS: support arrays this also needs dynamodb implementation - if ( Array.isArray(currVal) ){ - throw new Error('Current value is an array'); - } - if ( !currVal ){ - currVal = {}; - } - if ( typeof currVal !== 'object' ){ - throw new Error('Current value is not an object'); - } - // create or change values as needed - for ( const [path, amount] of Object.entries(pathAndAmountMap) ){ - const pathParts = path.split('.'); - let obj = currVal; - if ( obj === null ) continue; - for ( let i = 0; i < pathParts.length - 1; i++ ){ - const part = pathParts[i]; - if ( !obj[part] ){ - obj[part] = {}; - } - if ( typeof obj[part] !== 'object' || Array.isArray(currVal) ){ - throw new Error(`Path ${pathParts.slice(0, i + 1).join('.')} is not an object`); - } - obj = obj[part]; - } - if ( obj === null ) continue; - const lastPart = pathParts[pathParts.length - 1]; - if ( !obj[lastPart] ){ - obj[lastPart] = 0; - } - if ( typeof obj[lastPart] !== 'number' ){ - throw new Error(`Value at path ${path} is not a number`); - } - obj[lastPart] += amount; - } - this.set({ key, value: currVal }); - return currVal; - } - async decr({ key, path = '', amount = 1 }) { - return await this.incr({ key, path, amount: -amount }); - } - async #expireat(key, timestamp) { - const actor = Context.get('actor'); - - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( !user ) { - throw new Error('User not found'); - } - - const key_hash = murmurhash.v3(key); - - try { - await this.#db.write(`INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt) - VALUES (?, ?, ?, ?, ?, ?) ${ - this.#db.case({ - mysql: 'ON DUPLICATE KEY UPDATE expireAt = ?', - sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET expireAt = excluded.expireAt', - })}`, - [ - user.id, - app?.uid ?? GLOBAL_APP_KEY, - key_hash, - key, - null, // empty value - timestamp, - ...this.#db.case({ mysql: [timestamp], otherwise: [] }), - ]); - } catch ( e ) { - // I discovered that my .sqlite file was corrupted and the update - // above didn't work. The current database initialization does not - // cause this issue so I'm adding this log as a safeguard. - // - KernelDeimos / ED - const svc_error = this.services.get('error-service'); - svc_error.report('kvstore:sqlite_error', { - message: 'Broken database version - please contact maintainers', - source: e, - }); - } - } -} \ No newline at end of file diff --git a/src/backend/src/services/repositories/DBKVStore/DBKVStore.ts b/src/backend/src/services/repositories/DBKVStore/DBKVStore.ts new file mode 100644 index 00000000..115838f6 --- /dev/null +++ b/src/backend/src/services/repositories/DBKVStore/DBKVStore.ts @@ -0,0 +1,350 @@ +// TypeScript conversion of DBKVStore.mjs +import murmurhash from "murmurhash"; +// @ts-ignore +import APIError from '../../../api/APIError.js'; +// @ts-ignore +import { Context } from "../../../util/context.js"; + +const GLOBAL_APP_KEY = 'global'; + +export class DBKVStore { + #db: any; + #meteringService: any; + #global_config: any = {}; + + // TODO DS: make table name configurable + constructor({ sqlClient, meteringAndBillingService, globalConfig }: { sqlClient: any, meteringAndBillingService: any, globalConfig: any }) { + this.#db = sqlClient; + this.#meteringService = meteringAndBillingService; + this.#global_config = globalConfig; + } + + async get({ key }: { key: any }): Promise { + const actor = Context.get('actor'); + const app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + + if (!user) { + throw new Error('User not found'); + } + + const deleteExpired = async (rows: any[]) => { + const query = `DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (${rows.map(() => '?').join(',')})`; + const params = [user.id, app?.uid ?? GLOBAL_APP_KEY, ...rows.map((r: any) => r.kkey_hash)]; + return await this.#db.write(query, params); + }; + + if (Array.isArray(key)) { + const keys = key; + const key_hashes = keys.map((key: any) => murmurhash.v3(key)); + const rows = app + ? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (?)', [user.id, app.uid, key_hashes]) + : await this.#db.read( + `SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') AND kkey_hash IN (${key_hashes.map(() => '?').join(',')})`, + [user.id, key_hashes] + ); + + const kv: any = {}; + rows.forEach((row: any) => { + row.value = this.#db.case({ + mysql: () => row.value, + otherwise: () => JSON.parse(row.value ?? 'null'), + })(); + kv[row.kkey] = row.value; + }); + + const expiredKeys: any[] = []; + rows.forEach((row: any) => { + if (row?.expireAt && row.expireAt < Date.now() / 1000) { + expiredKeys.push(row); + kv[row.kkey] = null; + } else { + kv[row.kkey] = row.value ?? null; + } + }); + + // clean up expired keys asynchronously + if (expiredKeys.length) { + deleteExpired(expiredKeys); + } + + return keys.map((key: any) => kv[key]); + } + + const key_hash = murmurhash.v3(key); + const kv = app + ? await this.#db.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1', [user.id, app.uid, key_hash]) + : await this.#db.read( + `SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}') AND kkey_hash=? LIMIT 1`, + [user.id, key_hash] + ); + + if (kv[0]) { + kv[0].value = this.#db.case({ + mysql: () => kv[0].value, + otherwise: () => JSON.parse(kv[0].value ?? 'null'), + })(); + } + + if (kv[0]?.expireAt && kv[0].expireAt < Date.now() / 1000) { + // key has expired + // clean up asynchronously + deleteExpired([kv[0]]); + return null; + } + + await this.#meteringService.incrementUsage(actor, 'kv:read', Array.isArray(key) ? key.length : 1); + + return kv[0]?.value ?? null; + } + + async set({ key, value, expireAt }: { key: any, value: any, expireAt?: any }): Promise { + const actor = Context.get('actor'); + const config = this.#global_config; + + key = String(key); + if (Buffer.byteLength(key, 'utf8') > config.kv_max_key_size) { + throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`); + } + + value = value === undefined ? null : value; + if ( + value !== null && + Buffer.byteLength(JSON.stringify(value), 'utf8') > config.kv_max_value_size + ) { + throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`); + } + + const app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + if (!user) { + throw new Error('User not found'); + } + + const key_hash = murmurhash.v3(key); + + try { + await this.#db.write( + `INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt) + VALUES (?, ?, ?, ?, ?, ?) ${this.#db.case({ + mysql: 'ON DUPLICATE KEY UPDATE value = ?', + sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET value = excluded.value', + }) + }`, + [ + user.id, + app?.uid ?? GLOBAL_APP_KEY, + key_hash, + key, + JSON.stringify(value), + expireAt ?? null, + ...this.#db.case({ mysql: [value], otherwise: [] }), + ] + ); + } catch (e: any) { + console.error(e); + } + + await this.#meteringService.incrementUsage(actor, 'kv:write', 1); + + return true; + } + + async del({ key }: { key: any }): Promise { + const actor = Context.get('actor'); + const app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + if (!user) { + throw new Error('User not found'); + } + + const key_hash = murmurhash.v3(key); + + await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', [ + user.id, + app?.uid ?? GLOBAL_APP_KEY, + key_hash, + ]); + + await this.#meteringService.incrementUsage(actor, 'kv:write', 1); + + return true; + } + + async list({ as }: { as?: any }): Promise { + const actor = Context.get('actor'); + const app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + + if (!user) { + throw new Error('User not found'); + } + + let rows = app + ? await this.#db.read('SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND app=?', [user.id, app.uid]) + : await this.#db.read( + `SELECT kkey, value, expireAt FROM kv WHERE user_id=? AND (app IS NULL OR app = '${GLOBAL_APP_KEY}')`, + [user.id] + ); + + rows = rows.filter((row: any) => { + return !row?.expireAt || row?.expireAt > Date.now() / 1000; + }); + + rows = rows.map((row: any) => ({ + key: row.kkey, + value: this.#db.case({ + mysql: () => row.value, + otherwise: () => JSON.parse(row.value ?? 'null'), + })(), + })); + + as = as || 'entries'; + + if (!['keys', 'values', 'entries'].includes(as)) { + throw APIError.create('field_invalid', null, { + key: 'as', + expected: '"keys", "values", or "entries"', + }); + } + + if (as === 'keys') { + rows = rows.map((row: any) => row.key); + } else if (as === 'values') { + rows = rows.map((row: any) => row.value); + } + + await this.#meteringService.incrementUsage(actor, 'kv:read', rows.length); + + return rows; + } + + async flush(): Promise { + const actor = Context.get('actor'); + const app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + if (!user) { + throw new Error('User not found'); + } + + await this.#db.write('DELETE FROM kv WHERE user_id=? AND app=?', [ + user.id, + app?.uid ?? GLOBAL_APP_KEY, + ]); + + await this.#meteringService.incrementUsage(actor, 'kv:write', 1); + + return true; + } + + async expireAt({ key, timestamp }: { key: any, timestamp: any }): Promise { + if (key === '') { + throw APIError.create('field_empty', null, { + key: 'key', + }); + } + + timestamp = Number(timestamp); + + return await this.#expireat(key, timestamp); + } + + async expire({ key, ttl }: { key: any, ttl: any }): Promise { + if (key === '') { + throw APIError.create('field_empty', null, { + key: 'key', + }); + } + + ttl = Number(ttl); + + // timestamp in seconds + let timestamp = Math.floor(Date.now() / 1000) + ttl; + + return await this.#expireat(key, timestamp); + } + + async incr({ key, pathAndAmountMap }: { key: string, pathAndAmountMap: Record }): Promise { + let currVal = await this.get({ key }); + const pathEntries = Object.entries(pathAndAmountMap); + if (typeof currVal !== 'object' && pathEntries.length <= 1 && !pathEntries[0]?.[0]) { + const amount = pathEntries[0]?.[1] ?? 1; + this.set({ key, value: (Number(currVal) || 0) + amount }); + return (Number(currVal) || 0) + amount; + } + // TODO DS: support arrays this also needs dynamodb implementation + if (Array.isArray(currVal)) { + throw new Error('Current value is an array'); + } + if (!currVal) { + currVal = {}; + } + if (typeof currVal !== 'object') { + throw new Error('Current value is not an object'); + } + // create or change values as needed + for (const [path, amount] of Object.entries(pathAndAmountMap)) { + const pathParts = path.split('.'); + let obj = currVal; + if (obj === null) continue; + for (let i = 0; i < pathParts.length - 1; i++) { + const part = pathParts[i]; + if (!obj[part]) { + obj[part] = {}; + } + if (typeof obj[part] !== 'object' || Array.isArray(currVal)) { + throw new Error(`Path ${pathParts.slice(0, i + 1).join('.')} is not an object`); + } + obj = obj[part]; + } + if (obj === null) continue; + const lastPart = pathParts[pathParts.length - 1]; + if (!obj[lastPart]) { + obj[lastPart] = 0; + } + if (typeof obj[lastPart] !== 'number') { + throw new Error(`Value at path ${path} is not a number`); + } + obj[lastPart] += amount; + } + this.set({ key, value: currVal }); + return currVal; + } + + async decr(...params: Parameters): Promise { + return await this.incr(...params); + } + + async #expireat(key: any, timestamp: any): Promise { + const actor = Context.get('actor'); + const app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + if (!user) { + throw new Error('User not found'); + } + + const key_hash = murmurhash.v3(key); + + try { + await this.#db.write( + `INSERT INTO kv (user_id, app, kkey_hash, kkey, value, expireAt) + VALUES (?, ?, ?, ?, ?, ?) ${this.#db.case({ + mysql: 'ON DUPLICATE KEY UPDATE expireAt = ?', + sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET expireAt = excluded.expireAt', + }) + }`, + [ + user.id, + app?.uid ?? GLOBAL_APP_KEY, + key_hash, + key, + null, // empty value + timestamp, + ...this.#db.case({ mysql: [timestamp], otherwise: [] }), + ] + ); + } catch (e: any) { + console.error(e); + } + } +} \ No newline at end of file diff --git a/src/backend/src/services/repositories/DBKVStore/index.mjs b/src/backend/src/services/repositories/DBKVStore/index.mjs index c8ad9b41..0eff0335 100644 --- a/src/backend/src/services/repositories/DBKVStore/index.mjs +++ b/src/backend/src/services/repositories/DBKVStore/index.mjs @@ -1,6 +1,6 @@ import BaseService from '../../BaseService.js'; import { DB_READ } from '../../database/consts.js'; -import { DBKVStore } from './DBKVStore.mjs'; +import { DBKVStore } from './DBKVStore.js'; export class DBKVServiceWrapper extends BaseService { kvStore = undefined; diff --git a/src/puter-js/src/modules/Auth.js b/src/puter-js/src/modules/Auth.js index ec523fc2..20795b20 100644 --- a/src/puter-js/src/modules/Auth.js +++ b/src/puter-js/src/modules/Auth.js @@ -174,7 +174,7 @@ class Auth{ async getMonthlyUsage() { try { - const resp = await fetch(`${this.APIOrigin}/v2/usage`, { + const resp = await fetch(`${this.APIOrigin}/meteringAndBilling/usage`, { headers: { Authorization: `Bearer ${this.authToken}`, }, @@ -216,7 +216,7 @@ class Auth{ } try { - const resp = await fetch(`${this.APIOrigin}/v2/usage/${appId}`, { + const resp = await fetch(`${this.APIOrigin}/meteringAndBilling/usage/${appId}`, { headers: { Authorization: `Bearer ${this.authToken}`, }, diff --git a/tsconfig.json b/tsconfig.json index be896fb4..8b0703bd 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,7 +7,8 @@ "strict": true, "esModuleInterop": true, "forceConsistentCasingInFileNames": true, - "skipLibCheck": true + "skipLibCheck": true, + "sourceMap": true }, "exclude": [ "**/*.test.ts",