mirror/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2025-12-20 20:40:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Revert "Revert "Revert "feat: alert when going over usage limits sign…"

Browse Source 
This reverts commit f2265cca59.
This commit is contained in:
Daniel Salazar
2025-10-20 18:45:08 -07:00
committed by GitHub
parent f2265cca59
commit 0260a9cbce
27 changed files with 1496 additions and 1507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
eslint.config.js
View File

@@ -63,7 +63,7 @@ export default defineConfig([
'@stylistic/indent-binary-ops': ['error', 4],
'@stylistic/array-bracket-newline': ['error', 'consistent'],
'@stylistic/semi': ['error', 'always'],
'@stylistic/quotes': 'off',
'@stylistic/quotes': ['error', 'single', { 'avoidEscape': true }],
'@stylistic/function-call-argument-newline': ['error', 'consistent'],
'@stylistic/arrow-spacing': ['error', { before: true, after: true }],
'@stylistic/space-before-function-paren': ['error', { 'anonymous': 'never', 'named': 'never', 'asyncArrow': 'always', 'catch': 'always' }],
@@ -115,7 +115,7 @@ export default defineConfig([
'@stylistic/indent-binary-ops': ['error', 4],
'@stylistic/array-bracket-newline': ['error', 'consistent'],
'@stylistic/semi': ['error', 'always'],
'@stylistic/quotes': ['error', 'single'],
'@stylistic/quotes': ['error', 'single', { 'avoidEscape': true }],
'@stylistic/function-call-argument-newline': ['error', 'consistent'],
'@stylistic/arrow-spacing': ['error', { before: true, after: true }],
'@stylistic/space-before-function-paren': ['error', { 'anonymous': 'never', 'named': 'never', 'asyncArrow': 'always', 'catch': 'always' }],
@@ -166,7 +166,7 @@ export default defineConfig([
'@stylistic/indent-binary-ops': ['error', 4],
'@stylistic/array-bracket-newline': ['error', 'consistent'],
'@stylistic/semi': ['error', 'always'],
'@stylistic/quotes': ['error', 'single'],
'@stylistic/quotes': ['error', 'single', { 'avoidEscape': true }],
'@stylistic/function-call-argument-newline': ['error', 'consistent'],
'@stylistic/arrow-spacing': ['error', { before: true, after: true }],
'@stylistic/space-before-function-paren': ['error', { 'anonymous': 'never', 'named': 'never', 'asyncArrow': 'always', 'catch': 'never' }],
@@ -212,7 +212,7 @@ export default defineConfig([
'@stylistic/indent-binary-ops': ['error', 4],
'@stylistic/array-bracket-newline': ['error', 'consistent'],
'@stylistic/semi': ['error', 'always'],
'@stylistic/quotes': ['error', 'single'],
'@stylistic/quotes': ['error', 'single', { 'avoidEscape': true }],
'@stylistic/function-call-argument-newline': ['error', 'consistent'],
'@stylistic/arrow-spacing': ['error', { before: true, after: true }],
'@stylistic/space-before-function-paren': ['error', { 'anonymous': 'never', 'named': 'never', 'asyncArrow': 'always', 'catch': 'never' }],

271
src/backend/src/filesystem/ll_operations/ll_read.js
View File

@@ -16,153 +16,146 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const APIError = require("../../api/APIError");
const { Sequence } = require("../../codex/Sequence");
const { MemoryFSProvider } = require("../../modules/puterfs/customfs/MemoryFSProvider");
const APIError = require('../../api/APIError');
const { get_user } = require('../../helpers');
const { MemoryFSProvider } = require('../../modules/puterfs/customfs/MemoryFSProvider');
const { UserActorType } = require('../../services/auth/Actor');
const { Actor } = require('../../services/auth/Actor');
const { DB_WRITE } = require('../../services/database/consts');
const { Context } = require('../../util/context');
const { buffer_to_stream } = require('../../util/streamutil');
const { TYPE_SYMLINK, TYPE_DIRECTORY } = require('../FSNodeContext');
const { LLFilesystemOperation } = require('./definitions');
const { DB_WRITE } = require("../../services/database/consts");
const { buffer_to_stream } = require("../../util/streamutil");
const { TYPE_SYMLINK, TYPE_DIRECTORY } = require("../FSNodeContext");
const { LLFilesystemOperation } = require("./definitions");
const dry_checks = [
async function check_ACL_for_read (a) {
if ( a.get('no_acl') ) return;
const context = a.iget('context');
const svc_acl = context.get('services').get('acl');
const { fsNode, actor } = a.values();
if ( ! await svc_acl.check(actor, fsNode, 'read') ) {
throw await svc_acl.get_safe_acl_error(actor, fsNode, 'read');
}
},
async function type_check_for_read (a) {
const fsNode = a.get('fsNode');
if ( await fsNode.get('type') === TYPE_DIRECTORY ) {
throw APIError.create('cannot_read_a_directory');
}
},
];
const checkACLForRead = async (aclService, actor, fsNode) => {
if ( !await aclService.check(actor, fsNode, 'read') ) {
throw await aclService.get_safe_acl_error(actor, fsNode, 'read');
}
};
const typeCheckForRead = async (fsNode) => {
if ( await fsNode.get('type') === TYPE_DIRECTORY ) {
throw APIError.create('cannot_read_a_directory');
}
};
class LLRead extends LLFilesystemOperation {
static CONCERN = 'filesystem';
static METHODS = {
_run: new Sequence({
async before_each (a, step) {
const operation = a.iget();
operation.checkpoint('step:' + step.name);
async _run({ fsNode, no_acl, actor, offset, length, range, version_id } = {}){
// extract services from context
const aclService = Context.get('services').get('acl');
const db = Context.get('services')
.get('database').get(DB_WRITE, 'filesystem');
const fileCacheService = Context.get('services').get('file-cache');
// validate input
if ( !await fsNode.exists() ){
throw APIError.create('subject_does_not_exist');
}
if ( no_acl ) return;
// validate initial node
await checkACLForRead(aclService, actor, fsNode);
await typeCheckForRead(fsNode);
let type = await fsNode.get('type');
while ( type === TYPE_SYMLINK ) {
fsNode = await fsNode.getTarget();
type = await fsNode.get('type');
}
// validate symlink leaf node
await checkACLForRead(aclService, actor, fsNode);
await typeCheckForRead(fsNode);
// calculate range inputs
const has_range = (
offset !== undefined &&
offset !== 0
) || (
length !== undefined &&
length != await fsNode.get('size')
) || range !== undefined;
// timestamp access
await db.write('UPDATE `fsentries` SET `accessed` = ? WHERE `id` = ?',
[Date.now() / 1000, fsNode.mysql_id]);
const ownerId = await fsNode.get('user_id');
const ownerActor = new Actor({
type: new UserActorType({
user: await get_user({ id: ownerId }),
}),
});
//define metering service
/** @type {import("../../services/MeteringService/MeteringService").MeteringAndBillingService} */
const meteringService = Context.get('services').get('meteringService').meteringAndBillingService;
// check file cache
const maybe_buffer = await fileCacheService.try_get(fsNode); // TODO DS: do we need those cache hit logs?
if ( maybe_buffer ) {
// Meter cached egress
// return cached stream
if ( has_range && (length || offset) ) {
meteringService.incrementUsage(ownerActor, 'filesystem:cached-egress:bytes', length);
return buffer_to_stream(maybe_buffer.slice(offset, offset + length));
}
}, [
async function check_that_node_exists (a) {
if ( ! await a.get('fsNode').exists() ) {
throw APIError.create('subject_does_not_exist');
meteringService.incrementUsage(ownerActor, 'filesystem:cached-egress:bytes', await fsNode.get('size'));
return buffer_to_stream(maybe_buffer);
}
// if no cache attempt reading from storageProvider (s3)
const svc_mountpoint = Context.get('services').get('mountpoint');
const provider = await svc_mountpoint.get_provider(fsNode.selector);
const storage = svc_mountpoint.get_storage(provider.constructor.name);
// Empty object here is in the case of local fiesystem,
// where s3:location will return null.
// TODO: storage interface shouldn't have S3-specific properties.
const location = await fsNode.get('s3:location') ?? {};
const stream = (await storage.create_read_stream(await fsNode.get('uid'), {
// TODO: fs:decouple-s3
bucket: location.bucket,
bucket_region: location.bucket_region,
version_id,
key: location.key,
memory_file: fsNode.entry,
...(range ? { range } : (has_range ? {
range: `bytes=${offset}-${offset + length - 1}`,
} : {})),
}));
// Meter ingress
const size = await (async () => {
if ( range ){
const match = range.match(/bytes=(\d+)-(\d+)/);
if ( match ) {
const start = parseInt(match[1], 10);
const end = parseInt(match[2], 10);
return end - start + 1;
}
},
...dry_checks,
async function resolve_symlink (a) {
let fsNode = a.get('fsNode');
let type = await fsNode.get('type');
while ( type === TYPE_SYMLINK ) {
fsNode = await fsNode.getTarget();
type = await fsNode.get('type');
}
if ( has_range ) {
return length;
}
return await fsNode.get('size');
})();
meteringService.incrementUsage(ownerActor, 'filesystem:egress:bytes', size);
// cache if whole file read
if ( !has_range ) {
// only cache for non-memoryfs providers
if ( ! (fsNode.provider instanceof MemoryFSProvider) ) {
const res = await fileCacheService.maybe_store(fsNode, stream);
if ( res.stream ) {
// return with split cached stream
return res.stream;
}
a.set('fsNode', fsNode);
},
...dry_checks,
async function calculate_has_range (a) {
const { offset, length, range } = a.values();
const fsNode = a.get('fsNode');
const has_range = (
offset !== undefined &&
offset !== 0
) || (
length !== undefined &&
length != await fsNode.get('size')
) || range !== undefined;
a.set('has_range', has_range);
},
async function update_accessed (a) {
const context = a.iget('context');
const db = context.get('services')
.get('database').get(DB_WRITE, 'filesystem');
const fsNode = a.get('fsNode');
await db.write(
'UPDATE `fsentries` SET `accessed` = ? WHERE `id` = ?',
[Date.now()/1000, fsNode.mysql_id]
);
},
async function check_for_cached_copy (a) {
const context = a.iget('context');
const svc_fileCache = context.get('services').get('file-cache');
const { fsNode, offset, length } = a.values();
const maybe_buffer = await svc_fileCache.try_get(fsNode, a.log);
if ( maybe_buffer ) {
a.log.cache(true, 'll_read');
const { has_range } = a.values();
if ( has_range && (length || offset) ) {
return a.stop(
buffer_to_stream(maybe_buffer.slice(offset, offset+length))
);
}
return a.stop(
buffer_to_stream(maybe_buffer)
);
}
a.log.cache(false, 'll_read');
},
async function create_S3_read_stream (a) {
const context = a.iget('context');
const { fsNode, version_id, offset, length, has_range, range } = a.values();
const svc_mountpoint = context.get('services').get('mountpoint');
const provider = await svc_mountpoint.get_provider(fsNode.selector);
const storage = svc_mountpoint.get_storage(provider.constructor.name);
// Empty object here is in the case of local fiesystem,
// where s3:location will return null.
// TODO: storage interface shouldn't have S3-specific properties.
const location = await fsNode.get('s3:location') ?? {};
const stream = (await storage.create_read_stream(await fsNode.get('uid'), {
// TODO: fs:decouple-s3
bucket: location.bucket,
bucket_region: location.bucket_region,
version_id,
key: location.key,
memory_file: fsNode.entry,
...(range? {range} : (has_range ? {
range: `bytes=${offset}-${offset+length-1}`
} : {})),
}));
a.set('stream', stream);
},
async function store_in_cache (a) {
const context = a.iget('context');
const svc_fileCache = context.get('services').get('file-cache');
const { fsNode, stream, has_range, range} = a.values();
if ( ! has_range ) {
// only cache for non-memoryfs providers
if ( ! (fsNode.provider instanceof MemoryFSProvider) ) {
const res = await svc_fileCache.maybe_store(fsNode, stream);
if ( res.stream ) a.set('stream', res.stream);
}
}
},
async function return_stream (a) {
return a.get('stream');
},
]),
};
}
}
return stream;
}
}
module.exports = {
LLRead
LLRead,
};

65
src/backend/src/modules/puterfs/MountpointService.js
View File

@@ -17,9 +17,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// const Mountpoint = o => ({ ...o });
const { RootNodeSelector, NodeUIDSelector, NodeChildSelector, NodePathSelector, NodeInternalIDSelector, NodeSelector, try_infer_attributes } = require("../../filesystem/node/selectors");
const { RootNodeSelector, NodeUIDSelector, NodeChildSelector, NodePathSelector, try_infer_attributes } = require("../../filesystem/node/selectors");
const BaseService = require("../../services/BaseService");
/**
@@ -40,29 +38,16 @@ const BaseService = require("../../services/BaseService");
* and their associated storage backends in future implementations.
*/
class MountpointService extends BaseService {
_construct () {
this.mounters_ = {};
this.mountpoints_ = {};
#storage = {};
#mounters = {};
#mountpoints = {};
register_mounter(name, mounter) {
this.#mounters[name] = mounter;
}
register_mounter (name, mounter) {
this.mounters_[name] = mounter;
}
/**
* Initializes the MountpointService instance
* Sets up initial state with null storage backend
* @private
* @async
* @returns {Promise<void>}
*/
async _init () {
// key: name of provider class (e.g: "PuterFSProvider", "MemoryFSProvider")
// value: storage instance
this.storage_ = {};
}
async ['__on_boot.consolidation'] () {
async ['__on_boot.consolidation']() {
const mountpoints = this.config.mountpoints ?? {
'/': {
mounter: 'puterfs',
@@ -72,38 +57,38 @@ class MountpointService extends BaseService {
for ( const path of Object.keys(mountpoints) ) {
const { mounter: mounter_name, options } =
mountpoints[path];
const mounter = this.mounters_[mounter_name];
const mounter = this.#mounters[mounter_name];
const provider = await mounter.mount({
path,
options
options,
});
this.mountpoints_[path] = {
this.#mountpoints[path] = {
provider,
};
}
this.services.emit('filesystem.ready', {
mountpoints: Object.keys(this.mountpoints_),
mountpoints: Object.keys(this.#mountpoints),
});
}
async get_provider (selector) {
async get_provider(selector) {
// If there is only one provider, we don't need to do any of this,
// and that's a big deal because the current implementation requires
// fetching a filesystem entry before we even have operation-level
// transient memoization instantiated.
if ( Object.keys(this.mountpoints_).length === 1 ) {
return Object.values(this.mountpoints_)[0].provider;
if ( Object.keys(this.#mountpoints).length === 1 ) {
return Object.values(this.#mountpoints)[0].provider;
}
try_infer_attributes(selector);
if ( selector instanceof RootNodeSelector ) {
return this.mountpoints_['/'].provider;
return this.#mountpoints['/'].provider;
}
if ( selector instanceof NodeUIDSelector ) {
for ( const [path, { provider }] of Object.entries(this.mountpoints_) ) {
for ( const { provider } of Object.values(this.#mountpoints) ) {
const result = await provider.quick_check({
selector,
});
@@ -128,7 +113,7 @@ class MountpointService extends BaseService {
selector.setPropertiesKnownBySelector(probe);
if ( probe.path ) {
let longest_mount_path = '';
for ( const path of Object.keys(this.mountpoints_) ) {
for ( const path of Object.keys(this.#mountpoints) ) {
if ( ! probe.path.startsWith(path) ) {
continue;
}
@@ -138,25 +123,25 @@ class MountpointService extends BaseService {
}
if ( longest_mount_path ) {
return this.mountpoints_[longest_mount_path].provider;
return this.#mountpoints[longest_mount_path].provider;
}
}
// Use root mountpoint as fallback
return this.mountpoints_['/'].provider;
return this.#mountpoints['/'].provider;
}
// Temporary solution - we'll develop this incrementally
set_storage (provider, storage) {
this.storage_[provider] = storage;
set_storage(provider, storage) {
this.#storage[provider] = storage;
}
/**
* Gets the current storage backend instance
* @returns {Object} The storage backend instance
*/
get_storage (provider) {
const storage = this.storage_[provider];
get_storage(provider) {
const storage = this.#storage[provider];
if ( ! storage ) {
throw new Error(`MountpointService.get_storage: storage for provider "${provider}" not found`);
}

408
src/backend/src/modules/puterfs/lib/PuterFSProvider.js
View File

@@ -20,43 +20,48 @@
const putility = require('@heyputer/putility');
const { MultiDetachable } = putility.libs.listener;
const { TDetachable } = putility.traits;
const { TeePromise } = putility.libs.promise;
const { NodeInternalIDSelector, NodeChildSelector, NodeUIDSelector, RootNodeSelector, NodePathSelector, NodeSelector } = require("../../../filesystem/node/selectors");
const { Context } = require("../../../util/context");
const { NodeInternalIDSelector, NodeChildSelector, NodeUIDSelector } = require('../../../filesystem/node/selectors');
const { Context } = require('../../../util/context');
const fsCapabilities = require('../../../filesystem/definitions/capabilities');
const { UploadProgressTracker } = require('../../../filesystem/storage/UploadProgressTracker');
const FSNodeContext = require('../../../filesystem/FSNodeContext');
const { RESOURCE_STATUS_PENDING_CREATE } = require('../ResourceService');
const { ParallelTasks } = require('../../../util/otelutil');
const { TYPE_DIRECTORY } = require('../../../filesystem/FSNodeContext');
const APIError = require('../../../api/APIError');
const { MODE_WRITE } = require('../../../services/fs/FSLockService');
const { DB_WRITE } = require('../../../services/database/consts');
const { stuck_detector_stream, hashing_stream } = require('../../../util/streamutil');
const crypto = require('crypto');
const { OperationFrame } = require('../../../services/OperationTraceService');
const path = require('path');
const uuidv4 = require('uuid').v4;
const config = require('../../../config.js');
const { Actor } = require('../../../services/auth/Actor.js');
const { UserActorType } = require('../../../services/auth/Actor.js');
const { get_user } = require('../../../helpers.js');
const STUCK_STATUS_TIMEOUT = 10 * 1000;
const STUCK_ALARM_TIMEOUT = 20 * 1000;
class PuterFSProvider extends putility.AdvancedBase {
static MODULES = {
_path: require('path'),
uuidv4: require('uuid').v4,
config: require('../../../config.js'),
};
constructor (...a) {
get #services() { // we really should just pass services in constructor, global state is a bit messy
return Context.get('services');
}
/** @type {import('../../../services/MeteringService/MeteringService.js').MeteringAndBillingService} */
get #meteringService() {
return this.#services.get('meteringService').meteringAndBillingService;
}
constructor(...a) {
super(...a);
this.log_fsentriesNotFound = (this.modules.config.logging ?? [])
this.log_fsentriesNotFound = (config.logging ?? [])
.includes('fsentries-not-found');
}
get_capabilities () {
get_capabilities() {
return new Set([
fsCapabilities.THUMBNAIL,
fsCapabilities.UUID,
@@ -80,11 +85,11 @@ class PuterFSProvider extends putility.AdvancedBase {
* @param {NodeSelector} param.selector - The selector used for checking.
* @returns {Promise<boolean>} - True if the node exists, false otherwise.
*/
async quick_check ({
async quick_check({
selector,
}) {
// a wrapper that access underlying database directly
const fsEntryFetcher = Context.get('services').get('fsEntryFetcher');
const fsEntryFetcher = this.#services.get('fsEntryFetcher');
// shortcut: has full path
if ( selector?.path ) {
@@ -100,18 +105,14 @@ class PuterFSProvider extends putility.AdvancedBase {
// shortcut: parent uid + child name
if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeUIDSelector ) {
return await fsEntryFetcher.nameExistsUnderParent(
selector.parent.uid,
selector.name,
);
return await fsEntryFetcher.nameExistsUnderParent(selector.parent.uid,
selector.name);
}
// shortcut: parent id + child name
if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeInternalIDSelector ) {
return await fsEntryFetcher.nameExistsUnderParentID(
selector.parent.id,
selector.name,
);
return await fsEntryFetcher.nameExistsUnderParentID(selector.parent.id,
selector.name);
}
// TODO (xiaochen): we should fallback to stat but we cannot at this moment
@@ -119,7 +120,7 @@ class PuterFSProvider extends putility.AdvancedBase {
return false;
}
async stat ({
async stat({
selector,
options,
controls,
@@ -134,7 +135,7 @@ class PuterFSProvider extends putility.AdvancedBase {
fsEntryService,
fsEntryFetcher,
resourceService,
} = Context.get('services').values;
} = this.#services.values;
if ( options.tracer == null ) {
options.tracer = traceService.tracer;
@@ -151,10 +152,10 @@ class PuterFSProvider extends putility.AdvancedBase {
await new Promise (rslv => {
const detachables = new MultiDetachable();
const callback = (resolver) => {
const callback = (_resolver) => {
detachables.as(TDetachable).detach();
rslv();
}
};
// either the resource is free
{
@@ -162,9 +163,7 @@ class PuterFSProvider extends putility.AdvancedBase {
// Promise that will be resolved when the resource
// is free no matter what, and then it will be
// garbage collected.
resourceService.waitForResource(
selector
).then(callback.bind(null, 'resourceService'));
resourceService.waitForResource(selector).then(callback.bind(null, 'resourceService'));
}
// or pending information about the resource
@@ -176,8 +175,7 @@ class PuterFSProvider extends putility.AdvancedBase {
// is guaranteed to resolve eventually, and then this
// detachable will be detached by `callback` so the
// listener can be garbage collected.
const det = fsEntryService.waitForEntry(
node, callback.bind(null, 'fsEntryService'));
const det = fsEntryService.waitForEntry(node, callback.bind(null, 'fsEntryService'));
if ( det ) detachables.add(det);
}
});
@@ -187,8 +185,7 @@ class PuterFSProvider extends putility.AdvancedBase {
entry = await fsEntryService.get(maybe_uid, options);
controls.log.debug('got an entry from the future');
} else {
entry = await fsEntryFetcher.find(
selector, options);
entry = await fsEntryFetcher.find(selector, options);
}
if ( ! entry ) {
@@ -202,38 +199,33 @@ class PuterFSProvider extends putility.AdvancedBase {
}
if ( entry.id ) {
controls.provide_selector(
new NodeInternalIDSelector('mysql', entry.id, {
source: 'FSNodeContext optimization'
})
);
controls.provide_selector(new NodeInternalIDSelector('mysql', entry.id, {
source: 'FSNodeContext optimization',
}));
}
return entry;
}
async readdir ({ context, node }) {
async readdir({ node }) {
const uuid = await node.get('uid');
const services = context.get('services');
const svc_fsentry = services.get('fsEntryService');
const svc_fsentry = this.#services.get('fsEntryService');
const child_uuids = await svc_fsentry
.fast_get_direct_descendants(uuid);
return child_uuids;
}
async move ({ context, node, new_parent, new_name, metadata }) {
const { _path } = this.modules;
async move({ context, node, new_parent, new_name, metadata }) {
const services = context.get('services');
const old_path = await node.get('path');
const new_path = _path.join(await new_parent.get('path'), new_name);
const new_path = path.join(await new_parent.get('path'), new_name);
const svc_fsEntry = services.get('fsEntryService');
const svc_fsEntry = this.#services.get('fsEntryService');
const op_update = await svc_fsEntry.update(node.uid, {
...(
await node.get('parent_uid') !== await new_parent.get('uid')
? { parent_uid: await new_parent.get('uid') }
: {}
? { parent_uid: await new_parent.get('uid') }
: {}
),
path: new_path,
name: new_name,
@@ -250,10 +242,10 @@ class PuterFSProvider extends putility.AdvancedBase {
await op_update.awaitDone();
const svc_fs = services.get('filesystem');
const svc_fs = this.#services.get('filesystem');
await svc_fs.update_child_paths(old_path, node.entry.path, user_id);
const svc_event = services.get('event');
const svc_event = this.#services.get('event');
const promises = [];
promises.push(svc_event.emit('fs.move.file', {
@@ -269,22 +261,17 @@ class PuterFSProvider extends putility.AdvancedBase {
return node;
}
async copy_tree ({ context, source, parent, target_name }) {
return await this.copy_tree_(
{ context, source, parent, target_name });
async copy_tree({ context, source, parent, target_name }) {
return await this.#copy_tree({ context, source, parent, target_name });
}
async copy_tree_ ({ context, source, parent, target_name }) {
// Modules
const { _path, uuidv4 } = this.modules;
async #copy_tree({ context, source, parent, target_name }) {
// Services
const services = context.get('services');
const svc_event = services.get('event');
const svc_trace = services.get('traceService');
const svc_size = services.get('sizeService');
const svc_resource = services.get('resourceService');
const svc_fsEntry = services.get('fsEntryService');
const svc_fs = services.get('filesystem');
const svc_event = this.#services.get('event');
const svc_trace = this.#services.get('traceService');
const svc_size = this.#services.get('sizeService');
const svc_resource = this.#services.get('resourceService');
const svc_fsEntry = this.#services.get('fsEntryService');
const svc_fs = this.#services.get('filesystem');
// Context
const actor = Context.get('actor');
@@ -292,7 +279,7 @@ class PuterFSProvider extends putility.AdvancedBase {
const tracer = svc_trace.tracer;
const uuid = uuidv4();
const ts = Math.round(Date.now()/1000);
const timestamp = Math.round(Date.now() / 1000);
await parent.fetchEntry();
await source.fetchEntry({ thumbnail: true });
@@ -303,13 +290,13 @@ class PuterFSProvider extends putility.AdvancedBase {
...(source.entry.is_shortcut ? {
is_shortcut: source.entry.is_shortcut,
shortcut_to: source.entry.shortcut_to,
} :{}),
} : {}),
parent_uid: parent.uid,
name: target_name,
created: ts,
modified: ts,
created: timestamp,
modified: timestamp,
path: _path.join(await parent.get('path'), target_name),
path: path.join(await parent.get('path'), target_name),
// if property exists but the value is undefined,
// it will still be included in the INSERT, causing
@@ -323,7 +310,7 @@ class PuterFSProvider extends putility.AdvancedBase {
svc_event.emit('fs.pending.file', {
fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),
context: context,
})
});
if ( await source.get('has-s3') ) {
Object.assign(raw_fsentry, {
@@ -333,7 +320,7 @@ class PuterFSProvider extends putility.AdvancedBase {
bucket_region: source.entry.bucket_region,
});
await tracer.startActiveSpan(`fs:cp:storage-copy`, async span => {
await tracer.startActiveSpan('fs:cp:storage-copy', async span => {
let progress_tracker = new UploadProgressTracker();
svc_event.emit('fs.storage.progress.copy', {
@@ -342,7 +329,7 @@ class PuterFSProvider extends putility.AdvancedBase {
meta: {
item_uid: uuid,
item_path: raw_fsentry.path,
}
},
});
// const storage = new PuterS3StorageStrategy({ services: svc });
@@ -376,27 +363,19 @@ class PuterFSProvider extends putility.AdvancedBase {
let node;
const tasks = new ParallelTasks({ tracer, max: 4 });
await context.arun(`fs:cp:parallel-portion`, async () => {
await context.arun('fs:cp:parallel-portion', async () => {
// Add child copy tasks if this is a directory
if ( source.entry.is_dir ) {
const children = await svc_fsEntry.fast_get_direct_descendants(
source.uid
);
const children = await svc_fsEntry.fast_get_direct_descendants(source.uid);
for ( const child_uuid of children ) {
tasks.add(`fs:cp:copy-child`, async () => {
const child_node = await svc_fs.node(
new NodeUIDSelector(child_uuid)
);
tasks.add('fs:cp:copy-child', async () => {
const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid));
const child_name = await child_node.get('name');
// TODO: this should be LLCopy instead
await this.copy_tree_({
await this.#copy_tree({
context,
source: await svc_fs.node(
new NodeUIDSelector(child_uuid)
),
parent: await svc_fs.node(
new NodeUIDSelector(uuid)
),
source: await svc_fs.node(new NodeUIDSelector(child_uuid)),
parent: await svc_fs.node(new NodeUIDSelector(uuid)),
target_name: child_name,
});
});
@@ -404,7 +383,7 @@ class PuterFSProvider extends putility.AdvancedBase {
}
// Add task to await entry
tasks.add(`fs:cp:entry-op`, async () => {
tasks.add('fs:cp:entry-op', async () => {
await entryOp.awaitDone();
svc_resource.free(uuid);
const copy_fsNode = await svc_fs.node(new NodeUIDSelector(uuid));
@@ -417,7 +396,7 @@ class PuterFSProvider extends putility.AdvancedBase {
svc_event.emit('fs.create.file', {
node,
context,
})
});
}, { force: true });
await tasks.awaitAll();
@@ -429,66 +408,70 @@ class PuterFSProvider extends putility.AdvancedBase {
return node;
}
async unlink ({ context, node }) {
async unlink({ context, node }) {
if ( await node.get('type') === TYPE_DIRECTORY ) {
console.log(`\x1B[31;1m===N=====${await node.get('path')}=========\x1B[0m`)
console.log(`\x1B[31;1m===N=====${await node.get('path')}=========\x1B[0m`);
throw new APIError(409, 'Cannot unlink a directory.');
}
await this.rmnode_({ context, node });
await this.#rmnode({ context, node });
}
async rmdir ({ context, node, options = {} }) {
async rmdir({ context, node, options = {} }) {
if ( await node.get('type') !== TYPE_DIRECTORY ) {
console.log(`\x1B[31;1m===D1====${await node.get('path')}=========\x1B[0m`)
console.log(`\x1B[31;1m===D1====${await node.get('path')}=========\x1B[0m`);
throw new APIError(409, 'Cannot rmdir a file.');
}
if ( await node.get('immutable') ) {
console.log(`\x1B[31;1m===D2====${await node.get('path')}=========\x1B[0m`)
console.log(`\x1B[31;1m===D2====${await node.get('path')}=========\x1B[0m`);
throw APIError.create('immutable');
}
// Services
const services = context.get('services');
const svc_fsEntry = services.get('fsEntryService');
const svc_fsEntry = this.#services.get('fsEntryService');
const children = await svc_fsEntry.fast_get_direct_descendants(
await node.get('uid')
);
const children = await svc_fsEntry.fast_get_direct_descendants(await node.get('uid'));
if ( children.length > 0 && ! options.ignore_not_empty ) {
console.log(`\x1B[31;1m===D3====${await node.get('path')}=========\x1B[0m`)
console.log(`\x1B[31;1m===D3====${await node.get('path')}=========\x1B[0m`);
throw APIError.create('not_empty');
}
await this.rmnode_({ context, node, options });
await this.#rmnode({ context, node, options });
}
async rmnode_ ({ context, node, options }) {
async #rmnode({ node, options: _options }) {
// Services
const services = context.get('services');
const svc_size = services.get('sizeService');
const svc_fsEntry = services.get('fsEntryService');
const svc_size = this.#services.get('sizeService');
const svc_fsEntry = this.#services.get('fsEntryService');
if ( await node.get('immutable') ) {
throw new APIError(403, 'File is immutable.');
}
svc_size.change_usage(
await node.get('user_id'),
-1 * await node.get('size')
);
const userId = await node.get('user_id');
const fileSize = await node.get('size');
svc_size.change_usage(userId,
-1 * fileSize);
const tracer = services.get('traceService').tracer;
const ownerActor = new Actor({
type: new UserActorType({
user: await get_user({ id: userId }),
}),
});
this.#meteringService.incrementUsage(ownerActor, 'filesystem:delete:bytes', fileSize);
const tracer = this.#services.get('traceService').tracer;
const tasks = new ParallelTasks({ tracer, max: 4 });
tasks.add(`remove-fsentry`, async () => {
tasks.add('remove-fsentry', async () => {
await svc_fsEntry.delete(await node.get('uid'));
});
if ( await node.get('has-s3') ) {
tasks.add(`remove-from-s3`, async () => {
tasks.add('remove-from-s3', async () => {
// const storage = new PuterS3StorageStrategy({ services: svc });
const storage = Context.get('storage');
const state_delete = storage.create_delete();
@@ -511,41 +494,35 @@ class PuterFSProvider extends putility.AdvancedBase {
* @param {boolean} param.immutable
* @returns {Promise<FSNode>}
*/
async mkdir({ context, parent, name, immutable}) {
async mkdir({ context, parent, name, immutable }) {
const { actor, thumbnail } = context.values;
const svc_fslock = context.get('services').get('fslock');
const lock_handle = await svc_fslock.lock_child(
await parent.get('path'),
name,
MODE_WRITE,
);
const svc_fslock = this.#services.get('fslock');
const lock_handle = await svc_fslock.lock_child(await parent.get('path'),
name,
MODE_WRITE);
try {
const { _path, uuidv4 } = this.modules;
const ts = Math.round(Date.now() / 1000);
const uid = uuidv4();
const resourceService = context.get('services').get('resourceService');
const svc_fsEntry = context.get('services').get('fsEntryService');
const svc_event = context.get('services').get('event');
const fs = context.get('services').get('filesystem');
const resourceService = this.#services.get('resourceService');
const svc_fsEntry = this.#services.get('fsEntryService');
const svc_event = this.#services.get('event');
const fs = this.#services.get('filesystem');
const existing = await fs.node(
new NodeChildSelector(parent.selector, name)
);
const existing = await fs.node(new NodeChildSelector(parent.selector, name));
if (await existing.exists()) {
if ( await existing.exists() ) {
throw APIError.create('item_with_same_name_exists', null, {
entry_name: name,
});
}
const svc_acl = context.get('services').get('acl');
if (! await parent.exists()) {
const svc_acl = this.#services.get('acl');
if ( ! await parent.exists() ) {
throw APIError.create('subject_does_not_exist');
}
if (! await svc_acl.check(actor, parent, 'write')) {
if ( ! await svc_acl.check(actor, parent, 'write') ) {
throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
}
@@ -558,7 +535,7 @@ class PuterFSProvider extends putility.AdvancedBase {
is_dir: 1,
uuid: uid,
parent_uid: await parent.get('uid'),
path: _path.join(await parent.get('path'), name),
path: path.join(await parent.get('path'), name),
user_id: actor.type.user.id,
name,
created: ts,
@@ -582,7 +559,7 @@ class PuterFSProvider extends putility.AdvancedBase {
context: Context.get(),
});
return node
return node;
} finally {
await lock_handle.unlock();
}
@@ -599,25 +576,22 @@ class PuterFSProvider extends putility.AdvancedBase {
* @param {File} param.file: The file to write.
* @returns {Promise<FSNode>}
*/
async write_new({context, parent, name, file}) {
const { _path, uuidv4, config } = this.modules;
async write_new({ context, parent, name, file }) {
const {
tmp, fsentry_tmp, message, actor: actor_let, app_id,
tmp, fsentry_tmp, message, actor: inputActor, app_id,
} = context.values;
let actor = actor_let ?? Context.get('actor');
const actor = inputActor ?? Context.get('actor');
const svc = Context.get('services');
const sizeService = svc.get('sizeService');
const resourceService = svc.get('resourceService');
const svc_fsEntry = svc.get('fsEntryService');
const svc_event = svc.get('event');
const fs = svc.get('filesystem');
const sizeService = this.#services.get('sizeService');
const resourceService = this.#services.get('resourceService');
const svc_fsEntry = this.#services.get('fsEntryService');
const svc_event = this.#services.get('event');
const fs = this.#services.get('filesystem');
// TODO: fs:decouple-versions
// add version hook externally so LLCWrite doesn't
// need direct database access
const db = svc.get('database').get(DB_WRITE, 'filesystem');
const db = this.#services.get('database').get(DB_WRITE, 'filesystem');
const uid = uuidv4();
@@ -625,47 +599,47 @@ class PuterFSProvider extends putility.AdvancedBase {
let bucket_region = config.s3_region ?? config.region;
let bucket = config.s3_bucket;
const svc_acl = context.get('services').get('acl');
const svc_acl = this.#services.get('acl');
if ( ! await svc_acl.check(actor, parent, 'write') ) {
throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
}
const storage_resp = await this._storage_upload({
const storage_resp = await this.#storage_upload({
uuid: uid,
bucket, bucket_region, file,
bucket,
bucket_region,
file,
tmp: {
...tmp,
path: _path.join(await parent.get('path'), name),
}
path: path.join(await parent.get('path'), name),
},
});
fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;
delete fsentry_tmp.thumbnail_promise;
const ts = Math.round(Date.now() / 1000);
const timestamp = Math.round(Date.now() / 1000);
const raw_fsentry = {
uuid: uid,
is_dir: 0,
user_id: actor.type.user.id,
created: ts,
accessed: ts,
modified: ts,
created: timestamp,
accessed: timestamp,
modified: timestamp,
parent_uid: await parent.get('uid'),
name,
size: file.size,
path: _path.join(await parent.get('path'), name),
path: path.join(await parent.get('path'), name),
...fsentry_tmp,
bucket_region,
bucket,
associated_app_id: app_id ?? null,
};
svc_event.emit('fs.pending.file', {
fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),
context,
})
});
resourceService.register({
uid,
@@ -675,6 +649,16 @@ class PuterFSProvider extends putility.AdvancedBase {
const filesize = file.size;
sizeService.change_usage(actor.type.user.id, filesize);
// Meter ingress
const ownerId = await parent.get('user_id');
const ownerActor = new Actor({
type: new UserActorType({
user: await get_user({ id: ownerId }),
}),
});
this.#meteringService.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize);
const entryOp = await svc_fsEntry.insert(raw_fsentry);
(async () => {
@@ -684,20 +668,18 @@ class PuterFSProvider extends putility.AdvancedBase {
const new_item_node = await fs.node(new NodeUIDSelector(uid));
const new_item = await new_item_node.get('entry');
const store_version_id = storage_resp.VersionId;
if( store_version_id ){
if ( store_version_id ){
// insert version into db
db.write(
"INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)",
[
actor.type.user.id,
new_item.id,
new_item.uuid,
store_version_id,
message ?? null,
ts,
]
);
}
db.write('INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)',
[
actor.type.user.id,
new_item.id,
new_item.uuid,
store_version_id,
message ?? null,
timestamp,
]);
}
})();
const node = await fs.node(new NodeUIDSelector(uid));
@@ -722,22 +704,21 @@ class PuterFSProvider extends putility.AdvancedBase {
*/
async write_overwrite({ context, node, file }) {
const {
tmp, fsentry_tmp, message, actor: actor_let
tmp, fsentry_tmp, message, actor: inputActor,
} = context.values;
let actor = actor_let ?? Context.get('actor');
const actor = inputActor ?? Context.get('actor');
const svc = Context.get('services');
const sizeService = svc.get('sizeService');
const resourceService = svc.get('resourceService');
const svc_fsEntry = svc.get('fsEntryService');
const svc_event = svc.get('event');
const sizeService = this.#services.get('sizeService');
const resourceService = this.#services.get('resourceService');
const svc_fsEntry = this.#services.get('fsEntryService');
const svc_event = this.#services.get('event');
// TODO: fs:decouple-versions
// add version hook externally so LLCWrite doesn't
// need direct database access
const db = svc.get('database').get(DB_WRITE, 'filesystem');
const db = this.#services.get('database').get(DB_WRITE, 'filesystem');
const svc_acl = context.get('services').get('acl');
const svc_acl = this.#services.get('acl');
if ( ! await svc_acl.check(actor, node, 'write') ) {
throw await svc_acl.get_safe_acl_error(actor, node, 'write');
}
@@ -747,13 +728,15 @@ class PuterFSProvider extends putility.AdvancedBase {
const bucket_region = node.entry.bucket_region;
const bucket = node.entry.bucket;
const state_upload = await this._storage_upload({
const state_upload = await this.#storage_upload({
uuid: node.entry.uuid,
bucket, bucket_region, file,
bucket,
bucket_region,
file,
tmp: {
...tmp,
path: await node.get('path'),
}
},
});
if ( fsentry_tmp?.thumbnail_promise ) {
@@ -777,6 +760,15 @@ class PuterFSProvider extends putility.AdvancedBase {
const filesize = file.size;
sizeService.change_usage(actor.type.user.id, filesize);
// Meter ingress
const ownerId = await node.get('user_id');
const ownerActor = new Actor({
type: new UserActorType({
user: await get_user({ id: ownerId }),
}),
});
this.#meteringService.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize);
const entryOp = await svc_fsEntry.update(uid, raw_fsentry_delta);
// depends on fsentry, does not depend on S3
@@ -786,7 +778,7 @@ class PuterFSProvider extends putility.AdvancedBase {
})();
const cachePromise = (async () => {
const svc_fileCache = context.get('services').get('file-cache');
const svc_fileCache = this.#services.get('file-cache');
await svc_fileCache.invalidate(node);
})();
@@ -809,24 +801,26 @@ class PuterFSProvider extends putility.AdvancedBase {
return node;
}
async _storage_upload ({
/**
* @param {Object} param
* @param {File} param.file: The file to write.
* @returns
*/
async #storage_upload({
uuid,
bucket, bucket_region, file,
bucket,
bucket_region,
file,
tmp,
}) {
const { config } = this.modules;
const log = this.#services.get('log-service').create('fs.#storage_upload');
const errors = this.#services.get('error-service').create(log);
const svc_event = this.#services.get('event');
const svc = Context.get('services');
const log = svc.get('log-service').create('fs._storage_upload');
const errors = svc.get('error-service').create(log);
const svc_event = svc.get('event');
const svc_mountpoint = svc.get('mountpoint');
const svc_mountpoint = this.#services.get('mountpoint');
const storage = svc_mountpoint.get_storage(this.constructor.name);
bucket ??= config.s3_bucket;
bucket ??= config.s3_bucket;
bucket_region ??= config.s3_region ?? config.region;
let upload_tracker = new UploadProgressTracker();
@@ -837,10 +831,10 @@ class PuterFSProvider extends putility.AdvancedBase {
meta: {
item_uid: uuid,
item_path: tmp.path,
}
})
},
});
if ( ! file.buffer ) {
if ( !file.buffer ) {
let stream = file.stream;
let alarm_timeout = null;
stream = stuck_detector_stream(stream, {
@@ -867,9 +861,9 @@ class PuterFSProvider extends putility.AdvancedBase {
on_unstuck: () => {
clearTimeout(alarm_timeout);
this.frame.status = OperationFrame.FRAME_STATUS_WORKING;
}
},
});
file = { ...file, stream, };
file = { ...file, stream };
}
let hashPromise;
@@ -884,7 +878,7 @@ class PuterFSProvider extends putility.AdvancedBase {
}
hashPromise.then(hash => {
const svc_event = Context.get('services').get('event');
const svc_event = this.#services.get('event');
svc_event.emit('outer.fs.write-hash', {
hash, uuid,
});

146
src/backend/src/routers/file.js
View File

@@ -16,78 +16,84 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
"use strict"
'use strict';
const express = require('express');
const router = new express.Router();
const {validate_signature_auth, get_url_from_req, get_descendants, id2path, get_user, sign_file} = require('../helpers');
const { subdomain, validate_signature_auth, get_url_from_req, get_descendants, id2path, get_user, sign_file } = require('../helpers');
const { DB_WRITE } = require('../services/database/consts');
const { Context } = require('../util/context');
const { UserActorType } = require('../services/auth/Actor');
const { Actor } = require('../services/auth/Actor');
// -----------------------------------------------------------------------//
// GET /file
// -----------------------------------------------------------------------//
router.get('/file', async (req, res, next)=>{
router.get('/file', async (req, res, next) => {
// services and "services"
/** @type {import('../services/MeteringService/MeteringService').MeteringAndBillingService} */
const meteringService = req.services.get('meteringService').meteringAndBillingService;
const log = req.services.get('log-service').create('/file');
const errors = req.services.get('error-service').create(log);
const db = req.services.get('database').get(DB_WRITE, 'filesystem');
// check subdomain
if(require('../helpers').subdomain(req) !== 'api')
if ( subdomain(req) !== 'api' ){
next();
}
// validate URL signature
try{
try {
validate_signature_auth(get_url_from_req(req), 'read');
}
catch(e){
console.log(e)
} catch (e){
console.log(e);
return res.status(403).send(e);
}
let can_write = false;
try{
try {
validate_signature_auth(get_url_from_req(req), 'write');
can_write = true;
}catch(e){}
const log = req.services.get('log-service').create('/file');
const errors = req.services.get('error-service').create(log);
} catch ( _e ){
// slent fail
}
// modules
const db = req.services.get('database').get(DB_WRITE, 'filesystem');
const mime = require('mime-types')
const uid = req.query.uid;
let download = req.query.download ?? false;
if(download === 'true' || download === '1' || download === true)
if ( download === 'true' || download === '1' || download === true ){
download = true;
}
// retrieve FSEntry from db
const fsentry = await db.read(
`SELECT * FROM fsentries WHERE uuid = ? LIMIT 1`, [uid]
);
const fsentry = await db.read('SELECT * FROM fsentries WHERE uuid = ? LIMIT 1', [uid]);
// FSEntry not found
if(!fsentry[0])
return res.status(400).send({message: 'No entry found with this uid'})
if ( !fsentry[0] )
{
return res.status(400).send({ message: 'No entry found with this uid' });
}
// check if item owner is suspended
const user = await get_user({id: fsentry[0].user_id});
if(user.suspended)
return res.status(401).send({error: 'Account suspended'});
const user = await get_user({ id: fsentry[0].user_id });
if ( user.suspended )
{
return res.status(401).send({ error: 'Account suspended' });
}
// ---------------------------------------------------------------//
// FSEntry is dir
// ---------------------------------------------------------------//
if(fsentry[0].is_dir){
if ( fsentry[0].is_dir ){
// convert to path
const dirpath = await id2path(fsentry[0].id);
console.log(dirpath, fsentry[0].user_id)
// get all children of this dir
const children = await get_descendants(dirpath, await get_user({id: fsentry[0].user_id}), 1);
const children = await get_descendants(dirpath, await get_user({ id: fsentry[0].user_id }), 1);
const signed_children = [];
if(children.length>0){
for(const child of children){
if ( children.length > 0 ){
for ( const child of children ){
// sign file
const signed_child = await sign_file(child,
can_write ? 'write' : 'read');
can_write ? 'write' : 'read');
signed_children.push(signed_child);
}
}
@@ -96,108 +102,116 @@ router.get('/file', async (req, res, next)=>{
}
// force download?
if(download)
if ( download ){
res.attachment(fsentry[0].name);
}
// record fsentry owner
res.resource_owner = fsentry[0].user_id;
// try to deduce content-type
const contentType = "application/octet-stream";
const contentType = 'application/octet-stream';
// update `accessed`
db.write(
"UPDATE fsentries SET accessed = ? WHERE `id` = ?",
[Date.now()/1000, fsentry[0].id]
);
db.write('UPDATE fsentries SET accessed = ? WHERE `id` = ?',
[Date.now() / 1000, fsentry[0].id]);
const range = req.headers.range;
const ownerActor = new Actor({
type: new UserActorType({
user: user,
}),
});
const fileSize = fsentry[0].size;
//--------------------------------------------------
// No range
//--------------------------------------------------
if (!range) {
if ( !range ){
// set content-type, if available
if(contentType !== null)
if ( contentType !== null ){
res.setHeader('Content-Type', contentType);
}
const storage = req.ctx.get('storage');
// stream data from S3
try{
try {
let stream = await storage.create_read_stream(fsentry[0].uuid, {
bucket: fsentry[0].bucket,
bucket_region: fsentry[0].bucket_region,
});
meteringService.incrementUsage(ownerActor, 'filesystem:egress:bytes', fileSize);
return stream.pipe(res);
}catch(e){
} catch (e){
errors.report('read from storage', {
source: e,
trace: true,
alarm: true,
});
return res.type('application/json').status(500).send({message: 'There was an internal problem reading the file.'});
return res.type('application/json').status(500).send({ message: 'There was an internal problem reading the file.' });
}
}
//--------------------------------------------------
// Range
//--------------------------------------------------
else{
// get file size
const file_size = fsentry[0].size;
else {
const total = fsentry[0].size;
const user_agent = req.get('User-Agent');
let start, end, CHUNK_SIZE = 5000000;
let start, end, chunkSize = 5000000;
let is_safari = false;
// Parse range header
var parts = range.replace(/bytes=/, "").split("-");
var parts = range.replace(/bytes=/, '').split('-');
var partialstart = parts[0];
var partialend = parts[1];
start = parseInt(partialstart, 10);
end = partialend ? parseInt(partialend, 10) : total-1;
end = partialend ? parseInt(partialend, 10) : total - 1;
// Safari
if(user_agent && user_agent.toLowerCase().includes('safari') && !user_agent.includes('Chrome')){
if ( user_agent && user_agent.toLowerCase().includes('safari') && !user_agent.includes('Chrome') ){
// Safari
is_safari = true;
CHUNK_SIZE = (end-start)+1;
}
// All other user agents
else{
end = Math.min(start + CHUNK_SIZE, file_size - 1);
chunkSize = (end - start) + 1;
} else {
// All other user agents
end = Math.min(start + chunkSize, fileSize - 1);
}
// Create headers
const headers = {
"Content-Range": `bytes ${start}-${end}/${file_size}`,
"Accept-Ranges": "bytes",
"Content-Length": is_safari ? CHUNK_SIZE : (end-start+1),
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': is_safari ? chunkSize : (end - start + 1),
};
// Set Content-Type, if available
if(contentType)
headers["Content-Type"] = contentType;
if ( contentType ){
headers['Content-Type'] = contentType;
}
// HTTP Status 206 for Partial Content
res.writeHead(206, headers);
try{
try {
const storage = Context.get('storage');
let stream = await storage.create_read_stream(fsentry[0].uuid, {
bucket: fsentry[0].bucket,
bucket_region: fsentry[0].bucket_region,
});
meteringService.incrementUsage(ownerActor, 'filesystem:egress:bytes', chunkSize);
return stream.pipe(res);
}catch(e){
} catch (e){
errors.report('read from storage', {
source: e,
trace: true,
alarm: true,
});
return res.type('application/json').status(500).send({message: 'There was an internal problem reading the file.'});
return res.type('application/json').status(500).send({ message: 'There was an internal problem reading the file.' });
}
}
})
});
module.exports = router
module.exports = router;

2
src/backend/src/routers/login.js
View File

@@ -53,7 +53,7 @@ const complete_ = async ({ req, res, user }) => {
};
// -----------------------------------------------------------------------//
// POST /file
// POST /login
// -----------------------------------------------------------------------//
router.post('/login', express.json(), body_parser_error_handler,
// Add diagnostic middleware to log captcha data

15
src/backend/src/services/MeteringService/MeteringService.ts
View File

@@ -54,7 +54,7 @@ export class MeteringAndBillingService {
if ( totalCost === 0 && costOverride === undefined ) {
// could be something is off, there are some models that cost nothing from openrouter, but then our overrides should not be undefined, so will flag
this.#alarmService.create('metering-service-warning', "potential abuse vector", {
this.#alarmService.create('metering-service-warning', 'potential abuse vector', {
actor,
usageType,
usageAmount,
@@ -125,7 +125,7 @@ export class MeteringAndBillingService {
const actorSubscriptionPromise = this.getActorSubscription(actor);
const actorAddonsPromise = this.getActorAddons(actor);
const [actorUsages, actorSubscription, actorAddons] = (await Promise.all([actorUsagesPromise, actorSubscriptionPromise, actorAddonsPromise]));
if ( actorUsages.total > actorSubscription.monthUsageAllowance && actorAddons.purchasedCredits ) {
if ( actorUsages.total > actorSubscription.monthUsageAllowance && actorAddons.purchasedCredits && actorAddons.purchasedCredits > (actorAddons.consumedPurchaseCredits || 0) ) {
// if we are now over the allowance, start consuming purchased credits
const withinBoundsUsage = Math.max(0, actorSubscription.monthUsageAllowance - actorUsages.total + totalCost);
const overageUsage = totalCost - withinBoundsUsage;
@@ -139,6 +139,17 @@ export class MeteringAndBillingService {
});
}
}
// alert if significantly over allowance and no purchased credits left
if ( actorUsages.total > (actorSubscription.monthUsageAllowance) * 2 && (actorAddons.purchasedCredits || 0) <= (actorAddons.consumedPurchaseCredits || 0) ) {
this.#alarmService.create('metering-service-usage-limit-exceeded', `Actor ${actorId} has exceeded their usage allowance significantly`, {
actor,
usageType,
usageAmount,
costOverride,
totalUsage: actorUsages.total,
monthUsageAllowance: actorSubscription.monthUsageAllowance,
});
}
return actorUsages;
});
} catch (e) {

8
src/backend/src/services/MeteringService/costMaps/awsPollyCostMap.ts
View File

@@ -11,14 +11,14 @@
export const AWS_POLLY_COST_MAP = {
// Standard engine: $4.00 per 1M characters (400 microcents per character)
"aws-polly:standard:character": 400,
'aws-polly:standard:character': 400,
// Neural engine: $16.00 per 1M characters (1600 microcents per character)
"aws-polly:neural:character": 1600,
'aws-polly:neural:character': 1600,
// Long-form engine: $100.00 per 1M characters (10000 microcents per character)
"aws-polly:long-form:character": 10000,
'aws-polly:long-form:character': 10000,
// Generative engine: $30.00 per 1M characters (3000 microcents per character)
"aws-polly:generative:character": 3000,
'aws-polly:generative:character': 3000,
};

2
src/backend/src/services/MeteringService/costMaps/awsTextractCostMap.ts
View File

@@ -11,5 +11,5 @@
//
export const AWS_TEXTRACT_COST_MAP = {
// Detect Document Text API: $1.50 per 1,000 pages (150000 microcents per page)
"aws-textract:detect-document-text:page": 150000,
'aws-textract:detect-document-text:page': 150000,
};

86
src/backend/src/services/MeteringService/costMaps/claudeCostMap.ts
View File

@@ -19,64 +19,58 @@
export const CLAUDE_COST_MAP = {
// Claude Sonnet 4.5
"claude:claude-sonnet-4-5-20250929:input_tokens": 300,
"claude:claude-sonnet-4-5-20250929:ephemeral_5m_input_tokens": 300 * 1.25,
"claude:claude-sonnet-4-5-20250929:ephemeral_1h_input_tokens": 300 * 2,
"claude:claude-sonnet-4-5-20250929:cache_read_input_tokens": 300 * 0.1,
"claude:claude-sonnet-4-5-20250929:output_tokens": 1500,
'claude:claude-sonnet-4-5-20250929:input_tokens': 300,
'claude:claude-sonnet-4-5-20250929:ephemeral_5m_input_tokens': 300 * 1.25,
'claude:claude-sonnet-4-5-20250929:ephemeral_1h_input_tokens': 300 * 2,
'claude:claude-sonnet-4-5-20250929:cache_read_input_tokens': 300 * 0.1,
'claude:claude-sonnet-4-5-20250929:output_tokens': 1500,
// Claude Opus 4.1
"claude:claude-opus-4-1-20250805:input_tokens": 1500,
"claude:claude-opus-4-1-20250805:ephemeral_5m_input_tokens": 1500 * 1.25,
"claude:claude-opus-4-1-20250805:ephemeral_1h_input_tokens": 1500 * 2,
"claude:claude-opus-4-1-20250805:cache_read_input_tokens": 1500 * 0.1,
"claude:claude-opus-4-1-20250805:output_tokens": 7500,
'claude:claude-opus-4-1-20250805:input_tokens': 1500,
'claude:claude-opus-4-1-20250805:ephemeral_5m_input_tokens': 1500 * 1.25,
'claude:claude-opus-4-1-20250805:ephemeral_1h_input_tokens': 1500 * 2,
'claude:claude-opus-4-1-20250805:cache_read_input_tokens': 1500 * 0.1,
'claude:claude-opus-4-1-20250805:output_tokens': 7500,
// Claude Opus 4
"claude:claude-opus-4-20250514:input_tokens": 1500,
"claude:claude-opus-4-20250514:ephemeral_5m_input_tokens": 1500 * 1.25,
"claude:claude-opus-4-20250514:ephemeral_1h_input_tokens": 1500 * 2,
"claude:claude-opus-4-20250514:cache_read_input_tokens": 1500 * 0.1,
"claude:claude-opus-4-20250514:output_tokens": 7500,
'claude:claude-opus-4-20250514:input_tokens': 1500,
'claude:claude-opus-4-20250514:ephemeral_5m_input_tokens': 1500 * 1.25,
'claude:claude-opus-4-20250514:ephemeral_1h_input_tokens': 1500 * 2,
'claude:claude-opus-4-20250514:cache_read_input_tokens': 1500 * 0.1,
'claude:claude-opus-4-20250514:output_tokens': 7500,
// Claude Sonnet 4
"claude:claude-sonnet-4-20250514:input_tokens": 300,
"claude:claude-sonnet-4-20250514:ephemeral_5m_input_tokens": 300 * 1.25,
"claude:claude-sonnet-4-20250514:ephemeral_1h_input_tokens": 300 * 2,
"claude:claude-sonnet-4-20250514:cache_read_input_tokens": 300 * 0.1,
"claude:claude-sonnet-4-20250514:output_tokens": 1500,
'claude:claude-sonnet-4-20250514:input_tokens': 300,
'claude:claude-sonnet-4-20250514:ephemeral_5m_input_tokens': 300 * 1.25,
'claude:claude-sonnet-4-20250514:ephemeral_1h_input_tokens': 300 * 2,
'claude:claude-sonnet-4-20250514:cache_read_input_tokens': 300 * 0.1,
'claude:claude-sonnet-4-20250514:output_tokens': 1500,
// Claude 3.7 Sonnet
"claude:claude-3-7-sonnet-20250219:input_tokens": 300,
"claude:claude-3-7-sonnet-20250219:ephemeral_5m_input_tokens": 300 * 1.25,
"claude:claude-3-7-sonnet-20250219:ephemeral_1h_input_tokens": 300 * 2,
"claude:claude-3-7-sonnet-20250219:cache_read_input_tokens": 300 * 0.1,
"claude:claude-3-7-sonnet-20250219:output_tokens": 1500,
'claude:claude-3-7-sonnet-20250219:input_tokens': 300,
'claude:claude-3-7-sonnet-20250219:ephemeral_5m_input_tokens': 300 * 1.25,
'claude:claude-3-7-sonnet-20250219:ephemeral_1h_input_tokens': 300 * 2,
'claude:claude-3-7-sonnet-20250219:cache_read_input_tokens': 300 * 0.1,
'claude:claude-3-7-sonnet-20250219:output_tokens': 1500,
// Claude 3.5 Sonnet (Oct 2024)
"claude:claude-3-5-sonnet-20241022:input_tokens": 300,
"claude:claude-3-5-sonnet-20241022:ephemeral_5m_input_tokens": 300 * 1.25,
"claude:claude-3-5-sonnet-20241022:ephemeral_1h_input_tokens": 300 * 2,
"claude:claude-3-5-sonnet-20241022:cache_read_input_tokens": 300 * 0.1,
"claude:claude-3-5-sonnet-20241022:output_tokens": 1500,
'claude:claude-3-5-sonnet-20241022:input_tokens': 300,
'claude:claude-3-5-sonnet-20241022:ephemeral_5m_input_tokens': 300 * 1.25,
'claude:claude-3-5-sonnet-20241022:ephemeral_1h_input_tokens': 300 * 2,
'claude:claude-3-5-sonnet-20241022:cache_read_input_tokens': 300 * 0.1,
'claude:claude-3-5-sonnet-20241022:output_tokens': 1500,
// Claude 3.5 Sonnet (June 2024)
"claude:claude-3-5-sonnet-20240620:input_tokens": 300,
"claude:claude-3-5-sonnet-20240620:ephemeral_5m_input_tokens": 300 * 1.25,
"claude:claude-3-5-sonnet-20240620:ephemeral_1h_input_tokens": 300 * 2,
"claude:claude-3-5-sonnet-20240620:cache_read_input_tokens": 300 * 0.1,
"claude:claude-3-5-sonnet-20240620:output_tokens": 1500,
'claude:claude-3-5-sonnet-20240620:input_tokens': 300,
'claude:claude-3-5-sonnet-20240620:ephemeral_5m_input_tokens': 300 * 1.25,
'claude:claude-3-5-sonnet-20240620:ephemeral_1h_input_tokens': 300 * 2,
'claude:claude-3-5-sonnet-20240620:cache_read_input_tokens': 300 * 0.1,
'claude:claude-3-5-sonnet-20240620:output_tokens': 1500,
// Claude 3 Haiku
"claude:claude-3-haiku-20240307:input_tokens": 25,
"claude:claude-3-haiku-20240307:ephemeral_5m_input_tokens": 25 * 1.25,
"claude:claude-3-haiku-20240307:ephemeral_1h_input_tokens": 25 * 2,
"claude:claude-3-haiku-20240307:cache_read_input_tokens": 25 * 0.1,
"claude:claude-3-haiku-20240307:output_tokens": 125
'claude:claude-3-haiku-20240307:input_tokens': 25,
'claude:claude-3-haiku-20240307:ephemeral_5m_input_tokens': 25 * 1.25,
'claude:claude-3-haiku-20240307:ephemeral_1h_input_tokens': 25 * 2,
'claude:claude-3-haiku-20240307:cache_read_input_tokens': 25 * 0.1,
'claude:claude-3-haiku-20240307:output_tokens': 125,
};

8
src/backend/src/services/MeteringService/costMaps/deepSeekCostMap.ts
View File

@@ -19,10 +19,10 @@
export const DEEPSEEK_COST_MAP = {
// DeepSeek Chat
"deepseek:deepseek-chat:prompt_tokens": 56,
"deepseek:deepseek-chat:completion_tokens": 168,
'deepseek:deepseek-chat:prompt_tokens': 56,
'deepseek:deepseek-chat:completion_tokens': 168,
// DeepSeek Reasoner
"deepseek:deepseek-reasoner:prompt_tokens": 56,
"deepseek:deepseek-reasoner:completion_tokens": 168,
'deepseek:deepseek-reasoner:prompt_tokens': 56,
'deepseek:deepseek-reasoner:completion_tokens': 168,
};

8
src/backend/src/services/MeteringService/costMaps/fileSystemCostMap.ts Normal file
View File

@@ -0,0 +1,8 @@
import { toMicroCents } from '../utils';
export const FILE_SYSTEM_COST_MAP = {
'filesystem:ingress:bytes': 0,
'filesystem:delete:bytes': 0,
'filesystem:egress:bytes': toMicroCents(0.12 / 1024 / 1024 / 1024), // $0.11 per GB ~> 0.12 per GiB
'filesystem:cached-egress:bytes': toMicroCents(0.1 / 1024 / 1024 / 1024), // $0.09 per GB ~> 0.1 per GiB,
};

12
src/backend/src/services/MeteringService/costMaps/geminiCostMap.ts
View File

@@ -9,9 +9,9 @@
*/
export const GEMINI_COST_MAP = {
// Gemini api usage types (costs per token in microcents)
"gemini:gemini-2.0-flash:promptTokenCount": 10,
"gemini:gemini-2.0-flash:candidatesTokenCount": 40,
"gemini:gemini-1.5-flash:promptTokenCount": 3,
"gemini:gemini-1.5-flash:candidatesTokenCount": 2,
"gemini:gemini-2.5-flash-image-preview:1024x1024": 3_900_000
}
'gemini:gemini-2.0-flash:promptTokenCount': 10,
'gemini:gemini-2.0-flash:candidatesTokenCount': 40,
'gemini:gemini-1.5-flash:promptTokenCount': 3,
'gemini:gemini-1.5-flash:candidatesTokenCount': 2,
'gemini:gemini-2.5-flash-image-preview:1024x1024': 3_900_000,
};

68
src/backend/src/services/MeteringService/costMaps/groqCostMap.ts
View File

@@ -19,52 +19,52 @@
export const GROQ_COST_MAP = {
// Gemma models
"groq:gemma2-9b-it:prompt_tokens": 20,
"groq:gemma2-9b-it:completion_tokens": 20,
"groq:gemma-7b-it:prompt_tokens": 7,
"groq:gemma-7b-it:completion_tokens": 7,
'groq:gemma2-9b-it:prompt_tokens': 20,
'groq:gemma2-9b-it:completion_tokens': 20,
'groq:gemma-7b-it:prompt_tokens': 7,
'groq:gemma-7b-it:completion_tokens': 7,
// Llama 3 Groq Tool Use Preview
"groq:llama3-groq-70b-8192-tool-use-preview:prompt_tokens": 89,
"groq:llama3-groq-70b-8192-tool-use-preview:completion_tokens": 89,
"groq:llama3-groq-8b-8192-tool-use-preview:prompt_tokens": 19,
"groq:llama3-groq-8b-8192-tool-use-preview:completion_tokens": 19,
'groq:llama3-groq-70b-8192-tool-use-preview:prompt_tokens': 89,
'groq:llama3-groq-70b-8192-tool-use-preview:completion_tokens': 89,
'groq:llama3-groq-8b-8192-tool-use-preview:prompt_tokens': 19,
'groq:llama3-groq-8b-8192-tool-use-preview:completion_tokens': 19,
// Llama 3.1
"groq:llama-3.1-70b-versatile:prompt_tokens": 59,
"groq:llama-3.1-70b-versatile:completion_tokens": 79,
"groq:llama-3.1-70b-specdec:prompt_tokens": 59,
"groq:llama-3.1-70b-specdec:completion_tokens": 99,
"groq:llama-3.1-8b-instant:prompt_tokens": 5,
"groq:llama-3.1-8b-instant:completion_tokens": 8,
'groq:llama-3.1-70b-versatile:prompt_tokens': 59,
'groq:llama-3.1-70b-versatile:completion_tokens': 79,
'groq:llama-3.1-70b-specdec:prompt_tokens': 59,
'groq:llama-3.1-70b-specdec:completion_tokens': 99,
'groq:llama-3.1-8b-instant:prompt_tokens': 5,
'groq:llama-3.1-8b-instant:completion_tokens': 8,
// Llama Guard
"groq:meta-llama/llama-guard-4-12b:prompt_tokens": 20,
"groq:meta-llama/llama-guard-4-12b:completion_tokens": 20,
"groq:llama-guard-3-8b:prompt_tokens": 20,
"groq:llama-guard-3-8b:completion_tokens": 20,
'groq:meta-llama/llama-guard-4-12b:prompt_tokens': 20,
'groq:meta-llama/llama-guard-4-12b:completion_tokens': 20,
'groq:llama-guard-3-8b:prompt_tokens': 20,
'groq:llama-guard-3-8b:completion_tokens': 20,
// Prompt Guard
"groq:meta-llama/llama-prompt-guard-2-86m:prompt_tokens": 4,
"groq:meta-llama/llama-prompt-guard-2-86m:completion_tokens": 4,
'groq:meta-llama/llama-prompt-guard-2-86m:prompt_tokens': 4,
'groq:meta-llama/llama-prompt-guard-2-86m:completion_tokens': 4,
// Llama 3.2 Preview
"groq:llama-3.2-1b-preview:prompt_tokens": 4,
"groq:llama-3.2-1b-preview:completion_tokens": 4,
"groq:llama-3.2-3b-preview:prompt_tokens": 6,
"groq:llama-3.2-3b-preview:completion_tokens": 6,
"groq:llama-3.2-11b-vision-preview:prompt_tokens": 18,
"groq:llama-3.2-11b-vision-preview:completion_tokens": 18,
"groq:llama-3.2-90b-vision-preview:prompt_tokens": 90,
"groq:llama-3.2-90b-vision-preview:completion_tokens": 90,
'groq:llama-3.2-1b-preview:prompt_tokens': 4,
'groq:llama-3.2-1b-preview:completion_tokens': 4,
'groq:llama-3.2-3b-preview:prompt_tokens': 6,
'groq:llama-3.2-3b-preview:completion_tokens': 6,
'groq:llama-3.2-11b-vision-preview:prompt_tokens': 18,
'groq:llama-3.2-11b-vision-preview:completion_tokens': 18,
'groq:llama-3.2-90b-vision-preview:prompt_tokens': 90,
'groq:llama-3.2-90b-vision-preview:completion_tokens': 90,
// Llama 3 8k/70B
"groq:llama3-70b-8192:prompt_tokens": 59,
"groq:llama3-70b-8192:completion_tokens": 79,
"groq:llama3-8b-8192:prompt_tokens": 5,
"groq:llama3-8b-8192:completion_tokens": 8,
'groq:llama3-70b-8192:prompt_tokens': 59,
'groq:llama3-70b-8192:completion_tokens': 79,
'groq:llama3-8b-8192:prompt_tokens': 5,
'groq:llama3-8b-8192:completion_tokens': 8,
// Mixtral
"groq:mixtral-8x7b-32768:prompt_tokens": 24,
"groq:mixtral-8x7b-32768:completion_tokens": 24,
'groq:mixtral-8x7b-32768:prompt_tokens': 24,
'groq:mixtral-8x7b-32768:completion_tokens': 24,
};

28
src/backend/src/services/MeteringService/costMaps/index.ts
View File

@@ -1,16 +1,17 @@
import { AWS_POLLY_COST_MAP } from "./awsPollyCostMap";
import { AWS_TEXTRACT_COST_MAP } from "./awsTextractCostMap";
import { CLAUDE_COST_MAP } from "./claudeCostMap";
import { DEEPSEEK_COST_MAP } from "./deepSeekCostMap";
import { GEMINI_COST_MAP } from "./geminiCostMap";
import { GROQ_COST_MAP } from "./groqCostMap";
import { KV_COST_MAP } from "./kvCostMap";
import { MISTRAL_COST_MAP } from "./mistralCostMap";
import { OPENAI_COST_MAP } from "./openAiCostMap";
import { OPENAI_IMAGE_COST_MAP } from "./openaiImageCostMap";
import { OPENROUTER_COST_MAP } from "./openrouterCostMap";
import { TOGETHER_COST_MAP } from "./togetherCostMap";
import { XAI_COST_MAP } from "./xaiCostMap";
import { AWS_POLLY_COST_MAP } from './awsPollyCostMap';
import { AWS_TEXTRACT_COST_MAP } from './awsTextractCostMap';
import { CLAUDE_COST_MAP } from './claudeCostMap';
import { DEEPSEEK_COST_MAP } from './deepSeekCostMap';
import { FILE_SYSTEM_COST_MAP } from './fileSystemCostMap';
import { GEMINI_COST_MAP } from './geminiCostMap';
import { GROQ_COST_MAP } from './groqCostMap';
import { KV_COST_MAP } from './kvCostMap';
import { MISTRAL_COST_MAP } from './mistralCostMap';
import { OPENAI_COST_MAP } from './openAiCostMap';
import { OPENAI_IMAGE_COST_MAP } from './openaiImageCostMap';
import { OPENROUTER_COST_MAP } from './openrouterCostMap';
import { TOGETHER_COST_MAP } from './togetherCostMap';
import { XAI_COST_MAP } from './xaiCostMap';
export const COST_MAPS = {
...AWS_POLLY_COST_MAP,
@@ -26,4 +27,5 @@ export const COST_MAPS = {
...OPENROUTER_COST_MAP,
...TOGETHER_COST_MAP,
...XAI_COST_MAP,
...FILE_SYSTEM_COST_MAP,
};

2
src/backend/src/services/MeteringService/costMaps/kvCostMap.ts
View File

@@ -2,4 +2,4 @@ export const KV_COST_MAP = {
// Map with unit to cost measurements in microcent
'kv:read': 63,
'kv:write': 125,
}
};

76
src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts
View File

@@ -19,42 +19,42 @@
export const MISTRAL_COST_MAP = {
// Mistral models (values in microcents/token, from MistralAIService.js)
"mistral:mistral-large-latest:prompt_tokens": 200,
"mistral:mistral-large-latest:completion_tokens": 600,
"mistral:pixtral-large-latest:prompt_tokens": 200,
"mistral:pixtral-large-latest:completion_tokens": 600,
"mistral:mistral-small-latest:prompt_tokens": 20,
"mistral:mistral-small-latest:completion_tokens": 60,
"mistral:codestral-latest:prompt_tokens": 30,
"mistral:codestral-latest:completion_tokens": 90,
"mistral:ministral-8b-latest:prompt_tokens": 10,
"mistral:ministral-8b-latest:completion_tokens": 10,
"mistral:ministral-3b-latest:prompt_tokens": 4,
"mistral:ministral-3b-latest:completion_tokens": 4,
"mistral:pixtral-12b:prompt_tokens": 15,
"mistral:pixtral-12b:completion_tokens": 15,
"mistral:mistral-nemo:prompt_tokens": 15,
"mistral:mistral-nemo:completion_tokens": 15,
"mistral:open-mistral-7b:prompt_tokens": 25,
"mistral:open-mistral-7b:completion_tokens": 25,
"mistral:open-mixtral-8x7b:prompt_tokens": 7,
"mistral:open-mixtral-8x7b:completion_tokens": 7,
"mistral:open-mixtral-8x22b:prompt_tokens": 2,
"mistral:open-mixtral-8x22b:completion_tokens": 6,
"mistral:magistral-medium-latest:prompt_tokens": 200,
"mistral:magistral-medium-latest:completion_tokens": 500,
"mistral:magistral-small-latest:prompt_tokens": 10,
"mistral:magistral-small-latest:completion_tokens": 10,
"mistral:mistral-medium-latest:prompt_tokens": 40,
"mistral:mistral-medium-latest:completion_tokens": 200,
"mistral:mistral-moderation-latest:prompt_tokens": 10,
"mistral:mistral-moderation-latest:completion_tokens": 10,
"mistral:devstral-small-latest:prompt_tokens": 10,
"mistral:devstral-small-latest:completion_tokens": 10,
"mistral:mistral-saba-latest:prompt_tokens": 20,
"mistral:mistral-saba-latest:completion_tokens": 60,
"mistral:open-mistral-nemo:prompt_tokens": 10,
"mistral:open-mistral-nemo:completion_tokens": 10,
"mistral:mistral-ocr-latest:prompt_tokens": 100,
"mistral:mistral-ocr-latest:completion_tokens": 300,
'mistral:mistral-large-latest:prompt_tokens': 200,
'mistral:mistral-large-latest:completion_tokens': 600,
'mistral:pixtral-large-latest:prompt_tokens': 200,
'mistral:pixtral-large-latest:completion_tokens': 600,
'mistral:mistral-small-latest:prompt_tokens': 20,
'mistral:mistral-small-latest:completion_tokens': 60,
'mistral:codestral-latest:prompt_tokens': 30,
'mistral:codestral-latest:completion_tokens': 90,
'mistral:ministral-8b-latest:prompt_tokens': 10,
'mistral:ministral-8b-latest:completion_tokens': 10,
'mistral:ministral-3b-latest:prompt_tokens': 4,
'mistral:ministral-3b-latest:completion_tokens': 4,
'mistral:pixtral-12b:prompt_tokens': 15,
'mistral:pixtral-12b:completion_tokens': 15,
'mistral:mistral-nemo:prompt_tokens': 15,
'mistral:mistral-nemo:completion_tokens': 15,
'mistral:open-mistral-7b:prompt_tokens': 25,
'mistral:open-mistral-7b:completion_tokens': 25,
'mistral:open-mixtral-8x7b:prompt_tokens': 7,
'mistral:open-mixtral-8x7b:completion_tokens': 7,
'mistral:open-mixtral-8x22b:prompt_tokens': 2,
'mistral:open-mixtral-8x22b:completion_tokens': 6,
'mistral:magistral-medium-latest:prompt_tokens': 200,
'mistral:magistral-medium-latest:completion_tokens': 500,
'mistral:magistral-small-latest:prompt_tokens': 10,
'mistral:magistral-small-latest:completion_tokens': 10,
'mistral:mistral-medium-latest:prompt_tokens': 40,
'mistral:mistral-medium-latest:completion_tokens': 200,
'mistral:mistral-moderation-latest:prompt_tokens': 10,
'mistral:mistral-moderation-latest:completion_tokens': 10,
'mistral:devstral-small-latest:prompt_tokens': 10,
'mistral:devstral-small-latest:completion_tokens': 10,
'mistral:mistral-saba-latest:prompt_tokens': 20,
'mistral:mistral-saba-latest:completion_tokens': 60,
'mistral:open-mistral-nemo:prompt_tokens': 10,
'mistral:open-mistral-nemo:completion_tokens': 10,
'mistral:mistral-ocr-latest:prompt_tokens': 100,
'mistral:mistral-ocr-latest:completion_tokens': 300,
};

84
src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts
View File

@@ -20,58 +20,58 @@
export const OPENAI_COST_MAP = {
// GPT-5 models
"openai:gpt-5-2025-08-07:prompt_tokens": 125,
"openai:gpt-5-2025-08-07:cached_tokens": 13,
"openai:gpt-5-2025-08-07:completion_tokens": 1000,
"openai:gpt-5-mini-2025-08-07:prompt_tokens": 25,
"openai:gpt-5-mini-2025-08-07:cached_tokens": 3,
"openai:gpt-5-mini-2025-08-07:completion_tokens": 200,
"openai:gpt-5-nano-2025-08-07:prompt_tokens": 5,
"openai:gpt-5-nano-2025-08-07:cached_tokens": 1,
"openai:gpt-5-nano-2025-08-07:completion_tokens": 40,
"openai:gpt-5-chat-latest:prompt_tokens": 125,
"openai:gpt-5-chat-latest:cached_tokens": 13,
"openai:gpt-5-chat-latest:completion_tokens": 1000,
'openai:gpt-5-2025-08-07:prompt_tokens': 125,
'openai:gpt-5-2025-08-07:cached_tokens': 13,
'openai:gpt-5-2025-08-07:completion_tokens': 1000,
'openai:gpt-5-mini-2025-08-07:prompt_tokens': 25,
'openai:gpt-5-mini-2025-08-07:cached_tokens': 3,
'openai:gpt-5-mini-2025-08-07:completion_tokens': 200,
'openai:gpt-5-nano-2025-08-07:prompt_tokens': 5,
'openai:gpt-5-nano-2025-08-07:cached_tokens': 1,
'openai:gpt-5-nano-2025-08-07:completion_tokens': 40,
'openai:gpt-5-chat-latest:prompt_tokens': 125,
'openai:gpt-5-chat-latest:cached_tokens': 13,
'openai:gpt-5-chat-latest:completion_tokens': 1000,
// GPT-4o models
"openai:gpt-4o:prompt_tokens": 250,
"openai:gpt-4o:cached_tokens": 125,
"openai:gpt-4o:completion_tokens": 1000,
"openai:gpt-4o-mini:prompt_tokens": 15,
"openai:gpt-4o-mini:cached_tokens": 8,
"openai:gpt-4o-mini:completion_tokens": 60,
'openai:gpt-4o:prompt_tokens': 250,
'openai:gpt-4o:cached_tokens': 125,
'openai:gpt-4o:completion_tokens': 1000,
'openai:gpt-4o-mini:prompt_tokens': 15,
'openai:gpt-4o-mini:cached_tokens': 8,
'openai:gpt-4o-mini:completion_tokens': 60,
// O1 models
"openai:o1:prompt_tokens": 1500,
"openai:o1:cached_tokens": 750,
"openai:o1:completion_tokens": 6000,
"openai:o1-mini:prompt_tokens": 110,
"openai:o1-mini:completion_tokens": 440,
"openai:o1-pro:prompt_tokens": 15000,
"openai:o1-pro:completion_tokens": 60000,
'openai:o1:prompt_tokens': 1500,
'openai:o1:cached_tokens': 750,
'openai:o1:completion_tokens': 6000,
'openai:o1-mini:prompt_tokens': 110,
'openai:o1-mini:completion_tokens': 440,
'openai:o1-pro:prompt_tokens': 15000,
'openai:o1-pro:completion_tokens': 60000,
// O3 models
"openai:o3:prompt_tokens": 1000,
"openai:o3:completion_tokens": 4000,
"openai:o3-mini:prompt_tokens": 110,
"openai:o3-mini:completion_tokens": 440,
'openai:o3:prompt_tokens': 1000,
'openai:o3:completion_tokens': 4000,
'openai:o3-mini:prompt_tokens': 110,
'openai:o3-mini:completion_tokens': 440,
// O4 models
"openai:o4-mini:prompt_tokens": 110,
"openai:o4-mini:completion_tokens": 440,
'openai:o4-mini:prompt_tokens': 110,
'openai:o4-mini:completion_tokens': 440,
// GPT-4.1 models
"openai:gpt-4.1:prompt_tokens": 200,
"openai:gpt-4.1:cached_tokens": 50,
"openai:gpt-4.1:completion_tokens": 800,
"openai:gpt-4.1-mini:prompt_tokens": 40,
"openai:gpt-4.1-mini:cached_tokens": 10,
"openai:gpt-4.1-mini:completion_tokens": 160,
"openai:gpt-4.1-nano:prompt_tokens": 10,
"openai:gpt-4.1-nano:cached_tokens": 2,
"openai:gpt-4.1-nano:completion_tokens": 40,
'openai:gpt-4.1:prompt_tokens': 200,
'openai:gpt-4.1:cached_tokens': 50,
'openai:gpt-4.1:completion_tokens': 800,
'openai:gpt-4.1-mini:prompt_tokens': 40,
'openai:gpt-4.1-mini:cached_tokens': 10,
'openai:gpt-4.1-mini:completion_tokens': 160,
'openai:gpt-4.1-nano:prompt_tokens': 10,
'openai:gpt-4.1-nano:cached_tokens': 2,
'openai:gpt-4.1-nano:completion_tokens': 40,
// GPT-4.5 preview
"openai:gpt-4.5-preview:prompt_tokens": 7500,
"openai:gpt-4.5-preview:completion_tokens": 15000,
'openai:gpt-4.5-preview:prompt_tokens': 7500,
'openai:gpt-4.5-preview:completion_tokens': 15000,
};

38
src/backend/src/services/MeteringService/costMaps/openaiImageCostMap.ts
View File

@@ -3,30 +3,30 @@
// All costs are in microcents (1/1,000,000th of a cent). Example: 1,000,000 microcents = $0.01 USD.//
// Naming pattern: "openai:{model}:{size}" or "openai:{model}:hd:{size}" for HD images
import { toMicroCents } from "../utils";
import { toMicroCents } from '../utils';
export const OPENAI_IMAGE_COST_MAP = {
// DALL-E 3
"openai:dall-e-3:1024x1024": toMicroCents(0.04), // $0.04
"openai:dall-e-3:1024x1792": toMicroCents(0.08), // $0.08
"openai:dall-e-3:1792x1024": toMicroCents(0.08), // $0.08
"openai:dall-e-3:hd:1024x1024": toMicroCents(0.08), // $0.08
"openai:dall-e-3:hd:1024x1792": toMicroCents(0.12), // $0.12
"openai:dall-e-3:hd:1792x1024": toMicroCents(0.12), // $0.12
'openai:dall-e-3:1024x1024': toMicroCents(0.04), // $0.04
'openai:dall-e-3:1024x1792': toMicroCents(0.08), // $0.08
'openai:dall-e-3:1792x1024': toMicroCents(0.08), // $0.08
'openai:dall-e-3:hd:1024x1024': toMicroCents(0.08), // $0.08
'openai:dall-e-3:hd:1024x1792': toMicroCents(0.12), // $0.12
'openai:dall-e-3:hd:1792x1024': toMicroCents(0.12), // $0.12
// DALL-E 2
"openai:dall-e-2:1024x1024": toMicroCents(0.02), // $0.02
"openai:dall-e-2:512x512": toMicroCents(0.018), // $0.018
"openai:dall-e-2:256x256": toMicroCents(0.016), // $0.016
'openai:dall-e-2:1024x1024': toMicroCents(0.02), // $0.02
'openai:dall-e-2:512x512': toMicroCents(0.018), // $0.018
'openai:dall-e-2:256x256': toMicroCents(0.016), // $0.016
// gpt-image-1
"openai:gpt-image-1:low:1024x1024": toMicroCents(0.011),
"openai:gpt-image-1:low:1024x1536": toMicroCents(0.016),
"openai:gpt-image-1:low:1536x1024": toMicroCents(0.016),
"openai:gpt-image-1:medium:1024x1024": toMicroCents(0.042),
"openai:gpt-image-1:medium:1024x1536": toMicroCents(0.063),
"openai:gpt-image-1:medium:1536x1024": toMicroCents(0.063),
"openai:gpt-image-1:high:1024x1024": toMicroCents(0.167),
"openai:gpt-image-1:high:1024x1536": toMicroCents(0.25),
"openai:gpt-image-1:high:1536x1024": toMicroCents(0.25),
'openai:gpt-image-1:low:1024x1024': toMicroCents(0.011),
'openai:gpt-image-1:low:1024x1536': toMicroCents(0.016),
'openai:gpt-image-1:low:1536x1024': toMicroCents(0.016),
'openai:gpt-image-1:medium:1024x1024': toMicroCents(0.042),
'openai:gpt-image-1:medium:1024x1536': toMicroCents(0.063),
'openai:gpt-image-1:medium:1536x1024': toMicroCents(0.063),
'openai:gpt-image-1:high:1024x1024': toMicroCents(0.167),
'openai:gpt-image-1:high:1024x1536': toMicroCents(0.25),
'openai:gpt-image-1:high:1536x1024': toMicroCents(0.25),
};

1438
src/backend/src/services/MeteringService/costMaps/openrouterCostMap.ts
View File

File diff suppressed because it is too large Load Diff

4
src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts
View File

@@ -7,6 +7,6 @@
export const TOGETHER_COST_MAP = {
// Test model (hardcoded)
"together:model-fallback-test-1:input": 10,
"together:model-fallback-test-1:output": 10,
'together:model-fallback-test-1:input': 10,
'together:model-fallback-test-1:output': 10,
};

34
src/backend/src/services/MeteringService/costMaps/xaiCostMap.ts
View File

@@ -19,35 +19,35 @@
export const XAI_COST_MAP = {
// Grok Beta
"xai:grok-beta:prompt_tokens": 500,
"xai:grok-beta:completion-tokens": 1500,
'xai:grok-beta:prompt_tokens': 500,
'xai:grok-beta:completion-tokens': 1500,
// Grok Vision Beta
"xai:grok-vision-beta:prompt_tokens": 500,
"xai:grok-vision-beta:completion-tokens": 1500,
"xai:grok-vision-beta:image": 1000,
'xai:grok-vision-beta:prompt_tokens': 500,
'xai:grok-vision-beta:completion-tokens': 1500,
'xai:grok-vision-beta:image': 1000,
// Grok 3
"xai:grok-3:prompt_tokens": 300,
"xai:grok-3:completion-tokens": 1500,
'xai:grok-3:prompt_tokens': 300,
'xai:grok-3:completion-tokens': 1500,
// Grok 3 Fast
"xai:grok-3-fast:prompt_tokens": 500,
"xai:grok-3-fast:completion-tokens": 2500,
'xai:grok-3-fast:prompt_tokens': 500,
'xai:grok-3-fast:completion-tokens': 2500,
// Grok 3 Mini
"xai:grok-3-mini:prompt_tokens": 30,
"xai:grok-3-mini:completion-tokens": 50,
'xai:grok-3-mini:prompt_tokens': 30,
'xai:grok-3-mini:completion-tokens': 50,
// Grok 3 Mini Fast
"xai:grok-3-mini-fast:prompt_tokens": 60,
"xai:grok-3-mini-fast:completion-tokens": 400,
'xai:grok-3-mini-fast:prompt_tokens': 60,
'xai:grok-3-mini-fast:completion-tokens': 400,
// Grok 2 Vision
"xai:grok-2-vision:prompt_tokens": 200,
"xai:grok-2-vision:completion-tokens": 1000,
'xai:grok-2-vision:prompt_tokens': 200,
'xai:grok-2-vision:completion-tokens': 1000,
// Grok 2
"xai:grok-2:prompt_tokens": 200,
"xai:grok-2:completion-tokens": 1000,
'xai:grok-2:prompt_tokens': 200,
'xai:grok-2:completion-tokens': 1000,
};

2
src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts
View File

@@ -1,4 +1,4 @@
import { toMicroCents } from "../utils";
import { toMicroCents } from '../utils';
export const REGISTERED_USER_FREE = {
id: 'user_free',

2
src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts
View File

@@ -1,4 +1,4 @@
import { toMicroCents } from "../utils";
import { toMicroCents } from '../utils';
export const TEMP_USER_FREE = {
id: 'temp_free',

2
src/backend/src/services/MeteringService/utils.ts
View File

@@ -1 +1 @@
export const toMicroCents = (dollars: number) => Math.round(dollars * 1_000_000 * 100);
export const toMicroCents = (dollars: number) => dollars * 1_000_000 * 100;

117
src/backend/src/services/OperationTraceService.js
View File

@@ -28,14 +28,13 @@ const { AssignableMethodsFeature } = require("../traits/AssignableMethodsFeature
// and is utilized throughout the OperationTraceService to manage frames.
const CONTEXT_KEY = Context.make_context_key('operation-trace');
/**
* @class OperationFrame
* @description The `OperationFrame` class represents a frame within an operation trace. It is designed to manage the state, attributes, and hierarchy of frames within an operational context. This class provides methods to set status, calculate effective status, add tags, attributes, messages, errors, children, and describe the frame. It also includes methods to recursively search through frames to find attributes and handle frame completion.
*/
class OperationFrame {
static LOG_DEBUG = true;
constructor ({ parent, label, x }) {
constructor({ parent, label, x }) {
this.parent = parent;
this.label = label;
this.tags = [];
@@ -48,31 +47,28 @@ class OperationFrame {
this.id = require('uuid').v4();
this.log = (x ?? Context).get('services').get('log-service').create(
`frame:${this.id}`,
{ concern: 'filesystem' },
);
`frame:${this.id}`,
{ concern: 'filesystem' });
}
static FRAME_STATUS_PENDING = { label: 'pending' };
static FRAME_STATUS_WORKING = { label: 'working', };
static FRAME_STATUS_WORKING = { label: 'working' };
static FRAME_STATUS_STUCK = { label: 'stuck' };
static FRAME_STATUS_READY = { label: 'ready' };
static FRAME_STATUS_DONE = { label: 'done' };
set status (status) {
set status(status) {
this.status_ = status;
this._calc_effective_status();
this.log.debug(
`FRAME STATUS ${status.label} ` +
this.log.debug(`FRAME STATUS ${status.label} ` +
(status !== this.effective_status_
? `(effective: ${this.effective_status_.label}) `
: ''),
{
tags: this.tags,
...this.attributes,
}
);
{
tags: this.tags,
...this.attributes,
});
if ( this.parent ) {
this.parent._calc_effective_status();
@@ -84,7 +80,7 @@ class OperationFrame {
*
* @param {Object} status - The new status to set.
*/
_calc_effective_status () {
_calc_effective_status() {
for ( const child of this.children ) {
if ( child.status === OperationFrame.FRAME_STATUS_STUCK ) {
this.effective_status_ = OperationFrame.FRAME_STATUS_STUCK;
@@ -113,7 +109,6 @@ class OperationFrame {
}
}
/**
* Gets the effective status of the operation frame.
*
@@ -124,48 +119,47 @@ class OperationFrame {
*
* @return {Object} The effective status of the operation frame.
*/
get status () {
get status() {
return this.effective_status_;
}
tag (...tags) {
tag(...tags) {
this.tags.push(...tags);
return this;
}
attr (key, value) {
attr(key, value) {
this.attributes[key] = value;
return this;
}
// recursively go through frames to find the attribute
get_attr (key) {
get_attr(key) {
if ( this.attributes[key] ) return this.attributes[key];
if ( this.parent ) return this.parent.get_attr(key);
}
log (message) {
log(message) {
this.messages.push(message);
return this;
}
error (err) {
error(err) {
this.error_ = err;
return this;
}
push_child (frame) {
push_child(frame) {
this.children.push(frame);
return this;
}
/**
* Recursively traverses the frame hierarchy to find the root frame.
*
* @returns {OperationFrame} The root frame of the current frame hierarchy.
*/
get_root_frame () {
get_root_frame() {
let frame = this;
while ( frame.parent ) {
frame = frame.parent;
@@ -173,18 +167,17 @@ class OperationFrame {
return frame;
}
/**
* Marks the operation frame as done.
* This method sets the status of the operation frame to 'done' and updates
* the effective status accordingly. It triggers a recalculation of the
* effective status for parent frames if necessary.
*/
done () {
done() {
this.status = OperationFrame.FRAME_STATUS_DONE;
}
describe (show_tree, highlight_frame) {
describe(show_tree, highlight_frame) {
let s = this.label + ` (${this.children.length})`;
if ( this.tags.length ) {
s += ' ' + this.tags.join(' ');
@@ -201,7 +194,6 @@ class OperationFrame {
const prefix_deep = '│ ';
const prefix_deep_end = ' ';
/**
* Recursively builds a string representation of the frame and its children.
*
@@ -219,14 +211,13 @@ class OperationFrame {
if ( child === highlight_frame ) s += `\x1B[0m`;
recurse(child, prefix + (is_last ? prefix_deep_end : prefix_deep));
}
}
};
if ( show_tree ) recurse(this, '');
return s;
}
}
/**
* @class OperationTraceService
* @classdesc The OperationTraceService class manages operation frames and their statuses.
@@ -236,7 +227,7 @@ class OperationFrame {
class OperationTraceService {
static CONCERN = 'filesystem';
constructor ({ services }) {
constructor({ services }) {
this.log = services.get('log-service').create('operation-trace', {
concern: this.constructor.CONCERN,
});
@@ -245,7 +236,6 @@ class OperationTraceService {
this.ongoing = {};
}
/**
* Adds a new operation frame to the trace.
*
@@ -258,22 +248,20 @@ class OperationTraceService {
* @param {?Object} [x] - The context for the operation frame.
* @returns {OperationFrame} The new operation frame.
*/
async add_frame (label) {
async add_frame(label) {
return this.add_frame_sync(label);
}
add_frame_sync (label, x) {
add_frame_sync(label, x) {
if ( x ) {
this.log.debug(
'add_frame_sync() called with explicit context: ' +
x.describe()
);
this.log.debug('add_frame_sync() called with explicit context: ' +
x.describe());
}
let parent = (x ?? Context).get(this.ckey('frame'));
const frame = new OperationFrame({
parent: parent || null,
label,
x
x,
});
parent && parent.push_child(frame);
this.log.debug(`FRAME START ` + frame.describe());
@@ -286,12 +274,11 @@ class OperationTraceService {
return frame;
}
ckey (key) {
ckey(key) {
return CONTEXT_KEY + ':' + key;
}
}
/**
* @class BaseOperation
* @extends AdvancedBase
@@ -306,8 +293,7 @@ class BaseOperation extends AdvancedBase {
new ContextAwareFeature(),
new OtelFeature(['run']),
new AssignableMethodsFeature(),
]
];
/**
* Executes the operation with the provided values.
@@ -316,23 +302,23 @@ class BaseOperation extends AdvancedBase {
* executes the `_run` method, and handles post-run logic. It also manages the status of child frames
* and handles errors, updating the frame's attributes accordingly.
*
* @param {Object} values - The values to be used in the operation.
* @param {Object} firstArg - The values to be used in the operation. TODO DS: support multiple args with old state assignment?
* @param {...unknown} rest - rest of args passed in only to children
* @returns {Promise<*>} - The result of the operation.
* @throws {Error} - If the frame is missing or any other error occurs during the operation.
*/
async run (values) {
this.values = values;
async run(firstArg, ...rest) {
this.values = firstArg;
values.user = values.user ??
(values.actor ? values.actor.type.user : undefined);
firstArg.user = firstArg.user ??
(firstArg.actor ? firstArg.actor.type.user : undefined);
// getting context with a new operation frame
let x, frame; {
x = Context.get();
const operationTraceSvc = x.get('services').get('operationTrace');
frame = await operationTraceSvc.add_frame(this.constructor.name);
x = x.sub({ [operationTraceSvc.ckey('frame')]: frame });
}
let x, frame;
x = Context.get();
const operationTraceSvc = x.get('services').get('operationTrace');
frame = await operationTraceSvc.add_frame(this.constructor.name);
x = x.sub({ [operationTraceSvc.ckey('frame')]: frame });
// the frame will be an explicit property as well as being in context
// (for convenience)
@@ -340,12 +326,12 @@ class BaseOperation extends AdvancedBase {
// let's make the logger for it too
this.log = x.get('services').get('log-service').create(
this.constructor.name, {
operation: frame.id,
...(this.constructor.CONCERN ? {
concern: this.constructor.CONCERN,
} : {})
});
this.constructor.name, {
operation: frame.id,
...(this.constructor.CONCERN ? {
concern: this.constructor.CONCERN,
} : {}),
});
// Run operation in new context
try {
@@ -359,7 +345,7 @@ class BaseOperation extends AdvancedBase {
}
frame.status = OperationFrame.FRAME_STATUS_WORKING;
this.checkpoint('._run()');
const res = await this._run();
const res = await this._run(firstArg, ...rest); // TODO DS: simplify this, why are the passed in values being stored in class state?
this.checkpoint('._post_run()');
const { any_async } = this._post_run();
this.checkpoint('delegate .run_() returned');
@@ -378,15 +364,14 @@ class BaseOperation extends AdvancedBase {
}
}
checkpoint (name) {
checkpoint(name) {
this.frame.checkpoint = name;
}
field (key, value) {
field(key, value) {
this.frame.attributes[key] = value;
}
/**
* Actions to perform after running.
*
@@ -394,7 +379,7 @@ class BaseOperation extends AdvancedBase {
* all child frames at least reach working state before the parent operation
* completes.
*/
_post_run () {
_post_run() {
let any_async = false;
for ( const child of this.frame.children ) {
if ( child.status === OperationFrame.FRAME_STATUS_PENDING ) {

15
src/backend/src/services/repositories/DBKVStore/DBKVStore.ts
View File

@@ -1,11 +1,11 @@
import murmurhash from "murmurhash";
import murmurhash from 'murmurhash';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import APIError from '../../../api/APIError.js';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import { Context } from "../../../util/context.js";
import type { MeteringAndBillingService } from "../../MeteringService/MeteringService.js";
import { Context } from '../../../util/context.js';
import type { MeteringAndBillingService } from '../../MeteringService/MeteringService.js';
const GLOBAL_APP_KEY = 'global';
@@ -258,13 +258,16 @@ export class DBKVStore {
return await this.#expireat(key, timestamp);
}
async incr<T extends Record<string, number>>({ key, pathAndAmountMap }: { key: string, pathAndAmountMap: T }): Promise<T extends { "": number } ? number : Record<string, number>> {
async incr<T extends Record<string, number>>({ key, pathAndAmountMap }: { key: string, pathAndAmountMap: T }): Promise<T extends { '': number } ? number : Record<string, number>> {
if ( Object.values(pathAndAmountMap).find((v) => typeof v !== 'number') ) {
throw new Error('All values in pathAndAmountMap must be numbers');
}
let currVal = await this.get({ key });
const pathEntries = Object.entries(pathAndAmountMap);
if ( typeof currVal !== 'object' && pathEntries.length <= 1 && !pathEntries[0]?.[0] ) {
const amount = pathEntries[0]?.[1] ?? 1;
this.set({ key, value: (Number(currVal) || 0) + amount });
return ((Number(currVal) || 0) + amount) as T extends { "": number } ? number : Record<string, number>;
return ((Number(currVal) || 0) + amount) as T extends { '': number } ? number : Record<string, number>;
}
// TODO DS: support arrays this also needs dynamodb implementation
if ( Array.isArray(currVal) ) {
@@ -302,7 +305,7 @@ export class DBKVStore {
obj[lastPart] += amount;
}
this.set({ key, value: currVal });
return currVal as T extends { "": number } ? number : Record<string, number>;
return currVal as T extends { '': number } ? number : Record<string, number>;
}
async decr(...params: Parameters<typeof DBKVStore.prototype.incr>): ReturnType<typeof DBKVStore.prototype.incr> {