From 117eda190f68b95a56b5a5285960f1cb2cc2a7bc Mon Sep 17 00:00:00 2001 From: KernelDeimos <7225168+KernelDeimos@users.noreply.github.com> Date: Wed, 5 Nov 2025 20:02:51 -0500 Subject: [PATCH] dev(puterfs): move copy_tree to extension --- extensions/puterfs/main.js | 144 +++++++++++++++++ .../src/modules/puterfs/PuterFSModule.js | 4 + .../modules/puterfs/lib/PuterFSProvider.js | 148 +----------------- 3 files changed, 151 insertions(+), 145 deletions(-) diff --git a/extensions/puterfs/main.js b/extensions/puterfs/main.js index a938dc91..32a7f7a7 100644 --- a/extensions/puterfs/main.js +++ b/extensions/puterfs/main.js @@ -68,6 +68,10 @@ const { NodeInternalIDSelector, } = extension.import('core').fs.selectors; +const { + FSNodeContext, +} = extension.import('fs'); + const { // MODE_READ, MODE_WRITE, @@ -81,6 +85,10 @@ const { RESOURCE_STATUS_PENDING_CREATE, } = extension.import('fs').resource; +const { + UploadProgressTracker, +} = extension.import('fs').util; + class PuterFSProvider { /** * Check if a given node exists. @@ -305,6 +313,142 @@ class PuterFSProvider { return entry; } + async copy_tree ({ context, source, parent, target_name }) { + // Context + const actor = (context ?? Context).get('actor'); + const user = actor.type.user; + + const tracer = svc_trace.tracer; + const uuid = uuidv4(); + const timestamp = Math.round(Date.now() / 1000); + await parent.fetchEntry(); + await source.fetchEntry({ thumbnail: true }); + + // New filesystem entry + const raw_fsentry = { + uuid, + is_dir: source.entry.is_dir, + ...(source.entry.is_shortcut ? { + is_shortcut: source.entry.is_shortcut, + shortcut_to: source.entry.shortcut_to, + } : {}), + parent_uid: parent.uid, + name: target_name, + created: timestamp, + modified: timestamp, + + path: path_.join(await parent.get('path'), target_name), + + // if property exists but the value is undefined, + // it will still be included in the INSERT, causing + // an error + ...(source.entry.thumbnail ? + { thumbnail: source.entry.thumbnail } : {}), + + user_id: user.id, + }; + + svc_event.emit('fs.pending.file', { + fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry), + context: context, + }); + + if ( await source.get('has-s3') ) { + Object.assign(raw_fsentry, { + size: source.entry.size, + associated_app_id: source.entry.associated_app_id, + bucket: source.entry.bucket, + bucket_region: source.entry.bucket_region, + }); + + await tracer.startActiveSpan('fs:cp:storage-copy', async span => { + let progress_tracker = new UploadProgressTracker(); + + svc_event.emit('fs.storage.progress.copy', { + upload_tracker: progress_tracker, + context, + meta: { + item_uid: uuid, + item_path: raw_fsentry.path, + }, + }); + + // const storage = new PuterS3StorageStrategy({ services: svc }); + const storage = context.get('storage'); + const state_copy = storage.create_copy(); + await state_copy.run({ + src_node: source, + dst_storage: { + key: uuid, + bucket: raw_fsentry.bucket, + bucket_region: raw_fsentry.bucket_region, + }, + storage_api: { progress_tracker }, + }); + + span.end(); + }); + } + + { + await svc_size.add_node_size(undefined, source, user); + } + + svc_resource.register({ + uid: uuid, + status: RESOURCE_STATUS_PENDING_CREATE, + }); + + const entryOp = await svc_fsEntry.insert(raw_fsentry); + + let node; + + const tasks = new ParallelTasks({ tracer, max: 4 }); + await context.arun('fs:cp:parallel-portion', async () => { + // Add child copy tasks if this is a directory + if ( source.entry.is_dir ) { + const children = await svc_fsEntry.fast_get_direct_descendants(source.uid); + for ( const child_uuid of children ) { + tasks.add('fs:cp:copy-child', async () => { + const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid)); + const child_name = await child_node.get('name'); + + await this.copy_tree({ + context, + source: await svc_fs.node(new NodeUIDSelector(child_uuid)), + parent: await svc_fs.node(new NodeUIDSelector(uuid)), + target_name: child_name, + }); + }); + } + } + + // Add task to await entry + tasks.add('fs:cp:entry-op', async () => { + await entryOp.awaitDone(); + svc_resource.free(uuid); + const copy_fsNode = await svc_fs.node(new NodeUIDSelector(uuid)); + copy_fsNode.entry = raw_fsentry; + copy_fsNode.found = true; + copy_fsNode.path = raw_fsentry.path; + + node = copy_fsNode; + + svc_event.emit('fs.create.file', { + node, + context, + }); + }, { force: true }); + + await tasks.awaitAll(); + }); + + node = node || await svc_fs.node(new NodeUIDSelector(uuid)); + + // TODO: What event do we emit? How do we know if we're overwriting? + return node; + } + async #rmnode ({ node, options }) { // Services if ( !options.override_immutable && await node.get('immutable') ) { diff --git a/src/backend/src/modules/puterfs/PuterFSModule.js b/src/backend/src/modules/puterfs/PuterFSModule.js index bcefc3d3..fb9b2faf 100644 --- a/src/backend/src/modules/puterfs/PuterFSModule.js +++ b/src/backend/src/modules/puterfs/PuterFSModule.js @@ -24,6 +24,7 @@ const selectors = require('../../filesystem/node/selectors'); const { RuntimeModule } = require('../../extension/RuntimeModule'); const { TmpProxyFSProvider } = require('./TmpProxyFSProvider'); const { MODE_READ, MODE_WRITE } = require('../../services/fs/FSLockService'); +const { UploadProgressTracker } = require('../../filesystem/storage/UploadProgressTracker'); class PuterFSModule extends AdvancedBase { async install (context) { @@ -46,6 +47,9 @@ class PuterFSModule extends AdvancedBase { resource: { RESOURCE_STATUS_PENDING_CREATE, }, + util: { + UploadProgressTracker, + }, }; context.get('runtime-modules').register(runtimeModule); } diff --git a/src/backend/src/modules/puterfs/lib/PuterFSProvider.js b/src/backend/src/modules/puterfs/lib/PuterFSProvider.js index 0f80fd1d..f43b42e9 100644 --- a/src/backend/src/modules/puterfs/lib/PuterFSProvider.js +++ b/src/backend/src/modules/puterfs/lib/PuterFSProvider.js @@ -157,151 +157,9 @@ class PuterFSProvider extends putility.AdvancedBase { return node; } - async copy_tree ({ context, source, parent, target_name }) { - return await this.#copy_tree({ context, source, parent, target_name }); - } - async #copy_tree ({ context, source, parent, target_name }) { - // Services - const svc_event = this.#services.get('event'); - const svc_trace = this.#services.get('traceService'); - const svc_size = this.#services.get('sizeService'); - const svc_resource = this.#services.get('resourceService'); - const svc_fsEntry = this.#services.get('fsEntryService'); - const svc_fs = this.#services.get('filesystem'); - - // Context - const actor = Context.get('actor'); - const user = actor.type.user; - - const tracer = svc_trace.tracer; - const uuid = uuidv4(); - const timestamp = Math.round(Date.now() / 1000); - await parent.fetchEntry(); - await source.fetchEntry({ thumbnail: true }); - - // New filesystem entry - const raw_fsentry = { - uuid, - is_dir: source.entry.is_dir, - ...(source.entry.is_shortcut ? { - is_shortcut: source.entry.is_shortcut, - shortcut_to: source.entry.shortcut_to, - } : {}), - parent_uid: parent.uid, - name: target_name, - created: timestamp, - modified: timestamp, - - path: path.join(await parent.get('path'), target_name), - - // if property exists but the value is undefined, - // it will still be included in the INSERT, causing - // an error - ...(source.entry.thumbnail ? - { thumbnail: source.entry.thumbnail } : {}), - - user_id: user.id, - }; - - svc_event.emit('fs.pending.file', { - fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry), - context: context, - }); - - if ( await source.get('has-s3') ) { - Object.assign(raw_fsentry, { - size: source.entry.size, - associated_app_id: source.entry.associated_app_id, - bucket: source.entry.bucket, - bucket_region: source.entry.bucket_region, - }); - - await tracer.startActiveSpan('fs:cp:storage-copy', async span => { - let progress_tracker = new UploadProgressTracker(); - - svc_event.emit('fs.storage.progress.copy', { - upload_tracker: progress_tracker, - context, - meta: { - item_uid: uuid, - item_path: raw_fsentry.path, - }, - }); - - // const storage = new PuterS3StorageStrategy({ services: svc }); - const storage = context.get('storage'); - const state_copy = storage.create_copy(); - await state_copy.run({ - src_node: source, - dst_storage: { - key: uuid, - bucket: raw_fsentry.bucket, - bucket_region: raw_fsentry.bucket_region, - }, - storage_api: { progress_tracker }, - }); - - span.end(); - }); - } - - { - await svc_size.add_node_size(undefined, source, user); - } - - svc_resource.register({ - uid: uuid, - status: RESOURCE_STATUS_PENDING_CREATE, - }); - - const entryOp = await svc_fsEntry.insert(raw_fsentry); - - let node; - - const tasks = new ParallelTasks({ tracer, max: 4 }); - await context.arun('fs:cp:parallel-portion', async () => { - // Add child copy tasks if this is a directory - if ( source.entry.is_dir ) { - const children = await svc_fsEntry.fast_get_direct_descendants(source.uid); - for ( const child_uuid of children ) { - tasks.add('fs:cp:copy-child', async () => { - const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid)); - const child_name = await child_node.get('name'); - // TODO: this should be LLCopy instead - await this.#copy_tree({ - context, - source: await svc_fs.node(new NodeUIDSelector(child_uuid)), - parent: await svc_fs.node(new NodeUIDSelector(uuid)), - target_name: child_name, - }); - }); - } - } - - // Add task to await entry - tasks.add('fs:cp:entry-op', async () => { - await entryOp.awaitDone(); - svc_resource.free(uuid); - const copy_fsNode = await svc_fs.node(new NodeUIDSelector(uuid)); - copy_fsNode.entry = raw_fsentry; - copy_fsNode.found = true; - copy_fsNode.path = raw_fsentry.path; - - node = copy_fsNode; - - svc_event.emit('fs.create.file', { - node, - context, - }); - }, { force: true }); - - await tasks.awaitAll(); - }); - - node = node || await svc_fs.node(new NodeUIDSelector(uuid)); - - // TODO: What event do we emit? How do we know if we're overwriting? - return node; + async copy_tree ({ context, node, options = {} }) { + console.error('This .copy_tree should not be called!'); + throw new Error('This .copy_tree should not be called!'); } async unlink ({ context, node, options = {} }) {