From e37166dae0b0fc0a5a155b143e6140139dc9163e Mon Sep 17 00:00:00 2001 From: Nariman Jelveh Date: Wed, 17 Sep 2025 15:36:42 -0700 Subject: [PATCH] =?UTF-8?q?Cache=20is=20king=20baby!=20Let's=20go=20?= =?UTF-8?q?=F0=9F=9A=80=20(#1574)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement the first naive version of `readdir` cache * Purge the entire cache on every single mutation Right now we're going to use the very naive, but safe, approach to purge the entire cache whenever there is change in the user's fs. We're going to incrementally improve this; but for now, better safe than sorry! * Add socket event listeners to flush cache on file system item changes This update introduces event listeners for 'item.added', 'item.renamed', and 'item.moved' events, triggering a cache flush on each event to ensure data consistency in the file system module. * increase exp time for the cache * Update readdir.js * Update index.js --- package-lock.json | 9 +++-- src/gui/src/UI/UIDesktop.js | 2 +- src/gui/src/UI/UIWindow.js | 5 ++- src/gui/src/helpers/refresh_item_container.js | 2 +- src/puter-js/package.json | 3 ++ src/puter-js/src/index.js | 4 ++ src/puter-js/src/modules/FileSystem/index.js | 17 +++++++++ .../src/modules/FileSystem/operations/copy.js | 3 ++ .../modules/FileSystem/operations/mkdir.js | 3 ++ .../src/modules/FileSystem/operations/move.js | 4 ++ .../modules/FileSystem/operations/readdir.js | 38 ++++++++++++++++++- .../modules/FileSystem/operations/rename.js | 2 + .../modules/FileSystem/operations/upload.js | 3 ++ .../modules/FileSystem/operations/write.js | 3 ++ src/puter-js/webpack.config.js | 2 - 15 files changed, 91 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0ec059a6..1c51b7aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2278,9 +2278,9 @@ "link": true }, "node_modules/@heyputer/kv.js": { - "version": "0.1.91", - "resolved": "https://registry.npmjs.org/@heyputer/kv.js/-/kv.js-0.1.91.tgz", - "integrity": "sha512-TzgPFVicgaxkz4mIavE8UdfICQ2Oql9BWkFlJAWEQ9cl+EXWmV1f7sQ0NRJxhzLahfVUdgoWsTXqx/ndr/9KBg==", + "version": "0.1.92", + "resolved": "https://registry.npmjs.org/@heyputer/kv.js/-/kv.js-0.1.92.tgz", + "integrity": "sha512-D+trimrG/V6mU5zeQrKyH476WotvvRn0McttxiFxEzWLiMqR6aBmQ5apeKrZAheglHmwf0D3FO5ykmU2lCuLvQ==", "license": "MIT", "dependencies": { "minimatch": "^9.0.0" @@ -20419,6 +20419,9 @@ "name": "@heyputer/puterjs", "version": "1.0.0", "license": "Apache-2.0", + "dependencies": { + "@heyputer/kv.js": "^0.1.92" + }, "devDependencies": { "concurrently": "^8.2.2", "webpack-cli": "^5.1.4" diff --git a/src/gui/src/UI/UIDesktop.js b/src/gui/src/UI/UIDesktop.js index c4e0f0c5..9d04a6d2 100644 --- a/src/gui/src/UI/UIDesktop.js +++ b/src/gui/src/UI/UIDesktop.js @@ -954,7 +954,7 @@ async function UIDesktop(options) { { html: i18n('refresh'), onClick: function () { - refresh_item_container(el_desktop); + refresh_item_container(el_desktop, { consistency: 'strong' }); } }, // ------------------------------------------- diff --git a/src/gui/src/UI/UIWindow.js b/src/gui/src/UI/UIWindow.js index d777ab59..52dcce45 100644 --- a/src/gui/src/UI/UIWindow.js +++ b/src/gui/src/UI/UIWindow.js @@ -2340,7 +2340,10 @@ async function UIWindow(options) { menu_items.push({ html: i18n('refresh'), onClick: function(){ - refresh_item_container(el_window_body, options); + refresh_item_container(el_window_body, { + ...options, + consistency: 'strong', + }); } }) // ------------------------------------------- diff --git a/src/gui/src/helpers/refresh_item_container.js b/src/gui/src/helpers/refresh_item_container.js index fc36f4fd..29b5acfd 100644 --- a/src/gui/src/helpers/refresh_item_container.js +++ b/src/gui/src/helpers/refresh_item_container.js @@ -108,7 +108,7 @@ const refresh_item_container = function(el_item_container, options){ $(el_item_container).find('.item').removeItems() // get items - puter.fs.readdir(container_path).then((fsentries)=>{ + puter.fs.readdir({path: container_path, consistency: options.consistency }).then((fsentries)=>{ // Check if the same folder is still loading since el_item_container's // data-path might have changed by other operations while waiting for the response to this `readdir`. if($(el_item_container).attr('data-path') !== container_path) diff --git a/src/puter-js/package.json b/src/puter-js/package.json index 7451ae76..82e9d724 100644 --- a/src/puter-js/package.json +++ b/src/puter-js/package.json @@ -16,5 +16,8 @@ "devDependencies": { "concurrently": "^8.2.2", "webpack-cli": "^5.1.4" + }, + "dependencies": { + "@heyputer/kv.js": "^0.1.92" } } diff --git a/src/puter-js/src/index.js b/src/puter-js/src/index.js index 76ecaaa5..76d242f1 100644 --- a/src/puter-js/src/index.js +++ b/src/puter-js/src/index.js @@ -28,6 +28,7 @@ import { FilesystemService } from './services/Filesystem.js'; import { FSRelayService } from './services/FSRelay.js'; import { NoPuterYetService } from './services/NoPuterYet.js'; import { XDIncomingService } from './services/XDIncoming.js'; +import kvjs from '@heyputer/kv.js'; // TODO: This is for a safe-guard below; we should check if we can // generalize this behavior rather than hard-coding it. @@ -115,6 +116,9 @@ export default globalThis.puter = (function() { constructor(options) { options = options ?? {}; + // Initialize the cache using kv.js + this._cache = new kvjs(); + // "modules" in puter.js are external interfaces for the developer this.modules_ = []; // "services" in puter.js are used by modules and may interact with each other diff --git a/src/puter-js/src/modules/FileSystem/index.js b/src/puter-js/src/modules/FileSystem/index.js index ae3ea06e..cbb57eb4 100644 --- a/src/puter-js/src/modules/FileSystem/index.js +++ b/src/puter-js/src/modules/FileSystem/index.js @@ -104,6 +104,19 @@ export class PuterJSFileSystemModule extends AdvancedBase { } bindSocketEvents() { + this.socket.on('item.added', (item) => { + // todo: NAIVE PURGE + puter._cache.flushall(); + }); + this.socket.on('item.renamed', (item) => { + // todo: NAIVE PURGE + puter._cache.flushall(); + }); + this.socket.on('item.moved', (item) => { + // todo: NAIVE PURGE + puter._cache.flushall(); + }); + this.socket.on('connect', () => { if(puter.debugMode) console.log('FileSystem Socket: Connected', this.socket.id); @@ -112,6 +125,10 @@ export class PuterJSFileSystemModule extends AdvancedBase { this.socket.on('disconnect', () => { if(puter.debugMode) console.log('FileSystem Socket: Disconnected'); + + // todo: NAIVE PURGE + // purge cache on disconnect since we may have become out of sync + puter._cache.flushall(); }); this.socket.on('reconnect', (attempt) => { diff --git a/src/puter-js/src/modules/FileSystem/operations/copy.js b/src/puter-js/src/modules/FileSystem/operations/copy.js index 4e33162a..77d7d172 100644 --- a/src/puter-js/src/modules/FileSystem/operations/copy.js +++ b/src/puter-js/src/modules/FileSystem/operations/copy.js @@ -55,6 +55,9 @@ const copy = function (...args) { // if user is copying an item to where its source is, change the name so there is no conflict dedupe_name: (options.dedupe_name || options.dedupeName), })); + + // todo: EXTREMELY NAIVE CACHE PURGE + puter._cache.flushall(); }) } diff --git a/src/puter-js/src/modules/FileSystem/operations/mkdir.js b/src/puter-js/src/modules/FileSystem/operations/mkdir.js index c256ab65..534e948f 100644 --- a/src/puter-js/src/modules/FileSystem/operations/mkdir.js +++ b/src/puter-js/src/modules/FileSystem/operations/mkdir.js @@ -53,6 +53,9 @@ const mkdir = function (...args) { original_client_socket_id: this.socket.id, create_missing_parents: (options.recursive || options.createMissingParents) ?? false, })); + + // todo: EXTREMELY NAIVE CACHE PURGE + puter._cache.flushall(); }) } diff --git a/src/puter-js/src/modules/FileSystem/operations/move.js b/src/puter-js/src/modules/FileSystem/operations/move.js index a5a1b89d..c9453d3c 100644 --- a/src/puter-js/src/modules/FileSystem/operations/move.js +++ b/src/puter-js/src/modules/FileSystem/operations/move.js @@ -65,6 +65,10 @@ const move = function (...args) { new_metadata: (options.new_metadata || options.newMetadata), original_client_socket_id: options.excludeSocketID, })); + + // todo: EXTREMELY NAIVE CACHE PURGE + puter._cache.flushall(); + }) } diff --git a/src/puter-js/src/modules/FileSystem/operations/readdir.js b/src/puter-js/src/modules/FileSystem/operations/readdir.js index 1bb9a16e..952b9eb4 100644 --- a/src/puter-js/src/modules/FileSystem/operations/readdir.js +++ b/src/puter-js/src/modules/FileSystem/operations/readdir.js @@ -20,11 +20,33 @@ const readdir = async function (...args) { } return new Promise(async (resolve, reject) => { + // consistency levels + if(!options.consistency){ + options.consistency = 'strong'; + } + // Either path or uid is required if(!options.path && !options.uid){ throw new Error({ code: 'NO_PATH_OR_UID', message: 'Either path or uid must be provided.' }); } + // Generate cache key based on path or uid + let cacheKey; + if(options.path){ + cacheKey = 'readdir:' + options.path; + }else if(options.uid){ + cacheKey = 'readdir:' + options.uid; + } + + if(options.consistency === 'eventual'){ + // Check cache + const cachedResult = await puter._cache.get(cacheKey); + if(cachedResult){ + resolve(cachedResult); + return; + } + } + // If auth token is not provided and we are in the web environment, // try to authenticate with Puter if(!puter.authToken && puter.env === 'web'){ @@ -40,7 +62,21 @@ const readdir = async function (...args) { const xhr = utils.initXhr('/readdir', this.APIOrigin, this.authToken); // set up event handlers for load and error events - utils.setupXhrEventHandlers(xhr, options.success, options.error, resolve, reject); + utils.setupXhrEventHandlers(xhr, options.success, options.error, async (result) => { + // Calculate the size of the result for cache eligibility check + const resultSize = JSON.stringify(result).length; + + // Cache the result if it's not bigger than MAX_CACHE_SIZE + const MAX_CACHE_SIZE = 20 * 1024 * 1024; + const EXPIRE_TIME = 30; + + if(resultSize <= MAX_CACHE_SIZE){ + // UPSERT the cache + await puter._cache.set(cacheKey, result, { EX: EXPIRE_TIME }); + } + + resolve(result); + }, reject); // Build request payload - support both path and uid parameters const payload = { diff --git a/src/puter-js/src/modules/FileSystem/operations/rename.js b/src/puter-js/src/modules/FileSystem/operations/rename.js index c670e482..c02fc86a 100644 --- a/src/puter-js/src/modules/FileSystem/operations/rename.js +++ b/src/puter-js/src/modules/FileSystem/operations/rename.js @@ -50,6 +50,8 @@ const rename = function (...args) { } xhr.send(JSON.stringify(dataToSend)); + // todo: EXTREMELY NAIVE CACHE PURGE + puter._cache.flushall(); }) } diff --git a/src/puter-js/src/modules/FileSystem/operations/upload.js b/src/puter-js/src/modules/FileSystem/operations/upload.js index 849d1517..b5d16cda 100644 --- a/src/puter-js/src/modules/FileSystem/operations/upload.js +++ b/src/puter-js/src/modules/FileSystem/operations/upload.js @@ -429,6 +429,9 @@ const upload = async function(items, dirPath, options = {}){ options.start(); } + // todo: EXTREMELY NAIVE CACHE PURGE + puter._cache.flushall(); + // send request xhr.send(fd); }) diff --git a/src/puter-js/src/modules/FileSystem/operations/write.js b/src/puter-js/src/modules/FileSystem/operations/write.js index 878e6105..6bb9efa1 100644 --- a/src/puter-js/src/modules/FileSystem/operations/write.js +++ b/src/puter-js/src/modules/FileSystem/operations/write.js @@ -55,6 +55,9 @@ const write = async function (targetPath, data, options = {}) { throw new Error({ code: 'field_invalid', message: 'write() data parameter is an invalid type' }); } + // todo: EXTREMELY NAIVE CACHE PURGE + puter._cache.flushall(); + // perform upload return this.upload(data, parent, options); } diff --git a/src/puter-js/webpack.config.js b/src/puter-js/webpack.config.js index 8cd71caf..1eb4bd18 100644 --- a/src/puter-js/webpack.config.js +++ b/src/puter-js/webpack.config.js @@ -10,8 +10,6 @@ import webpack from 'webpack'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -console.log('ENV CHECK!!!', process.env.PUTER_ORIGIN, process.env.PUTER_API_ORIGIN); - export default { entry: './src/index.js', output: {