From 31fffb087dcec05cd304312c4c0c8362745061e1 Mon Sep 17 00:00:00 2001 From: Nariman Jelveh Date: Thu, 18 Sep 2025 17:13:39 -0700 Subject: [PATCH] Refactor file system operations in GUI and puter.js to use eventual consistency (#1579) * Refactor file system operations in GUI and puter.js to use eventual consistency for stat and readdir calls This update modifies multiple instances of file system operations to include a consistency option set to 'eventual'. This change aims to improve performance and responsiveness by allowing for eventual consistency in file system interactions across various components, including helpers, UI elements, and IPC handling. * Update cache expiration time for file system operations in readdir and stat to 1 hour * feat: add network connectivity monitoring and cache purging This update introduces a new feature that monitors network connectivity and purges the cache when the connection is lost. The implementation includes event listeners for online/offline changes and visibility changes to ensure cache consistency during network disruptions. * clean up logs --- src/gui/src/IPC.js | 4 +- src/gui/src/UI/UIDesktop.js | 2 +- src/gui/src/UI/UIItem.js | 6 +- src/gui/src/UI/UITaskbar.js | 2 +- src/gui/src/UI/UIWindow.js | 4 +- src/gui/src/UI/UIWindowItemProperties.js | 1 + src/gui/src/UI/UIWindowRequestPermission.js | 2 +- src/gui/src/UI/UIWindowShare.js | 1 + src/gui/src/helpers.js | 8 +-- src/gui/src/helpers/item_icon.js | 2 +- src/gui/src/helpers/launch_app.js | 2 +- src/gui/src/helpers/refresh_item_container.js | 4 +- src/gui/src/initgui.js | 4 +- src/gui/src/keyboard.js | 2 +- src/gui/src/services/ExecService.js | 2 +- src/puter-js/src/index.js | 62 +++++++++++++++++++ src/puter-js/src/modules/FileSystem/index.js | 19 +++--- .../modules/FileSystem/operations/readdir.js | 8 ++- .../src/modules/FileSystem/operations/stat.js | 39 +++++++++++- 19 files changed, 142 insertions(+), 32 deletions(-) diff --git a/src/gui/src/IPC.js b/src/gui/src/IPC.js index db4731e8..d5562961 100644 --- a/src/gui/src/IPC.js +++ b/src/gui/src/IPC.js @@ -1418,7 +1418,7 @@ const ipc_listener = async (event, handled) => { source_path, target_path, el_filedialog_window, }) => { // source path must be in appdata directory - const stat_info = await puter.fs.stat(source_path); + const stat_info = await puter.fs.stat({path: source_path, consistency: 'eventual'}); if ( ! stat_info.appdata_app || stat_info.appdata_app !== app_uuid ) { const source_file_owner = stat_info?.appdata_app ?? 'the user'; if ( stat_info.appdata_app && stat_info.appdata_app !== app_uuid ) { @@ -1471,7 +1471,7 @@ const ipc_listener = async (event, handled) => { } else { await puter.fs.move(source_path, target_path); } - node = await puter.fs.stat(target_path); + node = await puter.fs.stat({path: target_path, consistency: 'eventual'}); }, parent_uuid: $(el_filedialog_window).attr('data-element_uuid'), }); diff --git a/src/gui/src/UI/UIDesktop.js b/src/gui/src/UI/UIDesktop.js index 9d04a6d2..af9ad631 100644 --- a/src/gui/src/UI/UIDesktop.js +++ b/src/gui/src/UI/UIDesktop.js @@ -1414,7 +1414,7 @@ async function UIDesktop(options) { } } - const stat = await puter.fs.stat(item_path); + const stat = await puter.fs.stat({path: item_path, consistency: 'eventual'}); // TODO: DRY everything here with open_item. Unfortunately we can't // use open_item here because it's coupled with UI logic; diff --git a/src/gui/src/UI/UIItem.js b/src/gui/src/UI/UIItem.js index 3fb92c41..52abfba3 100644 --- a/src/gui/src/UI/UIItem.js +++ b/src/gui/src/UI/UIItem.js @@ -925,7 +925,7 @@ function UIItem(options){ const element = $selected_items[index]; await window.delete_item(element); } - const trash = await puter.fs.stat(window.trash_path); + const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'}); // update other clients if(window.socket){ @@ -1375,7 +1375,7 @@ function UIItem(options){ if((alert_resp) === 'Delete'){ await window.delete_item(el_item); // check if trash is empty - const trash = await puter.fs.stat(window.trash_path); + const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'}); // update other clients if(window.socket){ window.socket.emit('trash.is_empty', {is_empty: trash.is_empty}); @@ -1531,6 +1531,7 @@ $(document).on('click', '.item-has-website-badge', async function(e){ returnSubdomains: true, returnPermissions: false, returnVersions: false, + consistency: 'eventual', success: function (fsentry){ if(fsentry.subdomains) window.open(fsentry.subdomains[0].address, '_blank'); @@ -1544,6 +1545,7 @@ $(document).on('long-hover', '.item-has-website-badge', function(e){ returnSubdomains: true, returnPermissions: false, returnVersions: false, + consistency: 'eventual', success: function (fsentry){ var box = e.target.getBoundingClientRect(); diff --git a/src/gui/src/UI/UITaskbar.js b/src/gui/src/UI/UITaskbar.js index 6ef8a4ac..c0707b4e 100644 --- a/src/gui/src/UI/UITaskbar.js +++ b/src/gui/src/UI/UITaskbar.js @@ -260,7 +260,7 @@ async function UITaskbar(options){ //--------------------------------------------- // add `Trash` to the taskbar //--------------------------------------------- - const trash = await puter.fs.stat(window.trash_path); + const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'}); if(window.socket){ window.socket.emit('trash.is_empty', {is_empty: trash.is_empty}); } diff --git a/src/gui/src/UI/UIWindow.js b/src/gui/src/UI/UIWindow.js index 52dcce45..e7be4d40 100644 --- a/src/gui/src/UI/UIWindow.js +++ b/src/gui/src/UI/UIWindow.js @@ -1115,7 +1115,7 @@ async function UIWindow(options) { // SIDEBAR sharing // -------------------------------------------------------- if(options.is_dir && !isMobile.phone){ - puter.fs.readdir('/').then(function(shared_users){ + puter.fs.readdir({path: '/', consistency: 'eventual'}).then(function(shared_users){ let ht = ''; if(shared_users && shared_users.length - 1 > 0){ ht += `

Shared with me

`; @@ -3161,7 +3161,7 @@ window.update_window_path = async function(el_window, target_path){ // /stat if(target_path !== '/'){ try{ - puter.fs.stat(target_path, function(fsentry){ + puter.fs.stat({path: target_path, consistency: 'eventual'}).then(fsentry => { $(el_window).removeClass('window-' + $(el_window).attr('data-uid')); $(el_window).addClass('window-' + fsentry.id); $(el_window).attr('data-uid', fsentry.id); diff --git a/src/gui/src/UI/UIWindowItemProperties.js b/src/gui/src/UI/UIWindowItemProperties.js index c7a907d3..af588ca5 100644 --- a/src/gui/src/UI/UIWindowItemProperties.js +++ b/src/gui/src/UI/UIWindowItemProperties.js @@ -109,6 +109,7 @@ async function UIWindowItemProperties(item_name, item_path, item_uid, left, top, returnPermissions: true, returnVersions: true, returnSize: true, + consistency: 'eventual', success: function (fsentry){ // hide versions tab if item is a directory if(fsentry.is_dir){ diff --git a/src/gui/src/UI/UIWindowRequestPermission.js b/src/gui/src/UI/UIWindowRequestPermission.js index f34a37ee..f700dc87 100644 --- a/src/gui/src/UI/UIWindowRequestPermission.js +++ b/src/gui/src/UI/UIWindowRequestPermission.js @@ -152,7 +152,7 @@ async function get_permission_description(permission) { let fsentry; if (resource_type === "fs") { - fsentry = await puter.fs.stat({ uid: resource_id }); + fsentry = await puter.fs.stat({ uid: resource_id, consistency: 'eventual' }); } const permission_mappings = { diff --git a/src/gui/src/UI/UIWindowShare.js b/src/gui/src/UI/UIWindowShare.js index cf973ca7..28a40d34 100644 --- a/src/gui/src/UI/UIWindowShare.js +++ b/src/gui/src/UI/UIWindowShare.js @@ -156,6 +156,7 @@ async function UIWindowShare(items, recipient){ path: items[i].path, returnSubdomains: true, returnPermissions: true, + consistency: 'eventual', }).then((fsentry) => { let recipients = fsentry.shares?.users; let perm_list = ''; diff --git a/src/gui/src/helpers.js b/src/gui/src/helpers.js index 8c486167..2a06347c 100644 --- a/src/gui/src/helpers.js +++ b/src/gui/src/helpers.js @@ -879,7 +879,7 @@ window.available_templates = () => { const loadTemplates = async () => { try{ // Directly check the templates directory - const hasTemplateFiles = await puter.fs.readdir(templatesPath) + const hasTemplateFiles = await puter.fs.readdir(templatesPath, {consistency: 'eventual'}) if(hasTemplateFiles.length == 0) { window.file_templates = [] @@ -1646,7 +1646,7 @@ window.move_items = async function(el_items, dest_path, is_undo = false){ // check if trash is empty if(untrashed_at_least_one_item){ - const trash = await puter.fs.stat(window.trash_path); + const trash = await puter.fs.stat({path: window.trash_path,consistency: 'eventual'}); if(window.socket){ window.socket.emit('trash.is_empty', {is_empty: trash.is_empty}); } @@ -2196,7 +2196,7 @@ async function readDirectoryRecursive(path, baseDir = '') { let allFiles = []; // Read the directory - const entries = await puter.fs.readdir(path); + const entries = await puter.fs.readdir(path, {consistency: 'eventual'}); if (entries.length === 0) { allFiles.push({ path }); @@ -2708,7 +2708,7 @@ window.get_profile_picture = async function(username){ let icon; // try getting profile pic try{ - let stat = await puter.fs.stat('/' + username + '/Public/.profile'); + let stat = await puter.fs.stat({path: '/' + username + '/Public/.profile', consistency: 'eventual'}); if(stat.size > 0 && stat.is_dir === false && stat.size < 1000000){ let profile_json = await puter.fs.read('/' + username + '/Public/.profile'); profile_json = await blob2str(profile_json); diff --git a/src/gui/src/helpers/item_icon.js b/src/gui/src/helpers/item_icon.js index ec239021..d64e8684 100644 --- a/src/gui/src/helpers/item_icon.js +++ b/src/gui/src/helpers/item_icon.js @@ -74,7 +74,7 @@ const item_icon = async (fsentry)=>{ let trash_img = $(`.item[data-path="${html_encode(window.trash_path)}" i] .item-icon-icon`).attr('src') // if trash_img is undefined that's probably because trash wasn't added anywhere, do a direct lookup to see if trash is empty or no if(!trash_img){ - let trashstat = await puter.fs.stat(window.trash_path); + let trashstat = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'}); if(trashstat.is_empty !== undefined && trashstat.is_empty === true) trash_img = window.icons['trash.svg']; else diff --git a/src/gui/src/helpers/launch_app.js b/src/gui/src/helpers/launch_app.js index 396ca017..46bb53d4 100644 --- a/src/gui/src/helpers/launch_app.js +++ b/src/gui/src/helpers/launch_app.js @@ -141,7 +141,7 @@ const launch_app = async (options)=>{ // if options.args.path is provided, use it as the path if(options.args?.path){ // if args.path is provided, enforce the directory - let fsentry = await puter.fs.stat(options.args.path); + let fsentry = await puter.fs.stat({path: options.args.path, consistency: 'eventual'}); if(!fsentry.is_dir){ let parent = path.dirname(options.args.path); if(parent === options.args.path) diff --git a/src/gui/src/helpers/refresh_item_container.js b/src/gui/src/helpers/refresh_item_container.js index adb101e3..302f2040 100644 --- a/src/gui/src/helpers/refresh_item_container.js +++ b/src/gui/src/helpers/refresh_item_container.js @@ -69,7 +69,7 @@ const refresh_item_container = function(el_item_container, options){ // -------------------------------------------------------- // Folder's configs and properties // -------------------------------------------------------- - puter.fs.stat(container_path, function(fsentry){ + puter.fs.stat({path: container_path, consistency: options.consistency ?? 'eventual'}).then(fsentry => { if(el_window){ $(el_window).attr('data-uid', fsentry.id); $(el_window).attr('data-sort_by', fsentry.sort_by ?? 'name'); @@ -206,7 +206,7 @@ const refresh_item_container = function(el_item_container, options){ // if this is desktop, add Trash if($(el_item_container).hasClass('desktop')){ try{ - const trash = await puter.fs.stat(window.trash_path); + const trash = await puter.fs.stat({path: window.trash_path, consistency: options.consistency ?? 'eventual'}); UIItem({ appendTo: el_item_container, uid: trash.id, diff --git a/src/gui/src/initgui.js b/src/gui/src/initgui.js index 470f38f6..4edf2e1d 100644 --- a/src/gui/src/initgui.js +++ b/src/gui/src/initgui.js @@ -600,7 +600,7 @@ window.initgui = async function(options){ // ------------------------------------------------------------------------------------- if(!window.embedded_in_popup){ await window.get_auto_arrange_data() - puter.fs.stat(window.desktop_path, async function(desktop_fsentry){ + puter.fs.stat({path: window.desktop_path, consistency: 'eventual'}).then(desktop_fsentry => { UIDesktop({desktop_fsentry: desktop_fsentry}); }) } @@ -1059,7 +1059,7 @@ window.initgui = async function(options){ // ------------------------------------------------------------------------------------- if(!window.embedded_in_popup){ await window.get_auto_arrange_data(); - puter.fs.stat(window.desktop_path, function (desktop_fsentry) { + puter.fs.stat({path: window.desktop_path, consistency: 'eventual'}).then(desktop_fsentry => { UIDesktop({ desktop_fsentry: desktop_fsentry }); }) } diff --git a/src/gui/src/keyboard.js b/src/gui/src/keyboard.js index 646a692c..902ffe08 100644 --- a/src/gui/src/keyboard.js +++ b/src/gui/src/keyboard.js @@ -532,7 +532,7 @@ $(document).bind('keydown', async function(e){ const element = $selected_items[index]; await window.delete_item(element); } - const trash = await puter.fs.stat(window.trash_path); + const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'}); if(window.socket){ window.socket.emit('trash.is_empty', {is_empty: trash.is_empty}); } diff --git a/src/gui/src/services/ExecService.js b/src/gui/src/services/ExecService.js index 8c0bf7e4..47d95d2a 100644 --- a/src/gui/src/services/ExecService.js +++ b/src/gui/src/services/ExecService.js @@ -133,7 +133,7 @@ export class ExecService extends Service { try { // Get file stats to verify it exists - const file_stat = await puter.fs.stat(first_file_path); + const file_stat = await puter.fs.stat({path: first_file_path, consistency: 'eventual'}); // Create file signature for the target app const file_signature_result = await puter.fs.sign(target_app_info.uuid, { diff --git a/src/puter-js/src/index.js b/src/puter-js/src/index.js index 76d242f1..64e1cce7 100644 --- a/src/puter-js/src/index.js +++ b/src/puter-js/src/index.js @@ -398,6 +398,9 @@ export default globalThis.puter = (function() { } this.workers = new WorkersHandler(this.authToken); + + // Initialize network connectivity monitoring and cache purging + this.initNetworkMonitoring(); } /** @@ -630,6 +633,65 @@ export default globalThis.puter = (function() { } return this; } + + /** + * Initializes network connectivity monitoring to purge cache when connection is lost + * @private + */ + initNetworkMonitoring = function() { + // Only initialize in environments that support navigator.onLine and window events + if (typeof globalThis.navigator === 'undefined' || + typeof globalThis.addEventListener !== 'function') { + return; + } + + // Track previous online state + let wasOnline = navigator.onLine; + + // Function to handle network state changes + const handleNetworkChange = () => { + const isOnline = navigator.onLine; + + // If we went from online to offline, purge the cache + if (wasOnline && !isOnline) { + console.log('Network connection lost - purging cache'); + this.purgeCache(); + } + + // Update the previous state + wasOnline = isOnline; + }; + + // Listen for online/offline events + globalThis.addEventListener('online', handleNetworkChange); + globalThis.addEventListener('offline', handleNetworkChange); + + // Also listen for visibility change as an additional indicator + // (some browsers don't fire offline events reliably) + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', () => { + // Small delay to allow network state to update + setTimeout(handleNetworkChange, 100); + }); + } + } + + /** + * Purges all cached data + * @public + */ + purgeCache = function() { + try { + if (this._cache && typeof this._cache.flushall === 'function') { + this._cache.flushall(); + console.log('Cache purged successfully'); + } else { + console.warn('Cache purge failed: cache instance not available'); + } + } catch (error) { + console.error('Error purging cache:', error); + } + } } diff --git a/src/puter-js/src/modules/FileSystem/index.js b/src/puter-js/src/modules/FileSystem/index.js index d9a6ee27..f5a34aae 100644 --- a/src/puter-js/src/modules/FileSystem/index.js +++ b/src/puter-js/src/modules/FileSystem/index.js @@ -12,6 +12,7 @@ import write from "./operations/write.js"; import sign from "./operations/sign.js"; import symlink from './operations/symlink.js'; import readdir from './operations/readdir.js'; +import stat from './operations/stat.js'; // Why is this called deleteFSEntry instead of just delete? because delete is // a reserved keyword in javascript import deleteFSEntry from "./operations/deleteFSEntry.js"; @@ -36,18 +37,19 @@ export class PuterJSFileSystemModule extends AdvancedBase { symlink = symlink; getReadURL = getReadURL; readdir = readdir; + stat = stat; FSItem = FSItem static NARI_METHODS = { - stat: { - positional: ['path'], - firstarg_options: true, - async fn (parameters) { - const svc_fs = await this.context.services.aget('filesystem'); - return svc_fs.filesystem.stat(parameters); - } - }, + // stat: { + // positional: ['path'], + // firstarg_options: true, + // async fn (parameters) { + // const svc_fs = await this.context.services.aget('filesystem'); + // return svc_fs.filesystem.stat(parameters); + // } + // }, } /** @@ -129,7 +131,6 @@ export class PuterJSFileSystemModule extends AdvancedBase { // todo: NAIVE PURGE // purge cache on disconnect since we may have become out of sync puter._cache.flushall(); - console.log('Purged cache on socket disconnect'); }); this.socket.on('reconnect', (attempt) => { diff --git a/src/puter-js/src/modules/FileSystem/operations/readdir.js b/src/puter-js/src/modules/FileSystem/operations/readdir.js index 8fc7d5cb..ffabb555 100644 --- a/src/puter-js/src/modules/FileSystem/operations/readdir.js +++ b/src/puter-js/src/modules/FileSystem/operations/readdir.js @@ -65,12 +65,18 @@ const readdir = async function (...args) { // Cache the result if it's not bigger than MAX_CACHE_SIZE const MAX_CACHE_SIZE = 20 * 1024 * 1024; - const EXPIRE_TIME = 30; + const EXPIRE_TIME = 60 * 60; // 1 hour if(resultSize <= MAX_CACHE_SIZE){ // UPSERT the cache await puter._cache.set(cacheKey, result, { EX: EXPIRE_TIME }); } + + // set each individual item's cache + for(const item of result){ + await puter._cache.set('item:' + item.id, item, { EX: EXPIRE_TIME }); + await puter._cache.set('item:' + item.path, item, { EX: EXPIRE_TIME }); + } resolve(result); }, reject); diff --git a/src/puter-js/src/modules/FileSystem/operations/stat.js b/src/puter-js/src/modules/FileSystem/operations/stat.js index 8922b8f7..42689d43 100644 --- a/src/puter-js/src/modules/FileSystem/operations/stat.js +++ b/src/puter-js/src/modules/FileSystem/operations/stat.js @@ -19,6 +19,11 @@ const stat = async function (...args) { } return new Promise(async (resolve, reject) => { + // consistency levels + if(!options.consistency){ + options.consistency = 'strong'; + } + // If auth token is not provided and we are in the web environment, // try to authenticate with Puter if(!puter.authToken && puter.env === 'web'){ @@ -30,11 +35,43 @@ const stat = async function (...args) { } } + // Generate cache key based on path or uid + let cacheKey; + if(options.path){ + cacheKey = 'item:' + options.path; + }else if(options.uid){ + cacheKey = 'item:' + options.uid; + } + + if(options.consistency === 'eventual' && !options.returnSubdomains && !options.returnPermissions && !options.returnVersions && !options.returnSize){ + // Check cache + const cachedResult = await puter._cache.get(cacheKey); + if(cachedResult){ + resolve(cachedResult); + return; + } + } + // create xhr object const xhr = utils.initXhr('/stat', this.APIOrigin, undefined, "post", "text/plain;actually=json"); // set up event handlers for load and error events - utils.setupXhrEventHandlers(xhr, options.success, options.error, resolve, reject); + utils.setupXhrEventHandlers(xhr, options.success, options.error, async (result) => { + // Calculate the size of the result for cache eligibility check + const resultSize = JSON.stringify(result).length; + + // Cache the result if it's not bigger than MAX_CACHE_SIZE + const MAX_CACHE_SIZE = 20 * 1024 * 1024; + const EXPIRE_TIME = 60 * 60; // 1 hour + + if(resultSize <= MAX_CACHE_SIZE){ + // UPSERT the cache + await puter._cache.set('item:' + result.path, result, { EX: EXPIRE_TIME }); + await puter._cache.set('item:' + result.uid, result, { EX: EXPIRE_TIME }); + } + + resolve(result); + }, reject); let dataToSend = {}; if (options.uid !== undefined) {