mirror/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2025-12-30 17:50:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

Refactor file system operations in GUI and puter.js to use eventual consistency (#1579)

Browse Source 
* Refactor file system operations in GUI and puter.js to use eventual consistency for stat and readdir calls

This update modifies multiple instances of file system operations to include a consistency option set to 'eventual'. This change aims to improve performance and responsiveness by allowing for eventual consistency in file system interactions across various components, including helpers, UI elements, and IPC handling.

* Update cache expiration time for file system operations in readdir and stat to 1 hour

* feat: add network connectivity monitoring and cache purging

This update introduces a new feature that monitors network connectivity and purges the cache when the connection is lost. The implementation includes event listeners for online/offline changes and visibility changes to ensure cache consistency during network disruptions.

* clean up logs
This commit is contained in:
Nariman Jelveh
2025-09-18 17:13:39 -07:00
committed by GitHub
parent 89d5030ae3
commit 31fffb087d
19 changed files with 142 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/gui/src/IPC.js
View File

@@ -1418,7 +1418,7 @@ const ipc_listener = async (event, handled) => {
source_path, target_path, el_filedialog_window,
}) => {
// source path must be in appdata directory
const stat_info = await puter.fs.stat(source_path);
const stat_info = await puter.fs.stat({path: source_path, consistency: 'eventual'});
if ( ! stat_info.appdata_app || stat_info.appdata_app !== app_uuid ) {
const source_file_owner = stat_info?.appdata_app ?? 'the user';
if ( stat_info.appdata_app && stat_info.appdata_app !== app_uuid ) {
@@ -1471,7 +1471,7 @@ const ipc_listener = async (event, handled) => {
} else {
await puter.fs.move(source_path, target_path);
}
node = await puter.fs.stat(target_path);
node = await puter.fs.stat({path: target_path, consistency: 'eventual'});
},
parent_uuid: $(el_filedialog_window).attr('data-element_uuid'),
});

2
src/gui/src/UI/UIDesktop.js
View File

@@ -1414,7 +1414,7 @@ async function UIDesktop(options) {
}
}
const stat = await puter.fs.stat(item_path);
const stat = await puter.fs.stat({path: item_path, consistency: 'eventual'});
// TODO: DRY everything here with open_item. Unfortunately we can't
// use open_item here because it's coupled with UI logic;

6
src/gui/src/UI/UIItem.js
View File

@@ -925,7 +925,7 @@ function UIItem(options){
const element = $selected_items[index];
await window.delete_item(element);
}
const trash = await puter.fs.stat(window.trash_path);
const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'});
// update other clients
if(window.socket){
@@ -1375,7 +1375,7 @@ function UIItem(options){
if((alert_resp) === 'Delete'){
await window.delete_item(el_item);
// check if trash is empty
const trash = await puter.fs.stat(window.trash_path);
const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'});
// update other clients
if(window.socket){
window.socket.emit('trash.is_empty', {is_empty: trash.is_empty});
@@ -1531,6 +1531,7 @@ $(document).on('click', '.item-has-website-badge', async function(e){
returnSubdomains: true,
returnPermissions: false,
returnVersions: false,
consistency: 'eventual',
success: function (fsentry){
if(fsentry.subdomains)
window.open(fsentry.subdomains[0].address, '_blank');
@@ -1544,6 +1545,7 @@ $(document).on('long-hover', '.item-has-website-badge', function(e){
returnSubdomains: true,
returnPermissions: false,
returnVersions: false,
consistency: 'eventual',
success: function (fsentry){
var box = e.target.getBoundingClientRect();

2
src/gui/src/UI/UITaskbar.js
View File

@@ -260,7 +260,7 @@ async function UITaskbar(options){
//---------------------------------------------
// add `Trash` to the taskbar
//---------------------------------------------
const trash = await puter.fs.stat(window.trash_path);
const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'});
if(window.socket){
window.socket.emit('trash.is_empty', {is_empty: trash.is_empty});
}

4
src/gui/src/UI/UIWindow.js
View File

@@ -1115,7 +1115,7 @@ async function UIWindow(options) {
// SIDEBAR sharing
// --------------------------------------------------------
if(options.is_dir && !isMobile.phone){
puter.fs.readdir('/').then(function(shared_users){
puter.fs.readdir({path: '/', consistency: 'eventual'}).then(function(shared_users){
let ht = '';
if(shared_users && shared_users.length - 1 > 0){
ht += `<h2 class="window-sidebar-title disable-user-select">Shared with me</h2>`;
@@ -3161,7 +3161,7 @@ window.update_window_path = async function(el_window, target_path){
// /stat
if(target_path !== '/'){
try{
puter.fs.stat(target_path, function(fsentry){
puter.fs.stat({path: target_path, consistency: 'eventual'}).then(fsentry => {
$(el_window).removeClass('window-' + $(el_window).attr('data-uid'));
$(el_window).addClass('window-' + fsentry.id);
$(el_window).attr('data-uid', fsentry.id);

1
src/gui/src/UI/UIWindowItemProperties.js
View File

@@ -109,6 +109,7 @@ async function UIWindowItemProperties(item_name, item_path, item_uid, left, top,
returnPermissions: true,
returnVersions: true,
returnSize: true,
consistency: 'eventual',
success: function (fsentry){
// hide versions tab if item is a directory
if(fsentry.is_dir){

2
src/gui/src/UI/UIWindowRequestPermission.js
View File

@@ -152,7 +152,7 @@ async function get_permission_description(permission) {
let fsentry;
if (resource_type === "fs") {
fsentry = await puter.fs.stat({ uid: resource_id });
fsentry = await puter.fs.stat({ uid: resource_id, consistency: 'eventual' });
}
const permission_mappings = {

1
src/gui/src/UI/UIWindowShare.js
View File

@@ -156,6 +156,7 @@ async function UIWindowShare(items, recipient){
path: items[i].path,
returnSubdomains: true,
returnPermissions: true,
consistency: 'eventual',
}).then((fsentry) => {
let recipients = fsentry.shares?.users;
let perm_list = '';

8
src/gui/src/helpers.js
View File

@@ -879,7 +879,7 @@ window.available_templates = () => {
const loadTemplates = async () => {
try{
// Directly check the templates directory
const hasTemplateFiles = await puter.fs.readdir(templatesPath)
const hasTemplateFiles = await puter.fs.readdir(templatesPath, {consistency: 'eventual'})
if(hasTemplateFiles.length == 0) {
window.file_templates = []
@@ -1646,7 +1646,7 @@ window.move_items = async function(el_items, dest_path, is_undo = false){
// check if trash is empty
if(untrashed_at_least_one_item){
const trash = await puter.fs.stat(window.trash_path);
const trash = await puter.fs.stat({path: window.trash_path,consistency: 'eventual'});
if(window.socket){
window.socket.emit('trash.is_empty', {is_empty: trash.is_empty});
}
@@ -2196,7 +2196,7 @@ async function readDirectoryRecursive(path, baseDir = '') {
let allFiles = [];
// Read the directory
const entries = await puter.fs.readdir(path);
const entries = await puter.fs.readdir(path, {consistency: 'eventual'});
if (entries.length === 0) {
allFiles.push({ path });
@@ -2708,7 +2708,7 @@ window.get_profile_picture = async function(username){
let icon;
// try getting profile pic
try{
let stat = await puter.fs.stat('/' + username + '/Public/.profile');
let stat = await puter.fs.stat({path: '/' + username + '/Public/.profile', consistency: 'eventual'});
if(stat.size > 0 && stat.is_dir === false && stat.size < 1000000){
let profile_json = await puter.fs.read('/' + username + '/Public/.profile');
profile_json = await blob2str(profile_json);

2
src/gui/src/helpers/item_icon.js
View File

@@ -74,7 +74,7 @@ const item_icon = async (fsentry)=>{
let trash_img = $(`.item[data-path="${html_encode(window.trash_path)}" i] .item-icon-icon`).attr('src')
// if trash_img is undefined that's probably because trash wasn't added anywhere, do a direct lookup to see if trash is empty or no
if(!trash_img){
let trashstat = await puter.fs.stat(window.trash_path);
let trashstat = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'});
if(trashstat.is_empty !== undefined && trashstat.is_empty === true)
trash_img = window.icons['trash.svg'];
else

2
src/gui/src/helpers/launch_app.js
View File

@@ -141,7 +141,7 @@ const launch_app = async (options)=>{
// if options.args.path is provided, use it as the path
if(options.args?.path){
// if args.path is provided, enforce the directory
let fsentry = await puter.fs.stat(options.args.path);
let fsentry = await puter.fs.stat({path: options.args.path, consistency: 'eventual'});
if(!fsentry.is_dir){
let parent = path.dirname(options.args.path);
if(parent === options.args.path)

4
src/gui/src/helpers/refresh_item_container.js
View File

@@ -69,7 +69,7 @@ const refresh_item_container = function(el_item_container, options){
// --------------------------------------------------------
// Folder's configs and properties
// --------------------------------------------------------
puter.fs.stat(container_path, function(fsentry){
puter.fs.stat({path: container_path, consistency: options.consistency ?? 'eventual'}).then(fsentry => {
if(el_window){
$(el_window).attr('data-uid', fsentry.id);
$(el_window).attr('data-sort_by', fsentry.sort_by ?? 'name');
@@ -206,7 +206,7 @@ const refresh_item_container = function(el_item_container, options){
// if this is desktop, add Trash
if($(el_item_container).hasClass('desktop')){
try{
const trash = await puter.fs.stat(window.trash_path);
const trash = await puter.fs.stat({path: window.trash_path, consistency: options.consistency ?? 'eventual'});
UIItem({
appendTo: el_item_container,
uid: trash.id,

4
src/gui/src/initgui.js
View File

@@ -600,7 +600,7 @@ window.initgui = async function(options){
// -------------------------------------------------------------------------------------
if(!window.embedded_in_popup){
await window.get_auto_arrange_data()
puter.fs.stat(window.desktop_path, async function(desktop_fsentry){
puter.fs.stat({path: window.desktop_path, consistency: 'eventual'}).then(desktop_fsentry => {
UIDesktop({desktop_fsentry: desktop_fsentry});
})
}
@@ -1059,7 +1059,7 @@ window.initgui = async function(options){
// -------------------------------------------------------------------------------------
if(!window.embedded_in_popup){
await window.get_auto_arrange_data();
puter.fs.stat(window.desktop_path, function (desktop_fsentry) {
puter.fs.stat({path: window.desktop_path, consistency: 'eventual'}).then(desktop_fsentry => {
UIDesktop({ desktop_fsentry: desktop_fsentry });
})
}

2
src/gui/src/keyboard.js
View File

@@ -532,7 +532,7 @@ $(document).bind('keydown', async function(e){
const element = $selected_items[index];
await window.delete_item(element);
}
const trash = await puter.fs.stat(window.trash_path);
const trash = await puter.fs.stat({path: window.trash_path, consistency: 'eventual'});
if(window.socket){
window.socket.emit('trash.is_empty', {is_empty: trash.is_empty});
}

2
src/gui/src/services/ExecService.js
View File

@@ -133,7 +133,7 @@ export class ExecService extends Service {
try {
// Get file stats to verify it exists
const file_stat = await puter.fs.stat(first_file_path);
const file_stat = await puter.fs.stat({path: first_file_path, consistency: 'eventual'});
// Create file signature for the target app
const file_signature_result = await puter.fs.sign(target_app_info.uuid, {

62
src/puter-js/src/index.js
View File

@@ -398,6 +398,9 @@ export default globalThis.puter = (function() {
}
this.workers = new WorkersHandler(this.authToken);
// Initialize network connectivity monitoring and cache purging
this.initNetworkMonitoring();
}
/**
@@ -630,6 +633,65 @@ export default globalThis.puter = (function() {
}
return this;
}
/**
* Initializes network connectivity monitoring to purge cache when connection is lost
* @private
*/
initNetworkMonitoring = function() {
// Only initialize in environments that support navigator.onLine and window events
if (typeof globalThis.navigator === 'undefined' ||
typeof globalThis.addEventListener !== 'function') {
return;
}
// Track previous online state
let wasOnline = navigator.onLine;
// Function to handle network state changes
const handleNetworkChange = () => {
const isOnline = navigator.onLine;
// If we went from online to offline, purge the cache
if (wasOnline && !isOnline) {
console.log('Network connection lost - purging cache');
this.purgeCache();
}
// Update the previous state
wasOnline = isOnline;
};
// Listen for online/offline events
globalThis.addEventListener('online', handleNetworkChange);
globalThis.addEventListener('offline', handleNetworkChange);
// Also listen for visibility change as an additional indicator
// (some browsers don't fire offline events reliably)
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', () => {
// Small delay to allow network state to update
setTimeout(handleNetworkChange, 100);
});
}
}
/**
* Purges all cached data
* @public
*/
purgeCache = function() {
try {
if (this._cache && typeof this._cache.flushall === 'function') {
this._cache.flushall();
console.log('Cache purged successfully');
} else {
console.warn('Cache purge failed: cache instance not available');
}
} catch (error) {
console.error('Error purging cache:', error);
}
}
}

19
src/puter-js/src/modules/FileSystem/index.js
View File

@@ -12,6 +12,7 @@ import write from "./operations/write.js";
import sign from "./operations/sign.js";
import symlink from './operations/symlink.js';
import readdir from './operations/readdir.js';
import stat from './operations/stat.js';
// Why is this called deleteFSEntry instead of just delete? because delete is
// a reserved keyword in javascript
import deleteFSEntry from "./operations/deleteFSEntry.js";
@@ -36,18 +37,19 @@ export class PuterJSFileSystemModule extends AdvancedBase {
symlink = symlink;
getReadURL = getReadURL;
readdir = readdir;
stat = stat;
FSItem = FSItem
static NARI_METHODS = {
stat: {
positional: ['path'],
firstarg_options: true,
async fn (parameters) {
const svc_fs = await this.context.services.aget('filesystem');
return svc_fs.filesystem.stat(parameters);
}
},
// stat: {
// positional: ['path'],
// firstarg_options: true,
// async fn (parameters) {
// const svc_fs = await this.context.services.aget('filesystem');
// return svc_fs.filesystem.stat(parameters);
// }
// },
}
/**
@@ -129,7 +131,6 @@ export class PuterJSFileSystemModule extends AdvancedBase {
// todo: NAIVE PURGE
// purge cache on disconnect since we may have become out of sync
puter._cache.flushall();
console.log('Purged cache on socket disconnect');
});
this.socket.on('reconnect', (attempt) => {

8
src/puter-js/src/modules/FileSystem/operations/readdir.js
View File

@@ -65,12 +65,18 @@ const readdir = async function (...args) {
// Cache the result if it's not bigger than MAX_CACHE_SIZE
const MAX_CACHE_SIZE = 20 * 1024 * 1024;
const EXPIRE_TIME = 30;
const EXPIRE_TIME = 60 * 60; // 1 hour
if(resultSize <= MAX_CACHE_SIZE){
// UPSERT the cache
await puter._cache.set(cacheKey, result, { EX: EXPIRE_TIME });
}
// set each individual item's cache
for(const item of result){
await puter._cache.set('item:' + item.id, item, { EX: EXPIRE_TIME });
await puter._cache.set('item:' + item.path, item, { EX: EXPIRE_TIME });
}
resolve(result);
}, reject);

39
src/puter-js/src/modules/FileSystem/operations/stat.js
View File

@@ -19,6 +19,11 @@ const stat = async function (...args) {
}
return new Promise(async (resolve, reject) => {
// consistency levels
if(!options.consistency){
options.consistency = 'strong';
}
// If auth token is not provided and we are in the web environment,
// try to authenticate with Puter
if(!puter.authToken && puter.env === 'web'){
@@ -30,11 +35,43 @@ const stat = async function (...args) {
}
}
// Generate cache key based on path or uid
let cacheKey;
if(options.path){
cacheKey = 'item:' + options.path;
}else if(options.uid){
cacheKey = 'item:' + options.uid;
}
if(options.consistency === 'eventual' && !options.returnSubdomains && !options.returnPermissions && !options.returnVersions && !options.returnSize){
// Check cache
const cachedResult = await puter._cache.get(cacheKey);
if(cachedResult){
resolve(cachedResult);
return;
}
}
// create xhr object
const xhr = utils.initXhr('/stat', this.APIOrigin, undefined, "post", "text/plain;actually=json");
// set up event handlers for load and error events
utils.setupXhrEventHandlers(xhr, options.success, options.error, resolve, reject);
utils.setupXhrEventHandlers(xhr, options.success, options.error, async (result) => {
// Calculate the size of the result for cache eligibility check
const resultSize = JSON.stringify(result).length;
// Cache the result if it's not bigger than MAX_CACHE_SIZE
const MAX_CACHE_SIZE = 20 * 1024 * 1024;
const EXPIRE_TIME = 60 * 60; // 1 hour
if(resultSize <= MAX_CACHE_SIZE){
// UPSERT the cache
await puter._cache.set('item:' + result.path, result, { EX: EXPIRE_TIME });
await puter._cache.set('item:' + result.uid, result, { EX: EXPIRE_TIME });
}
resolve(result);
}, reject);
let dataToSend = {};
if (options.uid !== undefined) {