feat(server/client): I have no words, hopefully this lesser abstraction and webdav tracking works now

Cargo.lock

@@ -3591,9 +3591,10 @@ dependencies = [
 
 [[package]]
 name = "readur"
-version = "2.4.2"
+version = "2.5.3"
 dependencies = [
  "anyhow",
+ "async-trait",
  "aws-config",
  "aws-credential-types",
  "aws-sdk-s3",
@@ -3614,6 +3615,7 @@ dependencies = [
  "notify",
  "oauth2",
  "quick-xml",
+ "rand 0.8.5",
  "raw-cpuid",
  "readur",
  "regex",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "readur"
-version = "2.4.2"
+version = "2.5.3"
 edition = "2021"
 
 [[bin]]
@@ -48,6 +48,7 @@ dotenvy = "0.15"
 hostname = "0.4"
 walkdir = "2"
 clap = { version = "4", features = ["derive"] }
+async-trait = "0.1"
 utoipa = { version = "5", features = ["axum_extras", "chrono", "uuid"] }
 aws-config = { version = "1.8", optional = true }
 aws-sdk-s3 = { version = "1.92", optional = true }
@@ -69,6 +70,7 @@ tempfile = "3"
 wiremock = "0.6" 
 tokio-test = "0.4"
 futures = "0.3"
+rand = "0.8"
 # Database testing dependencies
 testcontainers = "0.24"
 testcontainers-modules = { version = "0.12", features = ["postgres"] }

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "readur-frontend",
-  "version": "2.4.2",
+  "version": "2.5.3",
   "private": true,
   "type": "module",
   "scripts": {

src/db/webdav.rs

@@ -442,4 +442,173 @@ impl Database {
 
         Ok(())
     }
+
+    /// Bulk create or update WebDAV directories in a single transaction
+    /// This ensures atomic updates and prevents race conditions during directory sync
+    pub async fn bulk_create_or_update_webdav_directories(&self, directories: &[crate::models::CreateWebDAVDirectory]) -> Result<Vec<crate::models::WebDAVDirectory>> {
+        if directories.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let mut tx = self.pool.begin().await?;
+        let mut results = Vec::new();
+
+        for directory in directories {
+            let row = sqlx::query(
+                r#"INSERT INTO webdav_directories (user_id, directory_path, directory_etag, 
+                   file_count, total_size_bytes, last_scanned_at, updated_at)
+                   VALUES ($1, $2, $3, $4, $5, NOW(), NOW())
+                   ON CONFLICT (user_id, directory_path) DO UPDATE SET
+                   directory_etag = EXCLUDED.directory_etag,
+                   file_count = EXCLUDED.file_count,
+                   total_size_bytes = EXCLUDED.total_size_bytes,
+                   last_scanned_at = NOW(),
+                   updated_at = NOW()
+                   RETURNING id, user_id, directory_path, directory_etag, last_scanned_at,
+                   file_count, total_size_bytes, created_at, updated_at"#
+            )
+            .bind(directory.user_id)
+            .bind(&directory.directory_path)
+            .bind(&directory.directory_etag)
+            .bind(directory.file_count)
+            .bind(directory.total_size_bytes)
+            .fetch_one(&mut *tx)
+            .await?;
+
+            results.push(crate::models::WebDAVDirectory {
+                id: row.get("id"),
+                user_id: row.get("user_id"),
+                directory_path: row.get("directory_path"),
+                directory_etag: row.get("directory_etag"),
+                last_scanned_at: row.get("last_scanned_at"),
+                file_count: row.get("file_count"),
+                total_size_bytes: row.get("total_size_bytes"),
+                created_at: row.get("created_at"),
+                updated_at: row.get("updated_at"),
+            });
+        }
+
+        tx.commit().await?;
+        Ok(results)
+    }
+
+    /// Delete directories that no longer exist on the WebDAV server
+    /// Returns the number of directories deleted
+    pub async fn delete_missing_webdav_directories(&self, user_id: Uuid, existing_paths: &[String]) -> Result<i64> {
+        if existing_paths.is_empty() {
+            // If no directories exist, delete all for this user
+            return self.clear_webdav_directories(user_id).await;
+        }
+
+        // Build the NOT IN clause with placeholders
+        let placeholders = (0..existing_paths.len())
+            .map(|i| format!("${}", i + 2))
+            .collect::<Vec<_>>()
+            .join(",");
+
+        let query = format!(
+            r#"DELETE FROM webdav_directories 
+               WHERE user_id = $1 AND directory_path NOT IN ({})"#,
+            placeholders
+        );
+
+        let mut query_builder = sqlx::query(&query);
+        query_builder = query_builder.bind(user_id);
+        
+        for path in existing_paths {
+            query_builder = query_builder.bind(path);
+        }
+
+        let result = query_builder.execute(&self.pool).await?;
+        Ok(result.rows_affected() as i64)
+    }
+
+    /// Perform a complete atomic sync of directory state
+    /// This combines creation/updates and deletion in a single transaction
+    pub async fn sync_webdav_directories(
+        &self, 
+        user_id: Uuid, 
+        discovered_directories: &[crate::models::CreateWebDAVDirectory]
+    ) -> Result<(Vec<crate::models::WebDAVDirectory>, i64)> {
+        let mut tx = self.pool.begin().await?;
+        let mut updated_directories = Vec::new();
+
+        // First, update/create all discovered directories
+        for directory in discovered_directories {
+            let row = sqlx::query(
+                r#"INSERT INTO webdav_directories (user_id, directory_path, directory_etag, 
+                   file_count, total_size_bytes, last_scanned_at, updated_at)
+                   VALUES ($1, $2, $3, $4, $5, NOW(), NOW())
+                   ON CONFLICT (user_id, directory_path) DO UPDATE SET
+                   directory_etag = EXCLUDED.directory_etag,
+                   file_count = EXCLUDED.file_count,
+                   total_size_bytes = EXCLUDED.total_size_bytes,
+                   last_scanned_at = NOW(),
+                   updated_at = NOW()
+                   RETURNING id, user_id, directory_path, directory_etag, last_scanned_at,
+                   file_count, total_size_bytes, created_at, updated_at"#
+            )
+            .bind(directory.user_id)
+            .bind(&directory.directory_path)
+            .bind(&directory.directory_etag)
+            .bind(directory.file_count)
+            .bind(directory.total_size_bytes)
+            .fetch_one(&mut *tx)
+            .await?;
+
+            updated_directories.push(crate::models::WebDAVDirectory {
+                id: row.get("id"),
+                user_id: row.get("user_id"),
+                directory_path: row.get("directory_path"),
+                directory_etag: row.get("directory_etag"),
+                last_scanned_at: row.get("last_scanned_at"),
+                file_count: row.get("file_count"),
+                total_size_bytes: row.get("total_size_bytes"),
+                created_at: row.get("created_at"),
+                updated_at: row.get("updated_at"),
+            });
+        }
+
+        // Then, delete directories that are no longer present
+        let discovered_paths: Vec<String> = discovered_directories
+            .iter()
+            .map(|d| d.directory_path.clone())
+            .collect();
+
+        let deleted_count = if discovered_paths.is_empty() {
+            // If no directories discovered, delete all for this user
+            let result = sqlx::query(
+                r#"DELETE FROM webdav_directories WHERE user_id = $1"#
+            )
+            .bind(user_id)
+            .execute(&mut *tx)
+            .await?;
+            result.rows_affected() as i64
+        } else {
+            // Build the NOT IN clause
+            let placeholders = (0..discovered_paths.len())
+                .map(|i| format!("${}", i + 2))
+                .collect::<Vec<_>>()
+                .join(",");
+
+            let query = format!(
+                r#"DELETE FROM webdav_directories 
+                   WHERE user_id = $1 AND directory_path NOT IN ({})"#,
+                placeholders
+            );
+
+            let mut query_builder = sqlx::query(&query);
+            query_builder = query_builder.bind(user_id);
+            
+            for path in &discovered_paths {
+                query_builder = query_builder.bind(path);
+            }
+
+            let result = query_builder.execute(&mut *tx).await?;
+            result.rows_affected() as i64
+        };
+
+        tx.commit().await?;
+        Ok((updated_directories, deleted_count))
+    }
 }

src/models/source.rs

@@ -307,7 +307,7 @@ pub struct WebDAVDirectory {
     pub updated_at: DateTime<Utc>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CreateWebDAVDirectory {
     pub user_id: Uuid,
     pub directory_path: String,

src/routes/sources/sync.rs

@@ -347,7 +347,7 @@ pub async fn trigger_deep_scan(
                                     progress.set_phase(SyncPhase::Completed);
                                     if let Some(stats) = progress.get_stats() {
                                         info!("📊 Manual deep scan statistics: {} files processed, {} errors, {} warnings, elapsed: {}s", 
-                                              stats.files_processed, stats.errors, stats.warnings, stats.elapsed_time.as_secs());
+                                              stats.files_processed, stats.errors.len(), stats.warnings, stats.elapsed_time.as_secs());
                                     }
                                     
                                     // Update source status to idle

src/routes/webdav/webdav_sync.rs

@@ -256,7 +256,7 @@ async fn perform_sync_internal(
     // Log final statistics
     if let Some(stats) = progress.get_stats() {
         info!("📊 Final Sync Statistics: {} files processed, {} errors, {} warnings, elapsed: {}s", 
-              stats.files_processed, stats.errors, stats.warnings, stats.elapsed_time.as_secs());
+              stats.files_processed, stats.errors.len(), stats.warnings, stats.elapsed_time.as_secs());
     }
     
     Ok(total_files_processed)

src/scheduling/source_scheduler.rs

@@ -222,6 +222,11 @@ impl SourceScheduler {
                     // Get user's OCR setting - simplified, you might want to store this in source config  
                     let enable_background_ocr = true; // Default to true, could be made configurable per source
                     
+                    // Create progress tracker for this sync and register it
+                    let progress = Arc::new(crate::services::webdav::SyncProgress::new());
+                    progress.set_phase(crate::services::webdav::SyncPhase::Initializing);
+                    state_clone.sync_progress_tracker.register_sync(source.id, progress.clone());
+                    
                     // Pass cancellation token to sync service
                     match sync_service.sync_source_with_cancellation(&source_clone, enable_background_ocr, cancellation_token.clone()).await {
                         Ok(files_processed) => {
@@ -290,11 +295,12 @@ impl SourceScheduler {
                         }
                     }
                     
-                    // Cleanup: Remove the sync from running list
+                    // Cleanup: Remove the sync from running list and unregister progress tracker
                     {
                         let mut running_syncs = running_syncs_clone.write().await;
                         running_syncs.remove(&source_clone.id);
                     }
+                    state_clone.sync_progress_tracker.unregister_sync(source_clone.id);
                 });
             }
         }
@@ -377,6 +383,11 @@ impl SourceScheduler {
             tokio::spawn(async move {
                 let enable_background_ocr = true; // Could be made configurable
                 
+                // Create progress tracker for this sync and register it
+                let progress = Arc::new(crate::services::webdav::SyncProgress::new());
+                progress.set_phase(crate::services::webdav::SyncPhase::Initializing);
+                state_clone.sync_progress_tracker.register_sync(source_id, progress.clone());
+                
                 match sync_service.sync_source_with_cancellation(&source, enable_background_ocr, cancellation_token).await {
                     Ok(files_processed) => {
                         info!("Manual sync completed for source {}: {} files processed", 
@@ -402,11 +413,12 @@ impl SourceScheduler {
                     }
                 }
                 
-                // Cleanup: Remove the sync from running list
+                // Cleanup: Remove the sync from running list and unregister progress tracker
                 {
                     let mut running_syncs = running_syncs_clone.write().await;
                     running_syncs.remove(&source.id);
                 }
+                state_clone.sync_progress_tracker.unregister_sync(source_id);
             });
             
             Ok(())
@@ -429,22 +441,41 @@ impl SourceScheduler {
             token.cancel();
             info!("Cancellation signal sent for source {}", source_id);
             
-            // Update source status to indicate cancellation
+            // Use a transaction to atomically update status and prevent race conditions
+            let mut tx = self.state.db.get_pool().begin().await
+                .map_err(|e| format!("Failed to start transaction: {}", e))?;
+            
+            // Update source status to indicate cancellation - this will persist even if sync task tries to update later
             if let Err(e) = sqlx::query(
-                r#"UPDATE sources SET status = 'idle', last_error = 'Sync cancelled by user', last_error_at = NOW(), updated_at = NOW() WHERE id = $1"#
+                r#"UPDATE sources 
+                   SET status = 'idle', 
+                       last_error = 'Sync cancelled by user', 
+                       last_error_at = NOW(), 
+                       updated_at = NOW() 
+                   WHERE id = $1 AND status = 'syncing'"#
             )
             .bind(source_id)
-            .execute(self.state.db.get_pool())
+            .execute(&mut *tx)
             .await {
+                tx.rollback().await.ok();
                 error!("Failed to update source status after cancellation: {}", e);
+            } else {
+                // Commit the status change
+                if let Err(e) = tx.commit().await {
+                    error!("Failed to commit cancellation status update: {}", e);
+                }
             }
             
+            // Immediately unregister from progress tracker to update UI
+            self.state.sync_progress_tracker.unregister_sync(source_id);
+            
             // Remove from running syncs list
             {
                 let mut running_syncs = self.running_syncs.write().await;
                 running_syncs.remove(&source_id);
             }
             
+            info!("Sync cancellation completed for source {}", source_id);
             Ok(())
         } else {
             Err("No running sync found for this source".into())

src/scheduling/source_sync.rs

@@ -58,12 +58,13 @@ impl SourceSyncService {
             Ok(files_processed) => {
                 if cancellation_token.is_cancelled() {
                     info!("Sync for source {} was cancelled during execution", source.name);
-                    if let Err(e) = self.update_source_status(source.id, SourceStatus::Idle, Some("Sync cancelled by user")).await {
+                    // Don't overwrite status if it's already been set to cancelled by stop_sync
+                    if let Err(e) = self.update_source_status_if_not_cancelled(source.id, SourceStatus::Idle, Some("Sync cancelled by user")).await {
                         error!("Failed to update source status after cancellation: {}", e);
                     }
                 } else {
                     info!("Sync completed for source {}: {} files processed", source.name, files_processed);
-                    if let Err(e) = self.update_source_status(source.id, SourceStatus::Idle, None).await {
+                    if let Err(e) = self.update_source_status_if_not_cancelled(source.id, SourceStatus::Idle, None).await {
                         error!("Failed to update source status after successful sync: {}", e);
                     }
                 }
@@ -71,13 +72,14 @@ impl SourceSyncService {
             Err(e) => {
                 if cancellation_token.is_cancelled() {
                     info!("Sync for source {} was cancelled: {}", source.name, e);
-                    if let Err(e) = self.update_source_status(source.id, SourceStatus::Idle, Some("Sync cancelled by user")).await {
+                    // Don't overwrite status if it's already been set to cancelled by stop_sync
+                    if let Err(e) = self.update_source_status_if_not_cancelled(source.id, SourceStatus::Idle, Some("Sync cancelled by user")).await {
                         error!("Failed to update source status after cancellation: {}", e);
                     }
                 } else {
                     error!("Sync failed for source {}: {}", source.name, e);
                     let error_msg = format!("Sync failed: {}", e);
-                    if let Err(e) = self.update_source_status(source.id, SourceStatus::Error, Some(&error_msg)).await {
+                    if let Err(e) = self.update_source_status_if_not_cancelled(source.id, SourceStatus::Error, Some(&error_msg)).await {
                         error!("Failed to update source status after error: {}", e);
                     }
                 }
@@ -176,7 +178,7 @@ impl SourceSyncService {
         progress.set_phase(SyncPhase::Completed);
         if let Some(stats) = progress.get_stats() {
             info!("📊 Scheduled sync completed for '{}': {} files processed, {} errors, {} warnings, elapsed: {}s", 
-                  source.name, stats.files_processed, stats.errors, stats.warnings, stats.elapsed_time.as_secs());
+                  source.name, stats.files_processed, stats.errors.len(), stats.warnings, stats.elapsed_time.as_secs());
         }
 
         sync_result
@@ -745,4 +747,36 @@ impl SourceSyncService {
         Ok(())
     }
 
+    /// Update source status only if it hasn't already been set to cancelled
+    /// This prevents race conditions where stop_sync sets status to idle and sync task overwrites it
+    async fn update_source_status_if_not_cancelled(&self, source_id: Uuid, status: SourceStatus, error_message: Option<&str>) -> Result<()> {
+        let query = if let Some(error) = error_message {
+            sqlx::query(
+                r#"UPDATE sources 
+                   SET status = $2, last_error = $3, last_error_at = NOW(), updated_at = NOW()
+                   WHERE id = $1 AND NOT (status = 'idle' AND last_error = 'Sync cancelled by user')"#
+            )
+            .bind(source_id)
+            .bind(status.to_string())
+            .bind(error)
+        } else {
+            sqlx::query(
+                r#"UPDATE sources 
+                   SET status = $2, last_error = NULL, last_error_at = NULL, updated_at = NOW()
+                   WHERE id = $1 AND NOT (status = 'idle' AND last_error = 'Sync cancelled by user')"#
+            )
+            .bind(source_id)
+            .bind(status.to_string())
+        };
+
+        let result = query.execute(self.state.db.get_pool()).await
+            .map_err(|e| anyhow!("Failed to update source status: {}", e))?;
+
+        if result.rows_affected() == 0 {
+            info!("Source {} status not updated - already cancelled by user", source_id);
+        }
+
+        Ok(())
+    }
+
 }

src/services/sync_progress_tracker.rs

@@ -145,7 +145,7 @@ impl SyncProgressTracker {
                 .map(|d| d.as_secs()),
             current_directory: stats.current_directory,
             current_file: stats.current_file,
-            errors: stats.errors,
+            errors: stats.errors.len(),
             warnings: stats.warnings,
             is_active,
         }
@@ -186,6 +186,10 @@ impl SyncProgressTracker {
                 "failed".to_string(),
                 format!("Sync failed: {}", error),
             ),
+            SyncPhase::Retrying { attempt, category, delay_ms } => (
+                "retrying".to_string(),
+                format!("Retry attempt {} for {:?} (delay: {}ms)", attempt, category, delay_ms),
+            ),
         }
     }
 }

src/services/webdav/config.rs

@@ -30,6 +30,23 @@ pub struct ConcurrencyConfig {
     pub adaptive_rate_limiting: bool,
 }
 
+/// Configuration for Depth infinity PROPFIND optimizations
+#[derive(Debug, Clone)]
+pub struct DepthInfinityConfig {
+    /// Whether to attempt Depth infinity PROPFIND requests
+    pub enabled: bool,
+    /// Maximum response size in bytes before falling back to recursive approach
+    pub max_response_size_bytes: usize,
+    /// Timeout for infinity depth requests in seconds
+    pub timeout_seconds: u64,
+    /// Cache server capability detection results for this duration (seconds)
+    pub capability_cache_duration_seconds: u64,
+    /// Whether to automatically fallback to recursive approach on failure
+    pub auto_fallback: bool,
+    /// Maximum directory depth to attempt infinity for (0 = no limit)
+    pub max_depth_for_infinity: u32,
+}
+
 impl Default for RetryConfig {
     fn default() -> Self {
         Self {
@@ -53,6 +70,19 @@ impl Default for ConcurrencyConfig {
     }
 }
 
+impl Default for DepthInfinityConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            max_response_size_bytes: 50 * 1024 * 1024, // 50MB
+            timeout_seconds: 120, // 2 minutes for large directories
+            capability_cache_duration_seconds: 3600, // 1 hour
+            auto_fallback: true,
+            max_depth_for_infinity: 0, // No limit by default
+        }
+    }
+}
+
 impl WebDAVConfig {
     /// Creates a new WebDAV configuration
     pub fn new(

src/services/webdav/connection.rs

@@ -1,309 +0,0 @@
-use anyhow::{anyhow, Result};
-use reqwest::{Client, Method};
-use std::time::Duration;
-use tokio::time::sleep;
-use tracing::{debug, error, info, warn};
-
-use crate::models::{WebDAVConnectionResult, WebDAVTestConnection};
-use super::config::{WebDAVConfig, RetryConfig};
-
-#[derive(Clone)]
-pub struct WebDAVConnection {
-    client: Client,
-    config: WebDAVConfig,
-    retry_config: RetryConfig,
-}
-
-impl WebDAVConnection {
-    pub fn new(config: WebDAVConfig, retry_config: RetryConfig) -> Result<Self> {
-        // Validate configuration first
-        config.validate()?;
-        let client = Client::builder()
-            .timeout(config.timeout())
-            .build()?;
-
-        Ok(Self {
-            client,
-            config,
-            retry_config,
-        })
-    }
-
-    /// Tests WebDAV connection with the provided configuration
-    pub async fn test_connection(&self) -> Result<WebDAVConnectionResult> {
-        info!("🔍 Testing WebDAV connection to: {}", self.config.server_url);
-
-        // Validate configuration first
-        if let Err(e) = self.config.validate() {
-            return Ok(WebDAVConnectionResult {
-                success: false,
-                message: format!("Configuration error: {}", e),
-                server_version: None,
-                server_type: None,
-            });
-        }
-
-        // Test basic connectivity with OPTIONS request
-        match self.test_options_request().await {
-            Ok((server_version, server_type)) => {
-                info!("✅ WebDAV connection successful");
-                Ok(WebDAVConnectionResult {
-                    success: true,
-                    message: "Connection successful".to_string(),
-                    server_version,
-                    server_type,
-                })
-            }
-            Err(e) => {
-                error!("❌ WebDAV connection failed: {}", e);
-                Ok(WebDAVConnectionResult {
-                    success: false,
-                    message: format!("Connection failed: {}", e),
-                    server_version: None,
-                    server_type: None,
-                })
-            }
-        }
-    }
-
-    /// Tests connection with provided credentials (for configuration testing)
-    pub async fn test_connection_with_config(test_config: &WebDAVTestConnection) -> Result<WebDAVConnectionResult> {
-        let config = WebDAVConfig {
-            server_url: test_config.server_url.clone(),
-            username: test_config.username.clone(),
-            password: test_config.password.clone(),
-            watch_folders: vec!["/".to_string()],
-            file_extensions: vec![],
-            timeout_seconds: 30,
-            server_type: test_config.server_type.clone(),
-        };
-
-        let connection = Self::new(config, RetryConfig::default())?;
-        connection.test_connection().await
-    }
-
-    /// Performs OPTIONS request to test basic connectivity
-    async fn test_options_request(&self) -> Result<(Option<String>, Option<String>)> {
-        let webdav_url = self.config.webdav_url();
-        
-        let response = self.client
-            .request(Method::OPTIONS, &webdav_url)
-            .basic_auth(&self.config.username, Some(&self.config.password))
-            .send()
-            .await?;
-
-        if !response.status().is_success() {
-            return Err(anyhow!(
-                "OPTIONS request failed with status: {} - {}",
-                response.status(),
-                response.text().await.unwrap_or_default()
-            ));
-        }
-
-        // Extract server information from headers
-        let server_version = response
-            .headers()
-            .get("server")
-            .and_then(|v| v.to_str().ok())
-            .map(|s| s.to_string());
-
-        let server_type = self.detect_server_type(&response, &server_version).await;
-
-        Ok((server_version, server_type))
-    }
-
-    /// Detects the WebDAV server type based on response headers and capabilities
-    async fn detect_server_type(
-        &self,
-        response: &reqwest::Response,
-        server_version: &Option<String>,
-    ) -> Option<String> {
-        // Check server header first
-        if let Some(ref server) = server_version {
-            let server_lower = server.to_lowercase();
-            if server_lower.contains("nextcloud") {
-                return Some("nextcloud".to_string());
-            }
-            if server_lower.contains("owncloud") {
-                return Some("owncloud".to_string());
-            }
-            if server_lower.contains("apache") || server_lower.contains("nginx") {
-                // Could be generic WebDAV
-            }
-        }
-
-        // Check DAV capabilities
-        if let Some(dav_header) = response.headers().get("dav") {
-            if let Ok(dav_str) = dav_header.to_str() {
-                debug!("DAV capabilities: {}", dav_str);
-                // Different servers expose different DAV levels
-                if dav_str.contains("3") {
-                    return Some("webdav_level_3".to_string());
-                }
-            }
-        }
-
-        // Test for Nextcloud/ownCloud specific endpoints
-        if self.test_nextcloud_capabilities().await.is_ok() {
-            return Some("nextcloud".to_string());
-        }
-
-        Some("generic".to_string())
-    }
-
-    /// Tests for Nextcloud-specific capabilities
-    async fn test_nextcloud_capabilities(&self) -> Result<()> {
-        let capabilities_url = format!("{}/ocs/v1.php/cloud/capabilities", 
-            self.config.server_url.trim_end_matches('/'));
-
-        let response = self.client
-            .get(&capabilities_url)
-            .basic_auth(&self.config.username, Some(&self.config.password))
-            .header("OCS-APIRequest", "true")
-            .send()
-            .await?;
-
-        if response.status().is_success() {
-            debug!("Nextcloud capabilities endpoint accessible");
-            Ok(())
-        } else {
-            Err(anyhow!("Nextcloud capabilities not accessible"))
-        }
-    }
-
-    /// Tests PROPFIND request on root directory
-    pub async fn test_propfind(&self, path: &str) -> Result<()> {
-        let url = format!("{}{}", self.config.webdav_url(), path);
-        
-        let propfind_body = r#"<?xml version="1.0" encoding="utf-8"?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:displayname/>
-                    <D:getcontentlength/>
-                    <D:getlastmodified/>
-                    <D:getetag/>
-                    <D:resourcetype/>
-                </D:prop>
-            </D:propfind>"#;
-
-        let response = self.client
-            .request(Method::from_bytes(b"PROPFIND")?, &url)
-            .basic_auth(&self.config.username, Some(&self.config.password))
-            .header("Depth", "1")
-            .header("Content-Type", "application/xml")
-            .body(propfind_body)
-            .send()
-            .await?;
-
-        if response.status().as_u16() == 207 {
-            debug!("PROPFIND successful for path: {}", path);
-            Ok(())
-        } else {
-            Err(anyhow!(
-                "PROPFIND failed for path '{}' with status: {} - {}",
-                path,
-                response.status(),
-                response.text().await.unwrap_or_default()
-            ))
-        }
-    }
-
-    /// Performs authenticated request with retry logic
-    pub async fn authenticated_request(
-        &self,
-        method: Method,
-        url: &str,
-        body: Option<String>,
-        headers: Option<Vec<(&str, &str)>>,
-    ) -> Result<reqwest::Response> {
-        let mut attempt = 0;
-        let mut delay = self.retry_config.initial_delay_ms;
-
-        loop {
-            let mut request = self.client
-                .request(method.clone(), url)
-                .basic_auth(&self.config.username, Some(&self.config.password));
-
-            if let Some(ref body_content) = body {
-                request = request.body(body_content.clone());
-            }
-
-            if let Some(ref headers_list) = headers {
-                for (key, value) in headers_list {
-                    request = request.header(*key, *value);
-                }
-            }
-
-            match request.send().await {
-                Ok(response) => {
-                    let status = response.status();
-                    
-                    if status.is_success() || status.as_u16() == 207 {
-                        return Ok(response);
-                    }
-
-                    // Handle rate limiting
-                    if status.as_u16() == 429 {
-                        warn!("Rate limited, backing off for {}ms", self.retry_config.rate_limit_backoff_ms);
-                        sleep(Duration::from_millis(self.retry_config.rate_limit_backoff_ms)).await;
-                        continue;
-                    }
-
-                    // Handle client errors (don't retry)
-                    if status.is_client_error() && status.as_u16() != 429 {
-                        return Err(anyhow!("Client error: {} - {}", status, 
-                            response.text().await.unwrap_or_default()));
-                    }
-
-                    // Handle server errors (retry)
-                    if status.is_server_error() && attempt < self.retry_config.max_retries {
-                        warn!("Server error {}, retrying in {}ms (attempt {}/{})", 
-                            status, delay, attempt + 1, self.retry_config.max_retries);
-                        
-                        sleep(Duration::from_millis(delay)).await;
-                        delay = std::cmp::min(
-                            (delay as f64 * self.retry_config.backoff_multiplier) as u64,
-                            self.retry_config.max_delay_ms
-                        );
-                        attempt += 1;
-                        continue;
-                    }
-
-                    return Err(anyhow!("Request failed: {} - {}", status,
-                        response.text().await.unwrap_or_default()));
-                }
-                Err(e) => {
-                    if attempt < self.retry_config.max_retries {
-                        warn!("Request error: {}, retrying in {}ms (attempt {}/{})", 
-                            e, delay, attempt + 1, self.retry_config.max_retries);
-                        
-                        sleep(Duration::from_millis(delay)).await;
-                        delay = std::cmp::min(
-                            (delay as f64 * self.retry_config.backoff_multiplier) as u64,
-                            self.retry_config.max_delay_ms
-                        );
-                        attempt += 1;
-                        continue;
-                    }
-
-                    return Err(anyhow!("Request failed after {} attempts: {}", 
-                        self.retry_config.max_retries, e));
-                }
-            }
-        }
-    }
-
-    /// Gets the WebDAV URL for a specific path
-    pub fn get_url_for_path(&self, path: &str) -> String {
-        let base_url = self.config.webdav_url();
-        let clean_path = path.trim_start_matches('/');
-        
-        if clean_path.is_empty() {
-            base_url
-        } else {
-            // Ensure no double slashes by normalizing the base URL
-            let normalized_base = base_url.trim_end_matches('/');
-            format!("{}/{}", normalized_base, clean_path)
-        }
-    }
-}

src/services/webdav/discovery.rs

@@ -1,601 +0,0 @@
-use anyhow::Result;
-use reqwest::Method;
-use std::collections::HashSet;
-use tokio::sync::Semaphore;
-use futures_util::stream::{self, StreamExt};
-use tracing::{debug, info, warn};
-
-use crate::models::{FileIngestionInfo, WebDAVCrawlEstimate, WebDAVFolderInfo};
-use crate::webdav_xml_parser::{parse_propfind_response, parse_propfind_response_with_directories};
-use super::config::{WebDAVConfig, ConcurrencyConfig};
-use super::connection::WebDAVConnection;
-use super::url_management::WebDAVUrlManager;
-use super::progress::{SyncProgress, SyncPhase};
-
-/// Results from WebDAV discovery including both files and directories
-#[derive(Debug, Clone)]
-pub struct WebDAVDiscoveryResult {
-    pub files: Vec<FileIngestionInfo>,
-    pub directories: Vec<FileIngestionInfo>,
-}
-
-pub struct WebDAVDiscovery {
-    connection: WebDAVConnection,
-    config: WebDAVConfig,
-    concurrency_config: ConcurrencyConfig,
-    url_manager: WebDAVUrlManager,
-}
-
-impl WebDAVDiscovery {
-    pub fn new(
-        connection: WebDAVConnection, 
-        config: WebDAVConfig, 
-        concurrency_config: ConcurrencyConfig
-    ) -> Self {
-        let url_manager = WebDAVUrlManager::new(config.clone());
-        Self { 
-            connection, 
-            config, 
-            concurrency_config,
-            url_manager
-        }
-    }
-
-    /// Discovers files in a directory with support for pagination and filtering
-    pub async fn discover_files(&self, directory_path: &str, recursive: bool) -> Result<Vec<FileIngestionInfo>> {
-        info!("🔍 Discovering files in directory: {}", directory_path);
-        
-        if recursive {
-            self.discover_files_recursive(directory_path).await
-        } else {
-            self.discover_files_single_directory(directory_path).await
-        }
-    }
-
-    /// Discovers both files and directories with their ETags for directory tracking
-    pub async fn discover_files_and_directories(&self, directory_path: &str, recursive: bool) -> Result<WebDAVDiscoveryResult> {
-        info!("🔍 Discovering files and directories in: {}", directory_path);
-        
-        if recursive {
-            self.discover_files_and_directories_recursive(directory_path).await
-        } else {
-            self.discover_files_and_directories_single(directory_path).await
-        }
-    }
-
-    /// Discovers both files and directories with progress tracking
-    pub async fn discover_files_and_directories_with_progress(
-        &self, 
-        directory_path: &str, 
-        recursive: bool, 
-        progress: Option<&SyncProgress>
-    ) -> Result<WebDAVDiscoveryResult> {
-        if let Some(progress) = progress {
-            if recursive {
-                progress.set_phase(SyncPhase::DiscoveringDirectories);
-            }
-            progress.set_current_directory(directory_path);
-        }
-        
-        info!("🔍 Discovering files and directories in: {}", directory_path);
-        
-        if recursive {
-            self.discover_files_and_directories_recursive_with_progress(directory_path, progress).await
-        } else {
-            let result = self.discover_files_and_directories_single(directory_path).await?;
-            if let Some(progress) = progress {
-                progress.add_directories_found(result.directories.len());
-                progress.add_files_found(result.files.len());
-            }
-            Ok(result)
-        }
-    }
-
-    /// Discovers files in a single directory (non-recursive)
-    async fn discover_files_single_directory(&self, directory_path: &str) -> Result<Vec<FileIngestionInfo>> {
-        let url = self.connection.get_url_for_path(directory_path);
-        
-        let propfind_body = r#"<?xml version="1.0" encoding="utf-8"?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:displayname/>
-                    <D:getcontentlength/>
-                    <D:getlastmodified/>
-                    <D:getetag/>
-                    <D:resourcetype/>
-                    <D:creationdate/>
-                </D:prop>
-            </D:propfind>"#;
-
-        let response = self.connection
-            .authenticated_request(
-                Method::from_bytes(b"PROPFIND")?,
-                &url,
-                Some(propfind_body.to_string()),
-                Some(vec![
-                    ("Depth", "1"),
-                    ("Content-Type", "application/xml"),
-                ]),
-            )
-            .await?;
-
-        let body = response.text().await?;
-        let files = parse_propfind_response(&body)?;
-        
-        // Process file paths using the centralized URL manager
-        let files = self.url_manager.process_file_infos(files);
-        
-        // Filter files based on supported extensions
-        let filtered_files: Vec<FileIngestionInfo> = files
-            .into_iter()
-            .filter(|file| {
-                !file.is_directory && self.config.is_supported_extension(&file.name)
-            })
-            .collect();
-
-        debug!("Found {} supported files in directory: {}", filtered_files.len(), directory_path);
-        Ok(filtered_files)
-    }
-
-    /// Discovers both files and directories in a single directory (non-recursive)
-    async fn discover_files_and_directories_single(&self, directory_path: &str) -> Result<WebDAVDiscoveryResult> {
-        let url = self.connection.get_url_for_path(directory_path);
-        
-        let propfind_body = r#"<?xml version="1.0" encoding="utf-8"?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:displayname/>
-                    <D:getcontentlength/>
-                    <D:getlastmodified/>
-                    <D:getetag/>
-                    <D:resourcetype/>
-                    <D:creationdate/>
-                </D:prop>
-            </D:propfind>"#;
-
-        let response = self.connection
-            .authenticated_request(
-                Method::from_bytes(b"PROPFIND")?,
-                &url,
-                Some(propfind_body.to_string()),
-                Some(vec![
-                    ("Depth", "1"),
-                    ("Content-Type", "application/xml"),
-                ]),
-            )
-            .await?;
-
-        let body = response.text().await?;
-        let all_items = parse_propfind_response_with_directories(&body)?;
-        
-        // Process file paths using the centralized URL manager
-        let all_items = self.url_manager.process_file_infos(all_items);
-        
-        // Separate files and directories
-        let mut files = Vec::new();
-        let mut directories = Vec::new();
-        
-        for item in all_items {
-            if item.is_directory {
-                directories.push(item);
-            } else if self.config.is_supported_extension(&item.name) {
-                files.push(item);
-            }
-        }
-
-        debug!("Single directory '{}': {} files, {} directories", 
-            directory_path, files.len(), directories.len());
-            
-        Ok(WebDAVDiscoveryResult { files, directories })
-    }
-
-    /// Discovers files recursively in directory tree
-    async fn discover_files_recursive(&self, root_directory: &str) -> Result<Vec<FileIngestionInfo>> {
-        let mut all_files = Vec::new();
-        let mut directories_to_scan = vec![root_directory.to_string()];
-        let semaphore = Semaphore::new(self.concurrency_config.max_concurrent_scans);
-
-        while !directories_to_scan.is_empty() {
-            let current_batch: Vec<String> = directories_to_scan
-                .drain(..)
-                .take(self.concurrency_config.max_concurrent_scans)
-                .collect();
-
-            let tasks = current_batch.into_iter().map(|dir| {
-                let semaphore = &semaphore;
-                async move {
-                    let _permit = semaphore.acquire().await.unwrap();
-                    self.scan_directory_with_subdirs(&dir).await
-                }
-            });
-
-            let results = stream::iter(tasks)
-                .buffer_unordered(self.concurrency_config.max_concurrent_scans)
-                .collect::<Vec<_>>()
-                .await;
-
-            for result in results {
-                match result {
-                    Ok((files, subdirs)) => {
-                        all_files.extend(files);
-                        directories_to_scan.extend(subdirs);
-                    }
-                    Err(e) => {
-                        warn!("Failed to scan directory: {}", e);
-                    }
-                }
-            }
-        }
-
-        info!("Recursive discovery found {} total files", all_files.len());
-        Ok(all_files)
-    }
-
-    /// Discovers both files and directories recursively in directory tree
-    async fn discover_files_and_directories_recursive(&self, root_directory: &str) -> Result<WebDAVDiscoveryResult> {
-        self.discover_files_and_directories_recursive_with_progress(root_directory, None).await
-    }
-
-    /// Discovers both files and directories recursively with progress tracking
-    async fn discover_files_and_directories_recursive_with_progress(
-        &self, 
-        root_directory: &str, 
-        progress: Option<&SyncProgress>
-    ) -> Result<WebDAVDiscoveryResult> {
-        let mut all_files = Vec::new();
-        let mut all_directories = Vec::new();
-        let mut directories_to_scan = vec![root_directory.to_string()];
-        let semaphore = Semaphore::new(self.concurrency_config.max_concurrent_scans);
-
-        while !directories_to_scan.is_empty() {
-            let current_batch: Vec<String> = directories_to_scan
-                .drain(..)
-                .take(self.concurrency_config.max_concurrent_scans)
-                .collect();
-
-            let tasks = current_batch.into_iter().map(|dir| {
-                let semaphore = &semaphore;
-                async move {
-                    let _permit = semaphore.acquire().await.unwrap();
-                    
-                    // Update progress with current directory
-                    if let Some(progress) = progress {
-                        progress.set_current_directory(&dir);
-                    }
-                    
-                    let result = self.scan_directory_with_all_info(&dir).await;
-                    
-                    // Update progress counts on successful scan
-                    if let (Ok((ref files, ref directories, _)), Some(progress)) = (&result, progress) {
-                        progress.add_directories_found(directories.len());
-                        progress.add_files_found(files.len());
-                        progress.add_directories_processed(1);
-                    }
-                    
-                    result
-                }
-            });
-
-            let results = stream::iter(tasks)
-                .buffer_unordered(self.concurrency_config.max_concurrent_scans)
-                .collect::<Vec<_>>()
-                .await;
-
-            for result in results {
-                match result {
-                    Ok((files, directories, subdirs_to_scan)) => {
-                        all_files.extend(files);
-                        all_directories.extend(directories);
-                        directories_to_scan.extend(subdirs_to_scan);
-                    }
-                    Err(e) => {
-                        warn!("Failed to scan directory: {}", e);
-                        if let Some(progress) = progress {
-                            progress.add_error(&format!("Directory scan failed: {}", e));
-                        }
-                    }
-                }
-            }
-        }
-
-        info!("Recursive discovery found {} total files and {} directories", 
-              all_files.len(), all_directories.len());
-        
-        // Update final phase when discovery is complete
-        if let Some(progress) = progress {
-            progress.set_phase(SyncPhase::DiscoveringFiles);
-        }
-        
-        Ok(WebDAVDiscoveryResult { 
-            files: all_files, 
-            directories: all_directories 
-        })
-    }
-
-    /// Scans a directory and returns both files and subdirectories
-    async fn scan_directory_with_subdirs(&self, directory_path: &str) -> Result<(Vec<FileIngestionInfo>, Vec<String>)> {
-        let url = self.connection.get_url_for_path(directory_path);
-        
-        let propfind_body = r#"<?xml version="1.0" encoding="utf-8"?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:displayname/>
-                    <D:getcontentlength/>
-                    <D:getlastmodified/>
-                    <D:getetag/>
-                    <D:resourcetype/>
-                    <D:creationdate/>
-                </D:prop>
-            </D:propfind>"#;
-
-        let response = self.connection
-            .authenticated_request(
-                Method::from_bytes(b"PROPFIND")?,
-                &url,
-                Some(propfind_body.to_string()),
-                Some(vec![
-                    ("Depth", "1"),
-                    ("Content-Type", "application/xml"),
-                ]),
-            )
-            .await?;
-
-        let body = response.text().await?;
-        let all_items = parse_propfind_response_with_directories(&body)?;
-        
-        // Process file paths using the centralized URL manager
-        let all_items = self.url_manager.process_file_infos(all_items);
-        
-        // Separate files and directories
-        let mut filtered_files = Vec::new();
-        let mut subdirectory_paths = Vec::new();
-        
-        for item in all_items {
-            if item.is_directory {
-                // Use the relative_path which is now properly set by url_manager
-                subdirectory_paths.push(item.relative_path.clone());
-            } else if self.config.is_supported_extension(&item.name) {
-                filtered_files.push(item);
-            }
-        }
-        
-        let full_dir_paths = subdirectory_paths;
-
-        debug!("Directory '{}': {} files, {} subdirectories", 
-            directory_path, filtered_files.len(), full_dir_paths.len());
-            
-        Ok((filtered_files, full_dir_paths))
-    }
-
-    /// Scans a directory and returns files, directories, and subdirectory paths for queue
-    async fn scan_directory_with_all_info(&self, directory_path: &str) -> Result<(Vec<FileIngestionInfo>, Vec<FileIngestionInfo>, Vec<String>)> {
-        let url = self.connection.get_url_for_path(directory_path);
-        
-        let propfind_body = r#"<?xml version="1.0" encoding="utf-8"?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:displayname/>
-                    <D:getcontentlength/>
-                    <D:getlastmodified/>
-                    <D:getetag/>
-                    <D:resourcetype/>
-                    <D:creationdate/>
-                </D:prop>
-            </D:propfind>"#;
-
-        let response = self.connection
-            .authenticated_request(
-                Method::from_bytes(b"PROPFIND")?,
-                &url,
-                Some(propfind_body.to_string()),
-                Some(vec![
-                    ("Depth", "1"),
-                    ("Content-Type", "application/xml"),
-                ]),
-            )
-            .await?;
-
-        let body = response.text().await?;
-        let all_items = parse_propfind_response_with_directories(&body)?;
-        
-        // Process file paths using the centralized URL manager
-        let all_items = self.url_manager.process_file_infos(all_items);
-        
-        // Separate files and directories
-        let mut filtered_files = Vec::new();
-        let mut directories = Vec::new();
-        let mut subdirectory_paths = Vec::new();
-        
-        for item in all_items {
-            if item.is_directory {
-                // Use the relative_path which is now properly set by url_manager
-                directories.push(item.clone());
-                subdirectory_paths.push(item.relative_path.clone());
-            } else if self.config.is_supported_extension(&item.name) {
-                filtered_files.push(item);
-            }
-        }
-
-        debug!("Directory '{}': {} files, {} directories, {} paths to scan", 
-            directory_path, filtered_files.len(), directories.len(), subdirectory_paths.len());
-            
-        Ok((filtered_files, directories, subdirectory_paths))
-    }
-
-    /// Estimates crawl time and file counts for watch folders
-    pub async fn estimate_crawl(&self) -> Result<WebDAVCrawlEstimate> {
-        info!("📊 Estimating crawl for WebDAV watch folders");
-        
-        let mut folders = Vec::new();
-        let mut total_files = 0;
-        let mut total_supported_files = 0;
-        let mut total_size_mb = 0.0;
-
-        for watch_folder in &self.config.watch_folders {
-            match self.estimate_folder(watch_folder).await {
-                Ok(folder_info) => {
-                    total_files += folder_info.total_files;
-                    total_supported_files += folder_info.supported_files;
-                    total_size_mb += folder_info.total_size_mb;
-                    folders.push(folder_info);
-                }
-                Err(e) => {
-                    warn!("Failed to estimate folder '{}': {}", watch_folder, e);
-                    // Add empty folder info for failed estimates
-                    folders.push(WebDAVFolderInfo {
-                        path: watch_folder.clone(),
-                        total_files: 0,
-                        supported_files: 0,
-                        estimated_time_hours: 0.0,
-                        total_size_mb: 0.0,
-                    });
-                }
-            }
-        }
-
-        // Estimate total time based on file count and average processing time
-        let avg_time_per_file_seconds = 2.0; // Conservative estimate
-        let total_estimated_time_hours = (total_supported_files as f32 * avg_time_per_file_seconds) / 3600.0;
-
-        Ok(WebDAVCrawlEstimate {
-            folders,
-            total_files,
-            total_supported_files,
-            total_estimated_time_hours,
-            total_size_mb,
-        })
-    }
-
-    /// Estimates file count and size for a specific folder
-    async fn estimate_folder(&self, folder_path: &str) -> Result<WebDAVFolderInfo> {
-        debug!("Estimating folder: {}", folder_path);
-        
-        // Sample a few subdirectories to estimate the total
-        let sample_files = self.discover_files_single_directory(folder_path).await?;
-        
-        // Get subdirectories for deeper estimation
-        let subdirs = self.get_subdirectories(folder_path).await?;
-        
-        let mut total_files = sample_files.len() as i64;
-        let mut total_size: i64 = sample_files.iter().map(|f| f.size).sum();
-        
-        // Sample a few subdirectories to extrapolate
-        let sample_size = std::cmp::min(5, subdirs.len());
-        if sample_size > 0 {
-            let mut sample_total = 0i64;
-            
-            for subdir in subdirs.iter().take(sample_size) {
-                if let Ok(subdir_files) = self.discover_files_single_directory(subdir).await {
-                    sample_total += subdir_files.len() as i64;
-                }
-            }
-            
-            // Extrapolate based on sample
-            if sample_total > 0 {
-                let avg_files_per_subdir = sample_total as f64 / sample_size as f64;
-                total_files += (avg_files_per_subdir * subdirs.len() as f64) as i64;
-            }
-        }
-
-        // Filter for supported files
-        let supported_files = (total_files as f64 * self.calculate_support_ratio(&sample_files)) as i64;
-        
-        let total_size_mb = total_size as f64 / (1024.0 * 1024.0);
-        let estimated_time_hours = (supported_files as f32 * 2.0) / 3600.0; // 2 seconds per file
-
-        Ok(WebDAVFolderInfo {
-            path: folder_path.to_string(),
-            total_files,
-            supported_files,
-            estimated_time_hours,
-            total_size_mb,
-        })
-    }
-
-    /// Gets subdirectories for a given path
-    async fn get_subdirectories(&self, directory_path: &str) -> Result<Vec<String>> {
-        let url = self.connection.get_url_for_path(directory_path);
-        
-        let propfind_body = r#"<?xml version="1.0" encoding="utf-8"?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:resourcetype/>
-                </D:prop>
-            </D:propfind>"#;
-
-        let response = self.connection
-            .authenticated_request(
-                Method::from_bytes(b"PROPFIND")?,
-                &url,
-                Some(propfind_body.to_string()),
-                Some(vec![
-                    ("Depth", "1"),
-                    ("Content-Type", "application/xml"),
-                ]),
-            )
-            .await?;
-
-        let body = response.text().await?;
-        let all_items = parse_propfind_response_with_directories(&body)?;
-        
-        // Process file paths using the centralized URL manager
-        let all_items = self.url_manager.process_file_infos(all_items);
-        
-        // Filter out only directories and extract their paths
-        let directory_paths: Vec<String> = all_items
-            .into_iter()
-            .filter(|item| item.is_directory)
-            .map(|item| item.relative_path)
-            .collect();
-        
-        Ok(directory_paths)
-    }
-
-    /// Calculates the ratio of supported files in a sample
-    fn calculate_support_ratio(&self, sample_files: &[FileIngestionInfo]) -> f64 {
-        if sample_files.is_empty() {
-            return 1.0; // Assume all files are supported if no sample
-        }
-
-        let supported_count = sample_files
-            .iter()
-            .filter(|file| self.config.is_supported_extension(&file.name))
-            .count();
-
-        supported_count as f64 / sample_files.len() as f64
-    }
-
-    /// Filters files by last modified date (for incremental syncs)
-    pub fn filter_files_by_date(&self, files: Vec<FileIngestionInfo>, since: chrono::DateTime<chrono::Utc>) -> Vec<FileIngestionInfo> {
-        files
-            .into_iter()
-            .filter(|file| {
-                file.last_modified
-                    .map(|modified| modified > since)
-                    .unwrap_or(true) // Include files without modification date
-            })
-            .collect()
-    }
-
-    /// Deduplicates files by ETag or path
-    pub fn deduplicate_files(&self, files: Vec<FileIngestionInfo>) -> Vec<FileIngestionInfo> {
-        let mut seen_etags = HashSet::new();
-        let mut seen_paths = HashSet::new();
-        let mut deduplicated = Vec::new();
-
-        for file in files {
-            let is_duplicate = if !file.etag.is_empty() {
-                !seen_etags.insert(file.etag.clone())
-            } else {
-                !seen_paths.insert(file.relative_path.clone())
-            };
-
-            if !is_duplicate {
-                deduplicated.push(file);
-            }
-        }
-
-        debug!("Deduplicated {} files", deduplicated.len());
-        deduplicated
-    }
-}

src/services/webdav/mod.rs

@@ -1,29 +1,26 @@
-// WebDAV service modules organized by functionality
+// Simplified WebDAV service modules - consolidated architecture
 
 pub mod config;
-pub mod connection;
-pub mod discovery;
-pub mod validation;
-pub mod service;
+pub mod service; 
 pub mod smart_sync;
-pub mod url_management;
-pub mod progress;
+pub mod progress_shim; // Backward compatibility shim for simplified progress tracking
 
 // Re-export main types for convenience
 pub use config::{WebDAVConfig, RetryConfig, ConcurrencyConfig};
-pub use connection::WebDAVConnection;
-pub use discovery::WebDAVDiscovery;
-pub use validation::{
-    WebDAVValidator, ValidationReport, ValidationIssue, ValidationIssueType, 
-    ValidationSeverity, ValidationRecommendation, ValidationAction, ValidationSummary
+pub use service::{
+    WebDAVService, WebDAVDiscoveryResult, ServerCapabilities, HealthStatus, test_webdav_connection,
+    ValidationReport, ValidationIssue, ValidationIssueType, ValidationSeverity, 
+    ValidationRecommendation, ValidationAction, ValidationSummary
 };
-pub use service::{WebDAVService, ServerCapabilities, HealthStatus, test_webdav_connection};
 pub use smart_sync::{SmartSyncService, SmartSyncDecision, SmartSyncStrategy, SmartSyncResult};
-pub use url_management::WebDAVUrlManager;
-pub use progress::{SyncProgress, SyncPhase, ProgressStats};
+
+// Backward compatibility exports for progress tracking (simplified)
+pub use progress_shim::{SyncProgress, SyncPhase, ProgressStats};
 
 // Test modules
 #[cfg(test)]
 mod url_construction_tests;
 #[cfg(test)]
-mod subdirectory_edge_cases_tests;
+mod subdirectory_edge_cases_tests;
+#[cfg(test)]
+mod tests;

src/services/webdav/progress.rs

@@ -1,431 +0,0 @@
-use std::sync::{Arc, Mutex};
-use std::time::{Duration, Instant};
-use tracing::{info, warn};
-
-/// Thread-safe progress tracking for WebDAV sync operations
-#[derive(Debug, Clone)]
-pub struct SyncProgress {
-    inner: Arc<Mutex<SyncProgressInner>>,
-}
-
-#[derive(Debug)]
-struct SyncProgressInner {
-    start_time: Instant,
-    last_update: Instant,
-    last_status_report: Instant,
-    
-    // Discovery phase
-    directories_found: usize,
-    files_found: usize,
-    
-    // Processing phase
-    directories_processed: usize,
-    files_processed: usize,
-    bytes_processed: u64,
-    
-    // Current state
-    current_directory: String,
-    current_file: Option<String>,
-    current_phase: SyncPhase,
-    
-    // Performance tracking
-    processing_rate_files_per_sec: f64,
-    
-    // Error tracking
-    errors: Vec<String>,
-    warnings: usize,
-    
-    // Configuration
-    update_interval: Duration,
-    status_report_interval: Duration,
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum SyncPhase {
-    Initializing,
-    Evaluating,
-    DiscoveringDirectories,
-    DiscoveringFiles,
-    ProcessingFiles,
-    SavingMetadata,
-    Completed,
-    Failed(String),
-}
-
-impl SyncProgress {
-    /// Create a new progress tracker
-    pub fn new() -> Self {
-        let now = Instant::now();
-        Self {
-            inner: Arc::new(Mutex::new(SyncProgressInner {
-                start_time: now,
-                last_update: now,
-                last_status_report: now,
-                directories_found: 0,
-                files_found: 0,
-                directories_processed: 0,
-                files_processed: 0,
-                bytes_processed: 0,
-                current_directory: String::new(),
-                current_file: None,
-                current_phase: SyncPhase::Initializing,
-                processing_rate_files_per_sec: 0.0,
-                errors: Vec::new(),
-                warnings: 0,
-                update_interval: Duration::from_secs(10),
-                status_report_interval: Duration::from_secs(60),
-            })),
-        }
-    }
-
-    /// Set the current sync phase
-    pub fn set_phase(&self, phase: SyncPhase) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.current_phase = phase.clone();
-            match phase {
-                SyncPhase::Evaluating => {
-                    info!("🧠 Smart sync: Evaluating directory changes...");
-                }
-                SyncPhase::DiscoveringDirectories => {
-                    info!("🔍 Discovering directories...");
-                }
-                SyncPhase::DiscoveringFiles => {
-                    info!("🔍 Discovering files...");
-                }
-                SyncPhase::ProcessingFiles => {
-                    info!("📁 Processing files...");
-                }
-                SyncPhase::SavingMetadata => {
-                    info!("💾 Saving directory metadata...");
-                }
-                SyncPhase::Completed => {
-                    self.log_completion_summary();
-                }
-                SyncPhase::Failed(ref error) => {
-                    warn!("❌ Sync failed: {}", error);
-                }
-                _ => {}
-            }
-        }
-    }
-
-    /// Set the current directory being processed
-    pub fn set_current_directory(&self, directory: &str) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.current_directory = directory.to_string();
-            inner.current_file = None;
-            
-            // Check if we should log an update
-            self.maybe_log_progress(&mut inner);
-        }
-    }
-
-    /// Set the current file being processed
-    pub fn set_current_file(&self, file: Option<&str>) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.current_file = file.map(|f| f.to_string());
-            
-            // Check if we should log an update
-            self.maybe_log_progress(&mut inner);
-        }
-    }
-
-    /// Increment directory count (discovered or processed)
-    pub fn add_directories_found(&self, count: usize) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.directories_found += count;
-            self.maybe_log_progress(&mut inner);
-        }
-    }
-
-    /// Increment processed directory count
-    pub fn add_directories_processed(&self, count: usize) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.directories_processed += count;
-            self.maybe_log_progress(&mut inner);
-        }
-    }
-
-    /// Increment file count (discovered or processed)
-    pub fn add_files_found(&self, count: usize) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.files_found += count;
-            self.maybe_log_progress(&mut inner);
-        }
-    }
-
-    /// Increment processed file count
-    pub fn add_files_processed(&self, count: usize, bytes: u64) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.files_processed += count;
-            inner.bytes_processed += bytes;
-            
-            // Update processing rate
-            let elapsed = inner.start_time.elapsed().as_secs_f64();
-            if elapsed > 0.0 {
-                inner.processing_rate_files_per_sec = inner.files_processed as f64 / elapsed;
-            }
-            
-            self.maybe_log_progress(&mut inner);
-        }
-    }
-
-    /// Add an error message
-    pub fn add_error(&self, error: &str) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.errors.push(error.to_string());
-            warn!("🚨 Sync error: {}", error);
-        }
-    }
-
-    /// Add a warning
-    pub fn add_warning(&self, warning: &str) {
-        if let Ok(mut inner) = self.inner.lock() {
-            inner.warnings += 1;
-            warn!("⚠️ Sync warning: {}", warning);
-        }
-    }
-
-    /// Force a progress update (useful for important milestones)
-    pub fn force_update(&self) {
-        if let Ok(mut inner) = self.inner.lock() {
-            self.log_progress_now(&mut inner);
-        }
-    }
-
-    /// Force a status report (detailed progress summary)
-    pub fn force_status_report(&self) {
-        if let Ok(mut inner) = self.inner.lock() {
-            self.log_status_report(&mut inner);
-        }
-    }
-
-    /// Get current progress statistics
-    pub fn get_stats(&self) -> Option<ProgressStats> {
-        self.inner.lock().ok().map(|inner| ProgressStats {
-            elapsed_time: inner.start_time.elapsed(),
-            phase: inner.current_phase.clone(),
-            directories_found: inner.directories_found,
-            directories_processed: inner.directories_processed,
-            files_found: inner.files_found,
-            files_processed: inner.files_processed,
-            bytes_processed: inner.bytes_processed,
-            processing_rate: inner.processing_rate_files_per_sec,
-            errors: inner.errors.len(),
-            warnings: inner.warnings,
-            current_directory: inner.current_directory.clone(),
-            current_file: inner.current_file.clone(),
-        })
-    }
-
-    /// Check if we should log progress and do it if needed
-    fn maybe_log_progress(&self, inner: &mut SyncProgressInner) {
-        let now = Instant::now();
-        
-        // Regular progress updates
-        if now.duration_since(inner.last_update) >= inner.update_interval {
-            self.log_progress_now(inner);
-        }
-        
-        // Status reports (more detailed)
-        if now.duration_since(inner.last_status_report) >= inner.status_report_interval {
-            self.log_status_report(inner);
-        }
-    }
-
-    /// Log progress immediately
-    fn log_progress_now(&self, inner: &mut SyncProgressInner) {
-        let elapsed = inner.start_time.elapsed();
-        let elapsed_secs = elapsed.as_secs();
-        
-        match inner.current_phase {
-            SyncPhase::DiscoveringDirectories | SyncPhase::DiscoveringFiles => {
-                if !inner.current_directory.is_empty() {
-                    info!(
-                        "📊 Discovery Progress: {} dirs, {} files found | 📁 Current: {} | ⏱️ {}m {}s",
-                        inner.directories_found,
-                        inner.files_found,
-                        inner.current_directory,
-                        elapsed_secs / 60,
-                        elapsed_secs % 60
-                    );
-                }
-            }
-            SyncPhase::ProcessingFiles => {
-                let progress_pct = if inner.files_found > 0 {
-                    (inner.files_processed as f64 / inner.files_found as f64 * 100.0) as u32
-                } else {
-                    0
-                };
-                
-                let rate_str = if inner.processing_rate_files_per_sec > 0.0 {
-                    format!(" | 🔄 {:.1} files/sec", inner.processing_rate_files_per_sec)
-                } else {
-                    String::new()
-                };
-                
-                let current_file_str = inner.current_file
-                    .as_ref()
-                    .map(|f| format!(" | 📄 {}", f))
-                    .unwrap_or_default();
-                
-                info!(
-                    "📊 Processing: {}/{} files ({}%){}{} | ⏱️ {}m {}s",
-                    inner.files_processed,
-                    inner.files_found,
-                    progress_pct,
-                    rate_str,
-                    current_file_str,
-                    elapsed_secs / 60,
-                    elapsed_secs % 60
-                );
-            }
-            _ => {
-                if !inner.current_directory.is_empty() {
-                    info!(
-                        "📊 Sync Progress | 📁 Current: {} | ⏱️ {}m {}s",
-                        inner.current_directory,
-                        elapsed_secs / 60,
-                        elapsed_secs % 60
-                    );
-                }
-            }
-        }
-        
-        inner.last_update = Instant::now();
-    }
-
-    /// Log detailed status report
-    fn log_status_report(&self, inner: &mut SyncProgressInner) {
-        let elapsed = inner.start_time.elapsed();
-        let elapsed_secs = elapsed.as_secs();
-        
-        let rate_str = if inner.processing_rate_files_per_sec > 0.0 {
-            format!(" | Rate: {:.1} files/sec", inner.processing_rate_files_per_sec)
-        } else {
-            String::new()
-        };
-        
-        let size_mb = inner.bytes_processed as f64 / (1024.0 * 1024.0);
-        
-        let eta_str = if inner.processing_rate_files_per_sec > 0.0 && inner.files_found > inner.files_processed {
-            let remaining_files = inner.files_found - inner.files_processed;
-            let eta_secs = (remaining_files as f64 / inner.processing_rate_files_per_sec) as u64;
-            format!(" | Est. remaining: {}m {}s", eta_secs / 60, eta_secs % 60)
-        } else {
-            String::new()
-        };
-        
-        info!(
-            "📊 Status Report ({}m {}s elapsed):\n\
-             📁 Directories: {} found, {} processed\n\
-             📄 Files: {} found, {} processed\n\
-             💾 Data: {:.1} MB processed{}{}\n\
-             ⚠️ Issues: {} errors, {} warnings",
-            elapsed_secs / 60,
-            elapsed_secs % 60,
-            inner.directories_found,
-            inner.directories_processed,
-            inner.files_found,
-            inner.files_processed,
-            size_mb,
-            rate_str,
-            eta_str,
-            inner.errors.len(),
-            inner.warnings
-        );
-        
-        inner.last_status_report = Instant::now();
-    }
-
-    /// Log completion summary
-    fn log_completion_summary(&self) {
-        if let Ok(inner) = self.inner.lock() {
-            let elapsed = inner.start_time.elapsed();
-            let elapsed_secs = elapsed.as_secs();
-            let size_mb = inner.bytes_processed as f64 / (1024.0 * 1024.0);
-            
-            let avg_rate = if elapsed.as_secs_f64() > 0.0 {
-                inner.files_processed as f64 / elapsed.as_secs_f64()
-            } else {
-                0.0
-            };
-            
-            info!(
-                "✅ Sync Complete!\n\
-                 📊 Summary:\n\
-                 📁 Directories: {} processed\n\
-                 📄 Files: {} processed\n\
-                 💾 Data: {:.1} MB\n\
-                 ⏱️ Duration: {}m {}s\n\
-                 🔄 Avg rate: {:.1} files/sec\n\
-                 ⚠️ Issues: {} errors, {} warnings",
-                inner.directories_processed,
-                inner.files_processed,
-                size_mb,
-                elapsed_secs / 60,
-                elapsed_secs % 60,
-                avg_rate,
-                inner.errors.len(),
-                inner.warnings
-            );
-            
-            if !inner.errors.is_empty() {
-                warn!("🚨 Errors encountered during sync:");
-                for (i, error) in inner.errors.iter().enumerate() {
-                    warn!("  {}. {}", i + 1, error);
-                }
-            }
-        }
-    }
-}
-
-impl Default for SyncProgress {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-/// Snapshot of progress statistics
-#[derive(Debug, Clone)]
-pub struct ProgressStats {
-    pub elapsed_time: Duration,
-    pub phase: SyncPhase,
-    pub directories_found: usize,
-    pub directories_processed: usize,
-    pub files_found: usize,
-    pub files_processed: usize,
-    pub bytes_processed: u64,
-    pub processing_rate: f64,
-    pub errors: usize,
-    pub warnings: usize,
-    pub current_directory: String,
-    pub current_file: Option<String>,
-}
-
-impl ProgressStats {
-    /// Get progress percentage for files (0-100)
-    pub fn files_progress_percent(&self) -> f64 {
-        if self.files_found > 0 {
-            (self.files_processed as f64 / self.files_found as f64) * 100.0
-        } else {
-            0.0
-        }
-    }
-    
-    /// Get estimated time remaining in seconds
-    pub fn estimated_time_remaining(&self) -> Option<Duration> {
-        if self.processing_rate > 0.0 && self.files_found > self.files_processed {
-            let remaining_files = self.files_found - self.files_processed;
-            let eta_secs = (remaining_files as f64 / self.processing_rate) as u64;
-            Some(Duration::from_secs(eta_secs))
-        } else {
-            None
-        }
-    }
-    
-    /// Get human-readable data size processed
-    pub fn data_size_mb(&self) -> f64 {
-        self.bytes_processed as f64 / (1024.0 * 1024.0)
-    }
-}

src/services/webdav/progress_shim.rs

@@ -0,0 +1,109 @@
+// Simplified progress tracking shim for backward compatibility
+// This provides basic types that do nothing but maintain API compatibility
+
+use std::time::Duration;
+
+/// Simplified progress tracker that just logs
+#[derive(Debug, Clone)]
+pub struct SyncProgress {
+    // Empty struct - all progress tracking is now just logging
+}
+
+/// Simplified sync phases for basic logging
+#[derive(Debug, Clone, PartialEq)]
+pub enum SyncPhase {
+    Initializing,
+    Evaluating,
+    DiscoveringDirectories,
+    DiscoveringFiles,
+    ProcessingFiles,
+    SavingMetadata,
+    Completed,
+    Failed(String),
+    Retrying { attempt: u32, category: String, delay_ms: u64 },
+}
+
+/// Empty progress stats for compatibility
+#[derive(Debug, Clone)]
+pub struct ProgressStats {
+    pub phase: SyncPhase,
+    pub elapsed_time: Duration,
+    pub directories_found: usize,
+    pub directories_processed: usize,
+    pub files_found: usize,
+    pub files_processed: usize,
+    pub bytes_processed: u64,
+    pub processing_rate: f64,
+    pub current_directory: String,
+    pub current_file: Option<String>,
+    pub errors: Vec<String>,
+    pub warnings: usize,
+}
+
+impl SyncProgress {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    pub fn set_phase(&self, _phase: SyncPhase) {
+        // Do nothing - progress tracking simplified to basic logging
+    }
+
+    pub fn set_current_directory(&self, _directory: &str) {
+        // Do nothing - progress tracking simplified to basic logging
+    }
+
+    pub fn set_current_file(&self, _file: Option<&str>) {
+        // Do nothing - progress tracking simplified to basic logging
+    }
+
+    pub fn add_directories_found(&self, _count: usize) {
+        // Do nothing - progress tracking simplified to basic logging
+    }
+
+    pub fn add_files_found(&self, _count: usize) {
+        // Do nothing - progress tracking simplified to basic logging
+    }
+
+    pub fn add_files_processed(&self, _count: usize, _bytes: u64) {
+        // Do nothing - progress tracking simplified to basic logging
+    }
+
+    pub fn add_error(&self, _error: &str) {
+        // Do nothing - progress tracking simplified to basic logging
+    }
+
+    pub fn get_stats(&self) -> Option<ProgressStats> {
+        // Return dummy stats for compatibility
+        Some(ProgressStats {
+            phase: SyncPhase::Completed,
+            elapsed_time: Duration::from_secs(0),
+            directories_found: 0,
+            directories_processed: 0,
+            files_found: 0,
+            files_processed: 0,
+            bytes_processed: 0,
+            processing_rate: 0.0,
+            current_directory: String::new(),
+            current_file: None,
+            errors: Vec::new(),
+            warnings: 0,
+        })
+    }
+}
+
+impl ProgressStats {
+    pub fn files_progress_percent(&self) -> f64 {
+        0.0 // Simplified - no real progress tracking
+    }
+
+    pub fn estimated_time_remaining(&self) -> Option<Duration> {
+        None // Simplified - no real progress tracking
+    }
+}
+
+impl Default for SyncProgress {
+    fn default() -> Self {
+        Self::new()
+    }
+}

src/services/webdav/smart_sync.rs

@@ -5,10 +5,12 @@ use tracing::{debug, info, warn};
 use uuid::Uuid;
 
 use crate::{AppState, models::{CreateWebDAVDirectory, FileIngestionInfo}};
-use super::{WebDAVService, SyncProgress, SyncPhase};
+use crate::webdav_xml_parser::compare_etags;
+use super::{WebDAVService, SyncProgress};
 
 /// Smart sync service that provides intelligent WebDAV synchronization
 /// by comparing directory ETags to avoid unnecessary scans
+#[derive(Clone)]
 pub struct SmartSyncService {
     state: Arc<AppState>,
 }
@@ -46,17 +48,19 @@ impl SmartSyncService {
         Self { state }
     }
 
+    /// Get access to the application state (primarily for testing)
+    pub fn state(&self) -> &Arc<AppState> {
+        &self.state
+    }
+
     /// Evaluates whether sync is needed and determines the best strategy
     pub async fn evaluate_sync_need(
         &self,
         user_id: Uuid,
         webdav_service: &WebDAVService,
         folder_path: &str,
-        progress: Option<&SyncProgress>,
+        _progress: Option<&SyncProgress>, // Simplified: no complex progress tracking
     ) -> Result<SmartSyncDecision> {
-        if let Some(progress) = progress {
-            progress.set_phase(SyncPhase::Evaluating);
-        }
         info!("🧠 Evaluating smart sync for folder: {}", folder_path);
         
         // Get all known directory ETags from database in bulk
@@ -87,7 +91,8 @@ impl SmartSyncService {
                 for directory in &root_discovery.directories {
                     match relevant_dirs.get(&directory.relative_path) {
                         Some(known_etag) => {
-                            if known_etag != &directory.etag {
+                            // Use proper ETag comparison that handles weak/strong semantics
+                            if !compare_etags(known_etag, &directory.etag) {
                                 info!("Directory changed: {} (old: {}, new: {})", 
                                       directory.relative_path, known_etag, directory.etag);
                                 changed_directories.push(directory.relative_path.clone());
@@ -99,22 +104,42 @@ impl SmartSyncService {
                         }
                     }
                 }
+
+                // Check for deleted directories (directories that were known but not discovered)
+                let discovered_paths: std::collections::HashSet<String> = root_discovery.directories
+                    .iter()
+                    .map(|d| d.relative_path.clone())
+                    .collect();
                 
-                // If no changes detected in immediate subdirectories, we can skip
-                if changed_directories.is_empty() && new_directories.is_empty() {
+                let mut deleted_directories = Vec::new();
+                for (known_path, _) in &relevant_dirs {
+                    if !discovered_paths.contains(known_path) {
+                        info!("Directory deleted: {}", known_path);
+                        deleted_directories.push(known_path.clone());
+                    }
+                }
+                
+                // If directories were deleted, we need to clean them up
+                if !deleted_directories.is_empty() {
+                    info!("Found {} deleted directories that need cleanup", deleted_directories.len());
+                    // We'll handle deletion in the sync operation itself
+                }
+                
+                // If no changes detected and no deletions, we can skip
+                if changed_directories.is_empty() && new_directories.is_empty() && deleted_directories.is_empty() {
                     info!("✅ Smart sync: No directory changes detected, sync can be skipped");
                     return Ok(SmartSyncDecision::SkipSync);
                 }
                 
                 // Determine strategy based on scope of changes
-                let total_changes = changed_directories.len() + new_directories.len();
+                let total_changes = changed_directories.len() + new_directories.len() + deleted_directories.len();
                 let total_known = relevant_dirs.len();
                 let change_ratio = total_changes as f64 / total_known.max(1) as f64;
                 
-                if change_ratio > 0.3 || new_directories.len() > 5 {
-                    // Too many changes, do full deep scan for efficiency
-                    info!("📁 Smart sync: Large changes detected ({} changed, {} new), using full deep scan", 
-                          changed_directories.len(), new_directories.len());
+                if change_ratio > 0.3 || new_directories.len() > 5 || !deleted_directories.is_empty() {
+                    // Too many changes or deletions detected, do full deep scan for efficiency
+                    info!("📁 Smart sync: Large changes detected ({} changed, {} new, {} deleted), using full deep scan", 
+                          changed_directories.len(), new_directories.len(), deleted_directories.len());
                     return Ok(SmartSyncDecision::RequiresSync(SmartSyncStrategy::FullDeepScan));
                 } else {
                     // Targeted scan of changed directories
@@ -138,16 +163,16 @@ impl SmartSyncService {
         webdav_service: &WebDAVService,
         folder_path: &str,
         strategy: SmartSyncStrategy,
-        progress: Option<&SyncProgress>,
+        _progress: Option<&SyncProgress>, // Simplified: no complex progress tracking
     ) -> Result<SmartSyncResult> {
         match strategy {
             SmartSyncStrategy::FullDeepScan => {
                 info!("🔍 Performing full deep scan for: {}", folder_path);
-                self.perform_full_deep_scan(user_id, webdav_service, folder_path, progress).await
+                self.perform_full_deep_scan(user_id, webdav_service, folder_path, _progress).await
             }
             SmartSyncStrategy::TargetedScan(target_dirs) => {
                 info!("🎯 Performing targeted scan of {} directories", target_dirs.len());
-                self.perform_targeted_scan(user_id, webdav_service, target_dirs, progress).await
+                self.perform_targeted_scan(user_id, webdav_service, target_dirs, _progress).await
             }
         }
     }
@@ -158,21 +183,19 @@ impl SmartSyncService {
         user_id: Uuid,
         webdav_service: &WebDAVService,
         folder_path: &str,
-        progress: Option<&SyncProgress>,
+        _progress: Option<&SyncProgress>, // Simplified: no complex progress tracking
     ) -> Result<Option<SmartSyncResult>> {
-        match self.evaluate_sync_need(user_id, webdav_service, folder_path, progress).await? {
+        match self.evaluate_sync_need(user_id, webdav_service, folder_path, _progress).await? {
             SmartSyncDecision::SkipSync => {
                 info!("✅ Smart sync: Skipping sync for {} - no changes detected", folder_path);
-                if let Some(progress) = progress {
-                    progress.set_phase(SyncPhase::Completed);
-                }
+                // Simplified: basic logging instead of complex progress tracking
+                info!("Smart sync completed - no changes detected");
                 Ok(None)
             }
             SmartSyncDecision::RequiresSync(strategy) => {
-                let result = self.perform_smart_sync(user_id, webdav_service, folder_path, strategy, progress).await?;
-                if let Some(progress) = progress {
-                    progress.set_phase(SyncPhase::Completed);
-                }
+                let result = self.perform_smart_sync(user_id, webdav_service, folder_path, strategy, _progress).await?;
+                // Simplified: basic logging instead of complex progress tracking
+                info!("Smart sync completed - changes processed");
                 Ok(Some(result))
             }
         }
@@ -184,42 +207,58 @@ impl SmartSyncService {
         user_id: Uuid,
         webdav_service: &WebDAVService,
         folder_path: &str,
-        progress: Option<&SyncProgress>,
+        _progress: Option<&SyncProgress>, // Simplified: no complex progress tracking
     ) -> Result<SmartSyncResult> {
-        let discovery_result = webdav_service.discover_files_and_directories_with_progress(folder_path, true, progress).await?;
+        let discovery_result = webdav_service.discover_files_and_directories_with_progress(folder_path, true, _progress).await?;
         
         info!("Deep scan found {} files and {} directories in folder {}", 
               discovery_result.files.len(), discovery_result.directories.len(), folder_path);
         
-        // Update progress phase for metadata saving
-        if let Some(progress) = progress {
-            progress.set_phase(SyncPhase::SavingMetadata);
-        }
+        // Simplified: basic logging instead of complex progress tracking
+        info!("Saving metadata for scan results");
         
-        // Save all discovered directories to database for ETag tracking
-        let mut directories_saved = 0;
-        for directory_info in &discovery_result.directories {
-            let webdav_directory = CreateWebDAVDirectory {
+        // Save all discovered directories atomically using bulk operations
+        let directories_to_save: Vec<CreateWebDAVDirectory> = discovery_result.directories
+            .iter()
+            .map(|directory_info| CreateWebDAVDirectory {
                 user_id,
                 directory_path: directory_info.relative_path.clone(),
                 directory_etag: directory_info.etag.clone(),
                 file_count: 0, // Will be updated by stats
                 total_size_bytes: 0, // Will be updated by stats
-            };
-            
-            match self.state.db.create_or_update_webdav_directory(&webdav_directory).await {
-                Ok(_) => {
-                    debug!("Saved directory ETag: {} -> {}", directory_info.relative_path, directory_info.etag);
-                    directories_saved += 1;
-                }
-                Err(e) => {
-                    warn!("Failed to save directory ETag for {}: {}", directory_info.relative_path, e);
+            })
+            .collect();
+
+        match self.state.db.sync_webdav_directories(user_id, &directories_to_save).await {
+            Ok((saved_directories, deleted_count)) => {
+                info!("✅ Atomic sync completed: {} directories updated/created, {} deleted", 
+                      saved_directories.len(), deleted_count);
+                
+                if deleted_count > 0 {
+                    info!("🗑️ Cleaned up {} orphaned directory records", deleted_count);
                 }
             }
+            Err(e) => {
+                warn!("Failed to perform atomic directory sync: {}", e);
+                // Fallback to individual saves if atomic operation fails
+                let mut directories_saved = 0;
+                for directory_info in &discovery_result.directories {
+                    let webdav_directory = CreateWebDAVDirectory {
+                        user_id,
+                        directory_path: directory_info.relative_path.clone(),
+                        directory_etag: directory_info.etag.clone(),
+                        file_count: 0,
+                        total_size_bytes: 0,
+                    };
+                    
+                    if let Ok(_) = self.state.db.create_or_update_webdav_directory(&webdav_directory).await {
+                        directories_saved += 1;
+                    }
+                }
+                info!("Fallback: Saved ETags for {}/{} directories", directories_saved, discovery_result.directories.len());
+            }
         }
         
-        info!("Saved ETags for {}/{} directories", directories_saved, discovery_result.directories.len());
-        
         Ok(SmartSyncResult {
             files: discovery_result.files,
             directories: discovery_result.directories.clone(),
@@ -235,7 +274,7 @@ impl SmartSyncService {
         user_id: Uuid,
         webdav_service: &WebDAVService,
         target_directories: Vec<String>,
-        progress: Option<&SyncProgress>,
+        _progress: Option<&SyncProgress>, // Simplified: no complex progress tracking
     ) -> Result<SmartSyncResult> {
         let mut all_files = Vec::new();
         let mut all_directories = Vec::new();
@@ -243,28 +282,48 @@ impl SmartSyncService {
 
         // Scan each target directory recursively
         for target_dir in &target_directories {
-            if let Some(progress) = progress {
-                progress.set_current_directory(target_dir);
-            }
+            // Simplified: basic logging instead of complex progress tracking
+            info!("Scanning target directory: {}", target_dir);
             
-            match webdav_service.discover_files_and_directories_with_progress(target_dir, true, progress).await {
+            match webdav_service.discover_files_and_directories_with_progress(target_dir, true, _progress).await {
                 Ok(discovery_result) => {
                     all_files.extend(discovery_result.files);
                     
-                    // Save directory ETags for this scan
-                    for directory_info in &discovery_result.directories {
-                        let webdav_directory = CreateWebDAVDirectory {
+                    // Collect directory info for bulk update later
+                    let directories_to_save: Vec<CreateWebDAVDirectory> = discovery_result.directories
+                        .iter()
+                        .map(|directory_info| CreateWebDAVDirectory {
                             user_id,
                             directory_path: directory_info.relative_path.clone(),
                             directory_etag: directory_info.etag.clone(),
                             file_count: 0,
                             total_size_bytes: 0,
-                        };
-                        
-                        if let Err(e) = self.state.db.create_or_update_webdav_directory(&webdav_directory).await {
-                            warn!("Failed to save directory ETag for {}: {}", directory_info.relative_path, e);
-                        } else {
-                            debug!("Updated directory ETag: {} -> {}", directory_info.relative_path, directory_info.etag);
+                        })
+                        .collect();
+
+                    // Save directories using bulk operation
+                    if !directories_to_save.is_empty() {
+                        match self.state.db.bulk_create_or_update_webdav_directories(&directories_to_save).await {
+                            Ok(saved_directories) => {
+                                debug!("Bulk updated {} directory ETags for target scan", saved_directories.len());
+                            }
+                            Err(e) => {
+                                warn!("Failed bulk update for target scan, falling back to individual saves: {}", e);
+                                // Fallback to individual saves
+                                for directory_info in &discovery_result.directories {
+                                    let webdav_directory = CreateWebDAVDirectory {
+                                        user_id,
+                                        directory_path: directory_info.relative_path.clone(),
+                                        directory_etag: directory_info.etag.clone(),
+                                        file_count: 0,
+                                        total_size_bytes: 0,
+                                    };
+                                    
+                                    if let Err(e) = self.state.db.create_or_update_webdav_directory(&webdav_directory).await {
+                                        warn!("Failed to save directory ETag for {}: {}", directory_info.relative_path, e);
+                                    }
+                                }
+                            }
                         }
                     }
                     
@@ -277,10 +336,8 @@ impl SmartSyncService {
             }
         }
 
-        // Update progress phase for metadata saving
-        if let Some(progress) = progress {
-            progress.set_phase(SyncPhase::SavingMetadata);
-        }
+        // Simplified: basic logging instead of complex progress tracking
+        info!("Saving metadata for scan results");
         
         info!("Targeted scan completed: {} directories scanned, {} files found", 
               directories_scanned, all_files.len());

src/services/webdav/tests/atomic_operations_tests.rs

@@ -0,0 +1,259 @@
+use std::sync::Arc;
+use uuid::Uuid;
+use tokio;
+use crate::models::CreateWebDAVDirectory;
+use crate::test_utils::TestContext;
+use crate::db::Database;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    async fn setup_test_database() -> Arc<Database> {
+        let ctx = TestContext::new().await;
+        Arc::new(ctx.state.db.clone())
+    }
+
+    #[tokio::test]
+    async fn test_bulk_create_or_update_atomic() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        let directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir1".to_string(),
+                directory_etag: "etag1".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir2".to_string(),
+                directory_etag: "etag2".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir3".to_string(),
+                directory_etag: "etag3".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+        ];
+
+        // Test bulk operation
+        let result = db.bulk_create_or_update_webdav_directories(&directories).await;
+        assert!(result.is_ok());
+        
+        let saved_directories = result.unwrap();
+        assert_eq!(saved_directories.len(), 3);
+        
+        // Verify all directories were saved with correct ETags
+        for (original, saved) in directories.iter().zip(saved_directories.iter()) {
+            assert_eq!(original.directory_path, saved.directory_path);
+            assert_eq!(original.directory_etag, saved.directory_etag);
+            assert_eq!(original.user_id, saved.user_id);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_sync_webdav_directories_atomic() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // First, create some initial directories
+        let initial_directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir1".to_string(),
+                directory_etag: "etag1".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir2".to_string(),
+                directory_etag: "etag2".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+        ];
+
+        let _ = db.bulk_create_or_update_webdav_directories(&initial_directories).await.unwrap();
+
+        // Now sync with a new set that has one update, one delete, and one new
+        let sync_directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir1".to_string(),
+                directory_etag: "etag1_updated".to_string(), // Updated
+                file_count: 5,
+                total_size_bytes: 1024,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir3".to_string(), // New
+                directory_etag: "etag3".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+            // dir2 is missing, should be deleted
+        ];
+
+        let result = db.sync_webdav_directories(user_id, &sync_directories).await;
+        assert!(result.is_ok());
+        
+        let (updated_directories, deleted_count) = result.unwrap();
+        
+        // Should have 2 directories (dir1 updated, dir3 new)
+        assert_eq!(updated_directories.len(), 2);
+        
+        // Should have deleted 1 directory (dir2)
+        assert_eq!(deleted_count, 1);
+        
+        // Verify the updated directory has the new ETag
+        let dir1 = updated_directories.iter()
+            .find(|d| d.directory_path == "/test/dir1")
+            .unwrap();
+        assert_eq!(dir1.directory_etag, "etag1_updated");
+        assert_eq!(dir1.file_count, 5);
+        assert_eq!(dir1.total_size_bytes, 1024);
+        
+        // Verify the new directory exists
+        let dir3 = updated_directories.iter()
+            .find(|d| d.directory_path == "/test/dir3")
+            .unwrap();
+        assert_eq!(dir3.directory_etag, "etag3");
+    }
+
+    #[tokio::test]
+    async fn test_delete_missing_directories() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // Create some directories
+        let directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir1".to_string(),
+                directory_etag: "etag1".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir2".to_string(),
+                directory_etag: "etag2".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir3".to_string(),
+                directory_etag: "etag3".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+        ];
+
+        let _ = db.bulk_create_or_update_webdav_directories(&directories).await.unwrap();
+
+        // Delete directories not in this list (should delete dir2 and dir3)
+        let existing_paths = vec!["/test/dir1".to_string()];
+        let deleted_count = db.delete_missing_webdav_directories(user_id, &existing_paths).await.unwrap();
+        
+        assert_eq!(deleted_count, 2);
+        
+        // Verify only dir1 remains
+        let remaining_directories = db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(remaining_directories.len(), 1);
+        assert_eq!(remaining_directories[0].directory_path, "/test/dir1");
+    }
+
+    #[tokio::test]
+    async fn test_atomic_rollback_on_failure() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // Create a directory that would conflict
+        let initial_dir = CreateWebDAVDirectory {
+            user_id,
+            directory_path: "/test/dir1".to_string(),
+            directory_etag: "etag1".to_string(),
+            file_count: 0,
+            total_size_bytes: 0,
+        };
+        
+        let _ = db.create_or_update_webdav_directory(&initial_dir).await.unwrap();
+
+        // Try to bulk insert with one invalid entry that should cause rollback
+        let directories_with_invalid = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir2".to_string(),
+                directory_etag: "etag2".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+            CreateWebDAVDirectory {
+                user_id: Uuid::nil(), // Invalid user ID should cause failure
+                directory_path: "/test/dir3".to_string(),
+                directory_etag: "etag3".to_string(),
+                file_count: 0,
+                total_size_bytes: 0,
+            },
+        ];
+
+        // This should fail and rollback
+        let result = db.bulk_create_or_update_webdav_directories(&directories_with_invalid).await;
+        assert!(result.is_err());
+        
+        // Verify that no partial changes were made (only original dir1 should exist)
+        let directories = db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(directories.len(), 1);
+        assert_eq!(directories[0].directory_path, "/test/dir1");
+    }
+
+    #[tokio::test]
+    async fn test_concurrent_directory_updates() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // Spawn multiple concurrent tasks that try to update the same directory
+        let mut handles = vec![];
+        
+        for i in 0..10 {
+            let db_clone = db.clone();
+            let handle = tokio::spawn(async move {
+                let directory = CreateWebDAVDirectory {
+                    user_id,
+                    directory_path: "/test/concurrent".to_string(),
+                    directory_etag: format!("etag_{}", i),
+                    file_count: i as i64,
+                    total_size_bytes: (i * 1024) as i64,
+                };
+                
+                db_clone.create_or_update_webdav_directory(&directory).await
+            });
+            handles.push(handle);
+        }
+        
+        // Wait for all tasks to complete
+        let results: Vec<_> = futures::future::join_all(handles).await;
+        
+        // All operations should succeed (last writer wins)
+        for result in results {
+            assert!(result.is_ok());
+            assert!(result.unwrap().is_ok());
+        }
+        
+        // Verify final state
+        let directories = db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(directories.len(), 1);
+        assert_eq!(directories[0].directory_path, "/test/concurrent");
+        // ETag should be from one of the concurrent updates
+        assert!(directories[0].directory_etag.starts_with("etag_"));
+    }
+}

src/services/webdav/tests/critical_fixes_tests.rs

@@ -0,0 +1,372 @@
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use uuid::Uuid;
+use tokio;
+use crate::models::CreateWebDAVDirectory;
+use crate::db::Database;
+use crate::test_utils::TestContext;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Integration test that validates the race condition fix
+    /// Tests that concurrent directory updates are atomic and consistent
+    #[tokio::test]
+    async fn test_race_condition_fix_atomic_updates() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // Create initial directories
+        let initial_directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir1".to_string(),
+                directory_etag: "initial_etag1".to_string(),
+                file_count: 5,
+                total_size_bytes: 1024,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/dir2".to_string(),
+                directory_etag: "initial_etag2".to_string(),
+                file_count: 10,
+                total_size_bytes: 2048,
+            },
+        ];
+        
+        let _ = db.bulk_create_or_update_webdav_directories(&initial_directories).await.unwrap();
+        
+        // Simulate race condition: multiple tasks trying to update directories simultaneously
+        let mut handles = vec![];
+        
+        for i in 0..5 {
+            let db_clone = Arc::clone(&db);
+            let handle = tokio::spawn(async move {
+                let updated_directories = vec![
+                    CreateWebDAVDirectory {
+                        user_id,
+                        directory_path: "/test/dir1".to_string(),
+                        directory_etag: format!("race_etag1_{}", i),
+                        file_count: 5 + i as i64,
+                        total_size_bytes: 1024 + (i * 100) as i64,
+                    },
+                    CreateWebDAVDirectory {
+                        user_id,
+                        directory_path: "/test/dir2".to_string(),
+                        directory_etag: format!("race_etag2_{}", i),
+                        file_count: 10 + i as i64,
+                        total_size_bytes: 2048 + (i * 200) as i64,
+                    },
+                    CreateWebDAVDirectory {
+                        user_id,
+                        directory_path: format!("/test/new_dir_{}", i),
+                        directory_etag: format!("new_etag_{}", i),
+                        file_count: i as i64,
+                        total_size_bytes: (i * 512) as i64,
+                    },
+                ];
+                
+                // Use the atomic sync operation
+                db_clone.sync_webdav_directories(user_id, &updated_directories).await
+            });
+            handles.push(handle);
+        }
+        
+        // Wait for all operations to complete
+        let results: Vec<_> = futures::future::join_all(handles).await;
+        
+        // All operations should succeed (transactions ensure atomicity)
+        for result in results {
+            assert!(result.is_ok());
+            let sync_result = result.unwrap();
+            assert!(sync_result.is_ok());
+        }
+        
+        // Final state should be consistent
+        let final_directories = db.list_webdav_directories(user_id).await.unwrap();
+        
+        // Should have 3 directories (dir1, dir2, and one of the new_dir_X)
+        assert_eq!(final_directories.len(), 3);
+        
+        // All ETags should be from one consistent transaction
+        let dir1 = final_directories.iter().find(|d| d.directory_path == "/test/dir1").unwrap();
+        let dir2 = final_directories.iter().find(|d| d.directory_path == "/test/dir2").unwrap();
+        
+        // ETags should be from the same transaction (both should end with same number)
+        let etag1_suffix = dir1.directory_etag.chars().last().unwrap();
+        let etag2_suffix = dir2.directory_etag.chars().last().unwrap();
+        assert_eq!(etag1_suffix, etag2_suffix, "ETags should be from same atomic transaction");
+    }
+
+    /// Test that validates directory deletion detection works correctly
+    #[tokio::test]
+    async fn test_deletion_detection_fix() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // Create initial directories
+        let initial_directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/documents/folder1".to_string(),
+                directory_etag: "etag1".to_string(),
+                file_count: 5,
+                total_size_bytes: 1024,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/documents/folder2".to_string(),
+                directory_etag: "etag2".to_string(),
+                file_count: 3,
+                total_size_bytes: 512,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/documents/folder3".to_string(),
+                directory_etag: "etag3".to_string(),
+                file_count: 8,
+                total_size_bytes: 2048,
+            },
+        ];
+        
+        let _ = db.bulk_create_or_update_webdav_directories(&initial_directories).await.unwrap();
+        
+        // Verify all 3 directories exist
+        let directories_before = db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(directories_before.len(), 3);
+        
+        // Simulate sync where folder2 and folder3 are deleted from WebDAV server
+        let current_directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/documents/folder1".to_string(),
+                directory_etag: "etag1_updated".to_string(), // Updated
+                file_count: 6,
+                total_size_bytes: 1200,
+            },
+            // folder2 and folder3 are missing (deleted from server)
+        ];
+        
+        // Use atomic sync which should detect and remove deleted directories
+        let (updated_directories, deleted_count) = db.sync_webdav_directories(user_id, &current_directories).await.unwrap();
+        
+        // Should have 1 updated directory and 2 deletions
+        assert_eq!(updated_directories.len(), 1);
+        assert_eq!(deleted_count, 2);
+        
+        // Verify only folder1 remains with updated ETag
+        let final_directories = db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(final_directories.len(), 1);
+        assert_eq!(final_directories[0].directory_path, "/documents/folder1");
+        assert_eq!(final_directories[0].directory_etag, "etag1_updated");
+        assert_eq!(final_directories[0].file_count, 6);
+    }
+
+    /// Test that validates proper ETag comparison handling
+    #[tokio::test]
+    async fn test_etag_comparison_fix() {
+        use crate::webdav_xml_parser::{compare_etags, weak_compare_etags, strong_compare_etags};
+        
+        // Test weak vs strong ETag comparison
+        let strong_etag = "\"abc123\"";
+        let weak_etag = "W/\"abc123\"";
+        let different_etag = "\"def456\"";
+        
+        // Smart comparison should handle weak/strong equivalence
+        assert!(compare_etags(strong_etag, weak_etag), "Smart comparison should match weak and strong with same content");
+        assert!(!compare_etags(strong_etag, different_etag), "Smart comparison should reject different content");
+        
+        // Weak comparison should match regardless of weak/strong
+        assert!(weak_compare_etags(strong_etag, weak_etag), "Weak comparison should match");
+        assert!(weak_compare_etags(weak_etag, strong_etag), "Weak comparison should be symmetrical");
+        
+        // Strong comparison should reject weak ETags
+        assert!(!strong_compare_etags(strong_etag, weak_etag), "Strong comparison should reject weak ETags");
+        assert!(!strong_compare_etags(weak_etag, strong_etag), "Strong comparison should reject weak ETags");
+        assert!(strong_compare_etags(strong_etag, "\"abc123\""), "Strong comparison should match strong ETags");
+        
+        // Test case sensitivity (ETags should be case-sensitive per RFC)
+        assert!(!compare_etags("\"ABC123\"", "\"abc123\""), "ETags should be case-sensitive");
+        
+        // Test various real-world formats
+        let nextcloud_etag = "\"5f3e7e8a9b2c1d4\"";
+        let apache_etag = "\"1234-567-890abcdef\"";
+        let nginx_weak = "W/\"5f3e7e8a\"";
+        
+        assert!(!compare_etags(nextcloud_etag, apache_etag), "Different ETag values should not match");
+        assert!(weak_compare_etags(nginx_weak, "\"5f3e7e8a\""), "Weak and strong with same content should match in weak comparison");
+    }
+
+    /// Test performance of bulk operations vs individual operations
+    #[tokio::test]
+    async fn test_bulk_operations_performance() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // Create test data
+        let test_directories: Vec<_> = (0..100).map(|i| CreateWebDAVDirectory {
+            user_id,
+            directory_path: format!("/test/perf/dir{}", i),
+            directory_etag: format!("etag{}", i),
+            file_count: i as i64,
+            total_size_bytes: (i * 1024) as i64,
+        }).collect();
+        
+        // Test individual operations (old way)
+        let start_individual = Instant::now();
+        for directory in &test_directories {
+            let _ = db.create_or_update_webdav_directory(directory).await;
+        }
+        let individual_duration = start_individual.elapsed();
+        
+        // Clear data
+        let _ = db.clear_webdav_directories(user_id).await;
+        
+        // Test bulk operation (new way)
+        let start_bulk = Instant::now();
+        let _ = db.bulk_create_or_update_webdav_directories(&test_directories).await;
+        let bulk_duration = start_bulk.elapsed();
+        
+        // Bulk should be faster
+        assert!(bulk_duration < individual_duration, 
+               "Bulk operations should be faster than individual operations. Bulk: {:?}, Individual: {:?}", 
+               bulk_duration, individual_duration);
+        
+        // Verify all data was saved correctly
+        let saved_directories = db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(saved_directories.len(), 100);
+    }
+
+    /// Test transaction rollback behavior
+    #[tokio::test]
+    async fn test_transaction_rollback_consistency() {
+        let db = setup_test_database().await;
+        let user_id = Uuid::new_v4();
+        
+        // Create some initial data
+        let initial_directory = CreateWebDAVDirectory {
+            user_id,
+            directory_path: "/test/initial".to_string(),
+            directory_etag: "initial_etag".to_string(),
+            file_count: 1,
+            total_size_bytes: 100,
+        };
+        
+        let _ = db.create_or_update_webdav_directory(&initial_directory).await.unwrap();
+        
+        // Try to create directories where one has invalid data that should cause rollback
+        let directories_with_failure = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/valid1".to_string(),
+                directory_etag: "valid_etag1".to_string(),
+                file_count: 2,
+                total_size_bytes: 200,
+            },
+            CreateWebDAVDirectory {
+                user_id: Uuid::nil(), // This should cause a constraint violation
+                directory_path: "/test/invalid".to_string(),
+                directory_etag: "invalid_etag".to_string(),
+                file_count: 3,
+                total_size_bytes: 300,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/test/valid2".to_string(),
+                directory_etag: "valid_etag2".to_string(),
+                file_count: 4,
+                total_size_bytes: 400,
+            },
+        ];
+        
+        // This should fail and rollback
+        let result = db.bulk_create_or_update_webdav_directories(&directories_with_failure).await;
+        assert!(result.is_err(), "Transaction should fail due to invalid user_id");
+        
+        // Verify that no partial changes were made - only initial directory should exist
+        let final_directories = db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(final_directories.len(), 1);
+        assert_eq!(final_directories[0].directory_path, "/test/initial");
+        assert_eq!(final_directories[0].directory_etag, "initial_etag");
+    }
+
+    /// Integration test simulating real WebDAV sync scenario
+    #[tokio::test]
+    async fn test_full_sync_integration() {
+        use crate::services::webdav::SmartSyncService;
+        
+        let app_state = Arc::new(setup_test_app_state().await);
+        let smart_sync = SmartSyncService::new(app_state.clone());
+        let user_id = Uuid::new_v4();
+        
+        // Simulate initial sync with some directories
+        let initial_directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/documents".to_string(),
+                directory_etag: "docs_etag_v1".to_string(),
+                file_count: 10,
+                total_size_bytes: 10240,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/pictures".to_string(),
+                directory_etag: "pics_etag_v1".to_string(),
+                file_count: 5,
+                total_size_bytes: 51200,
+            },
+        ];
+        
+        let (saved_dirs, _) = app_state.db.sync_webdav_directories(user_id, &initial_directories).await.unwrap();
+        assert_eq!(saved_dirs.len(), 2);
+        
+        // Simulate second sync with changes
+        let updated_directories = vec![
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/documents".to_string(),
+                directory_etag: "docs_etag_v2".to_string(), // Changed
+                file_count: 12,
+                total_size_bytes: 12288,
+            },
+            CreateWebDAVDirectory {
+                user_id,
+                directory_path: "/videos".to_string(), // New directory
+                directory_etag: "videos_etag_v1".to_string(),
+                file_count: 3,
+                total_size_bytes: 102400,
+            },
+            // /pictures directory was deleted from server
+        ];
+        
+        let (updated_dirs, deleted_count) = app_state.db.sync_webdav_directories(user_id, &updated_directories).await.unwrap();
+        
+        // Should have 2 directories (updated documents + new videos) and 1 deletion (pictures)
+        assert_eq!(updated_dirs.len(), 2);
+        assert_eq!(deleted_count, 1);
+        
+        // Verify final state
+        let final_dirs = app_state.db.list_webdav_directories(user_id).await.unwrap();
+        assert_eq!(final_dirs.len(), 2);
+        
+        let docs_dir = final_dirs.iter().find(|d| d.directory_path == "/documents").unwrap();
+        assert_eq!(docs_dir.directory_etag, "docs_etag_v2");
+        assert_eq!(docs_dir.file_count, 12);
+        
+        let videos_dir = final_dirs.iter().find(|d| d.directory_path == "/videos").unwrap();
+        assert_eq!(videos_dir.directory_etag, "videos_etag_v1");
+        assert_eq!(videos_dir.file_count, 3);
+    }
+
+    // Helper functions
+    async fn setup_test_database() -> Arc<Database> {
+        let ctx = TestContext::new().await;
+        Arc::new(ctx.state.db.clone())
+    }
+
+    async fn setup_test_app_state() -> crate::AppState {
+        let ctx = TestContext::new().await;
+        (*ctx.state).clone()
+    }
+}

src/services/webdav/tests/deletion_detection_tests.rs

@@ -0,0 +1,332 @@
+use std::sync::Arc;
+use uuid::Uuid;
+use tokio;
+
+use crate::test_utils::TestContext;
+use crate::models::{CreateWebDAVDirectory, CreateUser, UserRole};
+use crate::services::webdav::{SmartSyncService, SmartSyncDecision, SmartSyncStrategy, WebDAVService};
+use crate::services::webdav::config::WebDAVConfig;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Test that smart sync detects when directories are deleted from the WebDAV server
+    #[tokio::test]
+    async fn test_deletion_detection_triggers_full_scan() {
+        let test_ctx = TestContext::new().await;
+        let state = test_ctx.state.clone();
+        
+        // Create test user
+        let user_data = CreateUser {
+            username: "deletion_test".to_string(),
+            email: "deletion_test@example.com".to_string(),
+            password: "password123".to_string(),
+            role: Some(UserRole::User),
+        };
+        let user = state.db.create_user(user_data).await
+            .expect("Failed to create test user");
+
+        // Setup initial state: user has 3 directories known in database
+        let initial_directories = vec![
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/dir1".to_string(),
+                directory_etag: "etag1".to_string(),
+                file_count: 5,
+                total_size_bytes: 1024,
+            },
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/dir2".to_string(),
+                directory_etag: "etag2".to_string(),
+                file_count: 3,
+                total_size_bytes: 512,
+            },
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/dir3".to_string(),
+                directory_etag: "etag3".to_string(),
+                file_count: 2,
+                total_size_bytes: 256,
+            },
+        ];
+
+        // Save initial directories to database
+        state.db.bulk_create_or_update_webdav_directories(&initial_directories).await
+            .expect("Failed to create initial directories");
+
+        // Verify the directories are stored
+        let stored_dirs = state.db.list_webdav_directories(user.id).await
+            .expect("Failed to list directories");
+        assert_eq!(stored_dirs.len(), 3);
+
+        // Create SmartSyncService for testing
+        let smart_sync = SmartSyncService::new(state.clone());
+        
+        // Since we can't easily mock a WebDAV server in unit tests,
+        // we'll test the database-level deletion detection logic directly
+        
+        // Simulate what happens when WebDAV discovery returns fewer directories
+        // This tests the core logic without needing a real WebDAV server
+        
+        // Get current directories
+        let known_dirs = state.db.list_webdav_directories(user.id).await
+            .expect("Failed to fetch known directories");
+        
+        // Simulate discovered directories (missing dir3 - it was deleted)
+        let discovered_paths: std::collections::HashSet<String> = [
+            "/test/dir1".to_string(),
+            "/test/dir2".to_string(),
+            // dir3 is missing - simulates deletion
+        ].into_iter().collect();
+        
+        let known_paths: std::collections::HashSet<String> = known_dirs
+            .iter()
+            .map(|d| d.directory_path.clone())
+            .collect();
+        
+        // Test deletion detection logic
+        let deleted_paths: Vec<String> = known_paths
+            .difference(&discovered_paths)
+            .cloned()
+            .collect();
+        
+        assert_eq!(deleted_paths.len(), 1);
+        assert!(deleted_paths.contains(&"/test/dir3".to_string()));
+        
+        // This demonstrates the core deletion detection logic that would
+        // trigger a full scan in the real smart sync implementation
+        println!("✅ Deletion detection test passed - detected {} deleted directories", deleted_paths.len());
+    }
+
+    /// Test that smart sync handles the case where no directories are deleted
+    #[tokio::test]
+    async fn test_no_deletion_detection() {
+        let test_ctx = TestContext::new().await;
+        let state = test_ctx.state.clone();
+        
+        // Create test user
+        let user_data = CreateUser {
+            username: "no_deletion_test".to_string(),
+            email: "no_deletion_test@example.com".to_string(),
+            password: "password123".to_string(),
+            role: Some(UserRole::User),
+        };
+        let user = state.db.create_user(user_data).await
+            .expect("Failed to create test user");
+
+        // Setup initial state
+        let initial_directories = vec![
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/dir1".to_string(),
+                directory_etag: "etag1".to_string(),
+                file_count: 5,
+                total_size_bytes: 1024,
+            },
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/dir2".to_string(),
+                directory_etag: "etag2".to_string(),
+                file_count: 3,
+                total_size_bytes: 512,
+            },
+        ];
+
+        state.db.bulk_create_or_update_webdav_directories(&initial_directories).await
+            .expect("Failed to create initial directories");
+
+        // Get current directories
+        let known_dirs = state.db.list_webdav_directories(user.id).await
+            .expect("Failed to fetch known directories");
+        
+        // Simulate discovered directories (all present, some with changed ETags)
+        let discovered_paths: std::collections::HashSet<String> = [
+            "/test/dir1".to_string(),
+            "/test/dir2".to_string(),
+        ].into_iter().collect();
+        
+        let known_paths: std::collections::HashSet<String> = known_dirs
+            .iter()
+            .map(|d| d.directory_path.clone())
+            .collect();
+        
+        // Test no deletion scenario
+        let deleted_paths: Vec<String> = known_paths
+            .difference(&discovered_paths)
+            .cloned()
+            .collect();
+        
+        assert_eq!(deleted_paths.len(), 0);
+        println!("✅ No deletion test passed - no directories were deleted");
+    }
+
+    /// Test bulk directory operations for performance
+    #[tokio::test]
+    async fn test_bulk_directory_deletion_detection() {
+        let test_ctx = TestContext::new().await;
+        let state = test_ctx.state.clone();
+        
+        // Create test user
+        let user_data = CreateUser {
+            username: "bulk_deletion_test".to_string(),
+            email: "bulk_deletion_test@example.com".to_string(),
+            password: "password123".to_string(),
+            role: Some(UserRole::User),
+        };
+        let user = state.db.create_user(user_data).await
+            .expect("Failed to create test user");
+
+        // Create a large number of directories to test bulk operations
+        let mut initial_directories = Vec::new();
+        for i in 0..100 {
+            initial_directories.push(CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: format!("/test/bulk_dir_{}", i),
+                directory_etag: format!("etag_{}", i),
+                file_count: i % 10,
+                total_size_bytes: (i * 1024) as i64,
+            });
+        }
+
+        // Save all directories
+        let start = std::time::Instant::now();
+        state.db.bulk_create_or_update_webdav_directories(&initial_directories).await
+            .expect("Failed to create bulk directories");
+        let insert_time = start.elapsed();
+        
+        // Test bulk retrieval
+        let start = std::time::Instant::now();
+        let known_dirs = state.db.list_webdav_directories(user.id).await
+            .expect("Failed to list directories");
+        let query_time = start.elapsed();
+        
+        assert_eq!(known_dirs.len(), 100);
+        
+        // Simulate many deletions (keep only first 30 directories)
+        let discovered_paths: std::collections::HashSet<String> = (0..30)
+            .map(|i| format!("/test/bulk_dir_{}", i))
+            .collect();
+        
+        let known_paths: std::collections::HashSet<String> = known_dirs
+            .iter()
+            .map(|d| d.directory_path.clone())
+            .collect();
+        
+        // Test bulk deletion detection
+        let start = std::time::Instant::now();
+        let deleted_paths: Vec<String> = known_paths
+            .difference(&discovered_paths)
+            .cloned()
+            .collect();
+        let deletion_detection_time = start.elapsed();
+        
+        assert_eq!(deleted_paths.len(), 70); // 100 - 30 = 70 deleted
+        
+        println!("✅ Bulk deletion detection performance:");
+        println!("   - Insert time: {:?}", insert_time);
+        println!("   - Query time: {:?}", query_time);
+        println!("   - Deletion detection time: {:?}", deletion_detection_time);
+        println!("   - Detected {} deletions out of 100 directories", deleted_paths.len());
+        
+        // Performance assertions
+        assert!(insert_time.as_millis() < 1000, "Bulk insert took too long: {:?}", insert_time);
+        assert!(query_time.as_millis() < 100, "Query took too long: {:?}", query_time);
+        assert!(deletion_detection_time.as_millis() < 10, "Deletion detection took too long: {:?}", deletion_detection_time);
+    }
+
+    /// Test ETag change detection combined with deletion detection
+    #[tokio::test]
+    async fn test_etag_changes_and_deletions() {
+        let test_ctx = TestContext::new().await;
+        let state = test_ctx.state.clone();
+        
+        // Create test user
+        let user_data = CreateUser {
+            username: "etag_deletion_test".to_string(),
+            email: "etag_deletion_test@example.com".to_string(),
+            password: "password123".to_string(),
+            role: Some(UserRole::User),
+        };
+        let user = state.db.create_user(user_data).await
+            .expect("Failed to create test user");
+
+        // Setup initial state
+        let initial_directories = vec![
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/unchanged".to_string(),
+                directory_etag: "etag_unchanged".to_string(),
+                file_count: 5,
+                total_size_bytes: 1024,
+            },
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/changed".to_string(),
+                directory_etag: "etag_old".to_string(),
+                file_count: 3,
+                total_size_bytes: 512,
+            },
+            CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: "/test/deleted".to_string(),
+                directory_etag: "etag_deleted".to_string(),
+                file_count: 2,
+                total_size_bytes: 256,
+            },
+        ];
+
+        state.db.bulk_create_or_update_webdav_directories(&initial_directories).await
+            .expect("Failed to create initial directories");
+
+        // Get known directories with their ETags
+        let known_dirs = state.db.list_webdav_directories(user.id).await
+            .expect("Failed to fetch known directories");
+        
+        let known_etags: std::collections::HashMap<String, String> = known_dirs
+            .into_iter()
+            .map(|d| (d.directory_path, d.directory_etag))
+            .collect();
+
+        // Simulate discovery results: one unchanged, one changed, one deleted
+        let discovered_dirs = vec![
+            ("/test/unchanged", "etag_unchanged"), // Same ETag
+            ("/test/changed", "etag_new"),         // Changed ETag
+            // "/test/deleted" is missing - deleted
+        ];
+
+        let mut unchanged_count = 0;
+        let mut changed_count = 0;
+        let discovered_paths: std::collections::HashSet<String> = discovered_dirs
+            .iter()
+            .map(|(path, etag)| {
+                if let Some(known_etag) = known_etags.get(*path) {
+                    if known_etag == etag {
+                        unchanged_count += 1;
+                    } else {
+                        changed_count += 1;
+                    }
+                }
+                path.to_string()
+            })
+            .collect();
+
+        let known_paths: std::collections::HashSet<String> = known_etags.keys().cloned().collect();
+        let deleted_paths: Vec<String> = known_paths
+            .difference(&discovered_paths)
+            .cloned()
+            .collect();
+
+        // Verify detection results
+        assert_eq!(unchanged_count, 1);
+        assert_eq!(changed_count, 1);
+        assert_eq!(deleted_paths.len(), 1);
+        assert!(deleted_paths.contains(&"/test/deleted".to_string()));
+
+        println!("✅ Combined ETag and deletion detection:");
+        println!("   - Unchanged directories: {}", unchanged_count);
+        println!("   - Changed directories: {}", changed_count);
+        println!("   - Deleted directories: {}", deleted_paths.len());
+    }
+}

src/services/webdav/tests/etag_comparison_tests.rs

@@ -0,0 +1,138 @@
+use crate::webdav_xml_parser::{
+    compare_etags, weak_compare_etags, strong_compare_etags, 
+    ParsedETag, normalize_etag
+};
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_normalize_etag_handles_quotes() {
+        assert_eq!(normalize_etag("\"abc123\""), "abc123");
+        assert_eq!(normalize_etag("abc123"), "abc123");
+        assert_eq!(normalize_etag("\"\""), "");
+    }
+
+    #[test]
+    fn test_normalize_etag_handles_weak_indicators() {
+        assert_eq!(normalize_etag("W/\"abc123\""), "abc123");
+        assert_eq!(normalize_etag("w/\"abc123\""), "abc123");
+        assert_eq!(normalize_etag("W/abc123"), "abc123");
+    }
+
+    #[test]
+    fn test_normalize_etag_handles_multiple_weak_indicators() {
+        // Malformed but seen in the wild
+        assert_eq!(normalize_etag("W/W/\"abc123\""), "abc123");
+        assert_eq!(normalize_etag("w/W/\"abc123\""), "abc123");
+    }
+
+    #[test]
+    fn test_parsed_etag_weak_detection() {
+        let weak_etag = ParsedETag::parse("W/\"abc123\"");
+        assert!(weak_etag.is_weak);
+        assert_eq!(weak_etag.normalized, "abc123");
+
+        let strong_etag = ParsedETag::parse("\"abc123\"");
+        assert!(!strong_etag.is_weak);
+        assert_eq!(strong_etag.normalized, "abc123");
+    }
+
+    #[test]
+    fn test_strong_comparison_rejects_weak_etags() {
+        let weak1 = ParsedETag::parse("W/\"abc123\"");
+        let weak2 = ParsedETag::parse("W/\"abc123\"");
+        let strong1 = ParsedETag::parse("\"abc123\"");
+        let strong2 = ParsedETag::parse("\"abc123\"");
+
+        // Strong comparison should reject any weak ETags
+        assert!(!weak1.strong_compare(&weak2));
+        assert!(!weak1.strong_compare(&strong1));
+        assert!(!strong1.strong_compare(&weak1));
+        
+        // Only strong ETags should match in strong comparison
+        assert!(strong1.strong_compare(&strong2));
+    }
+
+    #[test]
+    fn test_weak_comparison_accepts_all_combinations() {
+        let weak1 = ParsedETag::parse("W/\"abc123\"");
+        let weak2 = ParsedETag::parse("W/\"abc123\"");
+        let strong1 = ParsedETag::parse("\"abc123\"");
+        let strong2 = ParsedETag::parse("\"abc123\"");
+
+        // Weak comparison should accept all combinations if values match
+        assert!(weak1.weak_compare(&weak2));
+        assert!(weak1.weak_compare(&strong1));
+        assert!(strong1.weak_compare(&weak1));
+        assert!(strong1.weak_compare(&strong2));
+    }
+
+    #[test]
+    fn test_smart_comparison_logic() {
+        let weak = ParsedETag::parse("W/\"abc123\"");
+        let strong = ParsedETag::parse("\"abc123\"");
+
+        // If either is weak, should use weak comparison
+        assert!(weak.smart_compare(&strong));
+        assert!(strong.smart_compare(&weak));
+        
+        // If both are strong, should use strong comparison
+        let strong2 = ParsedETag::parse("\"abc123\"");
+        assert!(strong.smart_compare(&strong2));
+    }
+
+    #[test]
+    fn test_utility_functions() {
+        // Test the utility functions that the smart sync will use
+        assert!(compare_etags("W/\"abc123\"", "\"abc123\""));
+        assert!(weak_compare_etags("W/\"abc123\"", "\"abc123\""));
+        assert!(!strong_compare_etags("W/\"abc123\"", "\"abc123\""));
+    }
+
+    #[test]
+    fn test_case_sensitivity_preservation() {
+        // ETags should be case sensitive per RFC
+        assert!(!compare_etags("\"ABC123\"", "\"abc123\""));
+        assert!(!weak_compare_etags("\"ABC123\"", "\"abc123\""));
+        assert!(!strong_compare_etags("\"ABC123\"", "\"abc123\""));
+    }
+
+    #[test]
+    fn test_real_world_etag_formats() {
+        // Test various real-world ETag formats
+        let nextcloud_etag = "\"5f3e7e8a9b2c1d4\"";
+        let apache_etag = "\"1234-567-890abcdef\"";
+        let nginx_etag = "W/\"5f3e7e8a\"";
+        let sharepoint_etag = "\"{12345678-1234-1234-1234-123456789012},1\"";
+
+        // All should normalize correctly
+        assert_eq!(normalize_etag(nextcloud_etag), "5f3e7e8a9b2c1d4");
+        assert_eq!(normalize_etag(apache_etag), "1234-567-890abcdef");
+        assert_eq!(normalize_etag(nginx_etag), "5f3e7e8a");
+        assert_eq!(normalize_etag(sharepoint_etag), "{12345678-1234-1234-1234-123456789012},1");
+    }
+
+    #[test]
+    fn test_etag_equivalence() {
+        let etag1 = ParsedETag::parse("\"abc123\"");
+        let etag2 = ParsedETag::parse("W/\"abc123\"");
+        
+        // Should be equivalent despite weak/strong difference
+        assert!(etag1.is_equivalent(&etag2));
+        
+        let etag3 = ParsedETag::parse("\"def456\"");
+        assert!(!etag1.is_equivalent(&etag3));
+    }
+
+    #[test]
+    fn test_comparison_string_safety() {
+        let etag_with_quotes = ParsedETag::parse("\"test\\\"internal\\\"quotes\"");
+        let comparison_str = etag_with_quotes.comparison_string();
+        
+        // Should handle internal quotes safely
+        assert!(!comparison_str.contains('"'));
+        assert!(!comparison_str.contains("\\"));
+    }
+}

src/services/webdav/tests/mod.rs

@@ -0,0 +1,4 @@
+pub mod critical_fixes_tests;
+pub mod etag_comparison_tests;
+pub mod atomic_operations_tests;
+pub mod deletion_detection_tests;

src/services/webdav/url_management.rs

@@ -1,186 +0,0 @@
-use anyhow::Result;
-use crate::models::FileIngestionInfo;
-use super::config::WebDAVConfig;
-
-/// Centralized URL and path management for WebDAV operations
-/// 
-/// This module handles all the messy WebDAV URL construction, path normalization,
-/// and conversion between full WebDAV paths and relative paths. It's designed to
-/// prevent the URL doubling issues that plague WebDAV integrations.
-pub struct WebDAVUrlManager {
-    config: WebDAVConfig,
-}
-
-impl WebDAVUrlManager {
-    pub fn new(config: WebDAVConfig) -> Self {
-        Self { config }
-    }
-
-    /// Get the base WebDAV URL for the configured server
-    /// Returns something like: "https://nas.example.com/remote.php/dav/files/username"
-    pub fn base_url(&self) -> String {
-        self.config.webdav_url()
-    }
-
-    /// Convert full WebDAV href (from XML response) to relative path
-    /// 
-    /// Input:  "/remote.php/dav/files/username/Photos/image.jpg"
-    /// Output: "/Photos/image.jpg"
-    pub fn href_to_relative_path(&self, href: &str) -> String {
-        match self.config.server_type.as_deref() {
-            Some("nextcloud") => {
-                let prefix = format!("/remote.php/dav/files/{}", self.config.username);
-                if href.starts_with(&prefix) {
-                    let relative = &href[prefix.len()..];
-                    if relative.is_empty() { "/" } else { relative }.to_string()
-                } else {
-                    href.to_string()
-                }
-            }
-            Some("owncloud") => {
-                if href.starts_with("/remote.php/webdav") {
-                    let relative = &href[18..]; // Remove "/remote.php/webdav"
-                    if relative.is_empty() { "/" } else { relative }.to_string()
-                } else {
-                    href.to_string()
-                }
-            }
-            Some("generic") => {
-                if href.starts_with("/webdav") {
-                    let relative = &href[7..]; // Remove "/webdav"
-                    if relative.is_empty() { "/" } else { relative }.to_string()
-                } else {
-                    href.to_string()
-                }
-            }
-            _ => href.to_string(),
-        }
-    }
-
-    /// Convert relative path to full URL for WebDAV requests
-    /// 
-    /// Input:  "/Photos/image.jpg"
-    /// Output: "https://nas.example.com/remote.php/dav/files/username/Photos/image.jpg"
-    pub fn relative_path_to_url(&self, relative_path: &str) -> String {
-        let base_url = self.base_url();
-        let clean_path = relative_path.trim_start_matches('/');
-        
-        if clean_path.is_empty() {
-            base_url
-        } else {
-            let normalized_base = base_url.trim_end_matches('/');
-            format!("{}/{}", normalized_base, clean_path)
-        }
-    }
-
-    /// Process FileIngestionInfo from XML parser to set correct paths
-    /// 
-    /// This takes the raw XML parser output and fixes the path fields:
-    /// - Sets relative_path from href conversion
-    /// - Keeps full_path as the original href
-    /// - Sets legacy path field for backward compatibility
-    pub fn process_file_info(&self, mut file_info: FileIngestionInfo) -> FileIngestionInfo {
-        // The XML parser puts the href in full_path (which is correct)
-        let href = &file_info.full_path;
-        
-        // Convert to relative path
-        file_info.relative_path = self.href_to_relative_path(href);
-        
-        // Legacy path field should be relative for backward compatibility
-        #[allow(deprecated)]
-        {
-            file_info.path = file_info.relative_path.clone();
-        }
-        
-        file_info
-    }
-
-    /// Process a collection of FileIngestionInfo items
-    pub fn process_file_infos(&self, file_infos: Vec<FileIngestionInfo>) -> Vec<FileIngestionInfo> {
-        file_infos.into_iter()
-            .map(|file_info| self.process_file_info(file_info))
-            .collect()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn create_nextcloud_config() -> WebDAVConfig {
-        WebDAVConfig {
-            server_url: "https://nas.example.com".to_string(),
-            username: "testuser".to_string(),
-            password: "password".to_string(),
-            watch_folders: vec!["/Photos".to_string()],
-            file_extensions: vec!["jpg".to_string(), "pdf".to_string()],
-            timeout_seconds: 30,
-            server_type: Some("nextcloud".to_string()),
-        }
-    }
-
-    #[test]
-    fn test_nextcloud_href_to_relative_path() {
-        let manager = WebDAVUrlManager::new(create_nextcloud_config());
-        
-        // Test file path conversion
-        let href = "/remote.php/dav/files/testuser/Photos/image.jpg";
-        let relative = manager.href_to_relative_path(href);
-        assert_eq!(relative, "/Photos/image.jpg");
-        
-        // Test directory path conversion
-        let href = "/remote.php/dav/files/testuser/Photos/";
-        let relative = manager.href_to_relative_path(href);
-        assert_eq!(relative, "/Photos/");
-        
-        // Test root path
-        let href = "/remote.php/dav/files/testuser";
-        let relative = manager.href_to_relative_path(href);
-        assert_eq!(relative, "/");
-    }
-
-    #[test]
-    fn test_relative_path_to_url() {
-        let manager = WebDAVUrlManager::new(create_nextcloud_config());
-        
-        // Test file URL construction
-        let relative = "/Photos/image.jpg";
-        let url = manager.relative_path_to_url(relative);
-        assert_eq!(url, "https://nas.example.com/remote.php/dav/files/testuser/Photos/image.jpg");
-        
-        // Test root URL
-        let relative = "/";
-        let url = manager.relative_path_to_url(relative);
-        assert_eq!(url, "https://nas.example.com/remote.php/dav/files/testuser");
-    }
-
-    #[test]
-    fn test_process_file_info() {
-        let manager = WebDAVUrlManager::new(create_nextcloud_config());
-        
-        let file_info = FileIngestionInfo {
-            relative_path: "TEMP".to_string(), // Will be overwritten
-            full_path: "/remote.php/dav/files/testuser/Photos/image.jpg".to_string(),
-            #[allow(deprecated)]
-            path: "OLD".to_string(), // Will be overwritten
-            name: "image.jpg".to_string(),
-            size: 1024,
-            mime_type: "image/jpeg".to_string(),
-            last_modified: None,
-            etag: "abc123".to_string(),
-            is_directory: false,
-            created_at: None,
-            permissions: None,
-            owner: None,
-            group: None,
-            metadata: None,
-        };
-        
-        let processed = manager.process_file_info(file_info);
-        
-        assert_eq!(processed.relative_path, "/Photos/image.jpg");
-        assert_eq!(processed.full_path, "/remote.php/dav/files/testuser/Photos/image.jpg");
-        #[allow(deprecated)]
-        assert_eq!(processed.path, "/Photos/image.jpg");
-    }
-}

src/services/webdav/validation.rs

@@ -1,352 +0,0 @@
-use anyhow::Result;
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use tracing::{debug, info, warn};
-
-use super::config::WebDAVConfig;
-use super::connection::WebDAVConnection;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ValidationReport {
-    pub overall_health_score: i32, // 0-100
-    pub issues: Vec<ValidationIssue>,
-    pub recommendations: Vec<ValidationRecommendation>,
-    pub summary: ValidationSummary,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ValidationIssue {
-    pub issue_type: ValidationIssueType,
-    pub severity: ValidationSeverity,
-    pub directory_path: String,
-    pub description: String,
-    pub details: Option<serde_json::Value>,
-    pub detected_at: chrono::DateTime<chrono::Utc>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, Hash, PartialEq)]
-pub enum ValidationIssueType {
-    /// Directory exists on server but not in our tracking
-    Untracked,
-    /// Directory in our tracking but missing on server  
-    Missing,
-    /// ETag mismatch between server and our cache
-    ETagMismatch,
-    /// Directory hasn't been scanned in a very long time
-    Stale,
-    /// Server errors when accessing directory
-    Inaccessible,
-    /// ETag support seems unreliable for this directory
-    ETagUnreliable,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum ValidationSeverity {
-    Info,    // No action needed, just FYI
-    Warning, // Should investigate but not urgent
-    Error,   // Needs immediate attention
-    Critical, // System integrity at risk
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ValidationRecommendation {
-    pub action: ValidationAction,
-    pub reason: String,
-    pub affected_directories: Vec<String>,
-    pub priority: ValidationSeverity,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum ValidationAction {
-    /// Run a deep scan of specific directories
-    DeepScanRequired,
-    /// Clear and rebuild directory tracking
-    RebuildTracking,
-    /// ETag support is unreliable, switch to periodic scans
-    DisableETagOptimization,
-    /// Clean up orphaned database entries
-    CleanupDatabase,
-    /// Server configuration issue needs attention
-    CheckServerConfiguration,
-    /// No action needed, system is healthy
-    NoActionRequired,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ValidationSummary {
-    pub total_directories_checked: usize,
-    pub healthy_directories: usize,
-    pub directories_with_issues: usize,
-    pub critical_issues: usize,
-    pub warning_issues: usize,
-    pub info_issues: usize,
-    pub validation_duration_ms: u64,
-}
-
-pub struct WebDAVValidator {
-    connection: WebDAVConnection,
-    config: WebDAVConfig,
-}
-
-impl WebDAVValidator {
-    pub fn new(connection: WebDAVConnection, config: WebDAVConfig) -> Self {
-        Self { connection, config }
-    }
-
-    /// Performs comprehensive validation of WebDAV setup and directory tracking
-    pub async fn validate_system(&self) -> Result<ValidationReport> {
-        let start_time = std::time::Instant::now();
-        info!("🔍 Starting WebDAV system validation");
-
-        let mut issues = Vec::new();
-        let mut total_checked = 0;
-
-        // Test basic connectivity
-        match self.connection.test_connection().await {
-            Ok(result) if !result.success => {
-                issues.push(ValidationIssue {
-                    issue_type: ValidationIssueType::Inaccessible,
-                    severity: ValidationSeverity::Critical,
-                    directory_path: "/".to_string(),
-                    description: format!("WebDAV server connection failed: {}", result.message),
-                    details: None,
-                    detected_at: chrono::Utc::now(),
-                });
-            }
-            Err(e) => {
-                issues.push(ValidationIssue {
-                    issue_type: ValidationIssueType::Inaccessible,
-                    severity: ValidationSeverity::Critical,
-                    directory_path: "/".to_string(),
-                    description: format!("WebDAV server connectivity test failed: {}", e),
-                    details: None,
-                    detected_at: chrono::Utc::now(),
-                });
-            }
-            _ => {
-                debug!("✅ Basic connectivity test passed");
-            }
-        }
-
-        // Validate each watch folder
-        for folder in &self.config.watch_folders {
-            total_checked += 1;
-            if let Err(e) = self.validate_watch_folder(folder, &mut issues).await {
-                warn!("Failed to validate watch folder '{}': {}", folder, e);
-            }
-        }
-
-        // Test ETag reliability
-        self.validate_etag_support(&mut issues).await?;
-
-        // Generate recommendations based on issues
-        let recommendations = self.generate_recommendations(&issues);
-
-        let validation_duration = start_time.elapsed().as_millis() as u64;
-        let health_score = self.calculate_health_score(&issues);
-
-        let summary = ValidationSummary {
-            total_directories_checked: total_checked,
-            healthy_directories: total_checked - issues.len(),
-            directories_with_issues: issues.len(),
-            critical_issues: issues.iter().filter(|i| matches!(i.severity, ValidationSeverity::Critical)).count(),
-            warning_issues: issues.iter().filter(|i| matches!(i.severity, ValidationSeverity::Warning)).count(),
-            info_issues: issues.iter().filter(|i| matches!(i.severity, ValidationSeverity::Info)).count(),
-            validation_duration_ms: validation_duration,
-        };
-
-        info!("✅ WebDAV validation completed in {}ms. Health score: {}/100", 
-            validation_duration, health_score);
-
-        Ok(ValidationReport {
-            overall_health_score: health_score,
-            issues,
-            recommendations,
-            summary,
-        })
-    }
-
-    /// Validates a specific watch folder
-    async fn validate_watch_folder(&self, folder: &str, issues: &mut Vec<ValidationIssue>) -> Result<()> {
-        debug!("Validating watch folder: {}", folder);
-
-        // Test PROPFIND access
-        match self.connection.test_propfind(folder).await {
-            Ok(_) => {
-                debug!("✅ Watch folder '{}' is accessible", folder);
-            }
-            Err(e) => {
-                issues.push(ValidationIssue {
-                    issue_type: ValidationIssueType::Inaccessible,
-                    severity: ValidationSeverity::Error,
-                    directory_path: folder.to_string(),
-                    description: format!("Cannot access watch folder: {}", e),
-                    details: Some(serde_json::json!({
-                        "error": e.to_string(),
-                        "folder": folder
-                    })),
-                    detected_at: chrono::Utc::now(),
-                });
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Tests ETag support reliability
-    async fn validate_etag_support(&self, issues: &mut Vec<ValidationIssue>) -> Result<()> {
-        debug!("Testing ETag support reliability");
-
-        // Test ETag consistency across multiple requests
-        for folder in &self.config.watch_folders {
-            if let Err(e) = self.test_etag_consistency(folder, issues).await {
-                warn!("ETag consistency test failed for '{}': {}", folder, e);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Tests ETag consistency for a specific folder
-    async fn test_etag_consistency(&self, folder: &str, issues: &mut Vec<ValidationIssue>) -> Result<()> {
-        // Make two consecutive PROPFIND requests and compare ETags
-        let etag1 = self.get_folder_etag(folder).await?;
-        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
-        let etag2 = self.get_folder_etag(folder).await?;
-
-        if etag1 != etag2 && etag1.is_some() && etag2.is_some() {
-            issues.push(ValidationIssue {
-                issue_type: ValidationIssueType::ETagUnreliable,
-                severity: ValidationSeverity::Warning,
-                directory_path: folder.to_string(),
-                description: "ETag values are inconsistent across requests".to_string(),
-                details: Some(serde_json::json!({
-                    "etag1": etag1,
-                    "etag2": etag2,
-                    "folder": folder
-                })),
-                detected_at: chrono::Utc::now(),
-            });
-        }
-
-        Ok(())
-    }
-
-    /// Gets the ETag for a folder
-    async fn get_folder_etag(&self, folder: &str) -> Result<Option<String>> {
-        let url = self.connection.get_url_for_path(folder);
-        
-        let propfind_body = r#"<?xml version="1.0" encoding="utf-8"?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:getetag/>
-                </D:prop>
-            </D:propfind>"#;
-
-        let response = self.connection
-            .authenticated_request(
-                reqwest::Method::from_bytes(b"PROPFIND")?,
-                &url,
-                Some(propfind_body.to_string()),
-                Some(vec![
-                    ("Depth", "0"),
-                    ("Content-Type", "application/xml"),
-                ]),
-            )
-            .await?;
-
-        let body = response.text().await?;
-        
-        // Parse ETag from XML response (simplified)
-        if let Some(start) = body.find("<D:getetag>") {
-            if let Some(end) = body[start..].find("</D:getetag>") {
-                let etag = &body[start + 11..start + end];
-                return Ok(Some(etag.trim_matches('"').to_string()));
-            }
-        }
-
-        Ok(None)
-    }
-
-    /// Generates recommendations based on detected issues
-    fn generate_recommendations(&self, issues: &Vec<ValidationIssue>) -> Vec<ValidationRecommendation> {
-        let mut recommendations = Vec::new();
-        let mut directories_by_issue: HashMap<ValidationIssueType, Vec<String>> = HashMap::new();
-
-        // Group directories by issue type
-        for issue in issues {
-            directories_by_issue
-                .entry(issue.issue_type.clone())
-                .or_insert_with(Vec::new)
-                .push(issue.directory_path.clone());
-        }
-
-        // Generate recommendations for each issue type
-        for (issue_type, directories) in directories_by_issue {
-            let recommendation = match issue_type {
-                ValidationIssueType::Inaccessible => ValidationRecommendation {
-                    action: ValidationAction::CheckServerConfiguration,
-                    reason: "Some directories are inaccessible. Check server configuration and permissions.".to_string(),
-                    affected_directories: directories,
-                    priority: ValidationSeverity::Critical,
-                },
-                ValidationIssueType::ETagUnreliable => ValidationRecommendation {
-                    action: ValidationAction::DisableETagOptimization,
-                    reason: "ETag support appears unreliable. Consider disabling ETag optimization.".to_string(),
-                    affected_directories: directories,
-                    priority: ValidationSeverity::Warning,
-                },
-                ValidationIssueType::Missing => ValidationRecommendation {
-                    action: ValidationAction::CleanupDatabase,
-                    reason: "Some tracked directories no longer exist on the server.".to_string(),
-                    affected_directories: directories,
-                    priority: ValidationSeverity::Warning,
-                },
-                ValidationIssueType::Stale => ValidationRecommendation {
-                    action: ValidationAction::DeepScanRequired,
-                    reason: "Some directories haven't been scanned recently.".to_string(),
-                    affected_directories: directories,
-                    priority: ValidationSeverity::Info,
-                },
-                _ => ValidationRecommendation {
-                    action: ValidationAction::DeepScanRequired,
-                    reason: "General validation issues detected.".to_string(),
-                    affected_directories: directories,
-                    priority: ValidationSeverity::Warning,
-                },
-            };
-            recommendations.push(recommendation);
-        }
-
-        if recommendations.is_empty() {
-            recommendations.push(ValidationRecommendation {
-                action: ValidationAction::NoActionRequired,
-                reason: "System validation passed successfully.".to_string(),
-                affected_directories: Vec::new(),
-                priority: ValidationSeverity::Info,
-            });
-        }
-
-        recommendations
-    }
-
-    /// Calculates overall health score based on issues
-    fn calculate_health_score(&self, issues: &Vec<ValidationIssue>) -> i32 {
-        if issues.is_empty() {
-            return 100;
-        }
-
-        let mut penalty = 0;
-        for issue in issues {
-            let issue_penalty = match issue.severity {
-                ValidationSeverity::Critical => 30,
-                ValidationSeverity::Error => 20,
-                ValidationSeverity::Warning => 10,
-                ValidationSeverity::Info => 5,
-            };
-            penalty += issue_penalty;
-        }
-
-        std::cmp::max(0, 100 - penalty)
-    }
-}

src/swagger.rs

@@ -172,7 +172,7 @@ use crate::{
     modifiers(&SecurityAddon),
     info(
         title = "Readur API",
-        version = "2.4.2",
+        version = "2.5.3",
         description = "Document management and OCR processing API",
         contact(
             name = "Readur Team",

src/webdav_xml_parser.rs

@@ -604,6 +604,30 @@ impl ParsedETag {
         self.normalized == other.normalized
     }
     
+    /// RFC 7232 compliant strong comparison - weak ETags never match strong comparison
+    pub fn strong_compare(&self, other: &ParsedETag) -> bool {
+        // Strong comparison: ETags match AND neither is weak
+        !self.is_weak && !other.is_weak && self.normalized == other.normalized
+    }
+    
+    /// RFC 7232 compliant weak comparison - considers weak and strong ETags equivalent if values match
+    pub fn weak_compare(&self, other: &ParsedETag) -> bool {
+        // Weak comparison: ETags match regardless of weak/strong
+        self.normalized == other.normalized
+    }
+    
+    /// Smart comparison that chooses the appropriate method based on context
+    /// For WebDAV sync, we typically want weak comparison since servers may return weak ETags
+    pub fn smart_compare(&self, other: &ParsedETag) -> bool {
+        // If either ETag is weak, use weak comparison
+        if self.is_weak || other.is_weak {
+            self.weak_compare(other)
+        } else {
+            // Both are strong, use strong comparison
+            self.strong_compare(other)
+        }
+    }
+    
     /// Get a safe string for comparison that handles edge cases
     pub fn comparison_string(&self) -> String {
         // For comparison, we normalize further by removing internal quotes and whitespace
@@ -615,6 +639,31 @@ impl ParsedETag {
     }
 }
 
+/// Utility function for comparing two ETag strings with proper RFC 7232 semantics
+pub fn compare_etags(etag1: &str, etag2: &str) -> bool {
+    let parsed1 = ParsedETag::parse(etag1);
+    let parsed2 = ParsedETag::parse(etag2);
+    
+    // Use smart comparison which handles weak/strong appropriately
+    parsed1.smart_compare(&parsed2)
+}
+
+/// Utility function for weak ETag comparison (most common in WebDAV)
+pub fn weak_compare_etags(etag1: &str, etag2: &str) -> bool {
+    let parsed1 = ParsedETag::parse(etag1);
+    let parsed2 = ParsedETag::parse(etag2);
+    
+    parsed1.weak_compare(&parsed2)
+}
+
+/// Utility function for strong ETag comparison 
+pub fn strong_compare_etags(etag1: &str, etag2: &str) -> bool {
+    let parsed1 = ParsedETag::parse(etag1);
+    let parsed2 = ParsedETag::parse(etag2);
+    
+    parsed1.strong_compare(&parsed2)
+}
+
 fn classify_etag_format(etag: &str) -> ETagFormat {
     let _lower = etag.to_lowercase();
     
@@ -860,4 +909,38 @@ mod tests {
         assert_eq!(normalize_etag("\"\""), "");
         assert_eq!(normalize_etag("W/\"\""), "");
     }
+
+    #[test]
+    fn test_utility_function_performance() {
+        // Test that utility functions work correctly under load
+        let test_etags = [
+            ("\"abc123\"", "W/\"abc123\""),
+            ("\"def456\"", "\"def456\""),
+            ("W/\"ghi789\"", "W/\"ghi789\""),
+            ("\"jkl012\"", "\"mno345\""),
+        ];
+
+        for (etag1, etag2) in &test_etags {
+            let result1 = compare_etags(etag1, etag2);
+            let result2 = compare_etags(etag2, etag1); // Should be symmetric
+            assert_eq!(result1, result2, "ETag comparison should be symmetric");
+        }
+    }
+
+    #[test]
+    fn test_rfc_compliance() {
+        // Test RFC 7232 compliance for various ETag scenarios
+        
+        // Example from RFC 7232: W/"1" and "1" should match in weak comparison
+        assert!(weak_compare_etags("W/\"1\"", "\"1\""));
+        assert!(!strong_compare_etags("W/\"1\"", "\"1\""));
+        
+        // Both weak should match
+        assert!(weak_compare_etags("W/\"1\"", "W/\"1\""));
+        assert!(!strong_compare_etags("W/\"1\"", "W/\"1\""));
+        
+        // Both strong should match in both comparisons
+        assert!(weak_compare_etags("\"1\"", "\"1\""));
+        assert!(strong_compare_etags("\"1\"", "\"1\""));
+    }
 }

tests/integration_auto_resume_tests.rs

@@ -62,12 +62,13 @@ async fn create_test_app_state() -> Arc<AppState> {
     ));
     
     Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     })
 }
 

tests/integration_cancellation_tests.rs

@@ -55,6 +55,7 @@ async fn create_test_app_state() -> Arc<AppState> {
     let db = Database::new(&config.database_url).await.unwrap();
     
     let queue_service = Arc::new(readur::ocr::queue::OcrQueueService::new(db.clone(), db.pool.clone(), 2));
+    let sync_progress_tracker = Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new());
     Arc::new(AppState {
         db,
         config,
@@ -62,6 +63,7 @@ async fn create_test_app_state() -> Arc<AppState> {
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker,
     })
 }
 

tests/integration_document_upload_hash_duplicate_tests.rs

@@ -100,12 +100,13 @@ async fn create_test_app_state() -> Result<Arc<AppState>> {
     );
     
     Ok(Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     }))
 }
 

tests/integration_ignored_files_integration_tests.rs

@@ -43,12 +43,13 @@ async fn create_test_app_state() -> Result<Arc<AppState>> {
     let queue_service = Arc::new(readur::ocr::queue::OcrQueueService::new(db.clone(), db.pool.clone(), 1));
     
     Ok(Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     }))
 }
 

tests/integration_oidc_tests.rs

@@ -65,6 +65,7 @@ mod tests {
                     2
                 )),
                 oidc_client: None,
+                sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
             }));
 
         (app, ())
@@ -153,6 +154,7 @@ mod tests {
                     2
                 )),
                 oidc_client,
+                sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
             }));
 
         (app, mock_server)

tests/integration_smart_sync_error_handling.rs

@@ -91,7 +91,7 @@ async fn test_webdav_error_fallback() {
         .expect("WebDAV service creation should not fail");
     
     // Test smart sync evaluation with failing WebDAV service
-    let decision = smart_sync_service.evaluate_sync_need(user.id, &failing_webdav_service, "/Documents").await;
+    let decision = smart_sync_service.evaluate_sync_need(user.id, &failing_webdav_service, "/Documents", None).await;
     
     // The system should handle the WebDAV error gracefully
     match decision {
@@ -131,7 +131,7 @@ async fn test_database_error_handling() {
     let invalid_user_id = uuid::Uuid::new_v4(); // Random UUID that doesn't exist
     let webdav_service = create_test_webdav_service();
     
-    let decision = smart_sync_service.evaluate_sync_need(invalid_user_id, &webdav_service, "/Documents").await;
+    let decision = smart_sync_service.evaluate_sync_need(invalid_user_id, &webdav_service, "/Documents", None).await;
     
     match decision {
         Ok(SmartSyncDecision::RequiresSync(SmartSyncStrategy::FullDeepScan)) => {

tests/integration_smart_sync_first_time.rs

@@ -59,7 +59,7 @@ async fn test_first_time_sync_full_deep_scan() {
     
     // Test evaluation for first-time sync
     let webdav_service = create_test_webdav_service();
-    let decision = smart_sync_service.evaluate_sync_need(user.id, &webdav_service, "/Documents").await;
+    let decision = smart_sync_service.evaluate_sync_need(user.id, &webdav_service, "/Documents", None).await;
     
     match decision {
         Ok(SmartSyncDecision::RequiresSync(SmartSyncStrategy::FullDeepScan)) => {

tests/integration_source_scheduler_simple_tests.rs

@@ -50,12 +50,13 @@ async fn create_test_app_state() -> Arc<AppState> {
     
     let queue_service = Arc::new(readur::ocr::queue::OcrQueueService::new(db.clone(), db.pool.clone(), 2));
     Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     })
 }
 

tests/integration_source_scheduler_tests.rs

@@ -195,12 +195,13 @@ async fn create_test_app_state() -> Arc<AppState> {
     let queue_service = std::sync::Arc::new(readur::ocr::queue::OcrQueueService::new(db.clone(), db.pool.clone(), 2));
     
     Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     })
 }
 

tests/integration_source_sync_hash_duplicate_tests.rs

@@ -149,12 +149,13 @@ async fn create_test_app_state() -> Result<Arc<AppState>> {
     );
     
     Ok(Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     }))
 }
 

tests/integration_stop_sync_functionality_tests.rs

@@ -61,12 +61,13 @@ async fn create_test_app_state() -> Arc<AppState> {
     ));
     
     Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     })
 }
 

tests/integration_universal_source_sync_tests.rs

@@ -164,6 +164,7 @@ async fn create_test_app_state() -> Arc<AppState> {
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     })
 }
 

tests/integration_webdav_concurrency_tests.rs

@@ -45,7 +45,7 @@ async fn create_test_webdav_source(
         name: name.to_string(),
         source_type: SourceType::WebDAV,
         config: serde_json::to_value(config).unwrap(),
-        enabled: true,
+        enabled: Some(true),
     };
 
     state.db.create_source(user_id, &create_source).await
@@ -73,7 +73,7 @@ impl MockWebDAVService {
         &self,
         directory_path: &str,
         _recursive: bool,
-    ) -> Result<readur::services::webdav::discovery::WebDAVDiscoveryResult, anyhow::Error> {
+    ) -> Result<readur::services::webdav::WebDAVDiscoveryResult, anyhow::Error> {
         // Simulate network delay
         sleep(Duration::from_millis(self.delay_ms)).await;
 
@@ -92,11 +92,19 @@ impl MockWebDAVService {
             readur::models::FileIngestionInfo {
                 name: format!("test-file-{}.pdf", etag),
                 relative_path: format!("{}/test-file-{}.pdf", directory_path, etag),
+                full_path: format!("{}/test-file-{}.pdf", directory_path, etag),
+                #[allow(deprecated)]
+                path: format!("{}/test-file-{}.pdf", directory_path, etag),
                 size: 1024,
-                modified: chrono::Utc::now(),
+                mime_type: "application/pdf".to_string(),
+                last_modified: Some(chrono::Utc::now()),
                 etag: etag.clone(),
                 is_directory: false,
-                content_type: Some("application/pdf".to_string()),
+                created_at: Some(chrono::Utc::now()),
+                permissions: Some(0o644),
+                owner: None,
+                group: None,
+                metadata: None,
             }
         ];
 
@@ -104,15 +112,23 @@ impl MockWebDAVService {
             readur::models::FileIngestionInfo {
                 name: "subdir".to_string(),
                 relative_path: format!("{}/subdir", directory_path),
+                full_path: format!("{}/subdir", directory_path),
+                #[allow(deprecated)]
+                path: format!("{}/subdir", directory_path),
                 size: 0,
-                modified: chrono::Utc::now(),
+                mime_type: "".to_string(),
+                last_modified: Some(chrono::Utc::now()),
                 etag: etag.clone(),
                 is_directory: true,
-                content_type: None,
+                created_at: Some(chrono::Utc::now()),
+                permissions: Some(0o755),
+                owner: None,
+                group: None,
+                metadata: None,
             }
         ];
 
-        Ok(readur::services::webdav::discovery::WebDAVDiscoveryResult {
+        Ok(readur::services::webdav::WebDAVDiscoveryResult {
             files: mock_files,
             directories: mock_directories,
         })
@@ -480,10 +496,11 @@ async fn test_concurrent_directory_etag_updates_during_smart_sync() {
     let smart_sync_updates = (0..15).map(|i| {
         let state_clone = state.clone();
         let user_id = user_id;
+        let base_dirs = base_directories.clone(); // Clone for use in async task
         tokio::spawn(async move {
             // Pick a directory to update
-            let dir_index = i % base_directories.len();
-            let (path, _) = &base_directories[dir_index];
+            let dir_index = i % base_dirs.len();
+            let (path, _) = &base_dirs[dir_index];
             
             // Simulate smart sync discovering changes
             sleep(Duration::from_millis(((i % 5) * 20) as u64)).await;
@@ -626,6 +643,9 @@ async fn test_concurrent_operations_with_partial_failures() {
     // Verify system resilience
     assert!(successful_operations > 0, "At least some operations should succeed");
     
+    // Save the first source ID for later use
+    let first_source_id = sources[0].id;
+    
     // Verify all sources are in consistent states
     for source in sources {
         let final_source = state.db.get_source(user_id, source.id).await
@@ -639,7 +659,7 @@ async fn test_concurrent_operations_with_partial_failures() {
     }
     
     // System should remain functional for new operations
-    let recovery_test = scheduler.trigger_sync(sources[0].id).await;
+    let recovery_test = scheduler.trigger_sync(first_source_id).await;
     // Recovery might succeed or fail, but shouldn't panic
     println!("Recovery test result: {:?}", recovery_test.is_ok());
 }

tests/integration_webdav_hash_duplicate_tests.rs

@@ -149,12 +149,13 @@ async fn create_test_app_state() -> Result<Arc<AppState>> {
     );
     
     Ok(Arc::new(AppState {
-        db,
+        db: db.clone(),
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     }))
 }
 

tests/integration_webdav_integration_tests.rs

@@ -109,12 +109,13 @@ async fn setup_test_app() -> (Router, Arc<AppState>) {
     let db = Database::new(&db_url).await.expect("Failed to connect to test database");
     let queue_service = Arc::new(readur::ocr::queue::OcrQueueService::new(db.clone(), db.pool.clone(), 2));
     let state = Arc::new(AppState { 
-        db, 
+        db: db.clone(), 
         config,
         webdav_scheduler: None,
         source_scheduler: None,
         queue_service,
         oidc_client: None,
+        sync_progress_tracker: std::sync::Arc::new(readur::services::sync_progress_tracker::SyncProgressTracker::new()),
     });
 
     let app = Router::new()

tests/performance/webdav_performance_tests.rs

@@ -0,0 +1,461 @@
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::time::sleep;
+use tracing::info;
+use uuid::Uuid;
+
+use readur::test_utils::TestContext;
+use readur::services::webdav::{SmartSyncService, SyncProgress};
+use readur::models::{CreateWebDAVDirectory, Source, SourceType, SourceConfig};
+
+/// Performance tests for WebDAV operations with large directory hierarchies
+/// These tests help identify bottlenecks and optimization opportunities
+
+#[tokio::test]
+async fn test_large_directory_hierarchy_performance() {
+    let test_ctx = TestContext::new().await;
+    let state = test_ctx.state.clone();
+    
+    // Create test user
+    let user = state.db.create_user("test@example.com", "password123").await
+        .expect("Failed to create test user");
+    
+    // Create WebDAV source
+    let source_config = SourceConfig::WebDAV {
+        server_url: "https://test.example.com".to_string(),
+        username: "test".to_string(),
+        password: "test".to_string(),
+        watch_folders: vec!["/".to_string()],
+    };
+    
+    let _source = state.db.create_source(
+        user.id,
+        "large_hierarchy_test",
+        SourceType::WebDAV,
+        source_config,
+        vec!["pdf".to_string(), "txt".to_string()],
+    ).await.expect("Failed to create WebDAV source");
+    
+    // Simulate large directory hierarchy in database
+    let start_time = Instant::now();
+    let num_directories = 1000;
+    let num_files_per_dir = 50;
+    
+    info!("🏗️ Creating test data: {} directories with {} files each", 
+          num_directories, num_files_per_dir);
+    
+    // Create directory structure
+    let mut directories = Vec::new();
+    for i in 0..num_directories {
+        let depth = i % 5; // Vary depth from 0-4
+        let path = if depth == 0 {
+            format!("/test_dir_{}", i)
+        } else {
+            format!("/test_dir_0/subdir_{}/deep_{}", depth, i)
+        };
+        
+        directories.push(CreateWebDAVDirectory {
+            user_id: user.id,
+            directory_path: path.clone(),
+            directory_etag: format!("etag_dir_{}", i),
+            file_count: num_files_per_dir,
+            total_size_bytes: (num_files_per_dir * 1024 * 10) as i64, // 10KB per file
+        });
+    }
+    
+    // Bulk insert directories
+    let insert_start = Instant::now();
+    let result = state.db.bulk_create_or_update_webdav_directories(&directories).await;
+    let insert_duration = insert_start.elapsed();
+    
+    assert!(result.is_ok(), "Failed to create test directories: {:?}", result.err());
+    info!("✅ Directory insertion completed in {:?}", insert_duration);
+    
+    // Test smart sync evaluation performance
+    let smart_sync = SmartSyncService::new(state.clone());
+    
+    // Test smart sync evaluation with created directories
+    
+    // Test evaluation performance with large dataset
+    let eval_start = Instant::now();
+    
+    // Since we don't have a real WebDAV server, we'll test the database query performance
+    let known_dirs = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to fetch directories");
+    let eval_duration = eval_start.elapsed();
+    
+    assert_eq!(known_dirs.len(), num_directories);
+    info!("📊 Directory listing query completed in {:?} for {} directories", 
+          eval_duration, known_dirs.len());
+    
+    // Performance assertions
+    assert!(insert_duration < Duration::from_secs(10), 
+            "Directory insertion took too long: {:?}", insert_duration);
+    assert!(eval_duration < Duration::from_secs(5), 
+            "Directory evaluation took too long: {:?}", eval_duration);
+    
+    let total_duration = start_time.elapsed();
+    info!("🎯 Total test duration: {:?}", total_duration);
+    
+    // Performance metrics
+    let dirs_per_sec = num_directories as f64 / insert_duration.as_secs_f64();
+    let query_rate = num_directories as f64 / eval_duration.as_secs_f64();
+    
+    info!("📈 Performance metrics:");
+    info!("   - Directory insertion rate: {:.1} dirs/sec", dirs_per_sec);
+    info!("   - Directory query rate: {:.1} dirs/sec", query_rate);
+    
+    // Ensure reasonable performance thresholds
+    assert!(dirs_per_sec > 100.0, "Directory insertion rate too slow: {:.1} dirs/sec", dirs_per_sec);
+    assert!(query_rate > 200.0, "Directory query rate too slow: {:.1} dirs/sec", query_rate);
+}
+
+#[tokio::test]
+async fn test_concurrent_directory_operations_performance() {
+    let test_ctx = TestContext::new().await;
+    let state = test_ctx.state.clone();
+    
+    // Create test user
+    let user = state.db.create_user("test2@example.com", "password123").await
+        .expect("Failed to create test user");
+    
+    info!("🔄 Testing concurrent directory operations");
+    
+    let num_concurrent_ops = 10;
+    let dirs_per_op = 100;
+    
+    let start_time = Instant::now();
+    
+    // Spawn concurrent tasks that create directories
+    let mut tasks = Vec::new();
+    for task_id in 0..num_concurrent_ops {
+        let state_clone = state.clone();
+        let user_id = user.id;
+        
+        let task = tokio::spawn(async move {
+            let mut directories = Vec::new();
+            for i in 0..dirs_per_op {
+                directories.push(CreateWebDAVDirectory {
+                    user_id,
+                    directory_path: format!("/concurrent_test_{}/dir_{}", task_id, i),
+                    directory_etag: format!("etag_{}_{}", task_id, i),
+                    file_count: 10,
+                    total_size_bytes: 10240,
+                });
+            }
+            
+            let task_start = Instant::now();
+            let result = state_clone.db.bulk_create_or_update_webdav_directories(&directories).await;
+            let task_duration = task_start.elapsed();
+            
+            (task_id, result, task_duration, directories.len())
+        });
+        
+        tasks.push(task);
+    }
+    
+    // Wait for all tasks to complete
+    let mut total_dirs_created = 0;
+    let mut max_task_duration = Duration::from_secs(0);
+    
+    for task in tasks {
+        let (task_id, result, duration, dirs_count) = task.await
+            .expect("Task panicked");
+        
+        assert!(result.is_ok(), "Task {} failed: {:?}", task_id, result.err());
+        total_dirs_created += dirs_count;
+        max_task_duration = max_task_duration.max(duration);
+        
+        info!("Task {} completed: {} dirs in {:?}", task_id, dirs_count, duration);
+    }
+    
+    let total_duration = start_time.elapsed();
+    
+    info!("🎯 Concurrent operations summary:");
+    info!("   - Total directories created: {}", total_dirs_created);
+    info!("   - Total duration: {:?}", total_duration);
+    info!("   - Longest task duration: {:?}", max_task_duration);
+    info!("   - Average throughput: {:.1} dirs/sec", 
+          total_dirs_created as f64 / total_duration.as_secs_f64());
+    
+    // Verify all directories were created
+    let final_count = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to count directories")
+        .len();
+    
+    assert_eq!(final_count, total_dirs_created);
+    
+    // Performance assertions
+    assert!(total_duration < Duration::from_secs(30), 
+            "Concurrent operations took too long: {:?}", total_duration);
+    assert!(max_task_duration < Duration::from_secs(15), 
+            "Individual task took too long: {:?}", max_task_duration);
+}
+
+#[tokio::test] 
+async fn test_etag_comparison_performance() {
+    let test_ctx = TestContext::new().await;
+    let state = test_ctx.state.clone();
+    
+    // Create test user
+    let user = state.db.create_user("test3@example.com", "password123").await
+        .expect("Failed to create test user");
+    
+    info!("🔍 Testing ETag comparison performance for large datasets");
+    
+    let num_directories = 5000;
+    let changed_percentage = 0.1; // 10% of directories have changed ETags
+    
+    // Create initial directory set
+    let mut directories = Vec::new();
+    for i in 0..num_directories {
+        directories.push(CreateWebDAVDirectory {
+            user_id: user.id,
+            directory_path: format!("/etag_test/dir_{}", i),
+            directory_etag: format!("original_etag_{}", i),
+            file_count: 5,
+            total_size_bytes: 5120,
+        });
+    }
+    
+    // Insert initial directories
+    let insert_start = Instant::now();
+    state.db.bulk_create_or_update_webdav_directories(&directories).await
+        .expect("Failed to create initial directories");
+    let insert_duration = insert_start.elapsed();
+    
+    info!("✅ Inserted {} directories in {:?}", num_directories, insert_duration);
+    
+    // Simulate changed directories (as would come from WebDAV server)
+    let num_changed = (num_directories as f64 * changed_percentage) as usize;
+    let mut discovered_directories = directories.clone();
+    
+    // Change ETags for some directories
+    for i in 0..num_changed {
+        discovered_directories[i].directory_etag = format!("changed_etag_{}", i);
+    }
+    
+    // Test smart sync evaluation performance
+    let smart_sync = SmartSyncService::new(state.clone());
+    
+    // Measure time to load known directories
+    let load_start = Instant::now();
+    let known_dirs = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to load directories");
+    let load_duration = load_start.elapsed();
+    
+    // Measure time to compare ETags
+    let compare_start = Instant::now();
+    let mut changed_dirs = Vec::new();
+    let mut unchanged_dirs = 0;
+    
+    // Convert to HashMap for O(1) lookup (simulating smart sync logic)
+    let known_etags: std::collections::HashMap<String, String> = known_dirs
+        .into_iter()
+        .map(|d| (d.directory_path, d.directory_etag))
+        .collect();
+    
+    for discovered_dir in &discovered_directories {
+        if let Some(known_etag) = known_etags.get(&discovered_dir.directory_path) {
+            if known_etag != &discovered_dir.directory_etag {
+                changed_dirs.push(discovered_dir.directory_path.clone());
+            } else {
+                unchanged_dirs += 1;
+            }
+        }
+    }
+    
+    let compare_duration = compare_start.elapsed();
+    
+    info!("📊 ETag comparison results:");
+    info!("   - Total directories: {}", num_directories);
+    info!("   - Changed directories: {}", changed_dirs.len());
+    info!("   - Unchanged directories: {}", unchanged_dirs);
+    info!("   - Load time: {:?}", load_duration);
+    info!("   - Compare time: {:?}", compare_duration);
+    info!("   - Comparison rate: {:.1} dirs/sec", 
+          num_directories as f64 / compare_duration.as_secs_f64());
+    
+    // Verify correctness
+    assert_eq!(changed_dirs.len(), num_changed);
+    assert_eq!(unchanged_dirs, num_directories - num_changed);
+    
+    // Performance assertions
+    assert!(load_duration < Duration::from_secs(2), 
+            "Directory loading took too long: {:?}", load_duration);
+    assert!(compare_duration < Duration::from_millis(500), 
+            "ETag comparison took too long: {:?}", compare_duration);
+    
+    let comparison_rate = num_directories as f64 / compare_duration.as_secs_f64();
+    assert!(comparison_rate > 10000.0, 
+            "ETag comparison rate too slow: {:.1} dirs/sec", comparison_rate);
+}
+
+#[tokio::test]
+async fn test_progress_tracking_overhead() {
+    let test_setup = TestSetup::new().await;
+    
+    info!("⏱️ Testing progress tracking performance overhead");
+    
+    let num_operations = 10000;
+    let progress = SyncProgress::new();
+    
+    // Test progress updates without progress tracking
+    let start_no_progress = Instant::now();
+    for i in 0..num_operations {
+        // Simulate work without progress tracking
+        let _dummy = format!("operation_{}", i);
+    }
+    let duration_no_progress = start_no_progress.elapsed();
+    
+    // Test progress updates with progress tracking
+    let start_with_progress = Instant::now();
+    for i in 0..num_operations {
+        // Simulate work with progress tracking
+        let _dummy = format!("operation_{}", i);
+        
+        if i % 100 == 0 {
+            progress.add_files_found(1);
+            progress.set_current_directory(&format!("/test/dir_{}", i / 100));
+        }
+    }
+    let duration_with_progress = start_with_progress.elapsed();
+    
+    let overhead = duration_with_progress.saturating_sub(duration_no_progress);
+    let overhead_percentage = (overhead.as_secs_f64() / duration_no_progress.as_secs_f64()) * 100.0;
+    
+    info!("📈 Progress tracking overhead:");
+    info!("   - Without progress: {:?}", duration_no_progress);
+    info!("   - With progress: {:?}", duration_with_progress);
+    info!("   - Overhead: {:?} ({:.1}%)", overhead, overhead_percentage);
+    
+    // Assert that progress tracking overhead is reasonable (< 50%)
+    assert!(overhead_percentage < 50.0, 
+            "Progress tracking overhead too high: {:.1}%", overhead_percentage);
+    
+    // Verify progress state
+    let stats = progress.get_stats().expect("Failed to get progress stats");
+    assert!(stats.files_processed > 0);
+    assert!(!stats.current_directory.is_empty());
+}
+
+#[tokio::test]
+async fn test_memory_usage_with_large_datasets() {
+    let test_setup = TestSetup::new().await;
+    let state = test_setup.app_state();
+    
+    // Create test user
+    let user = test_setup.create_test_user().await;
+    
+    info!("💾 Testing memory usage patterns with large datasets");
+    
+    let batch_size = 1000;
+    let num_batches = 10;
+    
+    for batch in 0..num_batches {
+        let batch_start = Instant::now();
+        
+        // Create batch of directories
+        let mut directories = Vec::new();
+        for i in 0..batch_size {
+            directories.push(CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: format!("/memory_test/batch_{}/dir_{}", batch, i),
+                directory_etag: format!("etag_{}_{}", batch, i),
+                file_count: 20,
+                total_size_bytes: 20480,
+            });
+        }
+        
+        // Process batch
+        state.db.bulk_create_or_update_webdav_directories(&directories).await
+            .expect("Failed to process batch");
+        
+        let batch_duration = batch_start.elapsed();
+        
+        // Check memory isn't growing linearly (basic heuristic)
+        if batch > 0 {
+            info!("Batch {} processed in {:?}", batch, batch_duration);
+        }
+        
+        // Small delay to prevent overwhelming the system
+        sleep(Duration::from_millis(10)).await;
+    }
+    
+    // Verify final count
+    let final_dirs = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to count final directories");
+    
+    let expected_count = batch_size * num_batches;
+    assert_eq!(final_dirs.len(), expected_count);
+    
+    info!("✅ Memory test completed with {} directories", final_dirs.len());
+}
+
+/// Benchmark directory hierarchy traversal patterns
+#[tokio::test]
+async fn test_hierarchy_traversal_patterns() {
+    let test_setup = TestSetup::new().await;
+    let state = test_setup.app_state();
+    
+    // Create test user
+    let user = test_setup.create_test_user().await;
+    
+    info!("🌳 Testing different directory hierarchy patterns");
+    
+    // Pattern 1: Wide and shallow (1000 dirs at depth 1)
+    let wide_start = Instant::now();
+    let mut wide_dirs = Vec::new();
+    for i in 0..1000 {
+        wide_dirs.push(CreateWebDAVDirectory {
+            user_id: user.id,
+            directory_path: format!("/wide/dir_{}", i),
+            directory_etag: format!("wide_etag_{}", i),
+            file_count: 10,
+            total_size_bytes: 10240,
+        });
+    }
+    
+    state.db.bulk_create_or_update_webdav_directories(&wide_dirs).await
+        .expect("Failed to create wide hierarchy");
+    let wide_duration = wide_start.elapsed();
+    
+    // Pattern 2: Deep and narrow (100 dirs at depth 10)
+    let deep_start = Instant::now();
+    let mut deep_dirs = Vec::new();
+    let mut current_path = "/deep".to_string();
+    
+    for depth in 0..10 {
+        for i in 0..10 {
+            current_path = format!("{}/level_{}_dir_{}", current_path, depth, i);
+            deep_dirs.push(CreateWebDAVDirectory {
+                user_id: user.id,
+                directory_path: current_path.clone(),
+                directory_etag: format!("deep_etag_{}_{}", depth, i),
+                file_count: 5,
+                total_size_bytes: 5120,
+            });
+        }
+    }
+    
+    state.db.bulk_create_or_update_webdav_directories(&deep_dirs).await
+        .expect("Failed to create deep hierarchy");
+    let deep_duration = deep_start.elapsed();
+    
+    info!("🎯 Hierarchy performance comparison:");
+    info!("   - Wide & shallow (1000 dirs): {:?}", wide_duration);
+    info!("   - Deep & narrow (100 dirs): {:?}", deep_duration);
+    
+    // Both should be reasonably fast
+    assert!(wide_duration < Duration::from_secs(5));
+    assert!(deep_duration < Duration::from_secs(5));
+    
+    // Query performance test
+    let query_start = Instant::now();
+    let all_dirs = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to query all directories");
+    let query_duration = query_start.elapsed();
+    
+    info!("   - Query all {} directories: {:?}", all_dirs.len(), query_duration);
+    assert!(query_duration < Duration::from_secs(2));
+}

tests/performance/webdav_simple_performance_tests.rs

@@ -0,0 +1,304 @@
+use std::time::{Duration, Instant};
+use tracing::info;
+
+use readur::test_utils::TestContext;
+use readur::services::webdav::{SmartSyncService, SyncProgress};
+use readur::models::{CreateWebDAVDirectory, CreateUser, UserRole};
+
+/// Simplified performance tests for WebDAV operations
+/// These tests establish baseline performance metrics for large-scale operations
+
+#[tokio::test]
+async fn test_directory_insertion_performance() {
+    let test_ctx = TestContext::new().await;
+    let state = test_ctx.state.clone();
+    
+    // Create test user
+    let user_data = CreateUser {
+        username: "perf_test".to_string(),
+        email: "perf_test@example.com".to_string(),
+        password: "password123".to_string(),
+        role: Some(UserRole::User),
+    };
+    let user = state.db.create_user(user_data).await
+        .expect("Failed to create test user");
+    
+    println!("🏗️ Testing directory insertion performance");
+    
+    let num_directories = 1000;
+    let start_time = Instant::now();
+    
+    // Create directory structure
+    let mut directories = Vec::new();
+    for i in 0..num_directories {
+        directories.push(CreateWebDAVDirectory {
+            user_id: user.id,
+            directory_path: format!("/perf_test/dir_{}", i),
+            directory_etag: format!("etag_{}", i),
+            file_count: 10,
+            total_size_bytes: 10240,
+        });
+    }
+    
+    // Bulk insert directories
+    let insert_start = Instant::now();
+    let result = state.db.bulk_create_or_update_webdav_directories(&directories).await;
+    let insert_duration = insert_start.elapsed();
+    
+    assert!(result.is_ok(), "Failed to create directories: {:?}", result.err());
+    
+    // Test directory listing performance
+    let query_start = Instant::now();
+    let fetched_dirs = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to fetch directories");
+    let query_duration = query_start.elapsed();
+    
+    let total_duration = start_time.elapsed();
+    
+    // Performance metrics
+    let insert_rate = num_directories as f64 / insert_duration.as_secs_f64();
+    let query_rate = fetched_dirs.len() as f64 / query_duration.as_secs_f64();
+    
+    println!("📊 Directory performance results:");
+    println!("   - Directories created: {}", num_directories);
+    println!("   - Directories fetched: {}", fetched_dirs.len());
+    println!("   - Insert time: {:?} ({:.1} dirs/sec)", insert_duration, insert_rate);
+    println!("   - Query time: {:?} ({:.1} dirs/sec)", query_duration, query_rate);
+    println!("   - Total time: {:?}", total_duration);
+    
+    // Verify correctness
+    assert_eq!(fetched_dirs.len(), num_directories);
+    
+    // Performance assertions (reasonable thresholds)
+    assert!(insert_duration < Duration::from_secs(5), 
+            "Insert took too long: {:?}", insert_duration);
+    assert!(query_duration < Duration::from_secs(2), 
+            "Query took too long: {:?}", query_duration);
+    assert!(insert_rate > 200.0, 
+            "Insert rate too slow: {:.1} dirs/sec", insert_rate);
+    assert!(query_rate > 500.0, 
+            "Query rate too slow: {:.1} dirs/sec", query_rate);
+}
+
+#[tokio::test]
+async fn test_etag_comparison_performance() {
+    let test_ctx = TestContext::new().await;
+    let state = test_ctx.state.clone();
+    
+    // Create test user
+    let user_data = CreateUser {
+        username: "etag_test".to_string(),
+        email: "etag_test@example.com".to_string(),
+        password: "password123".to_string(),
+        role: Some(UserRole::User),
+    };
+    let user = state.db.create_user(user_data).await
+        .expect("Failed to create test user");
+    
+    println!("🔍 Testing ETag comparison performance");
+    
+    let num_directories = 2000;
+    let changed_count = 200; // 10% changed
+    
+    // Create initial directories
+    let mut directories = Vec::new();
+    for i in 0..num_directories {
+        directories.push(CreateWebDAVDirectory {
+            user_id: user.id,
+            directory_path: format!("/etag_test/dir_{}", i),
+            directory_etag: format!("original_etag_{}", i),
+            file_count: 5,
+            total_size_bytes: 5120,
+        });
+    }
+    
+    // Insert directories
+    state.db.bulk_create_or_update_webdav_directories(&directories).await
+        .expect("Failed to insert directories");
+    
+    // Load directories for comparison
+    let load_start = Instant::now();
+    let known_dirs = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to load directories");
+    let load_duration = load_start.elapsed();
+    
+    // Create comparison data (simulating discovered directories)
+    let mut discovered_dirs = directories.clone();
+    for i in 0..changed_count {
+        discovered_dirs[i].directory_etag = format!("changed_etag_{}", i);
+    }
+    
+    // Perform ETag comparison
+    let compare_start = Instant::now();
+    
+    // Convert to HashMap for efficient lookup
+    let known_etags: std::collections::HashMap<String, String> = known_dirs
+        .into_iter()
+        .map(|d| (d.directory_path, d.directory_etag))
+        .collect();
+    
+    let mut changed_dirs = 0;
+    let mut unchanged_dirs = 0;
+    
+    for discovered in &discovered_dirs {
+        if let Some(known_etag) = known_etags.get(&discovered.directory_path) {
+            if known_etag != &discovered.directory_etag {
+                changed_dirs += 1;
+            } else {
+                unchanged_dirs += 1;
+            }
+        }
+    }
+    
+    let compare_duration = compare_start.elapsed();
+    
+    println!("📊 ETag comparison results:");
+    println!("   - Total directories: {}", num_directories);
+    println!("   - Changed detected: {}", changed_dirs);
+    println!("   - Unchanged detected: {}", unchanged_dirs);
+    println!("   - Load time: {:?}", load_duration);
+    println!("   - Compare time: {:?}", compare_duration);
+    println!("   - Comparison rate: {:.1} dirs/sec", 
+          num_directories as f64 / compare_duration.as_secs_f64());
+    
+    // Verify correctness
+    assert_eq!(changed_dirs, changed_count);
+    assert_eq!(unchanged_dirs, num_directories - changed_count);
+    
+    // Performance assertions
+    assert!(load_duration < Duration::from_secs(2));
+    assert!(compare_duration < Duration::from_millis(100)); // Very fast operation
+    
+    let comparison_rate = num_directories as f64 / compare_duration.as_secs_f64();
+    assert!(comparison_rate > 20000.0, 
+            "Comparison rate too slow: {:.1} dirs/sec", comparison_rate);
+}
+
+#[tokio::test]
+async fn test_progress_tracking_performance() {
+    println!("⏱️ Testing progress tracking performance overhead");
+    
+    let num_operations = 5000;
+    let progress = SyncProgress::new();
+    
+    // Test without progress tracking
+    let start_no_progress = Instant::now();
+    for i in 0..num_operations {
+        let _work = format!("operation_{}", i);
+    }
+    let duration_no_progress = start_no_progress.elapsed();
+    
+    // Test with progress tracking
+    let start_with_progress = Instant::now();
+    for i in 0..num_operations {
+        let _work = format!("operation_{}", i);
+        
+        if i % 50 == 0 {
+            progress.add_files_found(1);
+            progress.set_current_directory(&format!("/test/dir_{}", i / 50));
+        }
+    }
+    let duration_with_progress = start_with_progress.elapsed();
+    
+    let overhead = duration_with_progress.saturating_sub(duration_no_progress);
+    let overhead_percentage = if duration_no_progress.as_nanos() > 0 {
+        (overhead.as_nanos() as f64 / duration_no_progress.as_nanos() as f64) * 100.0
+    } else {
+        0.0
+    };
+    
+    println!("📈 Progress tracking overhead analysis:");
+    println!("   - Operations: {}", num_operations);
+    println!("   - Without progress: {:?}", duration_no_progress);
+    println!("   - With progress: {:?}", duration_with_progress);
+    println!("   - Overhead: {:?} ({:.1}%)", overhead, overhead_percentage);
+    
+    // Verify progress was tracked
+    let stats = progress.get_stats().expect("Failed to get progress stats");
+    assert!(stats.files_found > 0);
+    
+    // Performance assertion - overhead should be minimal
+    assert!(overhead_percentage < 100.0, 
+            "Progress tracking overhead too high: {:.1}%", overhead_percentage);
+}
+
+#[tokio::test] 
+async fn test_smart_sync_evaluation_performance() {
+    let test_ctx = TestContext::new().await;
+    let state = test_ctx.state.clone();
+    
+    // Create test user
+    let user_data = CreateUser {
+        username: "smart_sync_test".to_string(),
+        email: "smart_sync_test@example.com".to_string(),
+        password: "password123".to_string(),
+        role: Some(UserRole::User),
+    };
+    let user = state.db.create_user(user_data).await
+        .expect("Failed to create test user");
+    
+    println!("🧠 Testing smart sync evaluation performance");
+    
+    let num_directories = 3000;
+    
+    // Create directory structure
+    let mut directories = Vec::new();
+    for i in 0..num_directories {
+        let depth = i % 4; // Vary depth
+        let path = if depth == 0 {
+            format!("/smart_test/dir_{}", i)
+        } else {
+            format!("/smart_test/level_{}/dir_{}", depth, i)
+        };
+        
+        directories.push(CreateWebDAVDirectory {
+            user_id: user.id,
+            directory_path: path,
+            directory_etag: format!("etag_{}", i),
+            file_count: (i % 20) as i64, // Vary file counts
+            total_size_bytes: ((i % 20) * 1024) as i64,
+        });
+    }
+    
+    // Insert directories
+    let insert_start = Instant::now();
+    state.db.bulk_create_or_update_webdav_directories(&directories).await
+        .expect("Failed to insert directories");
+    let insert_duration = insert_start.elapsed();
+    
+    // Test smart sync service performance
+    let smart_sync = SmartSyncService::new(state.clone());
+    
+    // Test directory filtering performance (simulating smart sync logic)
+    let filter_start = Instant::now();
+    let known_dirs = state.db.list_webdav_directories(user.id).await
+        .expect("Failed to fetch directories");
+    
+    // Filter directories by path prefix (common smart sync operation)
+    let prefix = "/smart_test/";
+    let filtered_dirs: Vec<_> = known_dirs
+        .into_iter()
+        .filter(|d| d.directory_path.starts_with(prefix))
+        .collect();
+    
+    let filter_duration = filter_start.elapsed();
+    
+    println!("📊 Smart sync evaluation results:");
+    println!("   - Total directories: {}", num_directories);
+    println!("   - Filtered directories: {}", filtered_dirs.len());
+    println!("   - Insert time: {:?}", insert_duration);
+    println!("   - Filter time: {:?}", filter_duration);
+    println!("   - Filter rate: {:.1} dirs/sec", 
+          filtered_dirs.len() as f64 / filter_duration.as_secs_f64());
+    
+    // Verify filtering worked correctly
+    assert_eq!(filtered_dirs.len(), num_directories);
+    
+    // Performance assertions
+    assert!(insert_duration < Duration::from_secs(10));
+    assert!(filter_duration < Duration::from_millis(500));
+    
+    let filter_rate = filtered_dirs.len() as f64 / filter_duration.as_secs_f64();
+    assert!(filter_rate > 6000.0, 
+            "Filter rate too slow: {:.1} dirs/sec", filter_rate);
+}

tests/unit_smart_sync_service_tests.rs

@@ -58,7 +58,7 @@ async fn test_evaluate_sync_need_first_time_no_known_directories() {
     
     // Test evaluation - should detect no known directories and require deep scan
     let webdav_service = create_real_webdav_service();
-    let decision = smart_sync_service.evaluate_sync_need(user_id, &webdav_service, "/Documents").await;
+    let decision = smart_sync_service.evaluate_sync_need(user_id, &webdav_service, "/Documents", None).await;
     
     match decision {
         Ok(SmartSyncDecision::RequiresSync(SmartSyncStrategy::FullDeepScan)) => {
@@ -197,16 +197,16 @@ async fn test_directory_etag_comparison_logic() {
     let mut unchanged_directories = Vec::new();
     
     for current_dir in &current_dirs {
-        match known_map.get(&current_dir.path) {
+        match known_map.get(&current_dir.relative_path) {
             Some(known_etag) => {
                 if known_etag != &current_dir.etag {
-                    changed_directories.push(current_dir.path.clone());
+                    changed_directories.push(current_dir.relative_path.clone());
                 } else {
-                    unchanged_directories.push(current_dir.path.clone());
+                    unchanged_directories.push(current_dir.relative_path.clone());
                 }
             }
             None => {
-                new_directories.push(current_dir.path.clone());
+                new_directories.push(current_dir.relative_path.clone());
             }
         }
     }
@@ -295,7 +295,7 @@ async fn test_smart_sync_error_handling() {
     
     // This should not panic, but handle the error gracefully
     let webdav_service = create_real_webdav_service();
-    let decision = smart_sync_service.evaluate_sync_need(invalid_user_id, &webdav_service, "/Documents").await;
+    let decision = smart_sync_service.evaluate_sync_need(invalid_user_id, &webdav_service, "/Documents", None).await;
     
     match decision {
         Ok(SmartSyncDecision::RequiresSync(SmartSyncStrategy::FullDeepScan)) => {

tests/unit_webdav_url_management_tests.rs

@@ -1,5 +1,5 @@
 use readur::models::FileIngestionInfo;
-use readur::services::webdav::{WebDAVConfig, WebDAVUrlManager};
+use readur::services::webdav::{WebDAVConfig, WebDAVService};
 
 #[test]
 fn test_nextcloud_directory_path_handling() {
@@ -13,7 +13,7 @@ fn test_nextcloud_directory_path_handling() {
         server_type: Some("nextcloud".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Test a directory from Nextcloud WebDAV response
     let directory_info = FileIngestionInfo {
@@ -57,7 +57,7 @@ fn test_nextcloud_file_path_handling() {
         server_type: Some("nextcloud".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Test a file from Nextcloud WebDAV response
     let file_info = FileIngestionInfo {
@@ -101,7 +101,7 @@ fn test_webdav_root_path_handling() {
         server_type: Some("nextcloud".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Test root directory handling
     let root_info = FileIngestionInfo {
@@ -141,7 +141,7 @@ fn test_url_construction_from_relative_path() {
         server_type: Some("nextcloud".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Test URL construction for scanning subdirectories
     let subfolder_url = manager.relative_path_to_url("/Photos/Subfolder/");
@@ -166,7 +166,7 @@ fn test_owncloud_path_handling() {
         server_type: Some("owncloud".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Test ownCloud path conversion
     let file_info = FileIngestionInfo {
@@ -204,7 +204,7 @@ fn test_generic_webdav_path_handling() {
         server_type: Some("generic".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Test generic WebDAV path conversion
     let file_info = FileIngestionInfo {
@@ -243,7 +243,7 @@ fn test_download_path_resolution() {
         server_type: Some("nextcloud".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Test that processed file info has correct paths for download operations
     let file_info = FileIngestionInfo {
@@ -292,7 +292,7 @@ fn test_with_nextcloud_fixture_data() {
         server_type: Some("nextcloud".to_string()),
     };
 
-    let manager = WebDAVUrlManager::new(config);
+    let manager = WebDAVService::new(config).unwrap();
 
     // Load the real Nextcloud XML fixture
     let fixture_path = "tests/fixtures/webdav/nextcloud_photos_propfind_response.xml";

tests/webdav_production_flow_integration_tests.rs

@@ -13,7 +13,7 @@ use readur::{
         SmartSyncStrategy, 
         SyncProgress, 
         SyncPhase,
-        discovery::WebDAVDiscoveryResult,
+        WebDAVDiscoveryResult,
     },
 };
 
@@ -53,7 +53,7 @@ async fn create_production_webdav_source(
         name: name.to_string(),
         source_type: SourceType::WebDAV,
         config: serde_json::to_value(config).unwrap(),
-        enabled: true,
+        enabled: Some(true),
     };
 
     state.db.create_source(user_id, &create_source).await
@@ -80,20 +80,36 @@ impl ProductionMockWebDAVService {
                 FileIngestionInfo {
                     name: "report.pdf".to_string(),
                     relative_path: "/Documents/report.pdf".to_string(),
+                    full_path: "/remote.php/dav/files/user/Documents/report.pdf".to_string(),
+                    #[allow(deprecated)]
+                    path: "/Documents/report.pdf".to_string(),
                     size: 2048576, // 2MB
-                    modified: chrono::Utc::now() - chrono::Duration::hours(2),
+                    last_modified: Some(chrono::Utc::now() - chrono::Duration::hours(2)),
                     etag: "report-etag-1".to_string(),
                     is_directory: false,
-                    content_type: Some("application/pdf".to_string()),
+                    mime_type: "application/pdf".to_string(),
+                    created_at: None,
+                    permissions: None,
+                    owner: None,
+                    group: None,
+                    metadata: None,
                 },
                 FileIngestionInfo {
                     name: "notes.md".to_string(),
                     relative_path: "/Documents/notes.md".to_string(),
+                    full_path: "/remote.php/dav/files/user/Documents/notes.md".to_string(),
+                    #[allow(deprecated)]
+                    path: "/Documents/notes.md".to_string(),
                     size: 4096, // 4KB
-                    modified: chrono::Utc::now() - chrono::Duration::minutes(30),
+                    last_modified: Some(chrono::Utc::now() - chrono::Duration::minutes(30)),
                     etag: "notes-etag-1".to_string(),
                     is_directory: false,
-                    content_type: Some("text/markdown".to_string()),
+                    mime_type: "text/markdown".to_string(),
+                    created_at: None,
+                    permissions: None,
+                    owner: None,
+                    group: None,
+                    metadata: None,
                 },
             ]
         ));
@@ -104,11 +120,19 @@ impl ProductionMockWebDAVService {
                 FileIngestionInfo {
                     name: "spec.docx".to_string(),
                     relative_path: "/Projects/spec.docx".to_string(),
+                    full_path: "/remote.php/dav/files/user/Projects/spec.docx".to_string(),
+                    #[allow(deprecated)]
+                    path: "/Projects/spec.docx".to_string(),
                     size: 1024000, // 1MB
-                    modified: chrono::Utc::now() - chrono::Duration::days(1),
+                    last_modified: Some(chrono::Utc::now() - chrono::Duration::days(1)),
                     etag: "spec-etag-1".to_string(),
                     is_directory: false,
-                    content_type: Some("application/vnd.openxmlformats-officedocument.wordprocessingml.document".to_string()),
+                    mime_type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document".to_string(),
+                    created_at: None,
+                    permissions: None,
+                    owner: None,
+                    group: None,
+                    metadata: None,
                 },
             ]
         ));
@@ -154,11 +178,19 @@ impl ProductionMockWebDAVService {
             let directory_info = FileIngestionInfo {
                 name: directory_path.split('/').last().unwrap_or("").to_string(),
                 relative_path: directory_path.to_string(),
+                full_path: format!("/remote.php/dav/files/user{}", directory_path),
+                #[allow(deprecated)]
+                path: directory_path.to_string(),
                 size: 0,
-                modified: chrono::Utc::now(),
+                last_modified: Some(chrono::Utc::now()),
                 etag: etag.clone(),
                 is_directory: true,
-                content_type: None,
+                mime_type: "application/octet-stream".to_string(),
+                created_at: None,
+                permissions: None,
+                owner: None,
+                group: None,
+                metadata: None,
             };
 
             Ok(WebDAVDiscoveryResult {
@@ -211,27 +243,30 @@ async fn test_production_sync_flow_concurrent_sources() {
     ];
     
     // Simulate production workload: concurrent sync triggers from different sources
-    let production_sync_operations = sources.iter().zip(mock_services.iter()).enumerate().map(|(i, (source, mock_service))| {
-        let scheduler_clone = scheduler.clone();
+    let production_sync_operations: Vec<_> = sources.iter().zip(mock_services.iter()).enumerate().map(|(i, (source, mock_service))| {
         let state_clone = state.clone();
         let smart_sync_service = SmartSyncService::new(state_clone.clone());
         let source_id = source.id;
         let source_name = source.name.clone();
+        let source_config = source.config.clone(); // Clone the config to avoid borrowing the source
         let mock_service = mock_service.clone();
         let user_id = user_id;
         
         tokio::spawn(async move {
             println!("🚀 Starting production sync for source: {}", source_name);
             
+            // Create scheduler instance for this task
+            let scheduler_local = SourceScheduler::new(state_clone.clone());
+            
             // Step 1: Trigger sync via scheduler (Route Level simulation)
-            let trigger_result = scheduler_clone.trigger_sync(source_id).await;
+            let trigger_result = scheduler_local.trigger_sync(source_id).await;
             if trigger_result.is_err() {
                 println!("❌ Failed to trigger sync for {}: {:?}", source_name, trigger_result);
                 return (i, source_name, false, 0, 0);
             }
             
             // Step 2: Simulate smart sync evaluation and execution
-            let config: WebDAVSourceConfig = serde_json::from_value(source.config.clone()).unwrap();
+            let config: WebDAVSourceConfig = serde_json::from_value(source_config).unwrap();
             let mut total_files_discovered = 0;
             let mut total_directories_processed = 0;
             
@@ -277,7 +312,7 @@ async fn test_production_sync_flow_concurrent_sources() {
             
             (i, source_name, true, total_files_discovered, total_directories_processed)
         })
-    });
+    }).collect();
     
     // Wait for all production sync operations
     let sync_results: Vec<_> = join_all(production_sync_operations).await;
@@ -366,20 +401,23 @@ async fn test_production_concurrent_user_actions() {
     ];
     
     let user_action_tasks = user_actions.into_iter().map(|(delay_ms, action, source_id, _)| {
-        let scheduler_clone = scheduler.clone();
+        let state_clone = state.clone();
         let action = action.to_string();
         tokio::spawn(async move {
             // Wait for scheduled time
             sleep(Duration::from_millis(delay_ms)).await;
             
+            // Create scheduler instance for this task
+            let scheduler_local = SourceScheduler::new(state_clone);
+            
             let result = match action.as_str() {
                 "trigger" => {
                     println!("🎯 User action: trigger sync for source {}", source_id);
-                    scheduler_clone.trigger_sync(source_id).await
+                    scheduler_local.trigger_sync(source_id).await
                 }
                 "stop" => {
                     println!("🛑 User action: stop sync for source {}", source_id);
-                    scheduler_clone.stop_sync(source_id).await
+                    scheduler_local.stop_sync(source_id).await
                 }
                 _ => Ok(()),
             };
@@ -477,7 +515,6 @@ async fn test_production_resource_management() {
     
     // Test concurrent operations under memory pressure
     let memory_stress_operations = (0..50).map(|i| {
-        let scheduler_clone = scheduler.clone();
         let smart_sync_clone = smart_sync_service.clone();
         let state_clone = state.clone();
         let source_id = sources[i % sources.len()].id;
@@ -492,7 +529,8 @@ async fn test_production_resource_management() {
                 }
                 1 => {
                     // Sync trigger operation
-                    scheduler_clone.trigger_sync(source_id).await.is_ok() as usize
+                    let scheduler_local = SourceScheduler::new(state_clone.clone());
+                    scheduler_local.trigger_sync(source_id).await.is_ok() as usize
                 }
                 2 => {
                     // Multiple directory updates
@@ -513,7 +551,8 @@ async fn test_production_resource_management() {
                 }
                 3 => {
                     // Stop operation
-                    scheduler_clone.stop_sync(source_id).await.is_ok() as usize
+                    let scheduler_local = SourceScheduler::new(state_clone.clone());
+                    scheduler_local.stop_sync(source_id).await.is_ok() as usize
                 }
                 4 => {
                     // Batch directory read and update

tests/webdav_smart_sync_integration_tests.rs

@@ -12,7 +12,7 @@ use readur::{
         SmartSyncStrategy, 
         SyncProgress, 
         SyncPhase,
-        discovery::WebDAVDiscoveryResult,
+        WebDAVDiscoveryResult,
     },
 };
 
@@ -82,22 +82,36 @@ impl MockWebDAVServiceForSmartSync {
                     FileIngestionInfo {
                         name: "default.pdf".to_string(),
                         relative_path: format!("{}/default.pdf", directory_path),
+                        full_path: format!("{}/default.pdf", directory_path),
+                        path: format!("{}/default.pdf", directory_path),
                         size: 1024,
-                        modified: chrono::Utc::now(),
+                        mime_type: "application/pdf".to_string(),
+                        last_modified: Some(chrono::Utc::now()),
                         etag: format!("default-etag-{}", directory_path.replace('/', "-")),
                         is_directory: false,
-                        content_type: Some("application/pdf".to_string()),
+                        created_at: None,
+                        permissions: None,
+                        owner: None,
+                        group: None,
+                        metadata: None,
                     }
                 ],
                 directories: vec![
                     FileIngestionInfo {
                         name: "subdir".to_string(),
                         relative_path: format!("{}/subdir", directory_path),
+                        full_path: format!("{}/subdir", directory_path),
+                        path: format!("{}/subdir", directory_path),
                         size: 0,
-                        modified: chrono::Utc::now(),
+                        mime_type: "inode/directory".to_string(),
+                        last_modified: Some(chrono::Utc::now()),
                         etag: format!("dir-etag-{}", directory_path.replace('/', "-")),
                         is_directory: true,
-                        content_type: None,
+                        created_at: None,
+                        permissions: None,
+                        owner: None,
+                        group: None,
+                        metadata: None,
                     }
                 ],
             })
@@ -166,20 +180,34 @@ async fn test_concurrent_smart_sync_etag_evaluation() {
                         FileIngestionInfo {
                             name: "subdir1".to_string(),
                             relative_path: "/test/subdir1".to_string(),
+                            full_path: "/test/subdir1".to_string(),
+                            path: "/test/subdir1".to_string(),
                             size: 0,
-                            modified: chrono::Utc::now(),
+                            mime_type: "inode/directory".to_string(),
+                            last_modified: Some(chrono::Utc::now()),
                             etag: "old-etag-2".to_string(), // Same as database
                             is_directory: true,
-                            content_type: None,
+                            created_at: None,
+                            permissions: None,
+                            owner: None,
+                            group: None,
+                            metadata: None,
                         },
                         FileIngestionInfo {
                             name: "subdir2".to_string(),
                             relative_path: "/test/subdir2".to_string(),
+                            full_path: "/test/subdir2".to_string(),
+                            path: "/test/subdir2".to_string(),
                             size: 0,
-                            modified: chrono::Utc::now(),
+                            mime_type: "inode/directory".to_string(),
+                            last_modified: Some(chrono::Utc::now()),
                             etag: "old-etag-3".to_string(), // Same as database
                             is_directory: true,
-                            content_type: None,
+                            created_at: None,
+                            permissions: None,
+                            owner: None,
+                            group: None,
+                            metadata: None,
                         },
                     ],
                 });
@@ -192,11 +220,18 @@ async fn test_concurrent_smart_sync_etag_evaluation() {
                         FileIngestionInfo {
                             name: "subdir1".to_string(),
                             relative_path: "/test/subdir1".to_string(),
+                            full_path: "/test/subdir1".to_string(),
+                            path: "/test/subdir1".to_string(),
                             size: 0,
-                            modified: chrono::Utc::now(),
+                            mime_type: "inode/directory".to_string(),
+                            last_modified: Some(chrono::Utc::now()),
                             etag: "new-etag-2".to_string(), // Changed
                             is_directory: true,
-                            content_type: None,
+                            created_at: None,
+                            permissions: None,
+                            owner: None,
+                            group: None,
+                            metadata: None,
                         },
                     ],
                 });
@@ -209,11 +244,18 @@ async fn test_concurrent_smart_sync_etag_evaluation() {
                         FileIngestionInfo {
                             name: "new_subdir".to_string(),
                             relative_path: "/test/new_subdir".to_string(),
+                            full_path: "/test/new_subdir".to_string(),
+                            path: "/test/new_subdir".to_string(),
                             size: 0,
-                            modified: chrono::Utc::now(),
+                            mime_type: "inode/directory".to_string(),
+                            last_modified: Some(chrono::Utc::now()),
                             etag: "new-dir-etag".to_string(),
                             is_directory: true,
-                            content_type: None,
+                            created_at: None,
+                            permissions: None,
+                            owner: None,
+                            group: None,
+                            metadata: None,
                         },
                     ],
                 });
@@ -226,20 +268,34 @@ async fn test_concurrent_smart_sync_etag_evaluation() {
                         FileIngestionInfo {
                             name: "subdir1".to_string(),
                             relative_path: "/test/subdir1".to_string(),
+                            full_path: "/test/subdir1".to_string(),
+                            path: "/test/subdir1".to_string(),
                             size: 0,
-                            modified: chrono::Utc::now(),
+                            mime_type: "inode/directory".to_string(),
+                            last_modified: Some(chrono::Utc::now()),
                             etag: "updated-etag-2".to_string(), // Changed
                             is_directory: true,
-                            content_type: None,
+                            created_at: None,
+                            permissions: None,
+                            owner: None,
+                            group: None,
+                            metadata: None,
                         },
                         FileIngestionInfo {
                             name: "another_new_dir".to_string(),
                             relative_path: "/test/another_new_dir".to_string(),
+                            full_path: "/test/another_new_dir".to_string(),
+                            path: "/test/another_new_dir".to_string(),
                             size: 0,
-                            modified: chrono::Utc::now(),
+                            mime_type: "inode/directory".to_string(),
+                            last_modified: Some(chrono::Utc::now()),
                             etag: "another-new-etag".to_string(), // New
                             is_directory: true,
-                            content_type: None,
+                            created_at: None,
+                            permissions: None,
+                            owner: None,
+                            group: None,
+                            metadata: None,
                         },
                     ],
                 });
@@ -261,7 +317,7 @@ async fn test_concurrent_smart_sync_etag_evaluation() {
             // that SmartSyncService would call
             
             // 1. Get known directories (what SmartSyncService.evaluate_sync_need does)
-            let known_dirs_result = smart_sync_clone.state.db.list_webdav_directories(user_id).await;
+            let known_dirs_result = smart_sync_clone.state().db.list_webdav_directories(user_id).await;
             
             // 2. Simulate discovery with delay (mock WebDAV call)
             let discovery_result = mock_service.mock_discover_files_and_directories("/test", false).await;
@@ -277,7 +333,7 @@ async fn test_concurrent_smart_sync_etag_evaluation() {
                         file_count: 0,
                         total_size_bytes: 0,
                     };
-                    let result = smart_sync_clone.state.db.create_or_update_webdav_directory(&update_dir).await;
+                    let result = smart_sync_clone.state().db.create_or_update_webdav_directory(&update_dir).await;
                     results.push(result.is_ok());
                 }
                 results
@@ -384,10 +440,10 @@ async fn test_concurrent_smart_sync_strategies() {
             println!("Starting strategy test {} ({}) for {}", i, test_name, base_path);
             
             // Simulate what perform_smart_sync would do for each strategy
-            let result = match strategy {
+            let result: Result<i32, anyhow::Error> = match strategy {
                 SmartSyncStrategy::FullDeepScan => {
                     // Simulate full deep scan - update all directories under base_path
-                    let all_dirs = smart_sync_clone.state.db.list_webdav_directories(user_id).await?;
+                    let all_dirs = smart_sync_clone.state().db.list_webdav_directories(user_id).await?;
                     let relevant_dirs: Vec<_> = all_dirs.into_iter()
                         .filter(|d| d.directory_path.starts_with(&base_path))
                         .collect();
@@ -402,7 +458,7 @@ async fn test_concurrent_smart_sync_strategies() {
                             total_size_bytes: dir.total_size_bytes + 100,
                         };
                         
-                        if smart_sync_clone.state.db.create_or_update_webdav_directory(&updated_dir).await.is_ok() {
+                        if smart_sync_clone.state().db.create_or_update_webdav_directory(&updated_dir).await.is_ok() {
                             update_count += 1;
                         }
                     }
@@ -420,7 +476,7 @@ async fn test_concurrent_smart_sync_strategies() {
                             total_size_bytes: 2048,
                         };
                         
-                        if smart_sync_clone.state.db.create_or_update_webdav_directory(&updated_dir).await.is_ok() {
+                        if smart_sync_clone.state().db.create_or_update_webdav_directory(&updated_dir).await.is_ok() {
                             update_count += 1;
                         }
                     }
@@ -503,7 +559,7 @@ async fn test_concurrent_smart_sync_progress_tracking() {
             // Simulate database operations
             sleep(Duration::from_millis(50)).await;
             
-            progress.set_phase(SyncPhase::Discovering);
+            progress.set_phase(SyncPhase::DiscoveringDirectories);
             progress.set_current_directory(&format!("/operation-{}/subdir", i));
             
             // Simulate discovery delay
@@ -520,7 +576,7 @@ async fn test_concurrent_smart_sync_progress_tracking() {
                 total_size_bytes: (i as i64) * 1024,
             };
             
-            let db_result = smart_sync_clone.state.db.create_or_update_webdav_directory(&directory).await;
+            let db_result = smart_sync_clone.state().db.create_or_update_webdav_directory(&directory).await;
             
             if db_result.is_ok() {
                 progress.set_phase(SyncPhase::Completed);
@@ -550,7 +606,7 @@ async fn test_concurrent_smart_sync_progress_tracking() {
         }
         
         if let Some(stats) = stats {
-            println!("Operation {}: Success: {}, Elapsed: {:?}, Errors: {}", 
+            println!("Operation {}: Success: {}, Elapsed: {:?}, Errors: {:?}", 
                      operation_id, db_success, stats.elapsed_time, stats.errors);
         }
     }
@@ -599,7 +655,7 @@ async fn test_concurrent_smart_sync_etag_conflicts() {
             println!("ETag conflict operation {} starting", i);
             
             // First, read the current directory state
-            let current_dirs = smart_sync_clone.state.db.list_webdav_directories(user_id).await?;
+            let current_dirs = smart_sync_clone.state().db.list_webdav_directories(user_id).await?;
             let shared_dir = current_dirs.iter()
                 .find(|d| d.directory_path == "/shared")
                 .ok_or_else(|| anyhow::anyhow!("Shared directory not found"))?;
@@ -616,7 +672,7 @@ async fn test_concurrent_smart_sync_etag_conflicts() {
                 total_size_bytes: shared_dir.total_size_bytes + (i as i64 * 100),
             };
             
-            let update_result = smart_sync_clone.state.db.create_or_update_webdav_directory(&updated_directory).await;
+            let update_result = smart_sync_clone.state().db.create_or_update_webdav_directory(&updated_directory).await;
             
             println!("ETag conflict operation {} completed: {:?}", i, update_result.is_ok());
             Result::<_, anyhow::Error>::Ok((i, update_result.is_ok()))