feat(client/server): add a new badge for each source that shows the number of documents stored from each source

This commit is contained in:
perf3ct
2025-06-27 21:32:50 +00:00
parent 341a91e1a7
commit a75fca0c28
4 changed files with 186 additions and 30 deletions

View File

@@ -65,6 +65,8 @@ import {
Storage as ServerIcon,
Pause as PauseIcon,
PlayArrow as ResumeIcon,
TextSnippet as DocumentIcon,
Visibility as OcrIcon,
} from '@mui/icons-material';
import { useNavigate } from 'react-router-dom';
import api, { queueService } from '../services/api';
@@ -84,6 +86,8 @@ interface Source {
total_files_synced: number;
total_files_pending: number;
total_size_bytes: number;
total_documents: number;
total_documents_ocr: number;
created_at: string;
updated_at: string;
}
@@ -699,7 +703,7 @@ const SourcesPage: React.FC = () => {
<Typography variant="h6" fontWeight="bold" gutterBottom>
{source.name}
</Typography>
<Stack direction="row" spacing={1} alignItems="center">
<Stack direction="row" spacing={1} alignItems="center" flexWrap="wrap">
<Chip
label={source.source_type.toUpperCase()}
size="small"
@@ -723,6 +727,32 @@ const SourcesPage: React.FC = () => {
fontWeight: 600,
}}
/>
<Chip
icon={<DocumentIcon sx={{ fontSize: '0.9rem !important' }} />}
label={`${source.total_documents} docs`}
size="small"
sx={{
borderRadius: 2,
bgcolor: alpha(theme.palette.info.main, 0.1),
color: theme.palette.info.main,
border: `1px solid ${alpha(theme.palette.info.main, 0.3)}`,
fontSize: '0.75rem',
fontWeight: 600,
}}
/>
<Chip
icon={<OcrIcon sx={{ fontSize: '0.9rem !important' }} />}
label={`${source.total_documents_ocr} OCR'd`}
size="small"
sx={{
borderRadius: 2,
bgcolor: alpha(theme.palette.success.main, 0.1),
color: theme.palette.success.main,
border: `1px solid ${alpha(theme.palette.success.main, 0.3)}`,
fontSize: '0.75rem',
fontWeight: 600,
}}
/>
{!source.enabled && (
<Chip
label="Disabled"
@@ -814,34 +844,25 @@ const SourcesPage: React.FC = () => {
{/* Stats Grid */}
<Grid container spacing={2} mb={3}>
<Grid item xs={6} sm={4} md={3}>
<Grid item xs={6} sm={4} md={2.4}>
<StatCard
icon={<TrendingUpIcon />}
label="Files Processed"
value={source.total_files_synced}
icon={<DocumentIcon />}
label="Documents Stored"
value={source.total_documents}
color="info"
tooltip="Total number of documents currently stored from this source"
/>
</Grid>
<Grid item xs={6} sm={4} md={2.4}>
<StatCard
icon={<OcrIcon />}
label="OCR Processed"
value={source.total_documents_ocr}
color="success"
tooltip="Files attempted to be synced, including duplicates and skipped files"
tooltip="Number of documents that have been successfully OCR'd"
/>
</Grid>
<Grid item xs={6} sm={4} md={3}>
<StatCard
icon={<SpeedIcon />}
label="Files Pending"
value={source.total_files_pending}
color="warning"
tooltip="Files discovered but not yet processed during sync"
/>
</Grid>
<Grid item xs={6} sm={4} md={3}>
<StatCard
icon={<StorageIcon />}
label="Total Size (Downloaded)"
value={formatBytes(source.total_size_bytes)}
color="primary"
tooltip="Total size of files successfully downloaded from this source"
/>
</Grid>
<Grid item xs={6} sm={4} md={3}>
<Grid item xs={6} sm={4} md={2.4}>
<StatCard
icon={<TimelineIcon />}
label="Last Sync"
@@ -852,6 +873,24 @@ const SourcesPage: React.FC = () => {
tooltip="When this source was last synchronized"
/>
</Grid>
<Grid item xs={6} sm={4} md={2.4}>
<StatCard
icon={<SpeedIcon />}
label="Files Pending"
value={source.total_files_pending}
color="warning"
tooltip="Files discovered but not yet processed during sync"
/>
</Grid>
<Grid item xs={6} sm={4} md={2.4}>
<StatCard
icon={<StorageIcon />}
label="Total Size"
value={formatBytes(source.total_size_bytes)}
color="primary"
tooltip="Total size of files successfully downloaded from this source"
/>
</Grid>
</Grid>
{/* Error Alert */}

View File

@@ -1509,4 +1509,59 @@ impl Database {
Ok(deleted_documents)
}
pub async fn count_documents_for_source(&self, source_id: Uuid) -> Result<(i64, i64)> {
let row = sqlx::query(
r#"
SELECT
COUNT(*) as total_documents,
COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr
FROM documents
WHERE source_id = $1
"#
)
.bind(source_id)
.fetch_one(&self.pool)
.await?;
let total_documents: i64 = row.get("total_documents");
let total_documents_ocr: i64 = row.get("total_documents_ocr");
Ok((total_documents, total_documents_ocr))
}
pub async fn count_documents_for_sources(&self, source_ids: &[Uuid]) -> Result<Vec<(Uuid, i64, i64)>> {
if source_ids.is_empty() {
return Ok(vec![]);
}
let query = format!(
r#"
SELECT
source_id,
COUNT(*) as total_documents,
COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr
FROM documents
WHERE source_id = ANY($1)
GROUP BY source_id
"#
);
let rows = sqlx::query(&query)
.bind(source_ids)
.fetch_all(&self.pool)
.await?;
let results = rows
.into_iter()
.map(|row| {
let source_id: Uuid = row.get("source_id");
let total_documents: i64 = row.get("total_documents");
let total_documents_ocr: i64 = row.get("total_documents_ocr");
(source_id, total_documents, total_documents_ocr)
})
.collect();
Ok(results)
}
}

View File

@@ -862,6 +862,12 @@ pub struct SourceResponse {
pub total_size_bytes: i64,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
/// Total number of documents/files currently stored from this source
#[serde(default)]
pub total_documents: i64,
/// Total number of documents that have been OCR'd from this source
#[serde(default)]
pub total_documents_ocr: i64,
}
#[derive(Debug, Serialize, Deserialize, ToSchema)]
@@ -903,6 +909,9 @@ impl From<Source> for SourceResponse {
total_size_bytes: source.total_size_bytes,
created_at: source.created_at,
updated_at: source.updated_at,
// These will be populated separately when needed
total_documents: 0,
total_documents_ocr: 0,
}
}
}

View File

@@ -50,7 +50,33 @@ async fn list_sources(
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let responses: Vec<SourceResponse> = sources.into_iter().map(|s| s.into()).collect();
// Get source IDs for batch counting
let source_ids: Vec<Uuid> = sources.iter().map(|s| s.id).collect();
// Get document counts for all sources in one query
let counts = state
.db
.count_documents_for_sources(&source_ids)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
// Create a map for quick lookup
let count_map: std::collections::HashMap<Uuid, (i64, i64)> = counts
.into_iter()
.map(|(id, total, ocr)| (id, (total, ocr)))
.collect();
let responses: Vec<SourceResponse> = sources
.into_iter()
.map(|s| {
let (total_docs, total_ocr) = count_map.get(&s.id).copied().unwrap_or((0, 0));
let mut response: SourceResponse = s.into();
response.total_documents = total_docs;
response.total_documents_ocr = total_ocr;
response
})
.collect();
Ok(Json(responses))
}
@@ -90,7 +116,12 @@ async fn create_source(
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(source.into()))
let mut response: SourceResponse = source.into();
// New sources have no documents yet
response.total_documents = 0;
response.total_documents_ocr = 0;
Ok(Json(response))
}
#[utoipa::path(
@@ -129,6 +160,13 @@ async fn get_source(
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
// Get document counts
let (total_documents, total_documents_ocr) = state
.db
.count_documents_for_source(source_id)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
// Calculate sync progress
let sync_progress = if source.total_files_pending > 0 {
Some(
@@ -140,8 +178,12 @@ async fn get_source(
None
};
let mut source_response: SourceResponse = source.into();
source_response.total_documents = total_documents;
source_response.total_documents_ocr = total_documents_ocr;
let response = SourceWithStats {
source: source.into(),
source: source_response,
recent_documents: recent_documents.into_iter().map(|d| d.into()).collect(),
sync_progress,
};
@@ -202,8 +244,19 @@ async fn update_source(
StatusCode::INTERNAL_SERVER_ERROR
})?;
info!("Successfully updated source {}: {}", source_id, source.name);
Ok(Json(source.into()))
// Get document counts
let (total_documents, total_documents_ocr) = state
.db
.count_documents_for_source(source_id)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let mut response: SourceResponse = source.into();
response.total_documents = total_documents;
response.total_documents_ocr = total_documents_ocr;
info!("Successfully updated source {}: {}", source_id, response.name);
Ok(Json(response))
}
#[utoipa::path(