mirror of
https://github.com/readur/readur.git
synced 2026-01-07 06:50:39 -06:00
feat(client/server): add a new badge for each source that shows the number of documents stored from each source
This commit is contained in:
@@ -65,6 +65,8 @@ import {
|
||||
Storage as ServerIcon,
|
||||
Pause as PauseIcon,
|
||||
PlayArrow as ResumeIcon,
|
||||
TextSnippet as DocumentIcon,
|
||||
Visibility as OcrIcon,
|
||||
} from '@mui/icons-material';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
import api, { queueService } from '../services/api';
|
||||
@@ -84,6 +86,8 @@ interface Source {
|
||||
total_files_synced: number;
|
||||
total_files_pending: number;
|
||||
total_size_bytes: number;
|
||||
total_documents: number;
|
||||
total_documents_ocr: number;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
@@ -699,7 +703,7 @@ const SourcesPage: React.FC = () => {
|
||||
<Typography variant="h6" fontWeight="bold" gutterBottom>
|
||||
{source.name}
|
||||
</Typography>
|
||||
<Stack direction="row" spacing={1} alignItems="center">
|
||||
<Stack direction="row" spacing={1} alignItems="center" flexWrap="wrap">
|
||||
<Chip
|
||||
label={source.source_type.toUpperCase()}
|
||||
size="small"
|
||||
@@ -723,6 +727,32 @@ const SourcesPage: React.FC = () => {
|
||||
fontWeight: 600,
|
||||
}}
|
||||
/>
|
||||
<Chip
|
||||
icon={<DocumentIcon sx={{ fontSize: '0.9rem !important' }} />}
|
||||
label={`${source.total_documents} docs`}
|
||||
size="small"
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
bgcolor: alpha(theme.palette.info.main, 0.1),
|
||||
color: theme.palette.info.main,
|
||||
border: `1px solid ${alpha(theme.palette.info.main, 0.3)}`,
|
||||
fontSize: '0.75rem',
|
||||
fontWeight: 600,
|
||||
}}
|
||||
/>
|
||||
<Chip
|
||||
icon={<OcrIcon sx={{ fontSize: '0.9rem !important' }} />}
|
||||
label={`${source.total_documents_ocr} OCR'd`}
|
||||
size="small"
|
||||
sx={{
|
||||
borderRadius: 2,
|
||||
bgcolor: alpha(theme.palette.success.main, 0.1),
|
||||
color: theme.palette.success.main,
|
||||
border: `1px solid ${alpha(theme.palette.success.main, 0.3)}`,
|
||||
fontSize: '0.75rem',
|
||||
fontWeight: 600,
|
||||
}}
|
||||
/>
|
||||
{!source.enabled && (
|
||||
<Chip
|
||||
label="Disabled"
|
||||
@@ -814,34 +844,25 @@ const SourcesPage: React.FC = () => {
|
||||
|
||||
{/* Stats Grid */}
|
||||
<Grid container spacing={2} mb={3}>
|
||||
<Grid item xs={6} sm={4} md={3}>
|
||||
<Grid item xs={6} sm={4} md={2.4}>
|
||||
<StatCard
|
||||
icon={<TrendingUpIcon />}
|
||||
label="Files Processed"
|
||||
value={source.total_files_synced}
|
||||
icon={<DocumentIcon />}
|
||||
label="Documents Stored"
|
||||
value={source.total_documents}
|
||||
color="info"
|
||||
tooltip="Total number of documents currently stored from this source"
|
||||
/>
|
||||
</Grid>
|
||||
<Grid item xs={6} sm={4} md={2.4}>
|
||||
<StatCard
|
||||
icon={<OcrIcon />}
|
||||
label="OCR Processed"
|
||||
value={source.total_documents_ocr}
|
||||
color="success"
|
||||
tooltip="Files attempted to be synced, including duplicates and skipped files"
|
||||
tooltip="Number of documents that have been successfully OCR'd"
|
||||
/>
|
||||
</Grid>
|
||||
<Grid item xs={6} sm={4} md={3}>
|
||||
<StatCard
|
||||
icon={<SpeedIcon />}
|
||||
label="Files Pending"
|
||||
value={source.total_files_pending}
|
||||
color="warning"
|
||||
tooltip="Files discovered but not yet processed during sync"
|
||||
/>
|
||||
</Grid>
|
||||
<Grid item xs={6} sm={4} md={3}>
|
||||
<StatCard
|
||||
icon={<StorageIcon />}
|
||||
label="Total Size (Downloaded)"
|
||||
value={formatBytes(source.total_size_bytes)}
|
||||
color="primary"
|
||||
tooltip="Total size of files successfully downloaded from this source"
|
||||
/>
|
||||
</Grid>
|
||||
<Grid item xs={6} sm={4} md={3}>
|
||||
<Grid item xs={6} sm={4} md={2.4}>
|
||||
<StatCard
|
||||
icon={<TimelineIcon />}
|
||||
label="Last Sync"
|
||||
@@ -852,6 +873,24 @@ const SourcesPage: React.FC = () => {
|
||||
tooltip="When this source was last synchronized"
|
||||
/>
|
||||
</Grid>
|
||||
<Grid item xs={6} sm={4} md={2.4}>
|
||||
<StatCard
|
||||
icon={<SpeedIcon />}
|
||||
label="Files Pending"
|
||||
value={source.total_files_pending}
|
||||
color="warning"
|
||||
tooltip="Files discovered but not yet processed during sync"
|
||||
/>
|
||||
</Grid>
|
||||
<Grid item xs={6} sm={4} md={2.4}>
|
||||
<StatCard
|
||||
icon={<StorageIcon />}
|
||||
label="Total Size"
|
||||
value={formatBytes(source.total_size_bytes)}
|
||||
color="primary"
|
||||
tooltip="Total size of files successfully downloaded from this source"
|
||||
/>
|
||||
</Grid>
|
||||
</Grid>
|
||||
|
||||
{/* Error Alert */}
|
||||
|
||||
@@ -1509,4 +1509,59 @@ impl Database {
|
||||
|
||||
Ok(deleted_documents)
|
||||
}
|
||||
|
||||
pub async fn count_documents_for_source(&self, source_id: Uuid) -> Result<(i64, i64)> {
|
||||
let row = sqlx::query(
|
||||
r#"
|
||||
SELECT
|
||||
COUNT(*) as total_documents,
|
||||
COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr
|
||||
FROM documents
|
||||
WHERE source_id = $1
|
||||
"#
|
||||
)
|
||||
.bind(source_id)
|
||||
.fetch_one(&self.pool)
|
||||
.await?;
|
||||
|
||||
let total_documents: i64 = row.get("total_documents");
|
||||
let total_documents_ocr: i64 = row.get("total_documents_ocr");
|
||||
|
||||
Ok((total_documents, total_documents_ocr))
|
||||
}
|
||||
|
||||
pub async fn count_documents_for_sources(&self, source_ids: &[Uuid]) -> Result<Vec<(Uuid, i64, i64)>> {
|
||||
if source_ids.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let query = format!(
|
||||
r#"
|
||||
SELECT
|
||||
source_id,
|
||||
COUNT(*) as total_documents,
|
||||
COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr
|
||||
FROM documents
|
||||
WHERE source_id = ANY($1)
|
||||
GROUP BY source_id
|
||||
"#
|
||||
);
|
||||
|
||||
let rows = sqlx::query(&query)
|
||||
.bind(source_ids)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
let results = rows
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
let source_id: Uuid = row.get("source_id");
|
||||
let total_documents: i64 = row.get("total_documents");
|
||||
let total_documents_ocr: i64 = row.get("total_documents_ocr");
|
||||
(source_id, total_documents, total_documents_ocr)
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
@@ -862,6 +862,12 @@ pub struct SourceResponse {
|
||||
pub total_size_bytes: i64,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
/// Total number of documents/files currently stored from this source
|
||||
#[serde(default)]
|
||||
pub total_documents: i64,
|
||||
/// Total number of documents that have been OCR'd from this source
|
||||
#[serde(default)]
|
||||
pub total_documents_ocr: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, ToSchema)]
|
||||
@@ -903,6 +909,9 @@ impl From<Source> for SourceResponse {
|
||||
total_size_bytes: source.total_size_bytes,
|
||||
created_at: source.created_at,
|
||||
updated_at: source.updated_at,
|
||||
// These will be populated separately when needed
|
||||
total_documents: 0,
|
||||
total_documents_ocr: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +50,33 @@ async fn list_sources(
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
let responses: Vec<SourceResponse> = sources.into_iter().map(|s| s.into()).collect();
|
||||
// Get source IDs for batch counting
|
||||
let source_ids: Vec<Uuid> = sources.iter().map(|s| s.id).collect();
|
||||
|
||||
// Get document counts for all sources in one query
|
||||
let counts = state
|
||||
.db
|
||||
.count_documents_for_sources(&source_ids)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
// Create a map for quick lookup
|
||||
let count_map: std::collections::HashMap<Uuid, (i64, i64)> = counts
|
||||
.into_iter()
|
||||
.map(|(id, total, ocr)| (id, (total, ocr)))
|
||||
.collect();
|
||||
|
||||
let responses: Vec<SourceResponse> = sources
|
||||
.into_iter()
|
||||
.map(|s| {
|
||||
let (total_docs, total_ocr) = count_map.get(&s.id).copied().unwrap_or((0, 0));
|
||||
let mut response: SourceResponse = s.into();
|
||||
response.total_documents = total_docs;
|
||||
response.total_documents_ocr = total_ocr;
|
||||
response
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(Json(responses))
|
||||
}
|
||||
|
||||
@@ -90,7 +116,12 @@ async fn create_source(
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
Ok(Json(source.into()))
|
||||
let mut response: SourceResponse = source.into();
|
||||
// New sources have no documents yet
|
||||
response.total_documents = 0;
|
||||
response.total_documents_ocr = 0;
|
||||
|
||||
Ok(Json(response))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
@@ -129,6 +160,13 @@ async fn get_source(
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
// Get document counts
|
||||
let (total_documents, total_documents_ocr) = state
|
||||
.db
|
||||
.count_documents_for_source(source_id)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
// Calculate sync progress
|
||||
let sync_progress = if source.total_files_pending > 0 {
|
||||
Some(
|
||||
@@ -140,8 +178,12 @@ async fn get_source(
|
||||
None
|
||||
};
|
||||
|
||||
let mut source_response: SourceResponse = source.into();
|
||||
source_response.total_documents = total_documents;
|
||||
source_response.total_documents_ocr = total_documents_ocr;
|
||||
|
||||
let response = SourceWithStats {
|
||||
source: source.into(),
|
||||
source: source_response,
|
||||
recent_documents: recent_documents.into_iter().map(|d| d.into()).collect(),
|
||||
sync_progress,
|
||||
};
|
||||
@@ -202,8 +244,19 @@ async fn update_source(
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
info!("Successfully updated source {}: {}", source_id, source.name);
|
||||
Ok(Json(source.into()))
|
||||
// Get document counts
|
||||
let (total_documents, total_documents_ocr) = state
|
||||
.db
|
||||
.count_documents_for_source(source_id)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
let mut response: SourceResponse = source.into();
|
||||
response.total_documents = total_documents;
|
||||
response.total_documents_ocr = total_documents_ocr;
|
||||
|
||||
info!("Successfully updated source {}: {}", source_id, response.name);
|
||||
Ok(Json(response))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
|
||||
Reference in New Issue
Block a user