mirror/patchmon.net
1
0
Fork 0
mirror of https://github.com/9technologygroup/patchmon.net.git synced 2026-01-01 10:39:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

Fixed issues with the agent not sending apt data properly

Browse Source 
Added Indexing to the database for faster and efficient searching
Fixed some filtering from the hosts page relating to packages that need updating
Added buy me a coffee link (sorry and thank you <3)
This commit is contained in:
Muhammad Ibrahim
2025-10-07 20:52:46 +01:00
parent 840779844a
commit a305fe23d3
8 changed files with 197 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
agents/patchmon-agent.sh
View File

@@ -605,8 +605,24 @@ get_apt_packages() {
local -n packages_ref=$1
local -n first_ref=$2
# Update package lists (use apt-get for older distros; quieter output)
apt-get update -qq
# Update package lists with retry logic for lock conflicts
local retry_count=0
local max_retries=3
local retry_delay=5
while [[ $retry_count -lt $max_retries ]]; do
if apt-get update -qq 2>/dev/null; then
break
else
retry_count=$((retry_count + 1))
if [[ $retry_count -lt $max_retries ]]; then
warning "APT lock detected, retrying in ${retry_delay} seconds... (attempt $retry_count/$max_retries)"
sleep $retry_delay
else
warning "APT lock persists after $max_retries attempts, continuing without update..."
fi
fi
done
# Determine upgradable packages using apt-get simulation (compatible with Ubuntu 18.04)
# Example line format:
@@ -626,6 +642,11 @@ get_apt_packages() {
is_security_update=true
fi
# Escape JSON special characters in package data
package_name=$(echo "$package_name" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g')
current_version=$(echo "$current_version" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g')
available_version=$(echo "$available_version" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g')
if [[ "$first_ref" == true ]]; then
first_ref=false
else
@@ -642,7 +663,11 @@ get_apt_packages() {
while IFS=' ' read -r package_name version; do
if [[ -n "$package_name" && -n "$version" ]]; then
# Check if this package is not in the upgrade list
if ! echo "$upgradable" | grep -q "^$package_name/"; then
if ! echo "$upgradable_sim" | grep -q "^Inst $package_name "; then
# Escape JSON special characters in package data
package_name=$(echo "$package_name" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g')
version=$(echo "$version" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g')
if [[ "$first_ref" == true ]]; then
first_ref=false
else
@@ -883,6 +908,15 @@ send_update() {
local network_json=$(get_network_info)
local system_json=$(get_system_info)
# Validate JSON before sending
if ! echo "$packages_json" | jq empty 2>/dev/null; then
error "Invalid packages JSON generated: $packages_json"
fi
if ! echo "$repositories_json" | jq empty 2>/dev/null; then
error "Invalid repositories JSON generated: $repositories_json"
fi
info "Sending update to PatchMon server..."
# Merge all JSON objects into one
@@ -909,15 +943,27 @@ EOF
# Merge the base payload with the system information
local payload=$(echo "$base_payload $merged_json" | jq -s '.[0] * .[1]')
# Write payload to temporary file to avoid "Argument list too long" error
local temp_payload_file=$(mktemp)
echo "$payload" > "$temp_payload_file"
# Debug: Show payload size
local payload_size=$(wc -c < "$temp_payload_file")
echo -e "${BLUE}ℹ️ 📊 Payload size: $payload_size bytes${NC}"
local response=$(curl $CURL_FLAGS -X POST \
-H "Content-Type: application/json" \
-H "X-API-ID: $API_ID" \
-H "X-API-KEY: $API_KEY" \
-d "$payload" \
"$PATCHMON_SERVER/api/$API_VERSION/hosts/update")
-d @"$temp_payload_file" \
"$PATCHMON_SERVER/api/$API_VERSION/hosts/update" 2>&1)
if [[ $? -eq 0 ]]; then
local curl_exit_code=$?
# Clean up temporary file
rm -f "$temp_payload_file"
if [[ $curl_exit_code -eq 0 ]]; then
if echo "$response" | grep -q "success"; then
local packages_count=$(echo "$response" | grep -o '"packagesProcessed":[0-9]*' | cut -d':' -f2)
success "Update sent successfully (${packages_count} packages processed)"
@@ -953,7 +999,7 @@ EOF
error "Update failed: $response"
fi
else
error "Failed to send update"
error "Failed to send update (curl exit code: $curl_exit_code): $response"
fi
}

30
backend/prisma/migrations/add_host_packages_indexes/migration.sql Normal file
View File

@@ -0,0 +1,30 @@
-- Add indexes to host_packages table for performance optimization
-- These indexes will dramatically speed up queries filtering by host_id, package_id, needs_update, and is_security_update
-- Index for queries filtering by host_id (very common - used when viewing packages for a specific host)
CREATE INDEX IF NOT EXISTS "host_packages_host_id_idx" ON "host_packages"("host_id");
-- Index for queries filtering by package_id (used when finding hosts for a specific package)
CREATE INDEX IF NOT EXISTS "host_packages_package_id_idx" ON "host_packages"("package_id");
-- Index for queries filtering by needs_update (used when finding outdated packages)
CREATE INDEX IF NOT EXISTS "host_packages_needs_update_idx" ON "host_packages"("needs_update");
-- Index for queries filtering by is_security_update (used when finding security updates)
CREATE INDEX IF NOT EXISTS "host_packages_is_security_update_idx" ON "host_packages"("is_security_update");
-- Composite index for the most common query pattern: host_id + needs_update
-- This is optimal for "show me outdated packages for this host"
CREATE INDEX IF NOT EXISTS "host_packages_host_id_needs_update_idx" ON "host_packages"("host_id", "needs_update");
-- Composite index for host_id + needs_update + is_security_update
-- This is optimal for "show me security updates for this host"
CREATE INDEX IF NOT EXISTS "host_packages_host_id_needs_update_security_idx" ON "host_packages"("host_id", "needs_update", "is_security_update");
-- Index for queries filtering by package_id + needs_update
-- This is optimal for "show me hosts where this package needs updates"
CREATE INDEX IF NOT EXISTS "host_packages_package_id_needs_update_idx" ON "host_packages"("package_id", "needs_update");
-- Index on last_checked for cleanup/maintenance queries
CREATE INDEX IF NOT EXISTS "host_packages_last_checked_idx" ON "host_packages"("last_checked");

11
backend/prisma/schema.prisma
View File

@@ -44,6 +44,14 @@ model host_packages {
packages packages @relation(fields: [package_id], references: [id], onDelete: Cascade)
@@unique([host_id, package_id])
@@index([host_id])
@@index([package_id])
@@index([needs_update])
@@index([is_security_update])
@@index([host_id, needs_update])
@@index([host_id, needs_update, is_security_update])
@@index([package_id, needs_update])
@@index([last_checked])
}
model host_repositories {
@@ -108,6 +116,9 @@ model packages {
created_at DateTime @default(now())
updated_at DateTime
host_packages host_packages[]
@@index([name])
@@index([category])
}
model repositories {

46
backend/src/routes/packageRoutes.js
View File

@@ -14,6 +14,7 @@ router.get("/", async (req, res) => {
category = "",
needsUpdate = "",
isSecurityUpdate = "",
host = "",
} = req.query;
const skip = (parseInt(page, 10) - 1) * parseInt(limit, 10);
@@ -33,8 +34,27 @@ router.get("/", async (req, res) => {
: {},
// Category filter
category ? { category: { equals: category } } : {},
// Update status filters
needsUpdate
// Host filter - only return packages installed on the specified host
// Combined with update status filters if both are present
host
? {
host_packages: {
some: {
host_id: host,
// If needsUpdate or isSecurityUpdate filters are present, apply them here
...(needsUpdate
? { needs_update: needsUpdate === "true" }
: {}),
...(isSecurityUpdate
? { is_security_update: isSecurityUpdate === "true" }
: {}),
},
},
}
: {},
// Update status filters (only applied if no host filter)
// If host filter is present, these are already applied above
!host && needsUpdate
? {
host_packages: {
some: {
@@ -43,7 +63,7 @@ router.get("/", async (req, res) => {
},
}
: {},
isSecurityUpdate
!host && isSecurityUpdate
? {
host_packages: {
some: {
@@ -84,24 +104,32 @@ router.get("/", async (req, res) => {
// Get additional stats for each package
const packagesWithStats = await Promise.all(
packages.map(async (pkg) => {
// Build base where clause for this package
const baseWhere = { package_id: pkg.id };
// If host filter is specified, add host filter to all queries
const hostWhere = host ? { ...baseWhere, host_id: host } : baseWhere;
const [updatesCount, securityCount, packageHosts] = await Promise.all([
prisma.host_packages.count({
where: {
package_id: pkg.id,
...hostWhere,
needs_update: true,
},
}),
prisma.host_packages.count({
where: {
package_id: pkg.id,
...hostWhere,
needs_update: true,
is_security_update: true,
},
}),
prisma.host_packages.findMany({
where: {
package_id: pkg.id,
needs_update: true,
...hostWhere,
// If host filter is specified, include all packages for that host
// Otherwise, only include packages that need updates
...(host ? {} : { needs_update: true }),
},
select: {
hosts: {
@@ -112,6 +140,10 @@ router.get("/", async (req, res) => {
os_type: true,
},
},
current_version: true,
available_version: true,
needs_update: true,
is_security_update: true,
},
take: 10, // Limit to first 10 for performance
}),

28
frontend/src/components/Layout.jsx
View File

File diff suppressed because one or more lines are too long

6
frontend/src/pages/HostDetail.jsx
View File

@@ -1012,13 +1012,15 @@ const HostDetail = () => {
{host.stats.total_packages}
</p>
<p className="text-sm text-secondary-500 dark:text-secondary-300">
Total Packages
Total Installed
</p>
</button>
<button
type="button"
onClick={() => navigate(`/packages?host=${hostId}`)}
onClick={() =>
navigate(`/packages?host=${hostId}&filter=outdated`)
}
className="text-center p-4 bg-warning-50 dark:bg-warning-900/20 rounded-lg hover:bg-warning-100 dark:hover:bg-warning-900/30 transition-colors group"
title="View outdated packages for this host"
>

6
frontend/src/pages/Hosts.jsx
View File

@@ -870,9 +870,11 @@ const Hosts = () => {
return (
<button
type="button"
onClick={() => navigate(`/packages?host=${host.id}`)}
onClick={() =>
navigate(`/packages?host=${host.id}&filter=outdated`)
}
className="text-sm text-primary-600 hover:text-primary-900 dark:text-primary-400 dark:hover:text-primary-300 font-medium hover:underline"
title="View packages for this host"
title="View outdated packages for this host"
>
{host.updatesCount || 0}
</button>

52
frontend/src/pages/Packages.jsx
View File

@@ -115,8 +115,20 @@ const Packages = () => {
refetch,
isFetching,
} = useQuery({
queryKey: ["packages"],
queryFn: () => packagesAPI.getAll({ limit: 1000 }).then((res) => res.data),
queryKey: ["packages", hostFilter, updateStatusFilter],
queryFn: () => {
const params = { limit: 10000 }; // High limit to effectively get all packages
if (hostFilter && hostFilter !== "all") {
params.host = hostFilter;
}
// Pass update status filter to backend to pre-filter packages
if (updateStatusFilter === "needs-updates") {
params.needsUpdate = "true";
} else if (updateStatusFilter === "security-updates") {
params.isSecurityUpdate = "true";
}
return packagesAPI.getAll(params).then((res) => res.data);
},
staleTime: 5 * 60 * 1000, // Data stays fresh for 5 minutes
refetchOnWindowFocus: false, // Don't refetch when window regains focus
});
@@ -160,15 +172,13 @@ const Packages = () => {
const matchesUpdateStatus =
updateStatusFilter === "all-packages" ||
updateStatusFilter === "needs-updates" ||
(updateStatusFilter === "security-updates" && pkg.isSecurityUpdate) ||
(updateStatusFilter === "regular-updates" && !pkg.isSecurityUpdate);
// For "all-packages", we don't filter by update status
// For other filters, we only show packages that need updates
const matchesUpdateNeeded =
updateStatusFilter === "all-packages" ||
(pkg.stats?.updatesNeeded || 0) > 0;
(updateStatusFilter === "needs-updates" &&
(pkg.stats?.updatesNeeded || 0) > 0) ||
(updateStatusFilter === "security-updates" &&
(pkg.stats?.securityUpdates || 0) > 0) ||
(updateStatusFilter === "regular-updates" &&
(pkg.stats?.updatesNeeded || 0) > 0 &&
(pkg.stats?.securityUpdates || 0) === 0);
const packageHosts = pkg.packageHosts || [];
const matchesHost =
@@ -176,11 +186,7 @@ const Packages = () => {
packageHosts.some((host) => host.hostId === hostFilter);
return (
matchesSearch &&
matchesCategory &&
matchesUpdateStatus &&
matchesUpdateNeeded &&
matchesHost
matchesSearch && matchesCategory && matchesUpdateStatus && matchesHost
);
});
@@ -435,8 +441,16 @@ const Packages = () => {
});
const uniquePackageHostsCount = uniquePackageHosts.size;
// Calculate total packages available
const totalPackagesCount = packages?.length || 0;
// Calculate total packages installed
// When filtering by host, count each package once (since it can only be installed once per host)
// When not filtering, sum up all installations across all hosts
const totalPackagesCount =
hostFilter && hostFilter !== "all"
? packages?.length || 0
: packages?.reduce(
(sum, pkg) => sum + (pkg.stats?.totalInstalls || 0),
0,
) || 0;
// Calculate outdated packages
const outdatedPackagesCount =
@@ -517,7 +531,7 @@ const Packages = () => {
<Package className="h-5 w-5 text-primary-600 mr-2" />
<div>
<p className="text-sm text-secondary-500 dark:text-white">
Total Packages
Total Installed
</p>
<p className="text-xl font-semibold text-secondary-900 dark:text-white">
{totalPackagesCount}