From db9e92436fc2e41cdbbe11c0057156559afa9e9e Mon Sep 17 00:00:00 2001 From: ljm42 Date: Tue, 22 Apr 2025 11:32:38 -0700 Subject: [PATCH 1/6] feat: diagnostics: add error logging --- emhttp/plugins/dynamix/scripts/diagnostics | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/scripts/diagnostics b/emhttp/plugins/dynamix/scripts/diagnostics index 2c85a03e0..8ef5343a7 100755 --- a/emhttp/plugins/dynamix/scripts/diagnostics +++ b/emhttp/plugins/dynamix/scripts/diagnostics @@ -59,12 +59,28 @@ function write(...$messages){ curl_close($com); } +// Add error logging function +function log_error($message, $command = '') { + global $diag; + $error_log = "/$diag/logs/diagnostics.error.log"; + $timestamp = date('Y-m-d H:i:s'); + $log_message = "[$timestamp] $message"; + if ($command) { + $log_message .= " (Command: $command)"; + } + file_put_contents($error_log, $log_message . "\n", FILE_APPEND); +} + +// Modify run function to include error logging function run($cmd, &$save=null, $timeout=30) { global $cli,$diag; // output command for display write($cmd); // execute command with timeout of 30s - exec("timeout -s9 $timeout $cmd", $save); + exec("timeout -s9 $timeout $cmd 2>&1", $save, $return_code); + if ($return_code !== 0) { + log_error("Command failed with return code $return_code", $cmd); + } return implode("\n",$save); } From 119ca57db4015e6fbd7c4dc31f53efcbf25082ed Mon Sep 17 00:00:00 2001 From: ljm42 Date: Tue, 22 Apr 2025 11:52:57 -0700 Subject: [PATCH 2/6] feat: diagnostics: use existing nchan publish function --- emhttp/plugins/dynamix/scripts/diagnostics | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/emhttp/plugins/dynamix/scripts/diagnostics b/emhttp/plugins/dynamix/scripts/diagnostics index 8ef5343a7..c09941067 100755 --- a/emhttp/plugins/dynamix/scripts/diagnostics +++ b/emhttp/plugins/dynamix/scripts/diagnostics @@ -27,6 +27,7 @@ $cli = empty($zip); $docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp'); require_once "$docroot/webGui/include/Helpers.php"; require_once "$docroot/webGui/include/Wrappers.php"; +require_once "$docroot/webGui/include/publish.php"; if (is_file('/boot/syslinux/syslinux.cfg')) { $bootenv = '/boot/syslinux'; @@ -45,18 +46,9 @@ $pools = pools_filter($disks); require_once "$docroot/webGui/include/CustomMerge.php"; function write(...$messages){ - $com = curl_init(); - curl_setopt_array($com,[ - CURLOPT_URL => 'http://localhost/pub/diagnostics?buffer_length=1', - CURLOPT_UNIX_SOCKET_PATH => '/var/run/nginx.socket', - CURLOPT_POST => 1, - CURLOPT_RETURNTRANSFER => true - ]); foreach ($messages as $message) { - curl_setopt($com, CURLOPT_POSTFIELDS, $message); - curl_exec($com); + publish('diagnostics', $message); } - curl_close($com); } // Add error logging function From 6891effea36bbdff72d280e8befe75b9ba94158c Mon Sep 17 00:00:00 2001 From: ljm42 Date: Tue, 22 Apr 2025 11:57:35 -0700 Subject: [PATCH 3/6] refactor: diagnostics: remove redundant if (!$all) checks --- emhttp/plugins/dynamix/scripts/diagnostics | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/emhttp/plugins/dynamix/scripts/diagnostics b/emhttp/plugins/dynamix/scripts/diagnostics index c09941067..50f49386e 100755 --- a/emhttp/plugins/dynamix/scripts/diagnostics +++ b/emhttp/plugins/dynamix/scripts/diagnostics @@ -478,7 +478,7 @@ foreach ($ports as $port) { file_put_contents("/$diag/system/ethtool.txt", "--------------------------------\r\n", FILE_APPEND); } run("ip -br a|todos >".escapeshellarg("/$diag/system/ifconfig.txt")); -if (!$all) maskIP("/$diag/system/ifconfig.txt"); +maskIP("/$diag/system/ifconfig.txt"); // create system information (suppress errors) run("find /sys/kernel/iommu_groups/ -type l 2>/dev/null|sort -V|todos >".escapeshellarg("/$diag/system/iommu_groups.txt")); @@ -511,7 +511,7 @@ if (!$all) { // include listening interfaces run("$docroot/webGui/scripts/show_interfaces ip|tr ',' '\n' >".escapeshellarg("/$diag/config/listen.txt")); run("$docroot/webGui/scripts/error_interfaces|sed 's///' >>".escapeshellarg("/$diag/config/listen.txt")); -if (!$all) maskIP("/$diag/config/listen.txt"); +maskIP("/$diag/config/listen.txt"); // copy share information (anonymize if applicable) $files = glob("/boot/config/shares/*.cfg"); @@ -699,7 +699,7 @@ $dhcplog = "/var/log/dhcplog"; if (file_exists($dhcplog)) { $log = "/$diag/logs/dhcplog.txt"; run("todos <$dhcplog >".escapeshellarg($log)); - if (!$all) maskIP($log); + maskIP($log); } // copy phplog From 35a7170b217e2fee029e22c5c643bcab77abd5ed Mon Sep 17 00:00:00 2001 From: ljm42 Date: Tue, 22 Apr 2025 12:11:27 -0700 Subject: [PATCH 4/6] fix: diagnostics: use find with sed to properly handle wildcards in config files --- emhttp/plugins/dynamix/scripts/diagnostics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emhttp/plugins/dynamix/scripts/diagnostics b/emhttp/plugins/dynamix/scripts/diagnostics index 50f49386e..09424de0c 100755 --- a/emhttp/plugins/dynamix/scripts/diagnostics +++ b/emhttp/plugins/dynamix/scripts/diagnostics @@ -500,7 +500,7 @@ if (!$all) { } // anonymize configuration files if (!$all) { - run("sed -ri 's/^((disk|flash)(Read|Write)List.*=\")[^\"]+/\\1.../' ".escapeshellarg("/$diag/config/*.cfg")." 2>/dev/null"); + run("find ".escapeshellarg("/$diag/config")." -name '*.cfg' -exec sed -ri 's/^((disk|flash)(Read|Write)List.*=\")[^\"]+/\\1.../' {} \\; 2>/dev/null"); // anonymize IP addresses maskIP("/$diag/config/network.cfg"); // anonymize wireless credentials From 9134a35caebcd86e43021ed42e32956e8fc5380c Mon Sep 17 00:00:00 2001 From: ljm42 Date: Tue, 22 Apr 2025 12:29:03 -0700 Subject: [PATCH 5/6] fix: diagnostics: remove error suppression to enable proper error logging --- emhttp/plugins/dynamix/scripts/diagnostics | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/emhttp/plugins/dynamix/scripts/diagnostics b/emhttp/plugins/dynamix/scripts/diagnostics index 09424de0c..618e0d878 100755 --- a/emhttp/plugins/dynamix/scripts/diagnostics +++ b/emhttp/plugins/dynamix/scripts/diagnostics @@ -129,19 +129,19 @@ function maskIP($file) { // anonymize public IPv4 addresses $rfc1918 = "(127|10|172\.1[6-9]|172\.2[0-9]|172\.3[0-1]|192\.168)((\.[0-9]{1,3}){2,3}([/\" .]|$))"; - run("sed -ri 's/([\"\[ ]){$rfc1918}/\\1@@@\\2\\3/g; s/([\"\[ ][0-9]{1,3}\.)([0-9]{1,3}\.){2}([0-9]{1,3})([/\" .]|$)/\\1XXX.XXX.\\3\\4/g; s/@@@//g' ".escapeshellarg($file)." 2>/dev/null"); + run("sed -ri 's/([\"\[ ]){$rfc1918}/\\1@@@\\2\\3/g; s/([\"\[ ][0-9]{1,3}\.)([0-9]{1,3}\.){2}([0-9]{1,3})([/\" .]|$)/\\1XXX.XXX.\\3\\4/g; s/@@@//g' ".escapeshellarg($file)); // anonymize full IPv6 addresses $file_escaped = escapeshellarg($file); // Anonymize IPv6 addresses without brackets - run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' $file_escaped 2>/dev/null"); + run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' $file_escaped"); // Anonymize IPv6 addresses with brackets - run("sed -ri 's/(\[([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\])([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' $file_escaped 2>/dev/null"); + run("sed -ri 's/(\[([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\])([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' $file_escaped"); // Handle any remaining edge cases, e.g., addresses with subnet masks - run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\/[0-9]{1,3})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\7/g' $file_escaped 2>/dev/null"); + run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\/[0-9]{1,3})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\7/g' $file_escaped"); } function download_url($url, $path="", $bg=false, $timeout=15) { @@ -341,11 +341,11 @@ function anonymize_syslog($file) { foreach ($titles as $mover) { if (!$mover) continue; $title = "/{$mover[0]}..".substr($mover,-1)."/..."; - run("sed -i 's/".str_replace("/","\/",$mover)."/".str_replace("/","\/",$title)."/g' ".escapeshellarg("$log.txt")." 2>/dev/null"); - //run("sed -ri 's|(file: [.>cr].*)[ /]$mover/.*$|\\1 file: $title|' ".escapeshellarg("$log.txt")." 2>/dev/null"); + run("sed -i 's/".str_replace("/","\/",$mover)."/".str_replace("/","\/",$title)."/g' ".escapeshellarg("$log.txt")); + //run("sed -ri 's|(file: [.>cr].*)[ /]$mover/.*$|\\1 file: $title|' ".escapeshellarg("$log.txt")); } run("grep -n ' cache_dirs: -' ".escapeshellarg("$log.txt")." 2>/dev/null|cut -d: -f1", $rows); - for ($i = 0; $i < count($rows); $i += 2) for ($row = $rows[$i]+1; $row < $rows[$i+1]; $row++) run("sed -ri '$row s|(cache_dirs: \S).*(\S)|\\1..\\2|' ".escapeshellarg("$log.txt")." 2>/dev/null"); + for ($i = 0; $i < count($rows); $i += 2) for ($row = $rows[$i]+1; $row < $rows[$i+1]; $row++) run("sed -ri '$row s|(cache_dirs: \S).*(\S)|\\1..\\2|' ".escapeshellarg("$log.txt")); } // replace consecutive repeated lines in syslog run("awk -i inplace '{if(s!=substr(\$0,17)){if(x>0)print\"### [PREVIOUS LINE REPEATED \"x\" TIMES] ###\\r\";print;x=0}else{x++}s=substr(\$0,17)}END{if(x>0)print\"### [PREVIOUS LINE REPEATED \"x\" TIMES] ###\\r\"}' ".escapeshellarg("$log.txt")); @@ -491,8 +491,8 @@ foreach ($folders as $folder) { } // copy configuration files (suppress errors) -run("cp /boot/config/*.{cfg,conf,dat} ".escapeshellarg("/$diag/config")." 2>/dev/null"); -run("cp /boot/config/go ".escapeshellarg("/$diag/config/go.txt")." 2>/dev/null"); +run("cp /boot/config/*.{cfg,conf,dat} ".escapeshellarg("/$diag/config")); +run("cp /boot/config/go ".escapeshellarg("/$diag/config/go.txt")); // anonymize go file if (!$all) { @@ -500,12 +500,12 @@ if (!$all) { } // anonymize configuration files if (!$all) { - run("find ".escapeshellarg("/$diag/config")." -name '*.cfg' -exec sed -ri 's/^((disk|flash)(Read|Write)List.*=\")[^\"]+/\\1.../' {} \\; 2>/dev/null"); + run("find ".escapeshellarg("/$diag/config")." -name '*.cfg' -exec sed -ri 's/^((disk|flash)(Read|Write)List.*=\")[^\"]+/\\1.../' {} \\;"); // anonymize IP addresses maskIP("/$diag/config/network.cfg"); // anonymize wireless credentials if (file_exists("/$diag/config/wireless.cfg")) { - run("sed -ri 's/^((USERNAME|PASSWORD)=\")[^\"]+/\\1.../' ".escapeshellarg("/$diag/config/wireless.cfg")." 2>/dev/null"); + run("sed -ri 's/^((USERNAME|PASSWORD)=\")[^\"]+/\\1.../' ".escapeshellarg("/$diag/config/wireless.cfg")); } } // include listening interfaces @@ -521,7 +521,7 @@ foreach ($files as $file) { $share = basename($file,'.cfg'); if (!in_array($share,$showshares)) $dest = anonymize($dest,2); @copy($file, $dest); - if (!$all) run("sed -ri 's/^(share(Comment|ReadList|WriteList)=\")[^\"]+/\\1.../' ".escapeshellarg($dest)." 2>/dev/null"); + if (!$all) run("sed -ri 's/^(share(Comment|ReadList|WriteList)=\")[^\"]+/\\1.../' ".escapeshellarg($dest)); $home = shareDisks($share); $home = $home ? "# Share exists on $home\r\n" : "# Share does not exist\r\n"; $shareDisk[] = str_pad(basename($dest,'.cfg'),34).str_pad(exec("grep -m1 'shareUseCache' ".escapeshellarg($file)),24).$home; @@ -677,13 +677,13 @@ if ($qemu) { } // copy VM XML config files -run("cp /etc/libvirt/qemu/*.xml ".escapeshellarg("/$diag/xml")." 2>/dev/null"); +run("cp /etc/libvirt/qemu/*.xml ".escapeshellarg("/$diag/xml")); // anonymize MAC OSK info $all_xml = glob("/$diag/xml/*.xml"); foreach ($all_xml as $xml) { - run("sed -ri 's/(,osk=).+/\\1.../' ".escapeshellarg("$xml")." 2>/dev/null"); - run("sed -ri 's/(passwd=).+/\\1.../' ".escapeshellarg("$xml")." 2>/dev/null"); + run("sed -ri 's/(,osk=).+/\\1.../' ".escapeshellarg("$xml")); + run("sed -ri 's/(passwd=).+/\\1.../' ".escapeshellarg("$xml")); } // copy syslog information (anonymize if applicable) @@ -750,8 +750,8 @@ newline("/$diag/system/sshd.txt"); // copy servers.conf copy("/etc/nginx/conf.d/servers.conf", "/$diag/system/servers.conf.txt"); maskIP("/$diag/system/servers.conf.txt"); -run("sed -Ei 's/[01234567890abcdef]+\.((my)?unraid\.net)/hash.\\1/gm;t' ".escapeshellarg("/$diag/system/servers.conf.txt")." 2>/dev/null"); -run("sed -Ei 's/\.[^\.]*\.ts\.net/\.magicdns\.ts\.net/gm' ".escapeshellarg("/$diag/system/servers.conf.txt")." 2>/dev/null"); +run("sed -Ei 's/[01234567890abcdef]+\.((my)?unraid\.net)/hash.\\1/gm;t' ".escapeshellarg("/$diag/system/servers.conf.txt")); +run("sed -Ei 's/\.[^\.]*\.ts\.net/\.magicdns\.ts\.net/gm' ".escapeshellarg("/$diag/system/servers.conf.txt")); newline("/$diag/system/servers.conf.txt"); // show installed patches From 8909e16deb2fa6eb814e7a1fe01891d3eb54763f Mon Sep 17 00:00:00 2001 From: ljm42 Date: Tue, 22 Apr 2025 12:44:52 -0700 Subject: [PATCH 6/6] fix: diagnostics: use escapeshellarg() --- emhttp/plugins/dynamix/scripts/diagnostics | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/emhttp/plugins/dynamix/scripts/diagnostics b/emhttp/plugins/dynamix/scripts/diagnostics index 618e0d878..80547b4d4 100755 --- a/emhttp/plugins/dynamix/scripts/diagnostics +++ b/emhttp/plugins/dynamix/scripts/diagnostics @@ -131,17 +131,14 @@ function maskIP($file) { $rfc1918 = "(127|10|172\.1[6-9]|172\.2[0-9]|172\.3[0-1]|192\.168)((\.[0-9]{1,3}){2,3}([/\" .]|$))"; run("sed -ri 's/([\"\[ ]){$rfc1918}/\\1@@@\\2\\3/g; s/([\"\[ ][0-9]{1,3}\.)([0-9]{1,3}\.){2}([0-9]{1,3})([/\" .]|$)/\\1XXX.XXX.\\3\\4/g; s/@@@//g' ".escapeshellarg($file)); - // anonymize full IPv6 addresses - $file_escaped = escapeshellarg($file); - // Anonymize IPv6 addresses without brackets - run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' $file_escaped"); + run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' ".escapeshellarg($file)); // Anonymize IPv6 addresses with brackets - run("sed -ri 's/(\[([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\])([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' $file_escaped"); + run("sed -ri 's/(\[([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\])([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\6/g' ".escapeshellarg($file)); // Handle any remaining edge cases, e.g., addresses with subnet masks - run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\/[0-9]{1,3})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\7/g' $file_escaped"); + run("sed -ri 's/(([0-9a-f]{1,4}:){4})(([0-9a-f]{1,4}:){3}|:)([0-9a-f]{1,4})(\/[0-9]{1,3})([ .:/]|$)/\\1XXXX:XXXX:XXXX:\\5\\7/g' ".escapeshellarg($file)); } function download_url($url, $path="", $bg=false, $timeout=15) { @@ -350,7 +347,7 @@ function anonymize_syslog($file) { // replace consecutive repeated lines in syslog run("awk -i inplace '{if(s!=substr(\$0,17)){if(x>0)print\"### [PREVIOUS LINE REPEATED \"x\" TIMES] ###\\r\";print;x=0}else{x++}s=substr(\$0,17)}END{if(x>0)print\"### [PREVIOUS LINE REPEATED \"x\" TIMES] ###\\r\"}' ".escapeshellarg("$log.txt")); // remove SHA256 hashes - run("sed -ri 's/(SHA256:).+[^\s\b]/SHA256:***REMOVED***/gm' $log.txt"); + run("sed -ri 's/(SHA256:).+[^\s\b]/SHA256:***REMOVED***/gm' ".escapeshellarg("$log.txt")); // truncate syslog if too big if (basename($file)=='syslog' && filesize($file)>=$max) run("tail -n 200 ".escapeshellarg("$log.txt")." >".escapeshellarg("$log.last200.txt")); run("truncate -s '<$max' ".escapeshellarg("$log.txt"));