Files
webgui/plugins/dynamix/scripts/diagnostics
2021-07-21 12:48:37 -04:00

412 lines
19 KiB
PHP
Executable File

#!/usr/bin/php -q
<?PHP
/* Copyright 2005-2021, Lime Technology
* Copyright 2012-2021, Bergware International.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License version 2,
* as published by the Free Software Foundation.
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*/
?>
<?
$opt = getopt('a',['all']);
$all = isset($opt['a']) || isset($opt['all']);
$zip = $all ? $argv[2] : $argv[1];
$cli = empty($zip);
$get = "/var/local/emhttp";
$var = (array)@parse_ini_file("$get/var.ini");
$docroot = $docroot ?? $_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp';
$folders = ['/boot','/boot/config','/boot/config/plugins','/boot/extra','/boot/syslinux','/var/log','/var/log/plugins','/var/log/packages','/tmp'];
function curl_socket($socket, $url, $postdata = NULL) {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_UNIX_SOCKET_PATH, $socket);
if ($postdata !== NULL) {
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata);
}
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_exec($ch);
curl_close($ch);
}
function publish($endpoint, $message){
curl_socket("/var/run/nginx.socket", "http://localhost/pub/$endpoint?buffer_length=1", $message);
}
function exert($cmd, &$save=null) {
global $cli,$diag;
// execute command with timeout of 30s
publish("diagnostic",$cmd);
exec("timeout -s9 30 $cmd", $save);
return implode("\n",$save);
}
function shareDisks($share) {
return exec("shopt -s dotglob; getfattr --no-dereference --absolute-names --only-values -n system.LOCATIONS ".escapeshellarg("/usr/local/emhttp/mnt/user/$share")." 2>/dev/null") ?: "";
}
function anonymize($text,$select) {
global $all, $customShares;
if ($all) return $text;
switch ($select) {
case 1:
$rows = explode("\n", $text);
$regex = "/\b((disk|cache|parity|cpu|eth|dev)[0-9]+)|(smart|flash|flashbackup|cache|parity|cpu$customShares)\b/";
foreach ($rows as &$row) {
if (!preg_match($regex, $row)) {
$row = preg_replace("/^(\s*\[\S).*(\S\])( => Array)$/","$1..$2$3",$row);
$row = preg_replace("/^(\s*\[(cachePool|name|nameOrig|comment|flashGUID|regGUID|regTo|readList|writeList|csrf_token)\] => \S).*(\S)$/","$1..$3",$row);
}
}
return implode("\n", $rows);
case 2:
$name = basename($text,'.cfg');
if ($name !== "cache") {
$len = strlen($name);
if ($len>2) {
$dash = str_repeat('-',$len-2);
$name = preg_replace("/^(\S).*(\S)/","$1$dash$2",$name);
$i = 1;
while (file_exists(dirname($text)."/$name.cfg")) {$name = substr($name,0,$len)." ($i)"; $i++;}
}
}
return dirname($text)."/$name.cfg";
}
}
function prefix($key) {
return preg_replace('/\d+$/','',$key);
}
function cache_only($disk) {
return $disk['type']=='Cache';
}
function cache_filter($disks) {
return array_filter($disks,'cache_only');
}
function pools_filter($disks) {
return array_unique(array_map('prefix',array_keys(cache_filter($disks))));
}
function download_url($url, $path = "", $bg = false, $timeout = 15) {
$ch = curl_init();
curl_setopt($ch,CURLOPT_URL,$url);
curl_setopt($ch,CURLOPT_FRESH_CONNECT,true);
curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,10);
curl_setopt($ch,CURLOPT_TIMEOUT,$timeout);
curl_setopt($ch,CURLOPT_ENCODING,"");
curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
$out = curl_exec($ch);
curl_close($ch);
if ( $path )
file_put_contents($path,$out);
return $out ?: false;
}
exert("mkdir -p /boot/logs");
if ($cli) {
// script is called from CLI
echo "Starting diagnostics collection... ";
$server = isset($var['NAME']) ? str_replace(' ','_',strtolower($var['NAME'])) : 'tower';
$date = date('Ymd-Hi');
$diag = "$server-diagnostics-$date";
$zip = "/boot/logs/$diag.zip";
} else {
// script is called from GUI
$diag = basename($zip, '.zip');
$split = explode('-', $diag);
$date = "{$split[2]}-{$split[3]}";
}
// don't anonymize system share names
$vardomain = (array)@parse_ini_file('/boot/config/domain.cfg');
$vardocker = (array)@parse_ini_file('/boot/config/docker.cfg');
$showshares = [];
$showshares[] = current(array_slice(explode('/',$vardomain['IMAGE_FILE']), 3, 1)).'.cfg';
$showshares[] = current(array_slice(explode('/',$vardomain['DOMAINDIR']), 3, 1)).'.cfg';
$showshares[] = current(array_slice(explode('/',$vardomain['MEDIADIR']), 3, 1)).'.cfg';
$showshares[] = current(array_slice(explode('/',$vardomain['DISKDIR']), 3, 1)).'.cfg';
$showshares[] = current(array_slice(explode('/',$vardocker['DOCKER_IMAGE_FILE']), 3, 1)).'.cfg';
$showshares[] = current(array_slice(explode('/',$vardocker['DOCKER_APP_CONFIG_PATH']), 3, 1)).'.cfg';
$showshares[] = current(array_slice(explode('/',$vardocker['DOCKER_HOME']), 3, 1)).'.cfg';
foreach ($showshares as $show) {
$showme = str_replace(".cfg","",$show);
if ($showme)
$customShares .= "|$showme";
}
// create folder structure
exert("mkdir -p ".escapeshellarg("/$diag/system")." ".escapeshellarg("/$diag/config")." ".escapeshellarg("/$diag/logs")." ".escapeshellarg("/$diag/shares")." ".escapeshellarg("/$diag/smart")." ".escapeshellarg("/$diag/qemu")." ".escapeshellarg("/$diag/xml"));
// get utilization of running processes
exert("top -bn1 -o%CPU 2>/dev/null|todos >".escapeshellarg("/$diag/system/top.txt"));
// make Unraid version reference
$unraid = parse_ini_file('/etc/unraid-version');
file_put_contents("/$diag/unraid-".$unraid['version'].".txt",$unraid['version']);
// copy ini variables
foreach (glob("$get/*.ini") as $file) {
$ini = basename($file,".ini");
// skip users file in anonymized mode
if ($all || $ini != "users") file_put_contents("/$diag/system/vars.txt",preg_replace(["/\n/","/^Array/"],["\r\n",$ini],anonymize(print_r(parse_ini_file($file,true),true),1)),FILE_APPEND);
}
// Create loads.txt
$cpuload = exert("uptime")." Cores: ".exert("nproc")."\r\n".(string)@file_get_contents("$get/cpuload.ini")."\r\n";
$diskload = (array)@file("$get/diskload.ini");
$disks = (array)@parse_ini_file("$get/disks.ini",true);
$loadTxt = [];
foreach ($diskload as $loadLine) {
$load = explode('=',$loadLine);
foreach ($disks as $disk) {
if ($load[0]==$disk['device']) {
$loadTxt[] = "{$disk['device']} ({$disk['name']})=".trim($load[1]);
break;
}
}
}
file_put_contents("/$diag/system/loads.txt",$cpuload.implode("\r\n",$loadTxt));
// individual commands execution (suppress errors)
exert("lscpu 2>/dev/null|todos >".escapeshellarg("/$diag/system/lscpu.txt"));
exert("lsscsi -vgl 2>/dev/null|todos >".escapeshellarg("/$diag/system/lsscsi.txt"));
exert("lspci -knn 2>/dev/null|todos >".escapeshellarg("/$diag/system/lspci.txt"));
exert("lsusb 2>/dev/null|todos >".escapeshellarg("/$diag/system/lsusb.txt"));
exert("free -mth 2>/dev/null|todos >".escapeshellarg("/$diag/system/memory.txt"));
exert("ps -auxf --sort=-pcpu 2>/dev/null|todos >".escapeshellarg("/$diag/system/ps.txt"));
exert("lsof -Pni 2>/dev/null|todos >".escapeshellarg("/$diag/system/lsof.txt"));
exert("lsmod|sort 2>/dev/null|todos >".escapeshellarg("/$diag/system/lsmod.txt"));
exert("df -h 2>/dev/null|todos >".escapeshellarg("/$diag/system/df.txt"));
exert("ifconfig -a -s 2>/dev/null|grep -Po '^(eth|bond)[0-9]+'", $ports);
exert("dmidecode -qt2|awk -F: '/^\tManufacturer:/{m=\$2};/^\tProduct Name:/{p=\$2} END{print m\" -\"p}' 2>/dev/null|todos >".escapeshellarg("/$diag/system/motherboard.txt"));
exert("dmidecode -qt0 2>/dev/null|todos >>".escapeshellarg("/$diag/system/motherboard.txt"));
exert("cat /proc/meminfo 2>/dev/null|todos >".escapeshellarg("/$diag/system/meminfo.txt"));
exert("dmidecode --type 17 2>/dev/null|todos >>".escapeshellarg("/$diag/system/meminfo.txt"));
// create ethernet information information (suppress errors)
foreach ($ports as $port) {
exert("ethtool ".escapeshellarg($port)." 2>/dev/null|todos >>".escapeshellarg("/$diag/system/ethtool.txt"));
file_put_contents("/$diag/system/ethtool.txt", "\r\n", FILE_APPEND);
exert("ethtool -i ".escapeshellarg($port)." 2>/dev/null|todos >>".escapeshellarg("/$diag/system/ethtool.txt"));
file_put_contents("/$diag/system/ethtool.txt", "--------------------------------\r\n", FILE_APPEND);
}
exert("ifconfig -a 2>/dev/null|todos >".escapeshellarg("/$diag/system/ifconfig.txt"));
// create system information (suppress errors)
exert("find /sys/kernel/iommu_groups/ -type l 2>/dev/null|sort -V|todos >".escapeshellarg("/$diag/system/iommu_groups.txt"));
exert("todos </proc/cmdline >".escapeshellarg("/$diag/system/cmdline.txt"));
// create folder structure listing
$dest = "/$diag/system/folders.txt";
foreach ($folders as $folder) {
if (is_dir($folder)) exert("echo -ne ".escapeshellarg("\r\n$folder\r\n")." >>".escapeshellarg($dest).";ls -l ".escapeshellarg($folder)."|todos >>".escapeshellarg("$dest")); else exert("echo -ne ".escapeshellarg("\r\n$folder\r\nfolder does not exist\r\n")." >>".escapeshellarg("$dest"));
}
// copy configuration files (suppress errors)
exert("cp /boot/config/*.{cfg,conf,dat} ".escapeshellarg("/$diag/config")." 2>/dev/null");
exert("cp /boot/config/go ".escapeshellarg("/$diag/config/go.txt")." 2>/dev/null");
if (!$all)
exert("sed -i -e '/password/c ***line removed***' -e '/user/c ***line removed***' -e '/pass/c ***line removed***' ".escapeshellarg("/$diag/config/go.txt"));
// anonymize configuration files
if (!$all) exert("sed -ri 's/^((disk|flash)(Read|Write)List.*=\")[^\"]+/\\1.../' ".escapeshellarg("/$diag/config/*.cfg")." 2>/dev/null");
// copy share information (anonymize if applicable)
$files = glob("/boot/config/shares/*.cfg");
foreach ($files as $file) {
$dest = "/$diag/shares/".basename($file);
if (!in_array(basename($file),$showshares)) {
$dest = anonymize($dest,2);
}
@copy($file, $dest);
if (!$all) exert("sed -ri 's/^(share(Comment|ReadList|WriteList)=\")[^\"]+/\\1.../' ".escapeshellarg($dest)." 2>/dev/null");
$share = pathinfo(basename($file),PATHINFO_FILENAME);
$shareExists = explode(",",shareDisks($share));
unset($exists);
foreach ($shareExists as $anon) {
if ( !$all )
if (!preg_match("/\b((disk)[0-9]+)|(cache)\b/", $anon)) {
$len = strlen($anon);
if ($len>2) {
$dash = str_repeat('-',$len-2);
$exists[] = substr($anon,0,1).$dash.substr($anon,-1);
}
}
else
$exists[] = $anon;
}
$shareDisk .= $exists ? str_pad(pathinfo($dest,PATHINFO_FILENAME),34).str_pad(exec("cat ".escapeshellarg($file)." | grep shareUseCache"),22)." Exists on ".implode(", ",$exists)."\r\n" : "";
$exists = $exists ?: ["no drives"];
file_put_contents($dest,"# Share exists on ".implode(", ",$exists)."\r\n",FILE_APPEND);
}
file_put_contents("/$diag/shares/shareDisks.txt",$shareDisk);
// create default user shares information
$shares = (array)@parse_ini_file("$get/shares.ini", true);
foreach ($shares as $share) {
$name = $share['name'];
if (!in_array("/boot/config/shares/$name.cfg",$files)) file_put_contents(anonymize("/$diag/shares/$name.cfg",2),"# This share has default settings.\r\n# Share exists on ".shareDisks($name)."\r\n");
}
// copy pools information (anonymize)
$files = glob("/boot/config/pools/*.cfg");
@mkdir("/$diag/config/pools");
foreach ($files as $file) {
$dest = anonymize("/$diag/config/pools/".basename($file),2);
@copy($file,$dest);
}
// copy modprobe information
$files = glob("/boot/config/modprobe.d/*.conf");
if ($files) {
@mkdir("/$diag/config/modprobe.d");
foreach ($files as $file) {
$dest = "/$diag/config/modprobe.d/".basename($file);
@copy($file,$dest);
}
}
// copy docker information (if existing)
$max = 1*1024*1024; //=1MB
$docker = "/var/log/docker.log";
if (file_exists($docker)) {
$log = "/$diag/logs/docker";
exert("todos <$docker >".escapeshellarg("$log.txt"));
if (filesize($docker)>=$max) {
exert("tail -n 200 ".escapeshellarg("$log.txt")." >".escapeshellarg("$log.last200.txt"));
exert("truncate -s '<$max' ".escapeshellarg("$log.txt"));
}
}
// create SMART reports (suppress errors)
$disks = (array)@parse_ini_file("$get/disks.ini", true);
include_once "$docroot/webGui/include/CustomMerge.php";
include_once "$docroot/webGui/include/Wrappers.php";
exert("ls -l /dev/disk/by-id/[asun]* 2>/dev/null|sed '/-part/d;s|^.*/by-id/[^-]*-||;s|-> ../../||;s|:|-|'", $devices);
foreach ($devices as $device) {
list($name,$port) = explode(' ',$device);
$diskName = ''; $type = '';
foreach ($disks as $find) {
if ($find['device']==$port) {
$diskName = $find['name'];
$type = get_value($find,'smType','');
get_ctlr_options($type, $find);
$port = $find['smDevice'] ?? $port;
break;
}
}
$port = port_name($port);
$status = $find['status'] == "DISK_OK" ? "" : " - {$find['status']}";
exert("smartctl -x $type ".escapeshellarg("/dev/$port")." 2>/dev/null|todos >".escapeshellarg("/$diag/smart/$name-$date $diskName ($port)$status.txt"));
}
// create pool btrfs information
foreach (pools_filter($disks) as $pool) {
if (strpos($disks[$pool]['fsType'],'btrfs')!==false) {
exert("echo 'Pool: $pool'|todos >>".escapeshellarg("/$diag/system/btrfs-usage.txt"));
exert("/sbin/btrfs filesystem usage -T /mnt/$pool 2>/dev/null|todos >>".escapeshellarg("/$diag/system/btrfs-usage.txt"));
}
}
// create installed plugin information
$pluginList = json_decode(download_url("https://raw.githubusercontent.com/Squidly271/AppFeed/master/pluginList.json"),true);
if ( ! $pluginList )
$installedPlugins = "Could not download current plugin versions\r\n\r\n";
$plugins = glob("/var/log/plugins/*.plg");
foreach ($plugins as $plugin) {
$plgVer = exert("plugin version ".escapeshellarg($plugin));
$plgURL = exert("plugin pluginURL ".escapeshellarg($plugin));
$installedPlugins .= basename($plugin)." - $plgVer";
if ( $pluginList && ! $pluginList[$plgURL] && basename($plugin) !== "unRAIDServer.plg")
$installedPlugins .= " (Unknown to Community Applications)";
if ( $pluginList[$plgURL]['blacklist'] )
$installedPlugins .= " (Blacklisted)";
if ( $pluginList[$plgURL]['deprecated'] || ( $pluginList[$plgURL]['dmax'] && version_compare($unraid['version'],"<") ) )
$installedPlugins .= " (Deprecated)";
if ( $pluginList[$plgURL]['version'] > $plgVer )
$installedPlugins .= " (Update available: {$pluginList[$plgURL]['version']})";
elseif ($pluginList[$plgURL])
$installedPlugins .= " (Up to date)";
if ( $pluginList[$plgURL]['max'] && version_compare($pluginList[$plgURL]['max'],$unraid['version'],"<") )
$installedPlugins .= " (Incompatible)";
if ( $pluginList[$plgURL]['min'] && version_compare($pluginList[$plgURL]['min'],$unraid['version'],">") )
$installedPlugins .= " (Incompatible)";
$installedPlugins .= "\r\n";
}
$installedPlugins = $installedPlugins ?: "No additional Plugins Installed";
file_put_contents("/$diag/system/plugins.txt",$installedPlugins);
// copy libvirt information (if existing)
$libvirtd = "/var/log/libvirt/libvirtd.log";
if (file_exists($libvirtd)) {
$log = "/$diag/logs/libvirt";
exert("todos <$libvirtd >".escapeshellarg("$log.txt"));
if (filesize($libvirtd)>=$max) {
exert("tail -n 200 ".escapeshellarg("$log.txt")." >".escapeshellarg("$log.last200.txt"));
exert("truncate -s '<$max' ".escapeshellarg("$log.txt"));
}
}
// copy VMs information (if existing)
$qemu = glob("/var/log/libvirt/qemu/*.log*");
if ($qemu) {
foreach ($qemu as $file) {
$log = "/$diag/qemu/".basename($file,'.log');
exert("todos <".escapeshellarg($file)." >".escapeshellarg("$log.txt"));
if (filesize($file)>=$max) {
exert("tail -n 200 ".escapeshellarg("$log.txt")." >".escapeshellarg("$log.last200.txt"));
exert("truncate -s '<$max' ".escapeshellarg("$log.txt"));
}
}
} else {
file_put_contents("/$diag/qemu/no qemu log files","");
}
// copy VM XML config files
exert("cp /etc/libvirt/qemu/*.xml ".escapeshellarg("/$diag/xml")." 2>/dev/null");
// anonymize MAC OSK info
$all_xml = glob("/$diag/xml/*.xml");
foreach ($all_xml as $xml) {
exert("sed -ri 's/(,osk=).+/\\1.../' ".escapeshellarg("$xml")." 2>/dev/null");
exert("sed -ri 's/(passwd=).+/\\1.../' ".escapeshellarg("$xml")." 2>/dev/null");
}
// copy syslog information (anonymize if applicable)
$max = 2*1024*1024; //=2MB
foreach (glob("/var/log/syslog*") as $file) {
$log = "/$diag/logs/".basename($file);
exert("todos <".escapeshellarg($file)." >".escapeshellarg("$log.txt"));
if (!$all) {
unset($titles,$rows);
exert("grep -Po 'logger: moving \"\K[^\"]+' ".escapeshellarg("$log.txt")." 2>/dev/null|sort|uniq", $titles);
exert("sed -ri 's|\b\S+@\S+\.\S+\b|email@removed.com|;s|\b(username\|password)([=:])\S+\b|\\1\\2xxx|;s|(GUID: \S)\S+(\S) |\\1..\\2 |;s|(moving \"\S\|\"/mnt/user/\S).*(\S)\"|\\1..\\2\"|' ".escapeshellarg("$log.txt"));
exert("sed -ri 's|(host: \").+(\.unraid\.net:[0-9]+\")|\\1hash\\2|;s|(referrer: \"https?://).+(\.unraid\.net)|\\1hash\\2|' ".escapeshellarg("$log.txt"));
foreach ($titles as $mover) {
$title = "/{$mover[0]}..".substr($mover,-1)."/...";
exert("sed -ri 's|(logger: [.>cr].*)[ /]$mover/.*$|\\1 file: $title|' ".escapeshellarg("$log.txt")." 2>/dev/null");
}
exert("grep -n ' cache_dirs: -' ".escapeshellarg("$log.txt")." 2>/dev/null|cut -d: -f1", $rows);
for ($i = 0; $i < count($rows); $i += 2) for ($row = $rows[$i]+1; $row < $rows[$i+1]; $row++) exert("sed -ri '$row s|(cache_dirs: \S).*(\S)|\\1..\\2|' ".escapeshellarg("$log.txt")." 2>/dev/null");
}
// replace consecutive repeated lines in syslog
exert("awk -i inplace '{if(s!=substr(\$0,17)){if(x>0)print\"### [PREVIOUS LINE REPEATED \"x\" TIMES] ###\\r\";print;x=0}else{x++}s=substr(\$0,17)}END{if(x>0)print\"### [PREVIOUS LINE REPEATED \"x\" TIMES] ###\\r\"}' ".escapeshellarg("$log.txt"));
// remove SHA256 hashes
exert("sed -ri 's/(SHA256:).+[^\s\b]/SHA256:***REMOVED***/gm' $log.txt");
// truncate syslog if too big
if (basename($file)=='syslog' && filesize($file)>=$max) exert("tail -n 200 ".escapeshellarg("$log.txt")." >".escapeshellarg("$log.last200.txt"));
exert("truncate -s '<$max' ".escapeshellarg("$log.txt"));
}
// copy graphql-api.log
$graphql = "/var/log/graphql-api.log";
if (file_exists($graphql)) {
$log = "/$diag/logs/graphql-api.txt";
exert("todos <$graphql >".escapeshellarg($log));
}
// copy vfio-pci log
$vfiopci = "/var/log/vfio-pci";
if (file_exists($vfiopci)) {
$log = "/$diag/logs/vfio-pci.txt";
exert("todos <$vfiopci >".escapeshellarg($log));
}
// generate unraid-api.txt
if (file_exists("/usr/local/sbin/unraid-api")) {
$log = "/$diag/system/unraid-api.txt";
exert("unraid-api report | todos >".escapeshellarg($log));
}
// create resulting zip file and remove temp folder
exert("zip -qmr ".escapeshellarg($zip)." ".escapeshellarg("/$diag"));
if ($cli) {
echo "done.\nZIP file '$zip' created.\n";
} else {
copy($zip,"/boot/logs/".basename($zip));
}
publish("diagnostic",basename($zip)."FINISHED");
?>