mirror of
https://github.com/unraid/webgui.git
synced 2026-01-13 05:00:06 -06:00
355 lines
15 KiB
PHP
Executable File
355 lines
15 KiB
PHP
Executable File
#!/usr/bin/php -q
|
|
<?PHP
|
|
/* Copyright 2005-2020, Lime Technology
|
|
* Copyright 2012-2020, Bergware International.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License version 2,
|
|
* as published by the Free Software Foundation.
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*/
|
|
?>
|
|
<?
|
|
// Multi-language support
|
|
function _($text) {return $text;}
|
|
function my_lang($text) {return $text;}
|
|
|
|
$var = parse_ini_file("/var/local/emhttp/var.ini");
|
|
$devs = parse_ini_file("/var/local/emhttp/devs.ini",true);
|
|
$disks = parse_ini_file("/var/local/emhttp/disks.ini",true);
|
|
|
|
$docroot = $docroot ?? $_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp';
|
|
require_once "$docroot/webGui/include/Helpers.php";
|
|
require_once "$docroot/webGui/include/Preselect.php";
|
|
require_once "$docroot/webGui/include/CustomMerge.php";
|
|
|
|
extract(parse_plugin_cfg("dynamix",true));
|
|
|
|
$notify = "$docroot/webGui/scripts/notify";
|
|
$ram = "/var/local/emhttp/monitor.ini";
|
|
$rom = "/boot/config/plugins/dynamix/monitor.ini";
|
|
$saved = @parse_ini_file($ram,true);
|
|
$high1 = $display['critical'];
|
|
$high2 = $display['warning'];
|
|
$server = strtoupper($var['NAME']);
|
|
$pools = pools_filter($disks);
|
|
$errors = [];
|
|
$top = 120;
|
|
|
|
|
|
function read_write_parity_log($epoch,$duration,$speed,$status,$error) {
|
|
$log = '/boot/config/parity-checks.log';
|
|
$timestamp = str_replace(['.0','.'],[' ',' '],date('M.d H:i:s',$epoch));
|
|
if (file_exists($log)) {
|
|
$handle = fopen($log, 'r');
|
|
while (($line = fgets($handle))!==false) {
|
|
if (strpos($line,$timestamp)!==false) break;
|
|
}
|
|
fclose($handle);
|
|
}
|
|
if (empty($line)) {
|
|
$year = date('Y',$epoch);
|
|
$line = "$year $timestamp|$duration|$speed|$status|$error";
|
|
if ($status==0||file_exists($log)) file_put_contents($log,"$line\n",FILE_APPEND);
|
|
}
|
|
return str_replace("\n","",$line);
|
|
}
|
|
function check_temp($name,$temp,$text,$info) {
|
|
global $notify,$disks,$saved,$display,$server,$top;
|
|
$disk = &$disks[$name];
|
|
$hot = $disk['hotTemp'] ?? $display['hot'];
|
|
$max = $disk['maxTemp'] ?? $display['max'];
|
|
$warn = exceed($temp,$max,$top) ? 'alert' : (exceed($temp,$hot,$top) ? 'warning' : '');
|
|
$item = 'temp';
|
|
$last = $saved[$item][$name] ?? 0;
|
|
if ($warn) {
|
|
if ($temp>$last) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text temperature")." -s ".escapeshellarg(ucfirst($warn)." [$server] - $text ".($warn=='alert'?'overheated (':'is hot (').my_temp($temp).")")." -d ".escapeshellarg("$info")." -i \"$warn\"");
|
|
$saved[$item][$name] = $max>0 && $temp<=$max ? $max : $temp;
|
|
}
|
|
} else {
|
|
if ($last && $temp<=$top) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal temperature")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
}
|
|
function check_smart($name,$port,$text,$info) {
|
|
global $var,$disks,$notify,$saved,$server,$numbers;
|
|
$disk = &$disks[$name];
|
|
$select = get_value($disk,'smSelect',0);
|
|
$level = get_value($disk,'smLevel',1);
|
|
$events = explode('|',get_value($disk,'smEvents',$numbers));
|
|
$type = get_value($disk,'smType','');
|
|
get_ctlr_options($type, $disk);
|
|
$file = "/var/local/emhttp/smart/$name";
|
|
exec("awk 'NR>7{print $1,$2,$4,$6,$9,$10}' ".escapeshellarg($file)." 2>/dev/null", $codes);
|
|
$item = 'smart';
|
|
foreach ($codes as $code) {
|
|
if (!$code || !is_numeric($code[0])) continue;
|
|
list($id,$class,$value,$thres,$when,$raw) = explode(' ',$code);
|
|
$fail = strpos($when,'FAILING_NOW')!==false;
|
|
if (!$fail && !in_array($id,$events)) continue;
|
|
$word = str_replace(['_',' (-)'],[' ',''],strtolower("$class ($when)"));
|
|
$ack = "$name.ack";
|
|
switch ($select) {
|
|
case 0:
|
|
$attr = "$name.$id";
|
|
$last = ($saved[$item][$attr] ?? 0)*$level;
|
|
if ($raw>0 || $fail) {
|
|
if ($raw>$last) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text SMART health [$id]")." -s ".escapeshellarg("Warning [$server] - $word is $raw")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = $raw;
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
} else {
|
|
if ($last>0) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text SMART message [$id]")." -s ".escapeshellarg("Notice [$server] - $word returned to normal value")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$attr]);
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
}
|
|
break;
|
|
case 1:
|
|
$attr = "$name.${id}n";
|
|
$last = $saved[$item][$attr] ?? 255;
|
|
if (($thres>0 && $value<=$thres*$level) || $fail) {
|
|
if ($value*($value>$thres?$level:1)<$last) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text SMART health [$id]")." -s ".escapeshellarg("Warning [$server] - $word is $value")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = $value;
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
} else {
|
|
if ($last<255) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text SMART message [$id]")." -s ".escapeshellarg("Notice [$server] - $word returned to normal value")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$attr]);
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
$file .= '.ssa';
|
|
if (!file_exists($file) || (time()-filemtime($file)>$var['poll_attributes'])) exec("smartctl -n standby -H $type ".escapeshellarg("/dev/$port")."|grep -Pom1 '^SMART.*: \K[A-Z]+'|tr -d '\n' >".escapeshellarg($file));
|
|
}
|
|
function check_usage($name,$used,$text,$info) {
|
|
global $notify,$disks,$saved,$display,$server;
|
|
if ($used == -1) return;
|
|
$disk = &$disks[$name];
|
|
$warning = $disk['warning'] ?? $display['warning'];
|
|
$critical = $disk['critical'] ?? $display['critical'];
|
|
$warn = exceed($used,$critical) ? 'alert' : (exceed($used,$warning) ? 'warning' : '');
|
|
$item = 'used';
|
|
$last = $saved[$item][$name] ?? 0;
|
|
if ($warn) {
|
|
if ($used>$last) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text disk utilization")." -s ".escapeshellarg(ucfirst($warn)." [$server] - $text is ".($warn=='alert'?'low on space':'high on usage')." (${used}%)")." -d ".escapeshellarg("$info")." -i \"$warn\"");
|
|
$saved[$item][$name] = $critical>0 && $used<=$critical ? $critical : $used;
|
|
}
|
|
} else {
|
|
if ($last && $used<=100) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal utilization level")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// check array devices
|
|
foreach ($disks as $disk) {
|
|
$name = $disk['name'];
|
|
if ($name=='flash' || substr($disk['status'],-3)=='_NP') continue;
|
|
$text = my_disk($name).(in_array($name,$pools)||$name=='parity'?' disk':'');
|
|
$info = !empty($disk['id']) ? "{$disk['id']} ({$disk['device']})" : "No device identification ({$disk['device']})";
|
|
// process disk temperature notifications
|
|
check_temp($name,$disk['temp'],$text,$info);
|
|
// process disk SMART notifications
|
|
check_smart($name,port_name($disk['smDevice'] ?? $disk['device']),$text,$info);
|
|
// process disk usage notifications
|
|
check_usage($name,isset($disk['fsSize'])&&$disk['fsSize']>0?100-round(100*$disk['fsFree']/$disk['fsSize']):-1,$text,$info);
|
|
// process disk operation notifications
|
|
$warn = strtok($disk['color'],'-');
|
|
$item = 'disk';
|
|
$last = $saved[$item][$name] ?? '';
|
|
switch ($warn) {
|
|
case 'red':
|
|
if ($warn!=$last) {
|
|
if ($var['fsState']!='Stopped') {
|
|
$status = strtolower(str_replace(['NP_','_'],['',' '],$disk['status']));
|
|
exec("$notify -e ".escapeshellarg("Unraid $text error")." -s ".escapeshellarg("Alert [$server] - $text in error state ($status)")." -d ".escapeshellarg("$info")." -i \"alert\"");
|
|
}
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
break;
|
|
case 'yellow':
|
|
if ($warn!=$last) {
|
|
if ($var['fsState']!='Stopped') {
|
|
$status = $name=='parity' ? "parity-sync in progress" : "drive not ready, content being reconstructed";
|
|
exec("$notify -e ".escapeshellarg("Unraid $text error")." -s ".escapeshellarg("Warning [$server] - $text, $status")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
}
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
break;
|
|
default:
|
|
if ($last) {
|
|
if ($var['fsState']!='Stopped') {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal operation")." -d ".escapeshellarg("$info"));
|
|
}
|
|
unset($saved[$item][$name]);
|
|
}
|
|
break;}
|
|
// count disk errors
|
|
if ($disk['numErrors']>0) $errors[] = "$text - $info (errors {$disk['numErrors']})";
|
|
// check file system of cache pool
|
|
$item = 'pool';
|
|
if (in_array($name,$pools) && strpos($disk['fsType'],'btrfs')!==false) {
|
|
$attr = 'missing';
|
|
if (exec("/sbin/btrfs filesystem show {$disk['uuid']} 2>/dev/null|grep -c 'missing'")>0) {
|
|
if (empty($saved[$item][$attr])) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Warning [$server] - Cache pool BTRFS missing device(s)")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = 1;
|
|
}
|
|
} elseif (isset($saved[$item][$attr])) unset($saved[$item][$attr]);
|
|
$attr = "profile-$name";
|
|
if (exec("/sbin/btrfs filesystem df /mnt/$name 2>/dev/null|grep -c '^Data'")>1) {
|
|
if (empty($saved[$item][$attr])) {
|
|
exec("$notify -e ".escapeshellarg("Unraid $text message")." -s ".escapeshellarg("Warning [$server] - $pool pool BTRFS too many profiles (You can ignore this warning when a pool balance operation is in progress)")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = 1;
|
|
}
|
|
} elseif (isset($saved[$item][$attr])) unset($saved[$item][$attr]);
|
|
}
|
|
}
|
|
|
|
// check unassigned devices
|
|
foreach ($devs as $dev) {
|
|
$name = $dev['device'];
|
|
$smart = "/var/local/emhttp/smart/$name";
|
|
$type = $var['smType'] ?? '';
|
|
$port = port_name($name);
|
|
if (!file_exists($smart) || (time()-filectime($smart)>$var['poll_attributes'])) exec("smartctl -n standby -A $type ".escapeshellarg("/dev/$port")." >".escapeshellarg($smart));
|
|
$temp = exec("awk 'BEGIN{s=t=\"*\"}\$1==190{s=\$10};\$1==194{t=\$10;exit};\$1==\"Temperature:\"{t=\$2;exit};/^Current Drive Temperature:/{t=\$4;exit} END{if(t!=\"*\")print t; else print s}' ".escapeshellarg($smart)." 2>/dev/null");
|
|
$text = "device $name";
|
|
$info = !empty($dev['id']) ? "{$dev['id']} ($name)": "No device identification ($name)";
|
|
// process disk temperature notifications
|
|
check_temp($name,$temp,$text,$info);
|
|
// process disk SMART notifications
|
|
check_smart($name,$port,$text,$info);
|
|
}
|
|
|
|
// report array read errors
|
|
$item = 'array';
|
|
$name = 'errors';
|
|
$last = $saved[$item][$name] ?? 0;
|
|
$warn = count($errors);
|
|
$info = "Array has $warn disk".($warn==1 ? "" : "s")." with read errors";
|
|
if ($warn>0) {
|
|
if ($warn<>$last) {
|
|
$message = implode('\n', $errors);
|
|
exec("$notify -e \"Unraid array errors\" -s ".escapeshellarg("Warning [$server] - array has errors")." -d ".escapeshellarg("$info")." -m ".escapeshellarg("$message")." -i \"warning\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e \"Unraid array errors\" -s ".escapeshellarg("Notice [$server] - array turned good")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
|
|
// process parity check, parity sync and data-rebuild notifications
|
|
$name = 'parity';
|
|
$last = $saved[$item][$name] ?? '';
|
|
if ($var['mdResyncPos']) {
|
|
if (!$last) {
|
|
if (strstr($var['mdResyncAction'],"recon")) {
|
|
$last = 'Parity sync / Data rebuild';
|
|
} elseif (strstr($var['mdResyncAction'],"clear")) {
|
|
$last = 'Disk clear';
|
|
} elseif ($var['mdResyncAction']=="check") {
|
|
$last = 'Read check';
|
|
} elseif (strstr($var['mdResyncAction'],"check")) {
|
|
$last = 'Parity check';
|
|
}
|
|
$info = "Size: ".my_scale($var['mdResyncSize']*1024,$unit)." $unit";
|
|
exec("$notify -e ".escapeshellarg("Unraid $last")." -s ".escapeshellarg("Notice [$server] - $last started")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$name] = $last;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
$duration = $var['sbSynced2'] - $var['sbSynced'];
|
|
$status = $var['sbSyncExit'];
|
|
$speed = $status==0 ? my_scale($var['mdResyncSize']*1024/$duration,$unit,1)." $unit/s" : "Unavailable";
|
|
list($entry,$duration,$speed,$status,$error) = explode('|', read_write_parity_log($var['sbSynced2'],$duration,$speed,$status,$var['sbSyncErrs']));
|
|
$info = $status==0 ? "Duration: ".my_check($duration, $speed) : ($status==-4 ? "Canceled" : "Error code: $status");
|
|
$level = ($status==0 && $var['sbSyncErrs']==0) ? "normal" : "warning";
|
|
exec("$notify -e ".escapeshellarg("Unraid $last")." -s ".escapeshellarg("Notice [$server] - $last finished ($error errors)")." -d ".escapeshellarg("$info")." -i \"$level\"");
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
|
|
// check read-write status of USB flash drive
|
|
$name = 'flash';
|
|
$last = $saved[$item][$name] ?? '';
|
|
$warn = exec("grep -Pom1 '/boot \S+ \K\S{2}' /proc/mounts");
|
|
$info = "{$disks['flash']['id']} ({$disks['flash']['device']})";
|
|
if ($warn!="rw") {
|
|
if ($warn!=$last) {
|
|
exec("$notify -e \"USB flash drive failure\" -s ".escapeshellarg("Alert [$server] - USB drive is not read-write")." -d ".escapeshellarg("$info")." -i \"alert\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e \"USB flash drive operation\" -s ".escapeshellarg("Notice [$server] - USB drive returned to normal operation")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
|
|
// check docker image disk utilization
|
|
system('mountpoint -q /var/lib/docker', $retval);
|
|
if ($retval===0) {
|
|
$item = 'system';
|
|
$name = 'docker';
|
|
$last = $saved[$item][$name] ?? '';
|
|
if (file_exists("/boot/config/docker.cfg")) {
|
|
$cfg = parse_ini_file("/boot/config/docker.cfg");
|
|
$info = "Docker utilization of image file {$cfg['DOCKER_IMAGE_FILE']}";
|
|
} else
|
|
$info = "Docker image file not specified";
|
|
|
|
$warn = exec("df /var/lib/docker|awk '/^\//{print $5*1}'");
|
|
if ($warn>=$high1 && $high1>0) {
|
|
if ($warn>$last) {
|
|
exec("$notify -e \"Docker critical image disk utilization\" -s ".escapeshellarg("Alert [$server] - Docker image disk utilization of ${warn}%")." -d ".escapeshellarg("$info")." -i \"alert\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} elseif ($warn>=$high2 && $high2>0) {
|
|
if ($warn>$last) {
|
|
exec("$notify -e \"Docker high image disk utilization\" -s ".escapeshellarg("Warning [$server] - Docker image disk utilization of ${warn}%")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e \"Docker image disk utilization\" -s ".escapeshellarg("Notice [$server] - Docker image disk utilization returned to normal level")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// save new status
|
|
if ($saved) {
|
|
$text = '';
|
|
foreach ($saved as $item => $block) {
|
|
if ($block) $text .= "[$item]\n";
|
|
foreach ($block as $key => $value) $text .= "$key=\"$value\"\n";
|
|
}
|
|
if ($text) {
|
|
if ($text != @file_get_contents($ram)) file_put_contents($ram, $text);
|
|
if (!file_exists($rom) || exec("diff -q $ram $rom")) file_put_contents($rom, $text);
|
|
} else {
|
|
@unlink($ram);
|
|
@unlink($rom);
|
|
}
|
|
}
|
|
exit(0);
|
|
?>
|