mirror of
https://github.com/unraid/webgui.git
synced 2026-01-13 13:09:58 -06:00
389 lines
17 KiB
PHP
Executable File
389 lines
17 KiB
PHP
Executable File
#!/usr/bin/php -q
|
|
<?PHP
|
|
/* Copyright 2005-2017, Lime Technology
|
|
* Copyright 2012-2017, Bergware International.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License version 2,
|
|
* as published by the Free Software Foundation.
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*/
|
|
?>
|
|
<?
|
|
require_once "/usr/local/emhttp/webGui/include/publish.php";
|
|
refresh_emhttp_state();
|
|
$var = parse_ini_file("/var/local/emhttp/var.ini");
|
|
$devs = parse_ini_file("/var/local/emhttp/devs.ini",true);
|
|
$disks = parse_ini_file("/var/local/emhttp/disks.ini",true);
|
|
|
|
$docroot = $docroot ?: @$_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp';
|
|
require_once "$docroot/webGui/include/Wrappers.php";
|
|
require_once "$docroot/webGui/include/Preselect.php";
|
|
require_once "$docroot/webGui/include/CustomMerge.php";
|
|
|
|
$notify = "$docroot/webGui/scripts/notify";
|
|
$ram = "/var/local/emhttp/monitor.ini";
|
|
$rom = "/boot/config/plugins/dynamix/monitor.ini";
|
|
$saved = @parse_ini_file($ram,true);
|
|
$unraid = parse_plugin_cfg("dynamix",true);
|
|
$high1 = $unraid['display']['critical'];
|
|
$high2 = $unraid['display']['warning'];
|
|
$server = strtoupper($var['NAME']);
|
|
$errors = [];
|
|
|
|
function plus($val,$word,$last) {
|
|
return $val>0 ? (($val||$last) ? ($val.' '.$word.($val!=1?'s':'').($last ?'':', ')) : '') : '';
|
|
}
|
|
function my_temp($value) {
|
|
global $unraid;
|
|
$unit = $unraid['display']['unit'];
|
|
return ($unit=='F' ? round(9/5*$value+32) : str_replace('.', $unraid['display']['number'][0], $value))." $unit";
|
|
}
|
|
function my_disk($name) {
|
|
return ucfirst(preg_replace('/^(disk|cache)([0-9]+)/','$1 $2',$name));
|
|
}
|
|
function my_scale($value,&$unit,$precision = NULL) {
|
|
global $unraid;
|
|
$scale = $unraid['display']['scale'];
|
|
$number = $unraid['display']['number'];
|
|
$units = ['B','KB','MB','GB','TB','PB'];
|
|
if ($scale==0 && $precision===NULL) {
|
|
$unit = '';
|
|
return number_format($value, 0, $number[0], ($value>=10000 ? $number[1] : ''));
|
|
} else {
|
|
$base = $value ? floor(log($value, 1000)) : 0;
|
|
if ($scale>0 && $base>$scale) $base = $scale;
|
|
$value = round($value/pow(1000, $base), $precision===NULL ? 2 : $precision);
|
|
if ($value>=1000 && $scale<0) { $value = 1; $base++; }
|
|
$unit = $units[$base];
|
|
return number_format($value, $precision===NULL ? (($value-intval($value)==0 || $value>=100) ? 0 : ($value>=10 ? 1 : 2)) : $precision, $number[0], ($value>=10000 ? $number[1] : ''));
|
|
}
|
|
}
|
|
function my_check($time,$speed) {
|
|
if (!$time) return 'unavailable (no parity-check entries logged)';
|
|
$days = floor($time/86400);
|
|
$hmss = $time-$days*86400;
|
|
$hour = floor($hmss/3600);
|
|
$mins = $hmss/60%60;
|
|
$secs = $hmss%60;
|
|
return plus($days,'day',($hour|$mins|$secs)==0).plus($hour,'hour',($mins|$secs)==0).plus($mins,'minute',$secs==0).plus($secs,'second',true).". Average speed: $speed";
|
|
}
|
|
function read_write_parity_log($epoch,$duration,$speed,$status,$error) {
|
|
$log = '/boot/config/parity-checks.log';
|
|
$timestamp = str_replace(['.0','.'],[' ',' '],date('M.d H:i:s',$epoch));
|
|
if (file_exists($log)) {
|
|
$handle = fopen($log, 'r');
|
|
while (($line = fgets($handle))!==false) {
|
|
if (strpos($line,$timestamp)!==false) break;
|
|
}
|
|
fclose($handle);
|
|
}
|
|
if (empty($line)) {
|
|
$year = date('Y',$epoch);
|
|
$line = "$year $timestamp|$duration|$speed|$status|$error";
|
|
if ($status==0||file_exists($log)) file_put_contents($log,"$line\n",FILE_APPEND);
|
|
}
|
|
return $line;
|
|
}
|
|
function check_temp($name,$temp,$text,$info) {
|
|
global $notify,$disks,$saved,$unraid,$server;
|
|
$disk = &$disks[$name];
|
|
$hot = strlen($disk['hotTemp']) ? $disk['hotTemp'] : $unraid['display']['hot'];
|
|
$max = strlen($disk['maxTemp']) ? $disk['maxTemp'] : $unraid['display']['max'];
|
|
$warn = $temp>=$max && $max>0 ? 'alert' : ($temp>=$hot && $hot>0 ? 'warning' : '');
|
|
$item = 'temp';
|
|
$last = isset($saved[$item][$name]) ? $saved[$item][$name] : 0;
|
|
if ($warn) {
|
|
if ($temp>$last) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text temperature")." -s ".escapeshellarg(ucfirst($warn)." [$server] - $text ".($warn=='alert'?'overheated (':'is hot (').my_temp($temp).")")." -d ".escapeshellarg("$info")." -i \"$warn\"");
|
|
$saved[$item][$name] = $max>0 && $temp<=$max ? $max : $temp;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal temperature")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
}
|
|
function check_smart($name,$port,$text,$info) {
|
|
global $var,$disks,$notify,$saved,$server,$numbers;
|
|
$disk = &$disks[$name];
|
|
$select = $disk['smSelect'] ?? -1; if ($select==-1) $select = $var['smSelect'] ?? 0;
|
|
$level = $disk['smLevel'] ?? -1; if ($level==-1) $level = $var['smLevel'] ?? 1;
|
|
$events = explode('|',$disk['smEvents'] ?? $var['smEvents'] ?? $numbers);
|
|
$type = $disk['smType'] ?? -1; if ($type==-1) $type = $var['smType'] ?? '';
|
|
if ($type) {
|
|
$ports = [];
|
|
if (!empty($disk['smDevice'])) $port = $disk['smDevice'];
|
|
if (!empty($disk['smPort1'])) $ports[] = $disk['smPort1'];
|
|
if (!empty($disk['smPort2'])) $ports[] = $disk['smPort2'];
|
|
if (!empty($disk['smPort3'])) $ports[] = $disk['smPort3'];
|
|
if ($ports) {
|
|
$glue = isset($disk['smGlue']) ? $disk['smGlue'] : ',';
|
|
$type .= ','.implode($glue,$ports);
|
|
}
|
|
}
|
|
$file = "/var/local/emhttp/smart/$name";
|
|
exec("awk 'NR>7{print $1,$2,$4,$6,$9,$10}' ".escapeshellarg($file)." 2>/dev/null", $codes);
|
|
$item = 'smart';
|
|
foreach ($codes as $code) {
|
|
if (!$code) continue;
|
|
list($id,$class,$value,$thres,$when,$raw) = explode(' ',$code);
|
|
$fail = strpos($when,'FAILING_NOW')!==false;
|
|
if (!$fail && !in_array($id,$events)) continue;
|
|
$word = str_replace(['_',' (-)'],[' ',''],strtolower("$class ($when)"));
|
|
$ack = "$name.ack";
|
|
switch ($select) {
|
|
case 0:
|
|
$attr = "$name.$id";
|
|
$last = isset($saved[$item][$attr]) ? $saved[$item][$attr]*$level : 0;
|
|
if ($raw>0 || $fail) {
|
|
if ($raw>$last) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text SMART health [$id]")." -s ".escapeshellarg("Warning [$server] - $word is $raw")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = $raw;
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
} else {
|
|
if ($last>0) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text SMART message [$id]")." -s ".escapeshellarg("Notice [$server] - $word returned to normal value")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$attr]);
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
}
|
|
break;
|
|
case 1:
|
|
$attr = "$name.${id}n";
|
|
$last = isset($saved[$item][$attr]) ? $saved[$item][$attr] : 255;
|
|
if (($thres>0 && $value<=$thres*$level) || $fail) {
|
|
if ($value*($value>$thres?$level:1)<$last) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text SMART health [$id]")." -s ".escapeshellarg("Warning [$server] - $word is $value")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = $value;
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
} else {
|
|
if ($last<255) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text SMART message [$id]")." -s ".escapeshellarg("Notice [$server] - $word returned to normal value")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$attr]);
|
|
unset($saved[$item][$ack]);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
$file .= '.ssa';
|
|
if (!file_exists($file) || (time()-filemtime($file)>=$var['poll_attributes'])) exec("smartctl -n standby -H ".escapeshellarg($type)." ".escapeshellarg("/dev/$port")."|grep -Pom1 '^SMART.*: \K[A-Z]+'|tr -d '\n' > ".escapeshellarg($file));
|
|
}
|
|
function check_usage($name,$used,$text,$info) {
|
|
global $notify,$disks,$saved,$unraid,$server;
|
|
if ($used == -1) return;
|
|
$disk = &$disks[$name];
|
|
$warning = strlen($disk['warning']) ? $disk['warning'] : $unraid['display']['warning'];
|
|
$critical = strlen($disk['critical']) ? $disk['critical'] : $unraid['display']['critical'];
|
|
$warn = $used>=$critical && $critical>0 ? 'alert' : ($used>=$warning && $warning>0 ? 'warning' : '');
|
|
$item = 'used';
|
|
$last = isset($saved[$item][$name]) ? $saved[$item][$name] : 0;
|
|
if ($warn) {
|
|
if ($used>$last) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text disk utilization")." -s ".escapeshellarg(ucfirst($warn)." [$server] - $text is ".($warn=='alert'?'low on space':'high on usage')." (${used}%)")." -d ".escapeshellarg("$info")." -i \"$warn\"");
|
|
$saved[$item][$name] = $critical>0 && $used<=$critical ? $critical : $used;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal utilization level")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// check array devices
|
|
foreach ($disks as $disk) {
|
|
$name = $disk['name'];
|
|
if ($name=='flash' || substr($disk['status'],-3)=='_NP') continue;
|
|
$text = my_disk($name).($name=='cache'||$name=='parity'?' disk':'');
|
|
$info = !empty($disk['id']) ? "{$disk['id']} ({$disk['device']})" : "No device identification ({$disk['device']})";
|
|
// process disk temperature notifications
|
|
check_temp($name,$disk['temp'],$text,$info);
|
|
// process disk SMART notifications
|
|
check_smart($name,$disk['device'],$text,$info);
|
|
// process disk usage notifications
|
|
check_usage($name,($disk['fsSize']?100-round(100*$disk['fsFree']/$disk['fsSize']):-1),$text,$info);
|
|
// process disk operation notifications
|
|
$warn = strtok($disk['color'],'-');
|
|
$item = 'disk';
|
|
$last = isset($saved[$item][$name]) ? $saved[$item][$name] : "";
|
|
switch ($warn) {
|
|
case 'red':
|
|
if ($warn!=$last) {
|
|
$status = strtolower(str_replace(['NP_','_'],['',' '],$disk['status']));
|
|
exec("$notify -e ".escapeshellarg("unRAID $text error")." -s ".escapeshellarg("Alert [$server] - $text in error state ($status)")." -d ".escapeshellarg("$info")." -i \"alert\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
break;
|
|
case 'yellow':
|
|
if ($warn!=$last) {
|
|
$status = $name=='parity' ? "parity-sync in progress" : "drive not ready, content being reconstructed";
|
|
exec("$notify -e ".escapeshellarg("unRAID $text error")." -s ".escapeshellarg("Warning [$server] - $text, $status")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
break;
|
|
default:
|
|
if ($last) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text message")." -s ".escapeshellarg("Notice [$server] - $text returned to normal operation")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
break;}
|
|
// count disk errors
|
|
if ($disk['numErrors']>0) $errors[] = "$text - $info (errors {$disk['numErrors']})";
|
|
// check file system of cache pool
|
|
$item = 'pool';
|
|
if ($name=='cache' && $disk['fsType']=='btrfs') {
|
|
$attr = 'missing';
|
|
if (exec("/sbin/btrfs filesystem show {$disk['uuid']} 2>/dev/null|grep -c 'missing'")>0) {
|
|
if (empty($saved[$item][$attr])) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text message")." -s ".escapeshellarg("Warning [$server] - Cache pool BTRFS missing device(s)")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = 1;
|
|
}
|
|
} elseif (isset($saved[$item][$attr])) unset($saved[$item][$attr]);
|
|
$attr = 'profile';
|
|
if (exec("/sbin/btrfs filesystem df /mnt/cache 2>/dev/null|grep -c '^Data'")>1) {
|
|
if (empty($saved[$item][$attr])) {
|
|
exec("$notify -e ".escapeshellarg("unRAID $text message")." -s ".escapeshellarg("Warning [$server] - Cache pool BTRFS too many profiles")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$attr] = 1;
|
|
}
|
|
} elseif (isset($saved[$item][$attr])) unset($saved[$item][$attr]);
|
|
}
|
|
}
|
|
|
|
// check unassigned devices
|
|
foreach ($devs as $dev) {
|
|
$name = $dev['device'];
|
|
if (empty($name)) continue;
|
|
$smart = "/var/local/emhttp/smart/$name";
|
|
if (!file_exists($smart) || (time()-filemtime($smart)>=$var['poll_attributes'])) exec("smartctl -n standby -A ".escapeshellarg("/dev/$name")." > ".escapeshellarg($smart));
|
|
$temp = exec("awk '\$1==190||\$1==194{print \$10;exit}' ".escapeshellarg($smart));
|
|
$text = "device $name";
|
|
$info = !empty($dev['id']) ? "{$dev['id']} ($name)": "No device identification ($name)";
|
|
// process disk temperature notifications
|
|
check_temp($name,$temp,$text,$info);
|
|
// process disk SMART notifications
|
|
check_smart($name,$dev['device'],$text,$info);
|
|
}
|
|
|
|
// report array read errors
|
|
$item = 'array';
|
|
$name = 'errors';
|
|
$last = isset($saved[$item][$name]) ? $saved[$item][$name] : 0;
|
|
$warn = count($errors);
|
|
$info = "Array has $warn disk".($warn==1 ? "" : "s")." with read errors";
|
|
if ($warn>0) {
|
|
if ($warn<>$last) {
|
|
$message = implode('\n', $errors);
|
|
exec("$notify -e \"unRAID array errors\" -s ".escapeshellarg("Warning [$server] - array has errors")." -d ".escapeshellarg("$info")." -m ".escapeshellarg("$message")." -i \"warning\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e \"unRAID array errors\" -s ".escapeshellarg("Notice [$server] - array turned good")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
|
|
// process parity check, parity sync and data-rebuild notifications
|
|
$name = 'parity';
|
|
$last = isset($saved[$item][$name]) ? $saved[$item][$name] : '';
|
|
if ($var['mdResync']>0) {
|
|
if (!$last) {
|
|
if (strstr($var['mdResyncAction'],"recon")) {
|
|
$last = 'Parity sync / Data rebuild';
|
|
} elseif (strstr($var['mdResyncAction'],"clear")) {
|
|
$last = 'Disk clear';
|
|
} elseif ($var['mdResyncAction']=="check") {
|
|
$last = 'Read check';
|
|
} elseif (strstr($var['mdResyncAction'],"check")) {
|
|
$last = 'Parity check';
|
|
}
|
|
$info = "Size: ".my_scale($var['mdResyncSize']*1024, $unit)." $unit";
|
|
exec("$notify -e ".escapeshellarg("unRAID $last")." -s ".escapeshellarg("Notice [$server] - $last started")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$name] = $last;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
$duration = $var['sbSynced2'] - $var['sbSynced'];
|
|
$status = $var['sbSyncExit'];
|
|
$speed = $status==0 ? my_scale($var['mdResyncSize']*1024/$duration,$unit,1)." $unit/s" : "Unavailable";
|
|
list($entry,$duration,$speed,$status,$error) = explode('|', read_write_parity_log($var['sbSynced2'],$duration,$speed,$status,$var['sbSyncErrs']));
|
|
$info = $status==0 ? "Duration: ".my_check($duration, $speed) : ($status==-4 ? "Canceled" : "Error code: $status");
|
|
$level = ($status==0 && $var['sbSyncErrs']==0) ? "normal" : "warning";
|
|
exec("$notify -e ".escapeshellarg("unRAID $last")." -s ".escapeshellarg("Notice [$server] - $last finished ($error errors)")." -d ".escapeshellarg("$info")." -i \"$level\"");
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
|
|
// check read-write status of USB flash drive
|
|
$name = 'flash';
|
|
$last = isset($saved[$item][$name]) ? $saved[$item][$name] : '';
|
|
$warn = exec("grep -Pom1 '/boot \S+ \K\S{2}' /proc/mounts");
|
|
$info = "{$disks['flash']['id']} ({$disks['flash']['device']})";
|
|
if ($warn!="rw") {
|
|
if ($warn!=$last) {
|
|
exec("$notify -e \"USB flash drive failure\" -s ".escapeshellarg("Alert [$server] - USB drive is not read-write")." -d ".escapeshellarg("$info")." -i \"alert\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e \"USB flash drive operation\" -s ".escapeshellarg("Notice [$server] - USB drive returned to normal operation")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
|
|
// check docker image disk utilization
|
|
system('mountpoint -q /var/lib/docker', $retval);
|
|
if ($retval===0) {
|
|
$item = 'system';
|
|
$name = 'docker';
|
|
$last = isset($saved[$item][$name]) ? $saved[$item][$name] : '';
|
|
if (file_exists("/boot/config/docker.cfg")) {
|
|
$cfg = parse_ini_file("/boot/config/docker.cfg");
|
|
$info = "Docker utilization of image file {$cfg['DOCKER_IMAGE_FILE']}";
|
|
} else
|
|
$info = "Docker image file not specified";
|
|
|
|
$warn = exec("df /var/lib/docker|awk '/^\//{print $5*1}'");
|
|
if ($warn>=$high1 && $high1>0) {
|
|
if ($warn>$last) {
|
|
exec("$notify -e \"Docker critical image disk utilization\" -s ".escapeshellarg("Alert [$server] - Docker image disk utilization of ${warn}%")." -d ".escapeshellarg("$info")." -i \"alert\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} elseif ($warn>=$high2 && $high2>0) {
|
|
if ($warn>$last) {
|
|
exec("$notify -e \"Docker high image disk utilization\" -s ".escapeshellarg("Warning [$server] - Docker image disk utilization of ${warn}%")." -d ".escapeshellarg("$info")." -i \"warning\"");
|
|
$saved[$item][$name] = $warn;
|
|
}
|
|
} else {
|
|
if ($last) {
|
|
exec("$notify -e \"Docker image disk utilization\" -s ".escapeshellarg("Notice [$server] - Docker image disk utilization returned to normal level")." -d ".escapeshellarg("$info"));
|
|
unset($saved[$item][$name]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// save new status
|
|
if ($saved) {
|
|
$text = '';
|
|
foreach ($saved as $item => $block) {
|
|
if ($block) $text .= "[$item]\n";
|
|
foreach ($block as $key => $value) $text .= "$key=\"$value\"\n";
|
|
}
|
|
if ($text) {
|
|
if ($text != @file_get_contents($ram)) file_put_contents($ram, $text);
|
|
if (!file_exists($rom) || exec("diff -q $ram $rom")) file_put_contents($rom, $text);
|
|
} else {
|
|
@unlink($ram);
|
|
@unlink($rom);
|
|
}
|
|
}
|
|
exit(0);
|
|
?>
|