Warning: session_start(): Session cannot be started after headers have already been sent in /home/tvrreohg/public_html/manga.php on line 13
#!/bin/bash # # check_raid_all -- universal RAID health check for Nagios / NRPE. # # Checks in a single run whatever the host actually has: # * Linux software RAID (mdadm), parsed from /proc/mdstat # * LSI MegaRAID / Dell PERC, via storcli64, perccli64 or MegaCli64 # # A host with no RAID card and no md arrays is reported OK (use -r to turn that # into CRITICAL), so the very same check can be deployed on every server. # Supersedes check_mdadm and check_megaraid_sas. # # Exit codes: 0 = OK, 1 = WARNING, 2 = CRITICAL, 3 = UNKNOWN # # The controller CLIs need root (/proc/mdstat does not) -- run this via sudo. set -u export LC_ALL=C PATH='/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin' export PATH PROGNAME=${0##*/} E_OK=0 E_WARNING=1 E_CRITICAL=2 E_UNKNOWN=3 # Return codes of the check_* functions below. R_CHECKED=0 # controller/arrays found and evaluated R_ABSENT=1 # nothing of this kind on the host R_NOANSWER=2 # tool is installed but told us nothing require_raid=0 min_arrays=0 want_hotspares=0 media_allow=0 pred_allow=0 other_allow=0 cmd_timeout=30 verbose=0 # Overridable only to make the parsers testable with recorded output. In # production sudo drops the whole environment (Defaults env_reset, the sudoers # default), so neither is reachable from the nrpe command line. If env_reset # is disabled on a platform, add CHECK_RAID_MDSTAT and CHECK_RAID_TOOLDIR to # env_delete in sudoers. mdstat_path=${CHECK_RAID_MDSTAT:-/proc/mdstat} extra_tooldir=${CHECK_RAID_TOOLDIR:-} [ -n "$extra_tooldir" ] && PATH="$extra_tooldir:$PATH" usage() { cat <&2; exit $E_UNKNOWN ;; esac done # ---------------------------------------------------------------- aggregation # Nagios severity order is not the exit-code order: OK < UNKNOWN < WARNING < # CRITICAL. Track the worst as a rank and translate back at the very end. worst_rank=0 summary='' detail='' perfdata='' rank_of() { case $1 in 0) printf 0 ;; 3) printf 1 ;; 1) printf 2 ;; 2) printf 3 ;; *) printf 1 ;; esac } escalate() { local _rank _rank=$(rank_of "$1") if [ "$_rank" -gt "$worst_rank" ]; then worst_rank=$_rank fi return 0 } add_summary() { if [ -n "$summary" ]; then summary="$summary; $1" else summary=$1 fi return 0 } add_detail() { if [ -n "$detail" ]; then detail="$detail $1" else detail=$1 fi return 0 } # ------------------------------------------------------------------- runners if command -v timeout >/dev/null 2>&1; then TIMEOUT="timeout $cmd_timeout" else TIMEOUT='' fi if [ "$(id -u)" -ne 0 ]; then SUDO='sudo -n' else SUDO='' fi # Run a controller CLI: never let it hang, never let its noise reach stdout. # stderr goes to a scratch file rather than /dev/null: run_tool runs inside # command substitutions, so a variable could not carry the error text back, # but a file can -- and it costs no second, separately timeout-bounded # invocation when the tool is wedged. rt_errfile="${TMPDIR:-/tmp}/check_raid_all.$$.stderr" trap 'rm -f "$rt_errfile"' EXIT run_tool() { # shellcheck disable=SC2086 $TIMEOUT $SUDO "$@" 2>"$rt_errfile" } # First stderr line of the most recent run_tool call, for the UNKNOWN message. mute_reason='' save_tool_error() { mute_reason=$(head -1 "$rt_errfile" 2>/dev/null) } # Find the first existing binary out of the names given. find_tool() { local _name _path _dir for _name in "$@"; do for _dir in "$extra_tooldir" /opt/MegaRAID/storcli /opt/MegaRAID/perccli /opt/MegaRAID/MegaCli \ /opt/storcli /opt/perccli /opt/MegaCli /usr/local/sbin /usr/sbin /usr/bin; do [ -n "$_dir" ] || continue if [ -x "$_dir/$_name" ]; then printf '%s\n' "$_dir/$_name" return 0 fi done _path=$(command -v "$_name" 2>/dev/null) if [ -n "$_path" ] && [ -x "$_path" ]; then printf '%s\n' "$_path" return 0 fi done return 1 } # -------------------------------------------------------------- software RAID # Flatten /proc/mdstat into one pipe-separated record per array: # name|state|personality|have|want|map|faulty|spare|action|percent MDSTAT_AWK=' function emit() { if (name == "") return printf "%s|%s|%s|%s|%s|%s|%d|%d|%s|%s\n", name, state, pers, have, want, map, faulty, spare, action, pct } /^md[^ \t]*[ \t]*:/ { emit() name = $1 state = $3 pers = ($4 ~ /^\(/) ? $5 : $4 if (pers ~ /\[/) pers = "" # inactive array: no personality, just members have = ""; want = ""; map = ""; faulty = 0; spare = 0; action = ""; pct = "" for (i = 3; i <= NF; i++) { if ($i ~ /\(F\)/) faulty++ if ($i ~ /\(S\)/) spare++ } next } name != "" { if (match($0, /\[[0-9]+\/[0-9]+\]/)) { split(substr($0, RSTART + 1, RLENGTH - 2), a, "/") have = a[1]; want = a[2] } if (match($0, /\[[U_]+\]/)) map = substr($0, RSTART + 1, RLENGTH - 2) if ($0 ~ /=DELAYED/) { action = "delayed" } else if ($0 ~ /=PENDING/) { action = "pending" } else if ($0 ~ /(recovery|resync|reshape|check)[ \t]*=/) { if (match($0, /recovery|resync|reshape|check/)) action = substr($0, RSTART, RLENGTH) if (match($0, /[0-9]+\.[0-9]+%/)) pct = substr($0, RSTART, RLENGTH) } } END { emit() } ' check_md() { local _records='' if [ -r "$mdstat_path" ]; then _records=$(awk "$MDSTAT_AWK" "$mdstat_path" 2>/dev/null) fi if [ -z "$_records" ]; then # No arrays at all. That is only a problem if we were told to expect some # -- which is exactly the case a bare mdstat parser reports as OK. if [ "$min_arrays" -gt 0 ]; then escalate $E_CRITICAL add_summary "mdadm: no arrays present, $min_arrays expected" perfdata="$perfdata md_arrays=0 md_failed=0 md_missing=$min_arrays md_degraded=0" return $R_CHECKED fi return $R_ABSENT fi local _total=0 _bad=0 _warn=0 _faulty=0 _spare=0 _msg='' local name state pers have want map faulty spare action pct local _state _note _desc while IFS='|' read -r name state pers have want map faulty spare action pct; do [ -n "$name" ] || continue _total=$((_total + 1)) _faulty=$((_faulty + faulty)) _spare=$((_spare + spare)) _state=ok _note='' if [ "$state" != 'active' ]; then # inactive/unknown: the array is not serving data at all _state=crit _note="state=$state" elif [ "${map#*_}" != "$map" ] || { [ -n "$want" ] && [ "$have" -lt "$want" ]; }; then # missing member(s) -- only a WARNING while it is being rebuilt case $action in recovery | reshape | resync) _state=warn _note="degraded, $action ${pct:-in progress}" ;; *) _state=crit _note='degraded' ;; esac elif [ "$faulty" -gt 0 ]; then # optimal map but a member is still flagged faulty (spare took over) _state=warn _note="$faulty faulty member(s)" else case $action in recovery | reshape | resync) _state=warn _note="$action ${pct:-in progress}" ;; delayed | pending) _state=warn _note="resync $action" ;; check) # routine monthly scrub, not a fault _note="check ${pct:-in progress}" ;; esac fi case $_state in crit) _bad=$((_bad + 1)) escalate $E_CRITICAL ;; warn) _warn=$((_warn + 1)) escalate $E_WARNING ;; esac _desc=$name [ -n "$pers" ] && _desc="$_desc $pers" [ -n "$want" ] && _desc="$_desc $have/$want" [ -n "$map" ] && _desc="$_desc [$map]" [ -n "$_note" ] && _desc="$_desc: $_note" if [ "$_state" != 'ok' ]; then _msg="$_msg ($_desc)" elif [ "$verbose" -eq 1 ]; then add_detail "($_desc)" fi done < exit code vd_severity() { case $1 in Optl* | Optimal*) printf %s $E_OK ;; Pdgd* | Partially*) printf %s $E_WARNING ;; Dgrd* | Degraded* | OfLn* | Offln* | Offline* | Failed*) printf %s $E_CRITICAL ;; *) printf %s $E_UNKNOWN ;; esac } # storcli/perccli physical-drive state -> exit code pd_severity() { case $1 in Onln | Online | JBOD | UGood | UGUnsp | GHS | DHS) printf %s $E_OK ;; Rbld | Cpybck | UBad | UBUnsp | Shld | SntAs) printf %s $E_WARNING ;; Offln | Failed | Msng | Missing | Bad) printf %s $E_CRITICAL ;; *) printf %s $E_UNKNOWN ;; esac } # MegaCli firmware state -> exit code fw_severity() { case $1 in Online* | JBOD* | Hotspare* | 'Unconfigured(good)'*) printf %s $E_OK ;; Rebuild* | Copyback* | 'Unconfigured(bad)'* | Shield*) printf %s $E_WARNING ;; Failed* | Offline* | Missing*) printf %s $E_CRITICAL ;; *) printf %s $E_UNKNOWN ;; esac } # Common tail for both hardware paths: error counters and hotspare expectation. report_hw_counters() { local _media=$1 _pred=$2 _other=$3 _smart=$4 _hotspares=$5 local _text='' if [ $((_media + _pred + _other + _smart)) -gt 0 ]; then _text="errors: media=$_media predictive=$_pred other=$_other" [ "$_smart" -gt 0 ] && _text="$_text smart_alerts=$_smart" if [ "$_media" -gt "$media_allow" ] || [ "$_pred" -gt "$pred_allow" ] \ || [ "$_other" -gt "$other_allow" ] || [ "$_smart" -gt 0 ]; then escalate $E_WARNING fi fi if [ "$want_hotspares" -gt 0 ]; then [ -n "$_text" ] && _text="$_text, " _text="${_text}hotspares: $_hotspares of $want_hotspares" if [ "$_hotspares" -lt "$want_hotspares" ]; then escalate $E_WARNING fi fi [ -n "$_text" ] && add_summary "$_text" perfdata="$perfdata media_err=$_media pred_err=$_pred other_err=$_other hotspares=$_hotspares" return 0 } # --------------------------------------------------------- storcli / perccli check_storcli() { local _tool=$1 local _label=${1##*/} local _out _ctrl _out=$(run_tool "$_tool" show ctrlcount nolog) if [ -z "$_out" ]; then save_tool_error return $R_NOANSWER fi # storcli changed the ctrlcount wording between releases: 007.16xx prints # "Number of Controllers = N", 007.37xx prints "Controller Count = N". _ctrl=$(printf '%s\n' "$_out" | awk -F= '/Number of Controllers|Controller Count/ { gsub(/[^0-9]/, "", $2); print $2; exit }') is_num "${_ctrl:-}" || return $R_NOANSWER [ "$_ctrl" -gt 0 ] || return $R_ABSENT _out=$(run_tool "$_tool" /call show nolog) if [ -z "$_out" ]; then save_tool_error return $R_NOANSWER fi local _vd_total=0 _vd_bad=0 _pd_total=0 _pd_bad=0 _hotspares=0 _msg='' local _ctl _id _state _sev # Virtual drives, e.g. "0/0 RAID1 Optl RW Yes RWBD - ON 446.625 GB" while read -r _ctl _id _state; do [ -n "$_id" ] || continue _vd_total=$((_vd_total + 1)) _sev=$(vd_severity "$_state") if [ "$_sev" -ne $E_OK ]; then _vd_bad=$((_vd_bad + 1)) escalate "$_sev" _msg="$_msg [c$_ctl]VD$_id=$_state" elif [ "$verbose" -eq 1 ]; then add_detail "[c$_ctl]VD$_id=$_state" fi done </dev/null 2>&1 || return 1 # Same hang guard as run_tool: a wedged PCI enumeration must end in this # check's own UNKNOWN, not in nrpe's command_timeout. All matching cards # are reported, joined with " + ", each stripped of PCI ids and revision. # shellcheck disable=SC2086 $TIMEOUT lspci -nn 2>/dev/null | awk ' /RAID bus controller \[0104\]/ { sub(/^[^:]*: /, "") sub(/ *\[[0-9a-f][0-9a-f][0-9a-f][0-9a-f]:[0-9a-f][0-9a-f][0-9a-f][0-9a-f]\].*/, "") sub(/ *\(rev .*/, "") cards = cards (cards ? " + " : "") $0 } END { if (cards) print cards }' } # ---------------------------------------------------------------------- main check_md hw_rc=$R_ABSENT mute_tool='' hw_tool=$(find_tool storcli64 storcli perccli64 perccli) if [ -n "${hw_tool:-}" ]; then check_storcli "$hw_tool" hw_rc=$? [ "$hw_rc" -eq $R_NOANSWER ] && mute_tool=${hw_tool##*/} fi # storcli may be present but unusable (old firmware, wrong flavour) -- MegaCli # is still worth a try before giving up on the hardware side. if [ "$hw_rc" -ne $R_CHECKED ]; then mega_tool=$(find_tool MegaCli64 MegaCli megacli) if [ -n "${mega_tool:-}" ]; then check_megacli "$mega_tool" case $? in "$R_CHECKED") hw_rc=$R_CHECKED; mute_tool='' ;; # Keep both names when storcli was mute too -- the message must not # hide that the first tool also failed. "$R_NOANSWER") mute_tool="${mute_tool:+$mute_tool+}${mega_tool##*/}" ;; esac fi fi if [ -n "$mute_tool" ]; then # The tool is installed but gave us nothing: missing driver, no permission # or a hung controller. Never silently pass this off as healthy. escalate $E_UNKNOWN add_summary "$mute_tool: no usable answer from the controller${mute_reason:+ [$mute_reason]} (root required? driver loaded?)" fi if [ -z "$summary" ]; then pci_card=$(pci_raid_controller) if [ -n "${pci_card:-}" ]; then escalate $E_UNKNOWN add_summary "RAID controller present (${pci_card}) but no storcli/perccli/MegaCli installed" elif [ "$require_raid" -eq 1 ]; then escalate $E_CRITICAL add_summary 'no RAID controller and no mdadm array found' else add_summary 'no RAID configured on this host' fi fi case $worst_rank in 0) status='OK'; exitcode=$E_OK ;; 1) status='UNKNOWN'; exitcode=$E_UNKNOWN ;; 2) status='WARNING'; exitcode=$E_WARNING ;; *) status='CRITICAL'; exitcode=$E_CRITICAL ;; esac out="$status - $summary" [ "$verbose" -eq 1 ] && [ -n "$detail" ] && out="$out [$detail]" if [ -n "$perfdata" ]; then printf '%s |%s\n' "$out" "$perfdata" else printf '%s\n' "$out" fi exit "$exitcode"