#!/bin/bash
# set -x

function wait_other() {
    local cmd=$1
    for i in {0..30};do
        ps -e|grep -q $cmd
        if [[ $? -ne 0 ]];then
            return 0
        fi
        sleep 1
    done
    error_quit "other '$cmd' still running"
}

function get_cache() {
    local cmd=$@
    local pname=$(basename $(echo "$1"|awk '{print $1}'))
    local cache_name="monitor_"`echo $*|tr ' ' '_'|tr '/' '_'|sed 's/[:,]//g'`
    local cache_file=/dev/shm/$cache_name
    local cache_time='0'
    local code=0

    wait_other $pname

    if [[ -f $cache_file ]];then
        cache_time=`stat -c %Z $cache_file`
    fi
    if [[ $(($(date +%s) - $cache_time)) -gt 300 ]];then
        $cmd > $cache_file
        code=$?
    fi
    cat $cache_file

    return $code
}


function error_quit() {
    echo "$1" > /dev/null 2>&1
    exit 1
}

# detect vm
function detect_vm() {
    `cat /proc/cpuinfo | grep 'flags' | grep -q 'hypervisor'`
    if [ $? -eq 0 ]; then
        error_quit "Not Support VM"
    fi
}

function check_cmd_exist() {
    local cmd=$1
    type $cmd > /dev/null 2>&1 || {
        echo >&2 "require command [$cmd] but it's not installed. Aborting."
        exit 1
    }
}

function check_manufacturer() {
    local CHECK_COMMAND=`dmidecode -t1`
    manufacturer=$(get_value "$(grep "Manufacturer" <<< "$CHECK_COMMAND")" ":")
    echo -n $manufacturer
}

function collect_lsi_raid_info() {
    detect_vm
    check_cmd_exist $CMD_LSPCI
    local result=`$CMD_LSPCI`
    case $result in
        *"LSI"*)
            result=$(grep "LSI" <<< "$result")
            case $result in
                *"2208"* | *"3108"* | *"3316"* | *"3408"* )
                    if [[ $result =~ '3408' ]];then
                        skip_bbu=1
                    fi
                    lsi_raid_collect $@
                    # echo "Can not Support this device: $g_cur_raid_type!" && exit 1;
                    ;;
                * )
                    # echo "Can not Support this device: $result"
                    exit 1;
                    ;;
            esac
            ;;
        * )
            # echo "Can not Support this device: $result" &&
            exit 1;
            ;;
    esac
}

CMD_LSPCI="get_cache lspci"
CMD_MEGACLI="get_cache /opt/MegaRAID/MegaCli/MegaCli64"
CMD_SMARTCTL="get_cache smartctl"
CMD_STORCLI="get_cache /opt/MegaRAID/storcli/storcli64"

function join_string() {
    local arr=("${!2}")
    local result=$(printf "$1%s" "${arr[@]}")
    result=${result:1}
    echo -n $result
}

function get_value() {
    local ori_str="$1"
    local trim_cmd="awk"
    if [ $# -gt 1 ] && [ "$2" != " " ]; then
        trim_cmd="$trim_cmd -F$2"
    fi

    local result=$($trim_cmd '{
        gsub(/^[ \t]+/, "", $2);
        gsub(/[ \t]+$/, "", $2);
        print $2}' <<< "$ori_str")
    echo -n $result
}

function lsi_raid_status() {
    local flag=$1
    local slots=$(get_slots)
    local status=()
    local i=0

    for slot in ${slots[@]}; do
        local info_result=$($CMD_MEGACLI -AdpAllInfo -a$slot)
        local Battery_result=$($CMD_MEGACLI -AdpBbuCmd -GetBbuStatus -a$slot)
        local slotstatus
        case "$flag" in
            batterystatus)
                slotstatus=$(get_value "$(grep "Battery State" <<< "$Battery_result")" ":")
                ;;
            bbu)
                slotstatus=$(get_value "$(grep "^BBU              :" <<< "$info_result")" ":" )
                ;;
            memory)
                slotstatus=$(get_value "$(grep "^Memory           :" <<< "$info_result")" ":")
                ;;
            *)
                exit 1
                ;;
            esac

        if [[ $slotstatus == "Present" || $slotstatus == "Optimal" ]];then
            status_result=0
        else
            if [[ $skip_bbu -eq 1 ]];then
                status_result=0
            else
                status_result=1
            fi
        fi
        status[i]='{"value": '$status_result', "tags":"slot='$slot'"}'
        ((i++))
    done

    local laststatus=$(join_string "," status[@])
    echo -n "[$laststatus]"
}

function lsi_raid_physical() {
    local slots=$(get_slots)
    local pd_result=()
    local raid="LSI"
    local i=0

    for slot in ${slots[@]}; do
        CMD_RESULT=`$CMD_MEGACLI -PDList -a $slot`
        PHYSICAL_DISK_NUM=`$CMD_MEGACLI -pdGetNum -a $slot | grep "Number of Physical Drives on Adapter $slot:" | awk -F":" '{print $2}' | sed 's/ //g'`
        Enclosure_Device_ID=`grep "Enclosure Device ID:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Slot_Number=`grep "Slot Number:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Raw_Size=`grep "Raw Size:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | awk -F"[" '{print $1}' | sed 's/ //g' | xargs | sed 's/ /|/g'`
        Firmware_state=`grep "Firmware state:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | awk -F"," '{print $1}' | xargs | sed 's/ /|/g'`

        for((j=1; j<=$PHYSICAL_DISK_NUM; j++))
        do
            local jEnclosure_Device_ID=`echo $Enclosure_Device_ID | awk -F"|" '{print $'$j'}'`
            local jSlot_Number=`echo $Slot_Number | awk -F"|" '{print $'$j'}'`
            local jRaw_Size=`echo $Raw_Size | awk -F"|" '{print $'$j'}'`
            local jFirmware_state=`echo $Firmware_state | awk -F"|" '{print $'$j'}'`
            if [[ -z $jFirmware_state ]];then
                continue
            fi
            if [[ $jFirmware_state == "Online" ]];then
                local jValue=0
            else
                local jValue=1
            fi
            pd_result[i]='{"value":'$jValue',"tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number',Raw_Size='$jRaw_Size'"}'
            ((i++))
        done
    done
    local pd_col=$(join_string "," pd_result[@])

    echo -n "[$pd_col]"
}

function lsi_raid_logical() {
    local slots=$(get_slots)
    local ld_result=()
    local raid="LSI"
    local i=0

    for slot in ${slots[@]}; do
        VIRTUAL_DISK_NUM=`$CMD_MEGACLI -LDGetNum -a $slot | grep "Number of Virtual Drives Configured on Adapter $slot:" | awk -F":" '{print $2}' |sed 's/ //g'`
        for((j=0; j<$VIRTUAL_DISK_NUM; j++))
        do
            Virtual_Drive=`$CMD_MEGACLI -LDInfo -L $j -a $slot | grep "Virtual Drive:" | awk -F":" '{print $2}' | awk -F"(" '{print $1}' | sed 's/ //g'`
            Size=`$CMD_MEGACLI -LDInfo -L $j -a $slot | grep "Size                :" | awk -F":" '{print $2}' | sed 's/ //g'`
            State=`$CMD_MEGACLI -LDInfo -L $j -a $slot | grep "State               :" | awk -F":" '{print $2}' | sed 's/ //g'`
            if [ $State == "Optimal" ];then
                Value=0
            else
                Value=1
            fi
            ld_result[i]='{"value": '$Value', "tags":"raid_type='$raid',slot='$slot',Virtual_Drive='$Virtual_Drive',Size='$Size'"}'
            ((i++))
        done
    done
    local ld_col=$(join_string "," ld_result[@])

    echo -n "[$ld_col]"
}

function lsi_check_rebuild() {
    local slots=$(get_slots)
    local cr_result=()
    local result
    local i=0
    local raid="LSI"

    for slot in ${slots[@]}; do
        CMD_RESULT=`$CMD_MEGACLI -PDList -a $slot`
        PHYSICAL_DISK_NUM=`$CMD_MEGACLI -pdGetNum -a $slot | grep "Number of Physical Drives on Adapter $slot:" | awk -F":" '{print $2}' | sed 's/ //g'`
        Enclosure_Device_ID=`grep "Enclosure Device ID:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Slot_Number=`grep "Slot Number:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Firmware_state=`grep "Firmware state:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | awk -F"," '{print $1}' | xargs | sed 's/ /|/g'`

        for ((j=1; j<=$PHYSICAL_DISK_NUM; j++)); do
            CHECK_REBUILD=`echo $Firmware_state | awk -F"|" '{print $'$j'}' | grep "Rebuild"`
            result=$(check_rebuild_log $i $CHECK_REBUILD)
            jEnclosure_Device_ID=`echo $Enclosure_Device_ID | awk -F"|" '{print $'$j'}'`
            jSlot_Number=`echo $Slot_Number | awk -F"|" '{print $'$j'}'`
            cr_result[i]='{"value": '$result', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number'"}'
            ((i++))
        done
    done

    local cr_col=$(join_string "," cr_result[@])
    echo -n "[$cr_col]"
}

function check_rebuild_log() {
    local disk=$1
    local command_result=$2
    local log_file="/tmp/rebuild_$disk"
    if [ ! -e $log_file ];then
        touch $log_file
    fi

    if [ ! -s $command_result ];then
        unixtime_now=`date +%s`
        if [ ! -s $log_file ];then
            echo $unixtime_now > $log_file
            echo "0"
        else
            old_time=`cat < $log_file`
            duration=`expr $unixtime_now - $old_time`
            if [ $duration -ge 54000 ]; then
                echo "1"
                sed -i '1d' $log_file
            else
                echo "0"
            fi
        fi
    else
        if [ -s $log_file ];then
            sed -i '1d' $log_file
        fi
        echo "0"
    fi
}

function lsi_raid_log() {
    local log_result=()
    local i=0

    if [ -f /tmp/raid_events.log ];then
        echo '' > /tmp/raid_events.log
    fi

    if [ -f "/opt/MegaRAID/storcli/storcli64" ];then
        $CMD_STORCLI /c0 show events type=sincereboot > /tmp/raid_events.log
    else
        /opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetSinceReboot -f /tmp/raid_events.log -aALL > /dev/null
    fi
    sleep 1
    if grep -q "Status = Failure" /tmp/raid_events.log;then
        /opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog -GetSinceReboot -f /tmp/raid_events.log -aALL > /dev/null
    fi

    for err_disk in $(grep -e "Unexpected sense: PD" -e "Command timeout" /tmp/raid_events.log | sed -n 's/.*PD .*(\(.*\)).*/\1/p' | sort -n -u); do
        local latest_err_seq
        local check_online_seq

        raw_Enclosure_Device_ID=$(echo "$err_disk" | awk -F/ '{print $1}' | awk -F 'e0x' '{print $2}')
        Enclosure_Device_ID=$((16#$raw_Enclosure_Device_ID))
        slot=$(echo "$err_disk" | awk -F/ '{print $2}' | awk -Fs '{print $2}')

        latest_err_seq=$(grep -B 6 -e "Unexpected sense: PD.*$err_disk" -e "Command timeout.*$err_disk"  /tmp/raid_events.log | grep "seqNum" | tail -n 1)
        check_online_seq=$(grep -B 6 "$err_disk.*to ONLINE" /tmp/raid_events.log | grep "seqNum" | tail -n 1)
        if [[ $(get_value "$latest_err_seq" ":") -gt $(get_value "$check_online_seq" ":") ]];then
            log_result[i]='{"value":1,"tags":"Enclosure_Device_ID='"$Enclosure_Device_ID"',Slot_Number='"$slot"'"}'
        else
            log_result[i]='{"value":0,"tags":"Enclosure_Device_ID='"$Enclosure_Device_ID"',Slot_Number='"$slot"'"}'
        fi
        ((i++))
    done

    workdir="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
    echo '' > "$workdir"/MegaSAS.log
    echo '' > "$workdir"/CmdTool.log

    local log_col
    log_col=$(join_string "," log_result[@])

    if [ -z "$log_col" ];then
        echo -n '[{"value": 0, "tags":"raid_type='LSI'"}]'
    else
        echo -n "[$log_col]"
    fi
}

function lsi_check_ldpdinfo() {
    local result
    local flag=$1
    local slots
    slots=$(get_slots)
    local cd_result=()
    local raid="LSI"
    local i=0

    for slot in ${slots[@]}; do
        CMD_RESULT=`$CMD_MEGACLI -PDList -a $slot`
        PHYSICAL_DISK_NUM=`$CMD_MEGACLI -pdGetNum -a $slot | grep "Number of Physical Drives on Adapter $slot:" | awk -F":" '{print $2}' | sed 's/ //g'`
        Enclosure_Device_ID=`grep "Enclosure Device ID:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Slot_Number=`grep "Slot Number:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Predictive=`grep "Predictive Failure Count:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Media=`grep "Media Error Count:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`

        for((j=1; j<=$PHYSICAL_DISK_NUM; j++))
        do
            case "$flag" in
                media)
                    jEnclosure_Device_ID=`echo $Enclosure_Device_ID | awk -F"|" '{print $'$j'}'`
                    jSlot_Number=`echo $Slot_Number | awk -F"|" '{print $'$j'}'`
                    jMedia=`echo $Media | awk -F"|" '{print $'$j'}'`
                    cd_result[i]='{"value": '$jMedia', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number'"}'
                    ((i++))
                    ;;
                predictive)
                    jEnclosure_Device_ID=`echo $Enclosure_Device_ID | awk -F"|" '{print $'$j'}'`
                    jSlot_Number=`echo $Slot_Number | awk -F"|" '{print $'$j'}'`
                    jPredictive=`echo $Predictive | awk -F"|" '{print $'$j'}'`
                    cd_result[i]='{"value": '$jPredictive', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number'"}'
                    ((i++))
                    ;;
                *)
                    exit 1
                    ;;
                esac
        done
    done
    local pd_col=$(join_string "," cd_result[@])

    echo -n "[$pd_col]"
}

function lsi_sata_status() {
    check_cmd_exist $CMD_SMARTCTL

    local result
    local sata_result=()
    local flag=$1
    local slots=$(get_slots)
    local raid="LSI"
    local i=0

    for slot in ${slots[@]}; do
        CMD_RESULT=`$CMD_MEGACLI -PDList -a $slot`
        PHYSICAL_DISK_NUM=`$CMD_MEGACLI -pdGetNum -a $slot | grep "Number of Physical Drives on Adapter $slot:" | awk -F":" '{print $2}' | sed 's/ //g'`
        Enclosure_Device_ID=`grep "Enclosure Device ID:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Slot_Number=`grep "Slot Number:" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Device_ID=`grep "Device Id" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        PD_Type=`grep "PD Type" <<< "$CMD_RESULT" | awk -F":" '{print $2}' | xargs | sed 's/ /|/g'`
        Media_Type=`grep "Media Type" <<< "$CMD_RESULT" | awk -F":" '{print $2}'`

        for((j=1; j<=$PHYSICAL_DISK_NUM; j++))
        do
            local ID=`echo $Device_ID | awk -F"|" '{print $'$j'}'`
            jEnclosure_Device_ID=`echo $Enclosure_Device_ID | awk -F"|" '{print $'$j'}'`
            jSlot_Number=`echo $Slot_Number | awk -F"|" '{print $'$j'}'`
            jPD_Type=`echo $PD_Type | awk -F"|" '{print $'$j'}'`
            jMedia_Type=`echo "$Media_Type" | awk 'NR=='"$j"' {print; exit}'|xargs`
            if [[ $jPD_Type == 'SATA' ]];then
                smart_type='sat+megaraid'
            elif [[ $jPD_Type == 'SAS' ]];then
                smart_type='megaraid'
            fi
            case "$flag" in
                udma)
                    if [[ $jPD_Type == 'SAS' || $jMedia_Type == 'Solid State Device' ]];then
                        continue
                    fi
                    result=`$CMD_SMARTCTL -a -d $smart_type,$ID /dev/sda | grep "UDMA_CRC_Error_Count" | awk '{print $NF}'`
                    sata_result[i]='{"value": '$result', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number',PD_type='$jPD_Type'"}'
                    ((i++))
                    ;;
                pending)
                    if [[ $jPD_Type == 'SAS' ]];then
                        continue
                    fi
                    result=`$CMD_SMARTCTL -a -d $smart_type,$ID /dev/sda | grep "Current_Pending_Sector" | awk '{print $NF}'`
                    sata_result[i]='{"value": '$result', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number',PD_type='$jPD_Type'"}'
                    ((i++))
                    ;;
                reallocated)
                    if [[ $jPD_Type == 'SAS' ]];then
                        continue
                    fi
                    smart_result=`$CMD_SMARTCTL -a -d $smart_type,$ID /dev/sda`
                    result=`echo "$smart_result" | grep "Reallocated_Sector_Ct" | awk '{print $NF}'`
                    sata_result[i]='{"value": '$result', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number',PD_type='$jPD_Type'"}'
                    ((i++))
                    ;;
                udma_thresh)
                    if [[ $jPD_Type == 'SAS' ]];then
                        continue
                    fi
                    smart_result=`$CMD_SMARTCTL -a -d $smart_type,$ID /dev/sda`
                    result_VALUE=`echo "$smart_result" | grep "UDMA_CRC_Error_Count" | awk '{print $4}'`
                    result_THRESH=`echo "$smart_result" | grep "UDMA_CRC_Error_Count" | awk '{print $6}'`
                    VALUE_minus_THRESH=`expr $result_VALUE - $result_THRESH`
                    sata_result[i]='{"value": '$VALUE_minus_THRESH', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number'"}'
                    ((i++))
                    ;;
                reallocated_thresh)
                    if [[ $jPD_Type == 'SAS' ]];then
                        continue
                    fi
                    smart_result=`$CMD_SMARTCTL -a -d $smart_type,$ID /dev/sda`
                    result_VALUE=`echo "$smart_result" | grep "Reallocated_Sector_Ct" | awk '{print $4}'`
                    result_THRESH=`echo "$smart_result" | grep "Reallocated_Sector_Ct" | awk '{print $6}'`
                    VALUE_minus_THRESH=`expr $result_VALUE - $result_THRESH`
                    sata_result[i]='{"value": '$VALUE_minus_THRESH', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number'"}'
                    ((i++))
                    ;;
                smart_health)
                    smart_result=`$CMD_SMARTCTL -a -d $smart_type,$ID /dev/sda`
                    result_VALUE=`echo "$smart_result" | grep "SMART overall-health" | awk '{print $NF}'`
                    if [ -z $result_VALUE ];then
                        result_VALUE=`echo "$smart_result" | grep "SMART Health Status" | awk '{print $NF}'`
                    fi
                    if [[ $result_VALUE == "OK" && $jPD_Type == 'SAS' ]];then
                        local resultValue=0
                    elif [[ $result_VALUE == "PASSED" && $jPD_Type == 'SATA' ]];then
                        local resultValue=0
                    else
                        local resultValue=1
                    fi
                    sata_result[i]='{"value": '$resultValue', "tags":"raid_type='$raid',slot='$slot',Enclosure_Device_ID='$jEnclosure_Device_ID',Slot_Number='$jSlot_Number'"}'
                    ((i++))
                    ;;
                *)
                    exit 1
                    ;;
                esac
        done
    done
    local ss_col=$(join_string "," sata_result[@])

    echo -n "[$ss_col]"
}

function get_slots() {
    local result=$($CMD_MEGACLI -AdpAllInfo -aAll)
    local slots=$(get_value "$(grep "^Adapter" <<< "$result")" "#")
    echo -n $slots
}

# status: -s [batterystatus | bbu | memory]
function lsi_raid_collect() {
    check_cmd_exist $CMD_MEGACLI
    while getopts ps:d:t:lrhg o; do
        case "$o" in
            s)
                local opt=$OPTARG
                lsi_raid_status $opt
                ;;
            p)
                lsi_raid_physical
                ;;
            l)
                lsi_raid_logical
                ;;
            r)
                lsi_check_rebuild
                ;;
            d)
                local opt=$OPTARG
                lsi_check_ldpdinfo $opt
                ;;
            t)
                local opt=$OPTARG
                lsi_sata_status $opt
                ;;
            g)
                lsi_raid_log
                ;;
            *)
                exit 1
                ;;
            esac
        done
}

collect_lsi_raid_info "$@"
