#!/bin/bash

# support:
# 1. 25G RDMA for uhost: net2+bond0, like uhost
# 1. 25G RDMA for storage: bond0, like udisk
# 1. 25G RDMA for bluefield: bond0+p0m0
# 2. 100G RDMA for uhost: bond0+udisk0
# 2. 100G RDMA for udisk: bond0
# 3. normal DoubleUplink: bond0

source /ucloud/exports.ini
# VMFLAG: 1 true
# BFFLAG: 1 true
# BondIP
# HOSTIP

check_net_type() {
    if [[ $VMFLAG = 1 ]];then
        :
    elif [[ $BFFLAG = 1 ]];then
        # BF 单上联不需要处理,也就是没有BondIP
        if [[ -n $BondIP ]];then
            extend_ifname=p0m0
            UCLOUD_BOND_NIC=net2,net3
        fi
    elif [[ -n $BondIP ]];then
        # 临时根据网卡型号和硬盘配置来判断，后面要改成通过网络区域+配置方案来判断
        NETPF="$(lspci -Dnnmb -d 0x15b3::0200 | grep "Ethernet" | grep -v "Virtual")"
        if [ -z "$NETPF" ]; then
            # 组 bond 必须要有 mlx 网卡
            PRINT_ERR "[ERROR] Bond Net -> $BondIP <- Need Mellanox NIC"
            return 1
        fi

        MLX_25G="$(echo "$NETPF" | grep -P "\b(MT27800|MT27710|MT2894)\b" | head -n1)"
        MLX_100G="$(echo "$NETPF" | grep -P "\b(MT28800|MT2892)\b" | head -n1)"
        MLX_200G="$(echo "$NETPF" | grep -P "\b(MT28800|MT2892)\b" | head -n1)"

        if [ -z "$MLX_25G$MLX_100G" ]; then
            PRINT_ERR "$NETPF"
            PRINT_ERR "[ERROR] Unknown Mellanox Model"
            return 2
        fi

        if [[ -n $MLX_100G ]];then
            RDMA_SPEED='100G'
            UCLOUD_BOND_NIC=net2,net3
            if [[ ! $(uname -r) =~ '5.10.0' ]];then
                PRINT_ERR "[ERROR] 100G RDMA only support kernel-5.10.0"
                return 1
            fi
            # check eth speed
            if [[ ! $(cat /sys/class/net/net*/speed) =~ 100000 ]];then
                PRINT_ERR "[ERROR] netcard speed not 100G - 可能bond所需网卡名有误eth/net"
                return 1
            fi
            # 除系统盘外没有其他盘则需要创建业务网卡 - udisk0
            real_disk=$(lsblk -d|grep -E "^sd|df|nvme"|grep -v "\<sda\>"|grep -Ec 'G|T')
            if [[ $RDMA_SPEED = '100G' && $real_disk -eq 0 ]];then
                extend_ifname=udisk0
            fi
        elif [[ -n $MLX_25G ]];then
            # 存在两种情况 1. 系统IP组bond，管理口同时也是业务，则2+3组bond；
            # 2. 存在系统IP外的额外IP组bond，则0+1组bond，2做管理口
            RDMA_SPEED='25G'
            if [[ $HOSTIP != $BondIP ]];then
                extend_ifname=bond0
                UCLOUD_BOND_NIC=net0,net1
            else
                UCLOUD_BOND_NIC=net2,net3
            fi
        fi

    fi
}

_install() {
    check_net_type || return 1

    if [ -z $BondIP ];then
        if [[ -n $UCLOUD_BOND_IP || -n $UCLOUD_BOND_NIC || -n $UCLOUD_BOND_RANGE ]];then
            PRINT_ERR "[ERROR] BondIP Not Exist In CMDB, But Exist /ucloud/exports.ini"
            return 1
        else
            # No Need Config Bond
            return 0
        fi
    fi

    if [[ $BFFLAG = 1 ]];then
        # check bluefield bond ip
        if [[ $HOSTIP != $UCLOUD_BOND_IP ]];then
            PRINT_ERR "[ERROR] BlueField HostIP Not Equal BondIP"
            return 1
        fi
    fi

    # check BondIP alive
    if [[ $HOSTIP != $BondIP ]];then
        ping -c1 -w5 $BondIP 1>/dev/null 2>&1
        if [[ $? -eq 0 ]] && [[ $(ifconfig bond0|grep -c $BondIP) -eq 0 ]];then
            PRINT_ERR "[ERROR] BondIP ping alive, maybe not right."
            return 1
        fi
    fi

    # Step1: Prepare configfile
    if [[ $CHANNEL == "reinstall" || $CHANNEL == "upan" ]] || [[ -z $UCLOUD_BOND_IP ]] || [[ -z $UCLOUD_BOND_NIC ]] || [[ -z $UCLOUD_BOND_RANGE ]];then
        if [[ $BFFLAG = 0 ]];then
            sed -i "/UCLOUD_BOND_IP/d" /ucloud/exports.ini
            sed -i "/UCLOUD_BOND_RANGE/d" /ucloud/exports.ini
            sed -i "/UCLOUD_BOND_NIC/d" /ucloud/exports.ini
            UCLOUD_BOND_IP=$BondIP
            echo "UCLOUD_BOND_IP=$BondIP" >> /ucloud/exports.ini
            local bond_ipseg_info=$(curl -s -X GET -H 'Accept: application/json' -H 'api-key: f0ca152370904201a80de50ecae966fa' "http://api-gw.ucloudadmin.com/ipdb/intranet/ip?kind=0&address=%5B%22${UCLOUD_BOND_IP}%22%5D")
            local bond_ipseg=`echo $bond_ipseg_info | jq -r '.data[0].ipseg'`
            local bond_ipseg_range=`echo $bond_ipseg|awk -F '/' '{print $NF}'`
            if [ -z $bond_ipseg_range ];then
                PRINT_ERR "[ERROR] bond_range not define"
                return 1
            else
                UCLOUD_BOND_RANGE=$bond_ipseg_range
            fi
            echo "UCLOUD_BOND_RANGE=$UCLOUD_BOND_RANGE" >> /ucloud/exports.ini
        fi
    fi

    # check exist: UCLOUD_BOND_IP UCLOUD_BOND_NIC UCLOUD_BOND_RANGE
    if [[ -z $UCLOUD_BOND_IP ]] || [[ -z $UCLOUD_BOND_NIC ]] || [[ -z $UCLOUD_BOND_RANGE ]];then
        PRINT_ERR "[ERROR] UCLOUD_BOND_IP, UCLOUD_BOND_NIC or UCLOUD_BOND_RANGE Not Define"
        return 1
    fi

    bond_ipseg=`ipcalc -n $UCLOUD_BOND_IP/$UCLOUD_BOND_RANGE|awk -F '=' '{print $2}'`
    if [[ -z $bond_ipseg ]];then
        PRINT_ERR "[ERROR] Validate IP address $UCLOUD_BOND_IP/$UCLOUD_BOND_RANGE failed"
        return 1
    fi
    
    # check BondIP validity
    if [[ $BondIP != $UCLOUD_BOND_IP ]];then
        PRINT_ERR "[ERROR] BondIP In CMDB and BondIP In exports.ini Not Equal"
        return 1
    fi    

    # request extend ip ,3线方案bond0 IP是预先分配好的，不用在这申请
    if [[ -n $extend_ifname && $extend_ifname != 'bond0' && -z $UCLOUD_EXTEND_IP ]];then
        local ip_by_sn extend_ip_info extend_ip
        ip_by_sn=$(curl -s -X GET -H 'Accept: application/json' -H 'api-key: f0ca152370904201a80de50ecae966fa' "http://api-gw.ucloudadmin.com/ipdb/intranet/ip?bind_resource=${SN}")
        if [[ `echo $ip_by_sn | jq -r '.data[]|.address'| wc -l` -gt 2 ]];then
            PRINT_ERR "[ERROR] SN 绑定的IP过多，请检查ipdb"
            return 1
        fi
        # 打乱请求时间，防止IPDB重复IP
        sleep $(($RANDOM%30))

        extend_ip_info=$(curl -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'api-key: f0ca152370904201a80de50ecae966fa' -d '{"kind": 0,"ipseg": "'${bond_ipseg}/${UCLOUD_BOND_RANGE}'","ip_type": 2,"count":1,"apply_type": 0,"usage": "ExtendIP","bind_resource": "'${SN}'","remark": "apply_by_initos"}' 'http://api-gw.ucloudadmin.com/ipdb/intranet/ip')
        extend_ip=`echo $extend_ip_info | jq -r '.data[0].address'`
        if [[ -z $extend_ip || $extend_ip == 'null' ]];then
            PRINT_ERR "[ERROR] can not get extend_ip for '${bond_ipseg}/${UCLOUD_BOND_RANGE}'"
            return 1
        else
            # check extend ip alive
            ping -c1 -w5 $extend_ip 1>/dev/null 2>&1
            if [[ $? -eq 0 ]];then
                PRINT_ERR "[ERROR] ExtendIP $extend_ip ping alive, please check ipdb."
                return 1
            fi

            sed -i "/UCLOUD_EXTEND_IP/d" /ucloud/exports.ini
            UCLOUD_EXTEND_IP=$extend_ip
            echo "UCLOUD_EXTEND_IP=$extend_ip" >> /ucloud/exports.ini
            # save to cmdb
            #PRINT_ERR "[ERROR] extend_ip $extend_ip save to cmdb failed"
            #return 1
        fi
    fi
    if [[ -n $extend_ifname && $extend_ifname == 'bond0' && -z $UCLOUD_EXTEND_IP ]];then
        UCLOUD_EXTEND_IP=$BondIP
    fi

    # Step2: Config NIC
    if [[ $OS == "CentOS6" ]];then
        local UCLOUD_BOND_NIC=`echo $UCLOUD_BOND_NIC|sed "s/net/eth/g"`
    fi
    local bond_nic1=`echo $UCLOUD_BOND_NIC | cut -f1 -d','`
    local bond_nic2=`echo $UCLOUD_BOND_NIC | cut -f2 -d','`
cat > /etc/sysconfig/network-scripts/ifcfg-$bond_nic1 << EOF
DEVICE="$bond_nic1"
BOOTPROTO="static"
NM_CONTROLLED="no"
ONBOOT="yes"
TYPE="Ethernet"
SLAVE="yes"
MASTER="bond0"
EOF
cat > /etc/sysconfig/network-scripts/ifcfg-$bond_nic2 << EOF
DEVICE="$bond_nic2"
BOOTPROTO="static"
NM_CONTROLLED="no"
ONBOOT="yes"
TYPE="Ethernet"
SLAVE="yes"
MASTER="bond0"
EOF

    # 管理口做bond
    if [[ $HOSTIP == $UCLOUD_BOND_IP ]];then
        local result=`get_gateway_and_mask $HOSTIP $HOSTIP_RANGE`
        local UCLOUD_BOND_GATEWAY=`echo $result|cut -d',' -f1`
        local UCLOUD_BOND_NETMASK=`echo $result|cut -d',' -f2`
        cat > /etc/sysconfig/network-scripts/ifcfg-bond0 << EOF
DEVICE=bond0
ONBOOT=yes
BONDING_OPTS='mode=4 miimon=100 xmit_hash_policy=layer3+4 lacp_rate=fast'
NM_CONTROLLED=no
TYPE=Ethernet
BOOTPROTO=static
IPV6INIT=no
IPADDR=$UCLOUD_BOND_IP
NETMASK=$UCLOUD_BOND_NETMASK
GATEWAY=$UCLOUD_BOND_GATEWAY
EOF
        # BondIP和原net2 IP相同，这里不处理干净的话,不重启的情况下直接起network服务会断网
        ifdown $bond_nic1
        ifdown $bond_nic2
        ifup bond0
    fi

    # extend ip
    if [[ -n $UCLOUD_EXTEND_IP ]];then
        if [[ $BondIP == $UCLOUD_EXTEND_IP ]];then
            local result=`get_gateway_and_mask $UCLOUD_BOND_IP $UCLOUD_BOND_RANGE`
            local UCLOUD_BOND_GATEWAY=`echo $result|cut -d',' -f1`
            local UCLOUD_BOND_NETMASK=`echo $result|cut -d',' -f2`
cat > /etc/sysconfig/network-scripts/ifcfg-bond0 << EOF
DEVICE=bond0
ONBOOT=yes
BONDING_OPTS='mode=4 miimon=100 xmit_hash_policy=layer3+4 lacp_rate=fast'
NM_CONTROLLED=no
TYPE=Ethernet
BOOTPROTO=static
IPV6INIT=no
IPADDR=$UCLOUD_BOND_IP
NETMASK=$UCLOUD_BOND_NETMASK
EOF
        else
cat > /etc/sysconfig/network-scripts/ifcfg-$extend_ifname << EOF
DEVICE=$extend_ifname
NM_CONTROLLED=no
ONBOOT=yes
BOOTPROTO=static
IPADDR=$UCLOUD_EXTEND_IP
NETMASK=$UCLOUD_BOND_NETMASK
TYPE=Ethernet
EOF
        fi
    fi

    # Step3: Config route
    if [[ -n $extend_ifname ]];then
        if [[ -z $UCLOUD_EXTEND_IP ]];then
            PRINT_ERR "[ERROR] Can not get ExtendIP for set route"
            return 1
        fi
        # Policy Route
        sed -i "/t_${extend_ifname}/d" /etc/iproute2/rt_tables
        echo "251     t_${extend_ifname}" >> /etc/iproute2/rt_tables
        cat > /etc/sysconfig/network-scripts/rule-$extend_ifname << EOF
iif $extend_ifname table t_${extend_ifname}
oif $extend_ifname table t_${extend_ifname}
from $UCLOUD_EXTEND_IP table t_${extend_ifname}
EOF
        cat > /etc/sysconfig/network-scripts/route-$extend_ifname << EOF
default via $UCLOUD_BOND_GATEWAY dev ${extend_ifname} table t_${extend_ifname}
${bond_ipseg}/${UCLOUD_BOND_RANGE} dev ${extend_ifname} table t_${extend_ifname}
EOF
        ## 持久化抑制直连路由,BF卡有冲突，不修改
        if [[ $BFFLAG = 0 ]];then
            cat > /sbin/ifup-local << 'EOF'
#!/bin/bash
if [[ $1 == 'extend_ifname' ]];then
    route_list=$(ip r show | grep -P 'dev extend_ifname ')
    TMP_IFS=$IFS
    IFS=$'\n'
    for line in $route_list;
    do
        eval "ip r delete $line"
    done
    IFS=$TMP_IFS
fi
EOF
            sed -i "s/extend_ifname/$extend_ifname/g" /sbin/ifup-local
            chmod +x /sbin/ifup-local
        fi
    fi

    # Step4: Config OFED and others
    if [[ $BFFLAG = 1 ]];then
        # 003-bfinit.sh
        cat > /etc/rc.local.d/003-bfinit.sh <<'EOF'
#!/bin/bash

echo 1 > /sys/module/bonding/parameters/arp_broadcast_mode
echo 1 > /sys/class/net/bond0/bonding/xmit_hash_policy
echo 300 > /proc/sys/net/ovs/ovs_upcall_limit_rate
EOF
        chmod +x /etc/rc.local.d/003-bfinit.sh
    fi

    if [[ $BFFLAG = 1 ]];then
        return 0
    fi
    if [[ $VMFLAG = 1 ]];then
        return 0
    fi
    # 此处到结束主要给非bf卡使用
    if [[ $(uname -r) =~ '4.14.0' ]];then
        # OFED
        if [[ $(rpm -qa |grep -ic ofed) -ne 31 ]];then
            yum install -y redhat-rpm-config rpm-build createrepo tcl gcc-gfortran tcsh tk gtk2 atk cairo > /dev/null 2>&1
            wget -P $BASEDIR -c http://softlib.service.ucloud.cn/usa/hotpatch/$(uname -r)/ofed.tgz > /dev/null 2>&1
            tar xf $BASEDIR/ofed.tgz -C $BASEDIR/
            # remove kernel-debuginfo (临时解决4.14-27版系统镜像内软件包冲突问题)
            rpm -e mlnx-ofa_kernel-debuginfo-4.5-OFED.4.5.1.0.1.0.19.1.g843cdb6.x86_64
            cd $BASEDIR/ofed; echo y |./mlnxofedinstall --without-fw-update
            cd $BASEDIR/modules
        fi
        # 上一步会覆盖掉mlnx-ofa_kernel-modules包，需要重新安装
        kernel_release=$(uname -r|sed -n "s/4.14.0-\([0-9]*\).el7.ucloud.x86_64/\1/p")
        yum -y install mlnx-ofa_kernel-modules-4.5-OFED.4.5.1.0.1.0.19.1.g843cdb6.kver.4.14.0_${kernel_release}.el7.ucloud.x86_64 1>/dev/null
        if [[ $? -ne 0 ]];then
            PRINT_ERR "[ERROR] mlnx-ofa_kernel-modules 重新安装失败"
            return 1
        fi
    elif [[ $(uname -r) =~ '5.10.0' ]];then
        yum -y install rdma-core mlnx-tools mft libibverbs librdmacm libibumad > /dev/null 2>&1
    fi

    # 003-rdma.sh and vf
    if [[ $RDMA_SPEED = '25G' ]];then
        install $BASEDIR/configs/rclocal/025G_003-rdma.sh /etc/rc.local.d/003-rdma.sh
    elif [[ $RDMA_SPEED = '100G' ]];then
        if [[ -z $extend_ifname ]];then
            install $BASEDIR/configs/rclocal/100G_003-rdma.sh /etc/rc.local.d/003-rdma.sh
        else
            # 下面是vf的创建过程
            # add udev rules
            mkdir -p /opt/ucloud
            install $BASEDIR/configs/udev_rules/99-ucloud-net-name.rules /etc/udev/rules.d/99-ucloud-net-name.rules
            install $BASEDIR/configs/udev_rules/net-name.sh /opt/ucloud/net-name.sh
            udevadm control --reload-rules

            # init mlx
            systemctl start mst
            # 主要影响rdma 创建8个vf
            mlxconfig -d /dev/mst/mt*_pciconf0 -y set NUM_OF_VFS=1 NUM_VF_MSIX=8 SRIOV_EN=1

            install $BASEDIR/optools/startup_mlx.sh /opt/op/startup_mlx.sh
            # vf通过/opt/op/startup_mlx.sh启动，在network启动时是起不来的,需要关闭
            sed -i 's/ONBOOT=yes/ONBOOT=no/' /etc/sysconfig/network-scripts/ifcfg-$extend_ifname
            install $BASEDIR/configs/rclocal/100G-vf_003-rdma.sh /etc/rc.local.d/003-rdma.sh
        fi
    fi
    chmod +x /etc/rc.local.d/003-rdma.sh

}

_check() {
    _check="PASS"

    check_net_type || _check="FAIL"

    if [ -z $BondIP ];then
        if [[ -n $UCLOUD_BOND_IP || -n $UCLOUD_BOND_NIC || -n $UCLOUD_BOND_RANGE || -n $extend_ifname ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] BondIP Not Exist In CMDB, But Exist /ucloud/exports.ini"
            return 1
        else
            return 0
        fi
    fi

    # check exist: UCLOUD_BOND_IP UCLOUD_BOND_NIC UCLOUD_BOND_RANGE
    if [[ -z $UCLOUD_BOND_IP ]] || [[ -z $UCLOUD_BOND_NIC ]] || [[ -z $UCLOUD_BOND_RANGE ]];then
        _check="FAIL"
        PRINT_ERR "[ERROR] UCLOUD_BOND_IP, UCLOUD_BOND_NIC or UCLOUD_BOND_RANGE Not Define"
    fi
    if [[ $OS == 'CentOS6' ]];then
        local UCLOUD_BOND_NIC=`echo $UCLOUD_BOND_NIC|sed "s/net/eth/g"`
    fi

    # check BondIP validity
    if [[ $BondIP != $UCLOUD_BOND_IP ]];then
        _check="FAIL"
        PRINT_ERR "[ERROR] BondIP In CMDB and BondIP in exports.ini Not Equal"
    fi        

    # check bond gateway & netmask
    if [[ $HOSTIP == $UCLOUD_BOND_IP ]];then
        result=`get_gateway_and_mask $HOSTIP $HOSTIP_RANGE`
        UCLOUD_BOND_GATEWAY=`echo $result|cut -d',' -f1`
        UCLOUD_BOND_NETMASK=`echo $result|cut -d',' -f2`
    else
        result=`get_gateway_and_mask $UCLOUD_BOND_IP $UCLOUD_BOND_RANGE`
        UCLOUD_BOND_GATEWAY=`echo $result|cut -d',' -f1`
        UCLOUD_BOND_NETMASK=`echo $result|cut -d',' -f2`
    fi


    # check bond0 status
    ip a |grep bond0|grep -q UP
    if [[ $? -ne 0 ]];then
        _check="FAIL"
        PRINT_ERR "[ERROR] bond0 not UP"
    fi

    bond_ipseg=`ipcalc -n $UCLOUD_BOND_IP/$UCLOUD_BOND_RANGE|awk -F '=' '{print $2}'`
    if [[ -z $bond_ipseg ]];then
        _check="FAIL"
        PRINT_ERR "[ERROR] Validate IP address $UCLOUD_BOND_IP/$UCLOUD_BOND_RANGE failed"
    fi

    # check route
    if [[ -n $extend_ifname ]];then
        if [[ ! -f /etc/sysconfig/network-scripts/route-$extend_ifname ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] can not found /etc/sysconfig/network-scripts/route-$extend_ifname"
        fi
        grep -q $UCLOUD_BOND_GATEWAY /etc/sysconfig/network-scripts/route-$extend_ifname
        if [[ $? -ne 0 ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] $UCLOUD_BOND_GATEWAY not in /etc/sysconfig/network-scripts/route-$extend_ifname"
        fi
        grep -q "${bond_ipseg}/${UCLOUD_BOND_RANGE} dev ${extend_ifname}" /etc/sysconfig/network-scripts/route-$extend_ifname
        if [[ $? -ne 0 ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] ${bond_ipseg}/${UCLOUD_BOND_RANGE} dev ${extend_ifname} not in /etc/sysconfig/network-scripts/route-$extend_ifname"
        fi

        if [[ ! -f /etc/sysconfig/network-scripts/rule-$extend_ifname ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] can not found /etc/sysconfig/network-scripts/rule-$extend_ifname"
        fi
        grep -q $extend_ifname /etc/sysconfig/network-scripts/rule-$extend_ifname
        if [[ $? -ne 0 ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] $extend_ifname not in /etc/sysconfig/network-scripts/rule-$extend_ifname"
        fi

        # BF卡不需要检查这个脚本，BF1有，BF2没有，但是都不是初始化配置上去的，没有关系
        if [[ $BFFLAG = 0 ]];then
            if [[ ! -x /sbin/ifup-local ]] || [[ ! -e /sbin/ifup-local ]];then
                _check="FAIL"
                PRINT_ERR "[ERROR] can not found /sbin/ifup-local"
            fi
        fi
        ip r|grep -q $extend_ifname
        if [[ $? -eq 0 ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] default route table found $extend_ifname"
        fi

        # check bond0 alive
        if [[ $extend_ifname != 'p0m0' ]];then
            ping -I $extend_ifname $UCLOUD_BOND_GATEWAY -c 5 > /dev/null 2>&1
            if [[ $? -ne 0 ]];then
                _check="FAIL"
                PRINT_ERR "[ERROR] $extend_ifname network ping not ok"
            fi
            ip r show table t_$extend_ifname |grep -q $UCLOUD_BOND_GATEWAY
            if [[ $? -ne 0 ]];then
                _check="FAIL"
                PRINT_ERR "[ERROR] $UCLOUD_BOND_GATEWAY not in ip r show table t_$extend_ifname"
            fi
            ip r show table t_$extend_ifname| grep -q "${bond_ipseg}/${UCLOUD_BOND_RANGE} dev ${extend_ifname}"
            if [[ $? -ne 0 ]];then
                _check="FAIL"
                PRINT_ERR "[ERROR] ${bond_ipseg}/${UCLOUD_BOND_RANGE} dev ${extend_ifname} not in ip r show table t_$extend_ifname"
            fi
        fi
    fi

    # check 003 status
    if [[ $BFFLAG = 0 ]] && [[ ! -x /etc/rc.local.d/003-rdma.sh ]];then
        _check="FAIL"
        PRINT_ERR "[ERROR] can not found /etc/rc.local.d/003-rdma.sh"
    elif [[ $BFFLAG = 1 ]] && [[ ! -x /etc/rc.local.d/003-bfinit.sh ]];then
        _check="FAIL"
        PRINT_ERR "[ERROR] can not found /etc/rc.local.d/003-bfinit.sh"
    fi

    # check vf
    if [[ $RDMA_SPEED = '100G' && -n $extend_ifname ]];then
        if [[ ! -x /opt/ucloud/net-name.sh ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] can not found /opt/ucloud/net-name.sh"
        fi
        if [[ ! -f /etc/udev/rules.d/99-ucloud-net-name.rules ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] can not found /etc/udev/rules.d/99-ucloud-net-name.rules"
        fi
        if [[ ! -x /opt/op/startup_mlx.sh ]];then
            _check="FAIL"
            PRINT_ERR "[ERROR] can not found /opt/op/startup_mlx.sh"
        fi
    fi

    # check bond slave
    local bond_nic1=`echo $UCLOUD_BOND_NIC | cut -f1 -d','`
    local bond_nic2=`echo $UCLOUD_BOND_NIC | cut -f2 -d','`
    if [ $(cat /sys/class/net/$bond_nic1/operstate|grep -c up) -eq 0 ];then
        _check="FAIL"
        PRINT_ERR "[ERROR] $bond_nic1 link down"
    fi
    if [ $(cat /sys/class/net/$bond_nic2/operstate|grep -c up) -eq 0 ];then
        _check="FAIL"
        PRINT_ERR "[ERROR] $bond_nic2 link down"
    fi
    if [ `cat /sys/class/net/bond0/bonding/slaves|grep -c $bond_nic1` -ne 1 ];then
        _check="FAIL"
        PRINT_ERR "[ERROR] $bond_nic1 not in bond"
    fi
    if [ `cat /sys/class/net/bond0/bonding/slaves|grep -c $bond_nic2` -ne 1 ];then
        _check="FAIL"
        PRINT_ERR "[ERROR] $bond_nic2 not in bond"
    fi

    # check ofed
    if [[ $BFFLAG = 0 ]];then
        if [[ $(uname -r) =~ 4.14.0 ]];then
            if [[ $(rpm -qa |grep -ic ofed) -ne 31 ]] && [[ $ARCH = 'x86_64' ]];then
                _check="FAIL"
                PRINT_ERR "[ERROR] ofed 未安装"
            fi
            # check mlnx-ofa_kernel-modules package
            kernel_release=$(uname -r|sed -n "s/4.14.0-\([0-9]*\).el7.ucloud.x86_64/\1/p")
            rpm -q mlnx-ofa_kernel-modules-4.5-OFED.4.5.1.0.1.0.19.1.g843cdb6.kver.4.14.0_${kernel_release}.el7.ucloud.x86_64 1>/dev/null
            if [[ $? -ne 0 ]];then
                _check="FAIL"
                PRINT_ERR "[ERROR] mlnx-ofa_kernel-modules 未安装"
            fi
        elif [[ $(uname -r) =~ 5.10.0 ]];then
            # 树内: mft-4.16.0-105.x86_64 rdma-core-34.0-1.el7.x86_64
            # OFED: mft-4.22.1-307.x86_64 rdma-core-58mlnx43-1.58307.x86_64
            if ! rpm -q mft |grep -qE 'mft-4.16.0-105|mft-4.22.1-307' ;then
                _check="FAIL"
                PRINT_ERR "[ERROR] mft-4.16.0 or mft-4.22.1-307 is not installed"
            fi
            if ! rpm -q rdma-core |grep -qE 'rdma-core-34.0-1|rdma-core-58mlnx43-1' ;then
                _check="FAIL"
                PRINT_ERR "[ERROR] rdma-core-34.0 or 58mlnx43 is not installed"
            fi
            for pkg in rdma-core mlnx-tools mft libibverbs librdmacm libibumad
            do
                local pkg=$(echo $pkg | sed 's/\*//g')
                rpm -q $pkg > /dev/null 2>&1
                if [[ $? -ne 0 ]];then
                    _check="FAIL"
                    PRINT_ERR "[ERROR] $pkg Not Installed."
                fi
            done
        fi
    fi

    local ip_by_sn ipdb_ip_all
    ip_by_sn=$(curl -s -X GET -H 'Accept: application/json' -H 'api-key: f0ca152370904201a80de50ecae966fa' "http://api-gw.ucloudadmin.com/ipdb/intranet/ip?bind_resource=${SN}")
    ipdb_ip_all=$(echo "$ip_by_sn" | jq -r '.data[]|.address')
    # check ipdb bind SN
    PRINT_INFO ipdb_ip_all $ipdb_ip_all
    PRINT_INFO UCLOUD_IPMI $UCLOUD_IPMI
    PRINT_INFO UCLOUD_IPADDR $UCLOUD_IPADDR
    PRINT_INFO UCLOUD_BOND_IP $UCLOUD_BOND_IP
    PRINT_INFO UCLOUD_EXTEND_IP $UCLOUD_EXTEND_IP
    for i in $UCLOUD_IPMI $UCLOUD_IPADDR $UCLOUD_BOND_IP $UCLOUD_EXTEND_IP
    do
        if ! echo "$ipdb_ip_all" | grep -qw "$i" ;then
            _check="FAIL"
            PRINT_ERR "[ERROR] SN $SN 未绑定 $i 请检查IPDB"
        fi
    done
    # check extend ip
    if [[ -n $extend_ifname ]] && [[ `echo "$ipdb_ip_all"| wc -w` -ne 3 ]];then
        _check="FAIL"
        PRINT_ERR "[ERROR] SN 绑定的IP数量不为3，请检查IPDB"
    fi

}

