#!/bin/bash

# 配置
rep_prefix="mlx_"
speed_limit=20000 # Mbps

NETPF="$(lspci -Dnnm -d 0x15b3: | grep "Ethernet" | grep -v "Virtual")"
if [ -z "$NETPF" ]; then
    echo "No Mellanox NIC"
    exit 1
fi

MLX_25G="$(echo "$NETPF" | grep -o "\bMT27800\b" | head -n1)"
MLX_100G="$(echo "$NETPF" | grep -P "\b(MT28800|MT2892)\b" | head -n1)"

if [ -z "$MLX_25G$MLX_100G" ]; then
    echo "Unknown Mellanox Model"
    exit 2
fi

USE_DEVLINK=false
if [ $(uname -r | cut -d. -f1) -gt 5 ]; then
    USE_DEVLINK=true
elif [ $(uname -r | cut -d. -f1) -eq 5 -a $(uname -r | cut -d. -f2) -ge 10 ]; then
    USE_DEVLINK=true
fi
if ! $USE_DEVLINK; then
    echo "Require kernel >= 5.10"
    exit 3
fi

DEVS=""
for iface in $(ls /sys/class/net/ | grep -P '^net\d+'); do
    if [ -n "$(ethtool -i $iface | grep '^driver:' | grep -e mlx5_core -e mlx5e_rep)" ]; then
        DEVS="$DEVS $iface"
    fi
done

prepare_mlx_25g() {
    dev=$1

    # 固件服务拉起，配置固件，以及业务侧依据 /dev/mst 来识别网络
    /etc/init.d/mst start
    # TODO: 作用不明
    mcra /dev/mst/mt4119_pciconf0 0x31500.17 0
    mcra /dev/mst/mt4119_pciconf1 0x31500.17 0

    # 先禁掉自动绑定，不然后面的变更完成后还得再做一些解绑，节省时间
    echo 0 > /sys/class/net/$dev/device/sriov_drivers_autoprobe

    # 开启vf
    cfg_vfs=$(cat /sys/class/net/$dev/device/sriov_totalvfs)
    vfs=$cfg_vfs
    rep_last="$rep_prefix$((vfs-1))"
    echo $cfg_vfs > /sys/class/net/$dev/device/sriov_numvfs

    # 启用网卡的switchdev模式，暴露iface
    pci_dev=$(basename $(readlink -f /sys/class/net/$dev/device))
    devlink dev eswitch set pci/$pci_dev mode switchdev
    while ! [ -e /sys/class/net/$dev ]; do
        sleep 1
    done
    ifup $dev

    # waiting for mlx_* all ready
    while [ ! -L /sys/class/net/$rep_last ]; do
        sleep 1
    done

    devlink dev param set pci/$pci_dev name fdb_large_groups value 64 cmode driverinit

    # 恢复自动绑定
    echo 1 > /sys/class/net/$dev/device/sriov_drivers_autoprobe

    # 限速
    for ((i=0; i<cfg_vfs; i++)); do
        ip link set $dev vf $i max_tx_rate $speed_limit
    done

    # 优化, 开启 内核态miss upcall限制
    echo 300 > /proc/sys/net/ovs/ovs_upcall_limit_rate
}

prepare_mlx_100g() {
    dev1=$1
    dev2=$2

    # 固件服务拉起，配置固件，以及业务侧依据 /dev/mst 来识别网络
    systemctl start mst
    # TODO: 作用不明
    # Reads/writes a single word from/to a device configuration register space
    # mcra 读写一个 寄存器空间 的值
    # 给某个设备的 0x31500.17 写入 0
    # /dev/mst/mt4121_pciconf0 
    # /dev/mst/mt4125_pciconf0 开机默认就是 0x00000000
#    mcra /dev/mst/mt*_pciconf0   0x31500.17 0
#    mcra /dev/mst/mt*_pciconf0.1 0x31500.17 0

    # 先禁掉自动绑定，不然后面的变更完成后还得再做一些解绑，节省时间
    echo 0 > /sys/class/net/$dev1/device/sriov_drivers_autoprobe
    echo 0 > /sys/class/net/$dev2/device/sriov_drivers_autoprobe

    # 开启vf
    cfg_vfs_dev1=$(cat /sys/class/net/$dev1/device/sriov_totalvfs)
    cfg_vfs_dev2=$(cat /sys/class/net/$dev2/device/sriov_totalvfs)
    vfs=$((cfg_vfs_dev1+cfg_vfs_dev2))
    rep_last="$rep_prefix$((vfs-1))"
    echo $cfg_vfs_dev1 > /sys/class/net/$dev1/device/sriov_numvfs
    echo $cfg_vfs_dev2 > /sys/class/net/$dev2/device/sriov_numvfs

    # 启用网卡的switchdev模式，暴露iface
    pci_dev1=$(basename $(readlink -f /sys/class/net/$dev1/device))
    devlink dev eswitch set pci/$pci_dev1 mode switchdev
    while ! [ -e /sys/class/net/$dev1 ]; do
        sleep 1
    done
    ifup $dev1

    pci_dev2=$(basename $(readlink -f /sys/class/net/$dev2/device))
    devlink dev eswitch set pci/$pci_dev2 mode switchdev
    while ! [ -e /sys/class/net/$dev2 ]; do
        sleep 1
    done
    ifup $dev2

    # waiting for mlx_* all ready
    while [ ! -L /sys/class/net/$rep_last ]; do
        sleep 1
    done

    devlink dev param set pci/$pci_dev1 name fdb_large_groups value 64 cmode driverinit
    devlink dev param set pci/$pci_dev2 name fdb_large_groups value 64 cmode driverinit

    # 恢复自动绑定（否则手工bind也受影响）
    echo 1 > /sys/class/net/$dev1/device/sriov_drivers_autoprobe
    echo 1 > /sys/class/net/$dev2/device/sriov_drivers_autoprobe

    # 限速
    for ((i=0; i<cfg_vfs_dev1; i++)); do
        ip link set $dev1 vf $i max_tx_rate $speed_limit
    done
    for ((i=0; i<cfg_vfs_dev2; i++)); do
        ip link set $dev2 vf $i max_tx_rate $speed_limit
    done

    echo 300 > /proc/sys/net/ovs/ovs_upcall_limit_rate #开启 内核态miss upcall限制

    # 分配最后一张vf给udisk专用
    rep4udisk=$rep_last
    # a. 找出最后一张vf的pci地址
    pci4udisk="$(lspci -D -d 0x15b3: | grep "Ethernet" | grep "Virtual" | tail -n1 | awk '{print $1}')"
    # b. 绑定
    echo "binding the last vf for udisk rdma usage..."
    echo $pci4udisk > /sys/bus/pci/drivers/mlx5_core/bind
    while [ ! -L /sys/class/net/udisk0 ]; do
        sleep 1
    done
    ip link set dev $rep4udisk up

    # 配置tc规则
    ifup udisk0
    cidr4udisk="$(ip addr show dev udisk0 | grep -oP '(?<=inet )[^ ]+')"
    mac4udisk="$(ip link show dev udisk0 | grep -oP '(?<=link/ether )[^ ]+')"
    ip4udisk="$(echo "$cidr4udisk" | cut -d/ -f1)"
    if4man="$(ip route get 172.16.0.0 | grep -oP '(?<=dev )[^ ]+')"

    tc qdisc  del dev $rep4udisk ingress
    tc qdisc  del dev $if4man    ingress
    tc qdisc  del dev $dev1      ingress
    tc qdisc  del dev $dev2      ingress

    tc qdisc  add dev $rep4udisk ingress
    tc filter add dev $rep4udisk pref 1 ingress protocol ip  flower skip_sw action mirred egress redirect dev $if4man
    tc filter add dev $rep4udisk pref 2 ingress protocol arp flower skip_sw dst_mac ff:ff:ff:ff:ff:ff action mirred egress redirect dev $if4man
    tc filter add dev $rep4udisk pref 3 ingress protocol arp flower skip_hw action mirred egress redirect dev $if4man

    tc qdisc  add dev $if4man ingress_block 22 ingress
    tc qdisc  add dev $dev1   ingress_block 22 ingress
    tc qdisc  add dev $dev2   ingress_block 22 ingress

    tc filter add block 22 pref 1 protocol ip  flower dst_mac $mac4udisk action mirred egress redirect dev $rep4udisk
    tc filter add block 22 pref 2 protocol arp flower skip_hw dst_mac $mac4udisk action mirred egress redirect dev $rep4udisk
    tc filter add block 22 pref 3 protocol arp flower skip_hw dst_mac ff:ff:ff:ff:ff:ff arp_tip $ip4udisk action mirred egress redirect dev $rep4udisk
}

if [ -z "$1" ]; then
    echo "Usage: $0 <ifname>"
    exit 3
fi
if [ -z "$(echo "$DEVS" | grep "\b$1\b")" ]; then
    echo "Invalid interface name, must be one of $DEVS"
    exit 4
fi

if [ -n "$MLX_25G" ]; then
    prepare_mlx_25g $1
fi

if [ -n "$MLX_100G" ]; then
    prepare_mlx_100g $DEVS
fi

