#!/bin/bash
###
version=0.4.9
update_time="April 10, 2023."
## 0.2.0 版本更新：由于大交换机节点太多，不适合进行全部的批量测试，所以最好是只测对角线上的部分
## 0.3.0 版本更新：经过实际测试发现，速率下降的部分不一定会真的体现在对角线上，所以最好还是全部测完
## 0.3.1 版本更新：在0.3.0版本中，假如分块为20个一组，测试200个节点，那么就会产生100个结果文件，本版本将尝试解决此问题，将结果全部整合到一起
## 0.3.2 版本更新：引入getopt，方便脚本使用

function help(){
  echo ""
  echo " Description:"
  echo -e "   batchIB-${version} is a tool for testing the bandwidth of batch nodes in
   a high-performance cluster. This tool is based on pdsh and requires
   passwordless logins to be configured between the tested nodes.

   The test is based on the \033[1;3m ib_write_bw / ib_read_bw \033[0m command in the IB 
   driver, and collects the average rate when the message size is 8MB.

   The way to specify the test nodes is similar to pdsh. As an example:
      Command: \033[1;3m batchIB --range-x=bn[001-004,006] --range-y=bn[004-005] \033[0m
      Results similar to (Unit: MB/s):

      host   bn001    bn002     bn003     bn004     bn006
      bn004  11508.42 11513.21  11511.15  self      11192.83
      bn005  11514.25 11510.41  11510.82  11502.15  11514.36 

   The test will first execute \033[1;3m ib_write_bw --size=8388608 \033[0m on bn004, and then execute
   \033[1;3m ib_write_bw --size=8388608 bn004 \033[0m on bn001, bn002, bn003, bn004, and bn006 in sequence. 
   NOTE: This process is performed serially.  
 
   This process can be parallelized by setting the -j parameter to specify concurrently
   running tasks. NOTICE: Overmuch processes may cause incorrect results."
  echo ""
  echo " Usage: batchIB [options] parameter ... "
  echo ""
  echo " Options:"
  echo "   -h, --help                  Print this message and exit."
  echo "   -v, --version               Print the version number of batchIB and exit."
  echo "   --jobs=<N>          Allow N jobs at once, one job with no arg. "
  echo "   --range-x=<host1,host2...>  Set node list of client."
  echo "   --range-y=<host1,host2...>  Set node list of server, default is range-x with no arg."
  echo "   --output=<filename>
                               Set the output filename."
  echo "   --cmd=<ib_write_bw/ib_read_bw/ib_send_lat>
                               Set the IB command, default is ib_write_bw."
  echo "   --size=<value>              Set the Message size, if cmd=ib_write_bw/ib_read_bw, size default is 8388608 bytes,
                               if cmd is ib_send_lat/ib_recv_lat, size default is 32 bytes."
  echo "   --host-x=</to/your/path/nodelist.txt>
                               Set the client node range through the file, file format: bn001 bn002 bn040..."
  echo "   --host-y=</to/your/path/nodelist.txt>
                               Set the server node range through the file, default is host-x with no arg."
  echo "   --device=<value>            Set the device in the test command, default is mlx5_0."
  echo ""
  echo " Example:"
  echo "   1. batchIB --range-x=bn[001-004,006,010-020];cn[001-002,004] -j 5 --cmd=ib_write_bw --size=8388608"
  echo "   2. batchIB --range-x=node[001-004,006,009] --cmd=ib_send_lat --size=32 "
  echo "   3. batchIB --range-x=node[001-004,006,009] --rang-y==node[001-004] --cmd=ib_write_bw --size=8388608"
  echo ""
}

# getopt参数选项
options=$(getopt -o hrtvf --long range-x:,range-y:,help,reflash,version,standard::,output::,host-x::,host-y::,mail::,cmd::,size::,jobs::,device:: -- "$@")
if [ $? -ne 0 ]; then
   echo "提供了无效参数"
   exit 1
fi

eval set -- "$options"

function warning_message() {
    echo -e "\033[31mWaining: The jobs is excessive, it may cause IO blockage or high resource preemption rate,resulting in 
         a high error rate in the results. It is recommended to reduce the -j parameter to obtain more 
         trustworthy results. use -f option to ignore this warning and force execution. \033[0m"
}

input_range_x=""
input_range_y=""
input_jobs=1 # 不输入 -j 参数时，默认为 1
input_cmd=ib_write_bw
input_device=mlx5_0
output_filename=out_batchIB_${input_cmd}_$(date '+%Y-%m-%d_%H:%M:%S').txt
while true; do
  case "$1" in
    --range-x)
      input_range_x="$2"
      shift 2
      ;;
    --range-y)
      input_range_y="$2"
      shift 2
      ;;
    -j|--jobs)
      input_jobs="$2"
      shift 2 
      ;;
    -h|--help)
      help
#      warning_message
      exit 0
      ;;
    -r|--reflash)
      echo "reflash"
      if [[ ${input_jobs} -ne 1 ]];then
          echo "Require j=1. Exit."
          exit 0
      fi
      shift
      ;;
    -v|--version)
      echo "bachIB version: ${version}. Published on ${update_time}"
      exit 0
      ;;
    --standard)
      { echo "$options" | grep -E '\-\-reflash|\-r ' ;} &>/dev/null
      if [ $? -ne 0 ]; then
          echo "-r option is required before --standard"
          exit 0
      fi
      standard="$2"
      echo "standard=${standard}"
      shift
      ;;
    -m|--mail)
      input_mail="$2"
      echo "input_mail=${input_mail}"
      shift 2
      ;;
    -o|--output)
      output_filename="$2"
      #echo "output_filename=${output_filename}"
      shift 2
      ;;
    --extractt)
      extract_workdir="$2"
      echo "extract_workdir=${extract_workdir}"
      shift 2
      ;;
    --host-x)
      nodelist_x="$2"
      nodelist_y="$2"
      #echo "nodelist_x=${nodelist_x}"
      shift 2
      ;;
    --host-y)
      nodelist_x="$2"
      #echo "nodelist_y=${nodelist_y}"
      shift 2
      ;;
    --device)
      input_device="$2"
      echo "input_device=${input_device}"
      shift 2
      ;;
    --cmd)
      input_cmd="$2"
      #echo "input_cmd=${input_cmd}"
      output_filename=out_batchIB_${input_cmd}_$(date '+%Y-%m-%d_%H:%M:%S').txt
      shift 2
      ;;
    --size)
      input_size="$2"
      #echo "input_size=${input_size}"
      shift 2
      ;;
    -f|--force)
      is_force="$2"
      echo "is_force=${is_force}"
      shift
      ;;
    --)
      shift
      break
      ;;
    *)
      echo "无效参数：$1"
      exit 1
      ;;
   esac
done
#echo "nodelist_x=${nodelist_x[@]}"
#echo "nodelist_y=${nodelist_y[@]}"
#echo "input_jobs=${input_jobs}"
#echo "input_range_y=${input_range_y}"
#if [ -z "${input_range_x}" ]; then
#  echo "缺少必需选项：-x 或 --range_x"
#  exit 1
#else
#  # if [[ ! "${input_range_x}" =~ ^bn\[[0-9]+-[0-9]+(,[0-9]+-[0-9]+)*\]$ ]]; then
#  # if [[ ! 1 = 1 ]]; then
#  #  echo "参数格式错误：-x 或 --range_x 格式应为:bn[001-002,004-005]或bn[001-004]"
#  #   exit 1
#  #fi
#  if [ -z "${input_range_y}" ]; then
#    input_range_y="${input_range_x}"
#  # else
#    # if [[ ! "${input_range_y}" =~ ^bn\[[0-9]+-[0-9]+(,[0-9]+-[0-9]+)*\]$ ]]; then
#    #  echo "参数格式错误：-y 或 --range_y 格式应为:bn[001-002,004-005]或bn[001-004]"
#    #  exit 1
#    # fi
#  fi
#fi

###########@################
####  开始检查前置条件  ####
############################
echo ""
echo "============================================================================"
echo "*** Startup Prepare"
#if ! which pdsh > /dev/null 2>&1; then
#  echo "checking for pdsh... no."
#  echo -e "\033[31mError: pdsh is not installed. Exit. \033[0m"
#  exit 1
#fi
#echo "checking for pdsh... yes."
#if ! which getopt > /dev/null; then
#  echo -e "\033[31mError: getopt is not installed. Exit. \033[0m"
#  exit 1
#fi
echo "checking for getopt... yes."
if ! which ib_write_bw > /dev/null; then
  echo -e "\033[31mError: IB Driver is not installed. Exit. \033[0m"
  exit 1
fi
echo "checking for IB Driver... yes."
result=`pdsh -w bn[100-102] hostname | grep 255`
# echo "result=${result} "
if [ -n "$result" ]; then
  echo -e "\033[31mError: SSH passwordless configure are incorrect. Exit. \033[0m"
  exit 1
else
  echo "checking for SSH passwordless configure... yes."
fi
echo "*** Finish  Prepare"
echo "============================================================================"
echo ""

###### 
function read_host_by_hostlist(){
    IFS=$'\n'   # 按换行分割
    file_array=($(cat $1))   # 读取文件内容
    for item in "${file_array[@]}"; do
        IFS=$' '   # 对于每一行再按空格分割
        sub_array=($item)
        for sub_item in "${sub_array[@]}"; do
            echo "$sub_item"
        done
        unset sub_array
    done
    unset IFS
}
#####

###########################
#### 检查前置条件完成  ####
###########################

process=${input_jobs} # 进程数
# 解析主机范围，传入参数为字符串，只有一个参数
func_parse_str() {
    bc=()
    string=$1
    # 用分号分隔不同列
    array=(${string//;/ })
    for element in "${array[@]}"; do
        local input="$element"
        local start end current
    # 删除字符前缀
    sn=${input:0:2}
    #判断是否为bn[001-005,007]之类的简写形式
    if [[ "${input:2:1}" == "[" ]];
    then
        input="${input#$sn}"
        input="${input#[}"
        input="${input%]}"
        # 逗号分隔字符串
        IFS="," read -ra ranges <<< "$input"
        for range in "${ranges[@]}"; do
        # 判断是否为单个数字
        if [[ $range =~ ^[0-9]+$ ]]; then
            number=$( printf "%03d\t" $((10#$range)))
            bc+=${sn}${number}
            continue
        fi
    
        # 判断是否为范围
        if [[ $range =~ ^([0-9]+)-([0-9]+)$ ]]; then
            start="${BASH_REMATCH[1]}"
            end="${BASH_REMATCH[2]}"
            for ((current=$((10#$start)); current<=$((10#$end)); current++)); do
                number=$( printf "%03d\t" $((10#$current)))
                bc+=${sn}${number}
            done
        fi
        done
    else
        t=$( printf "\t")
        IFS=',' read -ra ranges <<< "$input"
        for range in "${ranges[@]}"; do
             bc+=${range}${t}
        done
    fi
    done
    echo $bc
}

if [ -z "$nodelist_x" ]
then
    range_x=($(func_parse_str "${input_range_x}"))
    range_y=($(func_parse_str "${input_range_y}"))
else
    range_x=($(read_host_by_hostlist "${nodelist_x}"))
    range_y=($(read_host_by_hostlist "${nodelist_y}"))
fi

num_nodes_x=${#range_x[@]}
num_nodes_y=${#range_y[@]}

model_cmd=${input_cmd}
message_size=${input_size}
device_name=${input_device}

#echo "range_x=${range_x[@]}"
#echo "range_y=${range_y[@]}"

#echo "model_cmd=${model_cmd}"
# 往函数里面传入的参数依次为： 客户端的范围，服务端的范围，输出文件名
funbatchIB(){
    local i
    local j
    local local_range_x=($1) # 客户端, 横着的那个
    local local_range_y=($2) # 服务端，竖着的那个
    local filename=$3 #
    local length_local_range_x=${#local_range_x[@]}
    local length_local_range_y=${#local_range_y[@]}
    rm -rf $filename
    touch $filename

    # title 做表头
    printf "%-20s" "host" >> $filename
    for (( i=0;i<${length_local_range_x};i++ ))
    do
        host=${local_range_x[$i]}
        printf "%-120s" "${host}" >> $filename
    done
    echo "" >> $filename
    # title

    for (( i=0;i<${length_local_range_y};i++ )) # 遍历每一行
    do
        host_i=${local_range_y[$i]};# 拿到服务端主机名
        printf "%-20s" "${host_i}" >> $filename
        for (( j=0;j<${length_local_range_x};j++ ))
        do
            #echo "local_range_x[i]=${local_range_x[$i]}"
            #echo " "
            if [[ ${local_range_x[$i]} == ${local_range_y[$j]} ]];
            then
                printf "%-20s" "self" >> $filename
            else
                host_j=${local_range_x[$j]};# 拿到客户端主机名
                pdsh -w ${host_i} kill -9 `lsof -i:18515 | grep ${model_cmd} | awk -F " " '{printf $2}'` 1>/dev/null 2>&1 
                (pdsh -w ${host_i} ${model_cmd} --size=${message_size} -d ${device_name} 1>/dev/null 2>&1 ) & # 服务端等待连接，也就是竖着那个，host i 来自 range_y
                pdsh -w ${host_j} kill -9 `lsof -i:18515 | grep ${model_cmd} | awk -F " " '{printf $2}'` 1>/dev/null 2>&1 # 客户端发起连接，也就是横着那个，host j 来自 local_range_x
                if [[ ${model_cmd} == ib_write_bw ]] || [[ ${model_cmd} == ib_read_bw ]];then
                    out=`pdsh -w ${host_j} ${model_cmd} --size=${message_size} ${host_i} -d ${device_name} --report_gbits | grep ${message_size} | awk -F " " '{printf $5}'` 1>/dev/null 2>&1 
                elif [[ ${model_cmd} == ib_send_lat ]];then
                    out=`pdsh -w ${host_j} ${model_cmd} --size=${message_size} ${host_i} -d ${device_name} --report_gbits | grep 1000 |  grep ${message_size} | awk -F " " '{printf $6}'` 1>/dev/null 2>&1
                else 
                    echo -e "\033[31mError: Unknown Test CMD. Exit. \033[0m"
                    exit 0;
                fi
                if [ ! -n "$out" ]; then
                    printf "%-20s" "error" >> $filename
                else
                    printf "%-20s" "$out" >> $filename
                fi
            fi
        done
        echo "" >> $filename
    done
    return 0;
}

function rand(){
    min=$1
    max=$(($2- $min + 1))
    num=$(date +%s%N)
    echo $(($num % $max + $min))
}

function tmp_sleep(){

    local time1=$(rand 1 2)
    sleep $time1
    return 0;
}

is_process_running() {
  local pid=$1
  if kill -0 $pid 2>/dev/null; then
    # Process is still running
    return 0
  else
    # Process has finished
    return 1
  fi
}

function is_finish(){
  local arr=($1)
  local i=0
  for (( i=0;i<${#arr[@]};i++)); do
    local cur_pid=${arr[$i]}
    if is_process_running "${cur_pid}"; then
      return 1
    fi
  done
  return 0 
}

# 往函数里面传入的参数依次为： 客户端的范围，服务端的范围，输出文件名，
function read_test(){
    tmp_sleep
    local i
    local j
    local local_range_x=($1)   # 客户端
    local local_range_y=($2)   # 服务端
    local filename=$3    # 文件名
    local length_local_range_x=${#local_range_x[@]}
    local length_local_range_y=${#local_range_y[@]}
    rm -rf $filename
    touch $filename

    # title 做表头
    printf "%-20s" "host" >> $filename
    for (( i=0;i<${length_local_range_x};i++ ))
    do
        host=${local_range_x[$i]}
        printf "%-20s" "${host}" >> $filename
    done
    echo "" >> $filename
    # title

    for (( i=0;i<${length_local_range_y};i++ )) # 遍历每一行
    do
        host_i=${local_range_y[$i]};# 拿到服务端主机名
        printf "%-20s" "${host_i}" >> $filename
        for (( j=0;j<${length_local_range_x};j++ ))
        do
            #echo "local_range_x[i]=${local_range_x[$i]}"
            #echo " "
            if [[ ${local_range_x[$i]} == ${local_range_y[$j]} ]];
            then
                printf "%-20s" "self" >> $filename
            else
                host_j=${local_range_x[$j]};# 拿到客户端主机名
                printf "%-20s" "$host_i$host_j" >> $filename
            fi
        done
        echo "" >> $filename
    done
    return 0;
}

function IB_test(){
    tmp_sleep
    local i
    local range_x=($1) # 客户端
    local range_y=($2) # 服务端
    local filename=$3 #
    length_range_x=${#range_x[@]}
    length_range_y=${#range_y[@]}

    rm -rf $filename
    touch $filename

    echo "range_x=${range_x[@]}" > $filename
    echo "range_y=${range_y[@]}" >> $filename
}

function unite(){
    # 定义输出文件名
    hboutfile=$1
    hbdir=$2/*.wxsc
    # 删除原先的合并后文件
    rm -rf $hboutfile
    # 定义进度条
    hbb=''
    hbiii=0
    
    jindu(){
        printf "progress:[%-50s]%d%%\r" $hbb $hbiii
        sleep 0.3
        hbiii=`expr 2 + $hbiii`    
        hbb=#$hbb
    }
    # 记录开始时间
    hbstart=$(date +%s)
    # 定义第一行上的主机名并且输出
    ######################################################################################
    # 创建字符串数组hosty用来保存每列的主机名
    hbhosty=()
    jindu && jindu
    sleep 0.5
    # 遍历目录下的所有.txt文件
    for hbfile in $hbdir; do
      # 从每个文件的第一行获取主机名
      hbhost_names=$(sed -n '1p' $hbfile | awk '{for(hbi=2; hbi<=NF; hbi++) print $hbi}')
      # 将主机名添加到hosty
      hbhosty=( "${hbhosty[@]}" "$hbhost_names" )
    done
    for hbxi in {1..3}
    do
        jindu
        sleep 0.5
    done
    # 在hosty中移除重复的主机名
    hbhosty=($(echo "${hbhosty[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
    # 按升序对主机名进行排序
    hbhosty=($(echo "${hbhosty[@]}" | tr ' ' '\n' | sort -n | tr '\n' ' '))
    jindu && sleep 1
    # 将行首输入到1234.out文件中
    hbbt="host"
    printf "%-20s" "$hbbt" > $hboutfile
    printf "%-20s" "${hbhosty[@]}" >> $hboutfile
    echo >> $hboutfile
    for hbxi in {1..5}
    do
        jindu && sleep 0.8
    done
    sleep 1
    jindu
    ######################################################################################
    # 将所有速率数据存入字典中
    ######################################################################################
    # 创建空字典
    declare -A hbdict
    jindu && sleep 0.3 && jindu && jindu && sleep 0.6
    for hbj in $hbdir
    do
    # 读取文件
    while read line
    do
      # 忽略第一行
      if [ "${line:0:4}" != "host" ]; then
        # 将每行按空格分割为数组
        hbarr=($line)
        # 取第一列字符串
        hbkey1=${hbarr[0]}
        # 遍历整个数组，从第二列开始
        for hbi in $(seq 2 ${#hbarr[@]}); do
          # 取第一行的字符串
          hbkey2=$(sed -n 1p $hbj | awk '{print $'"$hbi"'}')
          # 合并字符串
          hbnewkey="$hbkey1$hbkey2"
          # 将字符串作为键，数组元素作为值存入字典
          hbdict[$hbnewkey]=${hbarr[$((hbi-1))]}
        done
      fi
    done < $hbj
    done
    
    for hbxi in {1..15}
    do
        jindu && sleep 0.6
    done
    sleep 1
    
    # 遍历字典
    #for key in "${!dict[@]}"; do
    #  echo "Key: $key Value: ${dict[$key]}"
    #done
    ######################################################################################
    
    
    # 定义每行上的主机名
    ######################################################################################
    # 定义包含.txt文件的目录
    #hbdir01="/online1/wxsc/sunty/10process/test"
    
    # 创建字符串数组hostx用来保存每行的主机名
    declare -a hbhostx
    jindu && sleep 1
    # 遍历目录下的所有.txt文件
    for hbfile in $hbdir; do
        #跳过第一行将主机名存储在hostx数组中
        hbhostnames=($(tail -n +2 $hbfile | awk '{print $1}'))
        for hbhost in "${hbhostnames[@]}"; do
            hbhostx=("${hbhostx[@]}" "$hbhost")
        done
    done
    jindu && sleep 0.8 && jindu
    # 在hostx中移除重复的主机名
    hbhostx=($(echo "${hbhostx[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
    jindu && sleep 0.5
    # 输出hostx数组，每个主机名占用10个字符
    #printf "%-10s\n" "${hostx[@]}"
    ######################################################################################
    
    for hbxi in {1..6}
    do
        jindu && sleep 0.7
    done
    
    # 输出每行主机名和每行每列的数据
    ######################################################################################
    for hbhost in "${hbhostx[@]}"
    do
        printf "%-20s" "$hbhost" >> $hboutfile
        for hbhost01 in "${hbhosty[@]}"
        do
            hbkey01=${hbhost}
            hbkey02=${hbhost01}
            hbnewkey01="$hbkey01$hbkey02"
            hbvalue01=${hbdict[$hbnewkey01]}
            printf "%-20s" "$hbvalue01" >> $hboutfile
        done
        echo >> $hboutfile
    done
    ######################################################################################
    
    for hbxi in {1..9}
    do
        jindu && sleep 0.8
    done
    
    # 记录结束时间
    hbend=$(date +%s)
    # 计算用时
    hbelapsed=$((hbend-hbstart))
    hbminutes=$((hbelapsed / 60))
    
    jindu && sleep 1 && jindu
    echo ""
    echo "---   Generate End: $(date '+%Y-%m-%d %H:%M:%S')"
    echo "--- Cost Time: $hbelapsed s or $hbminutes min"
}

nc=${process} # 组数
nx=${process} # 循环次数，虽然跟组数一样  但是为了防止后面修改需求，还是单独写出来吧，免得以后改代码麻烦

normal_coupe_x=$(( (num_nodes_x + process - 1) / process )) # 最大的/一般的分块的x节点长度
normal_coupe_y=$(( (num_nodes_y + process - 1) / process )) # 最大的/一般的分块的y节点长度

last_coupe_x=$(( num_nodes_x - normal_coupe_x * (process - 1) )) # 最后一个分块的x节点长度
last_coupe_y=$(( num_nodes_y - normal_coupe_y * (process - 1) )) # 最后一个分块的y节点长度

integer_direction=0 # 取整数的方向，如果为0则是向上取整
# 在此判断 num_nodes_x / process 是向上取整还是向下取整
if [[ ${last_coupe_x} -lt 0 ]] || [[ ${last_coupe_y} -lt 0 ]];then
    integer_direction=1
fi

if [[ ${integer_direction} == 1 ]];then
    normal_coupe_x=$(( num_nodes_x / process )) # 最大的/一般的分块的x节点长度
    normal_coupe_y=$(( num_nodes_y / process )) # 最大的/一般的分块的y节点长度
    last_coupe_x=$(( num_nodes_x - normal_coupe_x * (process - 1) )) # 最后一个分块的x节点长度
    last_coupe_y=$(( num_nodes_y - normal_coupe_y * (process - 1) )) # 最后一个分块的y节点长度
fi

# 我的想法是，y不变，每次循环之后，整体向右挪，那么第一次的时候，就需要记录好y的值
# 汇总采用临时文件的方式，先在主进程创建好临时文件，由于临时文件的文件名是随机的，所以需要一个数组对文件名进行记录，后续的收集也需要通过这个文件名来完成
echo -e "\033[34m++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ \033[0m"
echo -e "\033[34m+++ Function: Batch Test IB Performance For HPC \033[0m"
echo "+++ Version: ${version}. Published on ${update_time}"
echo "+++ Client Node Range: ${input_range_x}"
echo "+++ Server Node Range: ${input_range_y}"
echo "+++ Test Cmd: ${model_cmd} --size=${message_size}"
echo -e "\033[34m+++ Write By National Supercomputing Wuxi Center & yancheng HPC. \033[0m"
echo -e "\033[34m++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ \033[0m"
echo ""
echo "############################################################################"
echo "### Test Start: $(date '+%Y-%m-%d %H:%M:%S')"
schedule=''
schedule_index=0
k=$nx
printf "progress:[%-25s]%d%%\r" ${schedule} ${schedule_index}


# 分组算法，先提前计算好每个分组的第一个索引(pre_coupe_index)和最后一个索引(suf_coupe_index)
pre_coupe_x_index=() # x 节点的分组首索引，此数组长度应该为 nc
pre_coupe_y_index=() # y 节点的分组首索引，此数组长度应该为 nc
suf_coupe_x_index=() # x 节点的分组尾索引，此数组长度应该为 nc
suf_coupe_y_index=() # y 节点的分组尾索引，此数组长度应该为 nc

# 计算分组索引
for (( i=0;i<$nc;i++ ));do
    pre_coupe_x_index[$i]=$((i*normal_coupe_x))
    suf_coupe_x_index[$i]=$((${pre_coupe_x_index[$i]} + normal_coupe_x - 1))

    pre_coupe_y_index[$i]=$((i*normal_coupe_y))
    suf_coupe_y_index[$i]=$((${pre_coupe_y_index[$i]} + normal_coupe_y - 1))
done
# 最后一个分组，需要单独进行修改
suf_coupe_x_index[$(( nc - 1 ))]=$((${pre_coupe_x_index[$(( nc - 1 ))]} + last_coupe_x - 1))
suf_coupe_y_index[$(( nc - 1 ))]=$((${pre_coupe_y_index[$(( nc - 1 ))]} + last_coupe_y - 1))

# 拿到索引之后，进行分组，在每次循环中，只要每个分块都去索引数组中去索引的头和尾就好了

###################
#### 开始测试  ####
###################

#echo "range_x=${range_x[@]}"
#echo "range_y=${range_y[@]}"

PWD=$(cd $(dirname $0); pwd)
workdir=.batchIB_Test$(date '+%Y%m%d%H%M%S')
mkdir -p ${workdir}
pre_start_time=$(date +%s)
for (( inde_nx=0;index_nx<$nx;index_nx++ )); do
    start_time=$(date +%s)
    # 分组之后，再进行分割，生成相应的范围和文件名，然后传递到函数进行调用
    # 每一组都放入到后台执行，然后将进程id放入到数组里面，主函数每隔十秒判断这些进程是否还存在，如果不存在，说明执行完了
    result_pid=() # 存放进程的 pid
    for (( i=0;i<$nc;i++ ));do
        current_index=$((i+index_nx)) # 当前分块的索引
        current_index=$((current_index >= nc ? current_index - nc : current_index)) # 判断当前分块的索引是否越界
        # 根据 current_index 这一索引，去首尾索引中拿值
        current_pre_coupe_x_index=${pre_coupe_x_index[${current_index}]}
        current_suf_coupe_x_index=${suf_coupe_x_index[${current_index}]}
        current_pre_coupe_y_index=${pre_coupe_y_index[${i}]}
        current_suf_coupe_y_index=${suf_coupe_y_index[${i}]}
        tmp_range_x=()
        tmp_range_y=()
        # x 的首尾索引需要一直变化，但 y 不需要
        tmp=0
        for (( j=${current_pre_coupe_x_index};j<=${current_suf_coupe_x_index};j++ ))
        do
            tmp_range_x_index=$j
            tmp_range_x[${tmp}]=${range_x[${tmp_range_x_index}]}
            tmp=$(( tmp + 1 ))
        done
        tmp=0
        for (( j=${current_pre_coupe_y_index};j<=${current_suf_coupe_y_index};j++ ))
        do
            tmp_range_y_index=$j
            tmp_range_y[${tmp}]=${range_y[${tmp_range_y_index}]}
            tmp=$(( tmp + 1 ))
        done

        length_tmp_index_x=$((${#tmp_range_x[@]}-1))
        length_tmp_index_y=$((${#tmp_range_y[@]}-1))
         
        filename=${workdir}/out_IBTest_x_${tmp_range_x[0]}-${tmp_range_x[${length_tmp_index_x}]}_y_${tmp_range_y[0]}-${tmp_range_y[${length_tmp_index_y}]}.wxsc
        funbatchIB "${tmp_range_x[*]}" "${tmp_range_y[*]}" "$filename" &
#       IB_test "${tmp_range_x[*]}" "${tmp_range_y[*]}" "$filename" &
#       read_test "${tmp_range_x[*]}" "${tmp_range_y[*]}" "$filename" &
        result_pid[$i]=$!
    done

    # 然后 每隔一段时间就检查result的值是否全为0
    a=1
    while [ $a -eq 1 ]
    do
        is_finish "${result_pid[*]}"
        res=$?
        if [[ $res -eq 0 ]]
        then
           # echo "跳出循环，当前时间：$(date '+%Y-%m-%d_%H:%M:%S')"
            break;
        else
            # 说明还有些块没有执行完，再等待一会儿
            sleep 1;
        fi
    done
    if [[ ${index_nx} == $(( nx-1 )) ]];then
        schedule_index=100
    else 
        schedule_index=`expr $(( 100/nx )) + ${schedule_index}`
    fi
    schedule=''
    for (( sch_index=0;sch_index<$(( schedule_index/4 ));sch_index++ ))
    do
        schedule=#${schedule}
    done

    cen_time=$(date +%s); time_diff=$((cen_time - start_time)); k=$((k-1)); end_time=$((k*time_diff))
    hours=$((end_time / 3600)); minutes=$((end_time % 3600 / 60)); seconds=$((end_time % 60))
    printf "progress:[%-25s]%d%% eta: %d hours %02d min %02d s\r" ${schedule} ${schedule_index} $hours $minutes $seconds
done
end_time=$(date +%s)
cost_time=$((end_time - pre_start_time))
hours=$((cost_time / 3600)); minutes=$((cost_time % 3600 / 60)); seconds=$((cost_time % 60))

echo ""
echo "###   Test End: $(date '+%Y-%m-%d %H:%M:%S')"
printf "###  Cost Time: %d hours %02d min %02d s" $hours $minutes $seconds
echo ""
echo "############################################################################"
echo ""
####################
####  测试结束  ####
####################


########################
#### 开始校验文件   ####
########################

# 文件校验，检查每个文件的行数和列数是否正确，以及格式是否正确
# 需要注意的是，不同文件的行数和列数可能不同

# for (())


# Verify Result File Integrity... 10%



## 假如有一个文件校验失败，需要抛出该文件的文件名，提醒用户进行手动修改


#######################
#### 校验文件结束  ####
#######################

echo "----------------------------------------------------------------------------"
echo "--- Current Directory: $(cd `dirname $0`; pwd)"
echo "--- Output File: ${output_filename}"
echo "--- Generate Start: $(date '+%Y-%m-%d %H:%M:%S')"
unite "${output_filename}" "${workdir}"
#echo "---   Generate End: $(date '+%Y-%m-%d %H:%M:%S')"
echo "----------------------------------------------------------------------------"

######################
#### 开始汇总结果 ####
######################





######################
#### 汇总结果结束 ####
######################

#echo "All IB tests have been completed"
echo ""
echo -e "\033[34m*** All IB tests have been completed! ***\033[0m"
echo ""
# END
