Script điều chỉnh weight cho các OSDs khi thêm storage node mới

#!/bin/bash
OSDs="1 2 3 4 5 6 7 8 9"
INCREMENT_WEIGHT=0.01 # IMPORTANT
TARGET_WEIGHT=1.45
INTERVAL_CHECK=1 # second
WORK_DIR=/opt/weight

### telegram var
#read -p "Enter TOKEN_BOT: " TELEGRAM_BOT_TOKEN
#TELEGRAM_BOT_TOKEN=$(echo "$TELEGRAM_BOT_TOKEN" | tr -d '"')
#read -p "Enter CHAT_ID: " TELEGRAM_CHAT_ID
TELEGRAM_BOT_TOKEN=''
TELEGRAM_CHAT_ID=''

# basic validation on weird number
validate_weight () {
  local w=$1
  re='^[+-]?[0-9]+\.?[0-9]*$'
  if ! [[ $w =~ $re && $INCREMENT_WEIGHT =~ $re && $TARGET_WEIGHT =~ $re && $INTERVAL_CHECK =~ $re ]] ; then
    echo "xx Not a number" >&2
    exit 1
  elif (( $(echo "$w > $TARGET_WEIGHT" | bc -l) )); then
    echo "xx Weight cannot higher than target"
    exit 1
  elif (( $(echo "$TARGET_WEIGHT > 1.5" | bc -l) )) || (( $(echo "$TARGET_WEIGHT < 0" | bc -l) )); then
    echo "xx TARGET_WEIGHT cannot be out of range [0;1.5]"
    exit 1
  elif (( $(echo "$INCREMENT_WEIGHT > 0.05" | bc -l) )) || (( $(echo "$INCREMENT_WEIGHT < 0" | bc -l) )); then
    echo "xx INCREMENT_WEIGHT cannot be higher than 0.05"
    exit 1
  elif (( $(echo "$INCREMENT_WEIGHT > $TARGET_WEIGHT" | bc -l) )); then
    echo "xx INCREMENT_WEIGHT cannot be higher than TARGET_WEIGHT"
    exit 1
  fi
  return 0
}

## init the weight for listed OSD
## store to /opt/weight.X
for OSD in $OSDs; do
  init_weight=$(printf '%.2f\n' $(ceph  osd df | grep "^$OSD" | awk '{ print $3}'))
  echo $init_weight > $WORK_DIR/weight.$OSD
  echo "=Init $OSD with weight $init_weight"
  validate_weight $init_weight
done

## idea: while loop for keep running
## for loop to loop through OSDs
## 	while loop to wait and reweight
## 	if backfill < 10 and weight < target then reweight

while true; do
  sleep 1 &&
  #if (( $(cat test.ceph-s | grep -E "active\+remapped\+backfilling|active\+remapped\+backfill_wait" | awk '{sum+=$1} END {print sum}') < 15)) 2>/dev/null; then
  if (( $(ceph  -s | grep -E "active\+remapped\+backfilling|active\+remapped\+backfill_wait" | awk '{sum+=$1} END {print sum}') < 15)) 2>/dev/null; then
    #TELERAM_MESSAGE=""
    for OSD in $OSDs; do  # 141 142
      echo "Waiting for osd.$OSD"
      while true; do
        #if (( $(cat test.ceph-s | grep -E "active\+remapped\+backfilling|active\+remapped\+backfill_wait" | awk '{sum+=$1} END {print sum}') < 15 )) 2>/dev/null; then
        if (( $(ceph  -s | grep -E "active\+remapped\+backfilling|active\+remapped\+backfill_wait" | awk '{sum+=$1} END {print sum}') < 15 )) 2>/dev/null; then
          echo "+++++ Start: $(date)"
          ceph  -s | grep backfill
          ## get current weight
          #w=$(cat $WORK_DIR/weight.$OSD)
          w=$(printf '%.2f\n' $(ceph  osd df | grep "^$OSD" | awk '{ print $3}'))

          ## only reweight if current < TARGET_WEIGHT
          if (( $(echo "$w < $TARGET_WEIGHT" | bc -l) )); then

            echo "++ osd.$OSD current weight: $w will increase by $INCREMENT_WEIGHT"

            ## increase the current weight
            w=$(echo $w + $INCREMENT_WEIGHT | bc)
            if (( $(echo "$w < 1" | bc -l) )); then w=0$w; fi  # .21 -> 0.21

            echo "++ reweight: $(date)"
            ## reweight
            #if echo osd.$OSD $w; then
            if ceph osd crush reweight osd.$OSD $w; then
              #echo $w > $WORK_DIR/weight.$OSD;
              #notify
              TELERAM_MESSAGE="== New osd.$OSD weight $w\n$TELERAM_MESSAGE"
              break # out while loop
            fi
          else  # if current >= TARGET_WEIGHT
            break # out while loop
          fi # target-weight check
        fi # backill check
        sleep $INTERVAL_CHECK
      done # inner while-loop
    done # for-loop
    echo "++ Notify Telegram!"
    #curl -X POST --silent --proxy http://192.168.100.100:3128 \
    #  -H "Content-Type: application/json" \
    #  -d "{\"chat_id\":\"$TELEGRAM_CHAT_ID\", \"text\": \"$TELERAM_MESSAGE\", 'disable_notification': true}" \
    #  https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage > /dev/null
  else
    ### one endless loop case is when pgbackfill cleaned so fast and at least one OSD is not get TARGET
    is_break=1
    ceph osd df > /tmp/osd-df # reduce load ceph api
    for OSD in $OSDs; do
      w=$(printf '%.2f\n' $(grep "^$OSD" /tmp/osd-df | awk '{ print $3}'))
      # if all OSD >= TARGET -> BREAK OUTER WHILE
      # if one OSD < TARGET -> BREAK FOR -> CONTINUE TO INCREASE
      if (( $(echo "$w < $TARGET_WEIGHT" | bc -l) )); then
        is_break=0
        break
      else
        OSDs=$(echo $OSDs | sed s/$OSD//)
        echo "++ osd is removed: $OSD. Current list: $OSDs"
      fi
    done

    ## value didn't changed thru for, no OSD < TARGET, then exit while
    if (( $is_break == 1 )); then break 2; fi
  fi
done

nohup ./reweight.sh &