0%

Docker 方式部署

  • 创建目录

    1
    2
    
    mkdir -p /data/etcd /etc/etcd
    chown -R 1001.1001 /data/etcd
  • 创建 etcd 配置文件

    vim /etc/etcd/etcd.conf

    1
    2
    3
    4
    5
    6
    
    # 节点名称
    name: 'etcd01'
    # 指定节点的数据存储目录
    data-dir: '/data'
    # 对外提供服务的地址,客户端会连接到这里和 etcd 交互
    listen-client-urls: 'http://0.0.0.0:2379'
  • 启动容器

环境

OS主机名IP
Kylin Linux Advanced Server V10etcd01172.20.10.129
Kylin Linux Advanced Server V10etcd02172.20.10.132
Kylin Linux Advanced Server V10etcd03172.20.10.134

安装

下载、解压安装包

1
2
wget https://github.com/etcd-io/etcd/releases/download/v3.5.14/etcd-v3.5.14-linux-amd64.tar.gz
tar -xf etcd-v3.5.14-linux-amd64.tar.gz -C /usr/local/

添加环境变量

vim /etc/profile.d/etcd.sh

1
2
export PATH=$PATH:/usr/local/etcd-v3.5.14-linux-amd64
source /etc/profile.d/etcd.sh

创建配置文件

  • 创建 etcd01 节点配置文件

    vim /etc/etcd/etcd.conf

     1
     2
     3
     4
     5
     6
     7
     8
     9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    
    # 节点名称
    name: "etcd01"
    # 数据存储目录
    data-dir: "/data/etcd"
    # 对外公告的该节点客户端监听地址,这个值会告诉集群中其他节点
    advertise-client-urls: "http://172.20.10.129:2379"
    # 监听客户端请求的地址列表
    listen-client-urls: "http://172.20.10.129:2379,http://127.0.0.1:2379"
    # 监听URL,用于节点之间通信监听地址
    listen-peer-urls: "http://172.20.10.129:2380"
    # 服务端之间通讯使用的地址列表,该节点同伴监听地址,这个值会告诉集群中其他节点
    initial-advertise-peer-urls: "http://172.20.10.129:2380"
    # etcd启动时,etcd集群的节点地址列表
    initial-cluster: "etcd01=http://172.20.10.129:2380,etcd02=http://172.20.10.132:2380,etcd03=http://172.20.10.134:2380"
    # etcd集群的初始集群令牌
    initial-cluster-token: 'etcd-cluster'
    # etcd集群初始化的状态,new代表新建集群,existing表示加入现有集群
    initial-cluster-state: 'new'
  • 创建 etcd02 节点配置文件

脚本功能:

  • 恢复时,需要指定备份的 DB_NAME 和备份日期;
  • 恢复失败时,发送企业微信告警;
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/bin/bash

DB_NAME=$1; TS=$2
DB_TYPE="mariadb"
DB_VERSION="10.1.46"
STREAM_TOOL="mbstream"
BACKUP_TOOL="mariabackup"
SYSTEMD_UNIT="mariadb.service"
DECOMPRESS_PATH="/data/decompress"
RESTORE_EXTRA_ARGS="--apply-log-only"
CONF_FOO_PATH="/etc"
CONF_FILES=("$CONF_FOO_PATH/my.cnf" "$CONF_FOO_PATH/my.cnf.d/*.cnf")
_result_datadir=$(grep -s '^datadir' ${CONF_FILES[@]} | awk -F[=\ ] '{print $NF}' | sort -u)
OLD_MYSQL_DATA_PATH=${_result_datadir:-'/var/lib/mysql'}
CPU_CORES=$(grep processor /proc/cpuinfo -c)
HALF_MEM=$(awk '/^MemTotal/{print int($2/1024/1024+0.9)*1024/2"M"}' /proc/meminfo)
RCLONE="/usr/local/bin/rclone"
RCLONE_REMOTE="huawei:backup-bucket/mysql"
WECHAT_WEBHOOK_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
LAN_IP=$(hostname -I | awk '{print $1}')

[[ $# != 2 ]] && echo -e "Usage:\n\t$0 (备份库名) (时间节点)" && exit 1

raise_error() {
  msg="### 恢复失败! \n#### $DB_NAME $TS $BACKUP_TYPE\n#### $1"
  for i in {1..3}; do
    curl -s "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=$WECHAT_WEBHOOK_KEY" \
      -H "Content-Type: application/json" \
      -d '{
        "msgtype": "markdown",
        "markdown": {
          "content": "'"$msg"'"
        }
      }'
    [[ $? == 0 ]] && break || sleep $i
  done
  echo "$msg"
  exit 1
}

REMOTE_BACKUP_FILES=$($RCLONE lsf $RCLONE_REMOTE | grep "$DB_NAME-.*\.gz")

_ts=$TS
file_list=()
for ((i=1;i<=14;i++)); do
  if echo "$REMOTE_BACKUP_FILES" | grep -q "$DB_NAME-$_ts-full.tar.gz"; then
    file_list+=("$DB_NAME-$_ts-full.tar.gz")
    break
  elif echo "$REMOTE_BACKUP_FILES" | grep -q "$DB_NAME-$_ts-incr.gz"; then
    file_list+=("$DB_NAME-$_ts-incr.gz")
  fi
  _ts=$(date -d"$_ts -1 day" "+%Y%m%d")
done

[[ "${file_list[@]}" =~ "full.tar.gz" ]] || raise_error '缺失full.tar.gz全量备份文件'

read -t5 -n1 -p "即将终止mysqld进程并清空$DECOMPRESS_PATH及$OLD_MYSQL_DATA_PATH目录,请确认 (5s后默认为Y) [Y/n]: " choice
choice=${choice:-Y}
echo ""
case $choice in
  Y|y)
    systemctl stop "$SYSTEMD_UNIT"
    rm -rf $OLD_MYSQL_DATA_PATH/* $DECOMPRESS_PATH/*
    rm -f ${CONF_FILES[@]}
    ;;
  N|n)
    exit 0
    ;;
  *)
    raise_error "选项错误,仅支持 [Y/y/N/n]"
    ;;
esac

ulimit -n 1024000

$RCLONE cat $RCLONE_REMOTE/$DB_NAME-$TS-conf.tgz | tar -xz -C $CONF_FOO_PATH

#sed -i "/^server[-_]id/c server_id = $(awk -F. '{print $3$4}' <<< $LAN_IP)" ${CONF_FILES[@]}
sed -i "/^server[-_]id/c server_id = $(awk -F. '{print $4}' <<< $LAN_IP)" ${CONF_FILES[@]}
[[ $(grep -i '^port' ${CONF_FILES[@]} | awk -F'[= ]' '{print $2}' | sort -un | head -n1) != 106* ]] && \
  sed -i "/^innodb[-_]buffer[-_]pool[-_]size/c innodb_buffer_pool_size = $HALF_MEM" ${CONF_FILES[@]}

_result_datadir=$(grep '^datadir' ${CONF_FILES[@]} | awk -F[=\ ] '{print $NF}' | sort -u)
NEW_MYSQL_DATA_PATH=${_result_datadir:-'/var/lib/mysql'}
[[ -e $NEW_MYSQL_DATA_PATH ]] && rm -rf $NEW_MYSQL_DATA_PATH/* || mkdir -p $NEW_MYSQL_DATA_PATH

STREAM_PIDLIST=()
PERPARE_PIDLIST=()
function twait () { tail --pid=$1 -f /dev/null; }

for ((index=${#file_list[*]}-1; index>=0; index--)); do
  filename=${file_list[$index]}
  [[ $index == 0 ]] && unset RESTORE_EXTRA_ARGS
  if [[ $filename =~ full.tar.gz$ ]]; then
    ($RCLONE cat $RCLONE_REMOTE/$filename | gzip -d -c | $STREAM_TOOL -x -C "$NEW_MYSQL_DATA_PATH") &
    STREAM_PIDLIST+=("$!")
    (twait ${STREAM_PIDLIST[-1]}; \
      $BACKUP_TOOL --prepare --target-dir="$NEW_MYSQL_DATA_PATH" $RESTORE_EXTRA_ARGS --parallel=$CPU_CORES --use-memory=$HALF_MEM) &
    PERPARE_PIDLIST+=("$!")
  else
    decompress_tmp_dir="$DECOMPRESS_PATH/$(basename $filename .gz)"; mkdir -p "$decompress_tmp_dir"
    (twait ${STREAM_PIDLIST[-1]}; $RCLONE cat $RCLONE_REMOTE/$filename | gzip -d -c | $STREAM_TOOL -x -C "$decompress_tmp_dir") &
    STREAM_PIDLIST+=("$!")
    (twait ${PERPARE_PIDLIST[-1]}; twait ${STREAM_PIDLIST[-1]}; \
      $BACKUP_TOOL --prepare --target-dir="$NEW_MYSQL_DATA_PATH" $RESTORE_EXTRA_ARGS --incremental-dir="$decompress_tmp_dir" --parallel=$CPU_CORES --use-memory=$HALF_MEM) &
    PERPARE_PIDLIST+=("$!")
  fi
done

wait ${STREAM_PIDLIST[@]} ${PERPARE_PIDLIST[@]} || raise_error "合并失败"

id -u mysql >/dev/null 2>&1 || useradd mysql
chown -R mysql:mysql $NEW_MYSQL_DATA_PATH
systemctl start "$SYSTEMD_UNIT" || raise_error "启动数据库失败"

echo "恢复成功!"

脚本功能:

  • 支持全量 | 增量备份(备份时指定 full | incr);
  • 支持备份到华为云对象存储中;
  • 流式上传到对象存储中,本地不落盘;
  • 备份完成后,会验证备份的有效性;
  • 备份失败发送企业微信告警;
  • 配合定时任务,每周六全量备份,周日至周五增量备份;
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
#!/bin/bash

DB_NAME=$1
BACKUP_TYPE=$2
TS=$(date "+%Y%m%d")
MYSQL_USER="backup"
MYSQL_PASSWORD="abcd1234"
CONF_FOO_PATH="/etc"
CONF_FILES=("my.cnf" "my.cnf.d")
_result_socket=$(grep -s '^socket' $CONF_FOO_PATH/${CONF_FILES[1]}/*.cnf | awk -F[=\ ] '{print $NF}' | sort -u)
PORT=$(grep -s '^port' $CONF_FOO_PATH/${CONF_FILES[1]}/*.cnf | awk -F[=\ ] '{print $NF}' | sort -u)
MYSQL_SOCKET=${_result_socket:-'/var/lib/mysql/mysql.sock'}
LOCAL_BACKUP_DIR="/data/backup"
RCLONE="/usr/local/bin/rclone"
RCLONE_REMOTE="huawei"
RCLONE_BUCKET="backup-bucket/mysql"
WECHAT_WEBHOOK_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

[[ $# != 2 ]] && echo -e "Usage:\n\t$0 (库名) (full|incr)" && exit 1

raise_error() {
  msg="### 备份失败! \n#### $DB_NAME $TS $BACKUP_TYPE\n#### $1"
  for i in {1..3}; do
    curl -s "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=$WECHAT_WEBHOOK_KEY" \
      -H "Content-Type: application/json" \
      -d '{
        "msgtype": "markdown",
        "markdown": {
          "content": "'"$msg"'"
        }
      }'
    [[ $? == 0 ]] && break || sleep $i
  done
  echo "$msg"
  exit 1
}

[[ -z "$MYSQL_SOCKET" ]] && MYSQL_SOCKET="/var/lib/mysql/mysql.sock"

mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" -S "$MYSQL_SOCKET" -e "select 1;" >/dev/null 2>&1 || raise_error "数据库账号或Socket连接失败"

ulimit -n 1024000
backup_start_sec=$(date +%s)

mkdir -p $LOCAL_BACKUP_DIR/{lastTimeCheckPoint,thisTimeCheckPoint,tmpDir}

# 备份版本和配置上传
mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" -S "$MYSQL_SOCKET" -e "select version()\G" | \
awk '/version/{print $2}' | grep -o '[0-9\.]*' > $LOCAL_BACKUP_DIR/${DB_NAME}-${TS}.version
$RCLONE copy $LOCAL_BACKUP_DIR/${DB_NAME}-${TS}.version ${RCLONE_REMOTE}:${RCLONE_BUCKET}/ || raise_error "版本文件上传失败"
rm -f $LOCAL_BACKUP_DIR/${DB_NAME}-${TS}.version

tar -zcf $LOCAL_BACKUP_DIR/${DB_NAME}-${TS}-conf.tgz -C $CONF_FOO_PATH my.cnf my.cnf.d
$RCLONE copy $LOCAL_BACKUP_DIR/${DB_NAME}-${TS}-conf.tgz ${RCLONE_REMOTE}:${RCLONE_BUCKET}/ || raise_error "配置文件上传失败"
rm -f $LOCAL_BACKUP_DIR/${DB_NAME}-${TS}-conf.tgz

BACKUP_OBJECT="${DB_NAME}-${TS}-${BACKUP_TYPE}.tar.gz"

if [[ $BACKUP_TYPE == "full" ]]; then
  mariabackup --backup --slave-info --tmpdir=$LOCAL_BACKUP_DIR/tmpDir \
    --user="$MYSQL_USER" --password="$MYSQL_PASSWORD" --socket="$MYSQL_SOCKET" \
    --extra-lsndir=$LOCAL_BACKUP_DIR/thisTimeCheckPoint \
    --stream=mbstream | gzip -c | $RCLONE rcat --s3-upload-cutoff 64M --s3-chunk-size 128M ${RCLONE_REMOTE}:${RCLONE_BUCKET}/${BACKUP_OBJECT} || raise_error "全量备份或上传失败"

  ### >>>>>>>>>>> 恢复测试 START <<<<<<<<<<<<<<
  echo "开始恢复测试..."
  RESTORE_TEST_DIR="$LOCAL_BACKUP_DIR/restore_test/$PORT"
  rm -rf "$RESTORE_TEST_DIR" && mkdir -p "$RESTORE_TEST_DIR"

  file_list=()
  full_ts=""
  for ((i=0;i<=7;i++)); do
    ts_check=$(date -d "$TS -$i day" +%Y%m%d)
    if $RCLONE lsf ${RCLONE_REMOTE}:${RCLONE_BUCKET}/ | grep -q "^${DB_NAME}-${ts_check}-full.tar.gz$"; then
      full_ts="$ts_check"
      file_list+=("${DB_NAME}-${ts_check}-full.tar.gz")
      break
    fi
  done

  if [[ -z "$full_ts" ]]; then
    raise_error "恢复测试失败:未找到全量备份"
  fi

  for ((i=0;i<=7;i++)); do
    ts_check=$(date -d "$full_ts +$i day" +%Y%m%d)
    if $RCLONE lsf ${RCLONE_REMOTE}:${RCLONE_BUCKET}/ | grep -q "^${DB_NAME}-${ts_check}-incr.tar.gz$"; then
      file_list+=("${DB_NAME}-${ts_check}-incr.tar.gz")
    fi
  done

  echo "下载恢复测试文件:${file_list[@]}"
  for f in "${file_list[@]}"; do
    $RCLONE copy ${RCLONE_REMOTE}:${RCLONE_BUCKET}/$f $RESTORE_TEST_DIR/ || raise_error "恢复测试失败:下载文件 $f 失败"
  done

  echo "开始测试解压和prepare..."
  full_dir="$RESTORE_TEST_DIR/full"
  mkdir -p "$full_dir"
  gzip -dc $RESTORE_TEST_DIR/${file_list[0]} | mbstream -x -C "$full_dir" || raise_error "全量解压失败"
  mariabackup --prepare --target-dir="$full_dir" --apply-log-only || raise_error "全量prepare失败"

  prev_lsn=$(awk -F'= ' '/to_lsn/ {print $2}' $full_dir/xtrabackup_checkpoints)

  for ((i=1;i<${#file_list[@]};i++)); do
    incr_dir="$RESTORE_TEST_DIR/incr_$i"
    mkdir -p "$incr_dir"
    gzip -dc $RESTORE_TEST_DIR/${file_list[$i]} | mbstream -x -C "$incr_dir" || raise_error "增量 $i 解压失败"

    from_lsn=$(awk -F'= ' '/from_lsn/ {print $2}' $incr_dir/xtrabackup_checkpoints)

    if [[ "$from_lsn" != "$prev_lsn" ]]; then
      echo "增量 $i from_lsn ($from_lsn) 与前一个 to_lsn ($prev_lsn) 不一致,跳过"
      continue
    fi

    mariabackup --prepare --target-dir="$full_dir" --incremental-dir="$incr_dir" --apply-log-only || raise_error "增量 $i prepare失败"
    prev_lsn=$(awk -F'= ' '/to_lsn/ {print $2}' $incr_dir/xtrabackup_checkpoints)
  done

  mariabackup --prepare --target-dir="$full_dir" || raise_error "最后apply-log失败"
  echo "恢复测试通过"
  rm -rf "$RESTORE_TEST_DIR"
  ### >>>>>>>>>>> 恢复测试 END <<<<<<<<<<<<<<

elif [[ $BACKUP_TYPE == "incr" ]]; then
  mariabackup --backup --slave-info --tmpdir=$LOCAL_BACKUP_DIR/tmpDir \
    --user="$MYSQL_USER" --password="$MYSQL_PASSWORD" --socket="$MYSQL_SOCKET" \
    --extra-lsndir=$LOCAL_BACKUP_DIR/thisTimeCheckPoint \
    --incremental-basedir=$LOCAL_BACKUP_DIR/lastTimeCheckPoint \
    --stream=mbstream | gzip -c | $RCLONE rcat ${RCLONE_REMOTE}:${RCLONE_BUCKET}/${BACKUP_OBJECT} || raise_error "增量备份或上传失败"
else
  raise_error "参数错误: 仅支持 full 或 incr"
fi

backup_end_sec=$(date +%s)
backup_spent_sec=$[ $backup_end_sec - $backup_start_sec ]
START=$(date -d@$backup_start_sec +"%Y-%m-%d %H:%M:%S")
END=$(date -d@$backup_end_sec +"%Y-%m-%d %H:%M:%S")
SPENT="$((backup_spent_sec / 3600))小时 $(((backup_spent_sec / 60) % 60))分钟 $((backup_spent_sec % 60))秒"
SIZE=$(/usr/local/bin/rclone lsf ${RCLONE_REMOTE}:${RCLONE_BUCKET}/ --format "ps" | grep "^${DB_NAME}-${TS}-${BACKUP_TYPE}.tar.gz;" | awk -F';' '{ printf "%d", $2 / 1024 / 1024 / 1024 }')

rm -rf $LOCAL_BACKUP_DIR/lastTimeCheckPoint/*
cp -a $LOCAL_BACKUP_DIR/thisTimeCheckPoint/* $LOCAL_BACKUP_DIR/lastTimeCheckPoint/

echo "备份成功: $DB_NAME-$TS-$BACKUP_TYPE 已上传到OBS"
echo -e "备份成功!\n Size: $SIZE GB \nStart: $START\n  End: $END\nSpent: $SPENT"

设置定时任务

一、安装

1.1 apt 方式安装

1
2
3
4
5
# 验证仓库版本
apt-cache madison containerd

# 安装 containerd
apt install containerd=1.6.12-0ubuntu1~22.04.1

生成 containerd 配置文件

1
2
3
4
5
6
root@k8s-node02:~# mkdir -p /etc/containerd/

root@k8s-node02:~# containerd config default > /etc/containerd/config.toml

root@k8s-node02:~# ll /etc/containerd/config.toml
-rw-r--r-- 1 root root 6994 Apr  9 13:36 /etc/containerd/config.toml

启动 containerd

1
systemctl enable --now containerd

通过命令行测试下载镜像

下载安装 ntttcp 工具

ntttcp 工具是微软开源的

1
2
3
4
5
git clone https://github.com/Microsoft/ntttcp-for-linux

cd ntttcp-for-linux/src/

make && make install

测试

1
2
3
4
5
# 在接收方运行,测试时间300秒,不指定-t,默认60秒
ntttcp -r -t 300

# 在发送方运行,测试时间300秒,不指定-t,默认60秒
ntttcp -s -t 300 接收方的IP地址

服务说明

RustDesk 是一个远程控制工具,开源跨平台,可以使用官方的服务器,也可以自建服务器使用

API/UI:监听21114(TCP)

hbbs:RustDesk ID 注册服务器,监听21115(TCP)、21116(TCP)、21118(TCP)

hbbr:RustDesk 中继服务器,监听21117(TCP)、21119(TCP)

21115 端口是 hbbs 用于 NAT 类型测试

21116 端口要同时开启 TCP 和 UDP,UDP 是 hbbs 用于 ID 注册与心跳服务,TCP 是 hbbs 用于 TCP 打洞与连接服务

21117 端口是 hbbr 用于中继服务

21118 和 21119 是为了支持网页客户端,如果不需要网页客户端,该端口可以不用开启

如果配置不加密的话,任何人配置上了我的服务器地址都可以使用我的服务了,建议开启加密连接。在启动 hbbr 和 hbbs时,增加启动参数 -k _,第一次启动时,会在目录下生成密钥对。

部署 WireGuard VPN,用于访问公司内网。

安装 Docker

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
# Ubuntu/Debian
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

echo \
"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu \
$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

apt update

apt install docker-ce -y

# CentOS/RHEL
curl https://download.docker.com/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker-ce.repo

dnf install docker-ce

安装 docker-compose

1
curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

安装 WireGuard

1
2
3
4
5
# ubuntu 22
apt install wireguard-tools wireguard

# Rocky Linux 9
dnf install wireguard-tools

安装 wg-ui

vim docker-compose.yml

环境准备

  • 检查系统是否支持 KVM

    1
    2
    
    # 安装 cpu-checker
    apt install cpu-checker -y

    执行kvm-ok,正常情况,可以看到如下输出

    1
    2
    3
    
    root@ubuntu:~# kvm-ok
    INFO: /dev/kvm exists
    KVM acceleration can be used