326 lines
11 KiB
Bash
Executable File
326 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
WORK_DIR=""
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--work-dir)
|
||
WORK_DIR="${2:-}"
|
||
shift 2
|
||
;;
|
||
*)
|
||
echo "[ERROR] unsupported argument: $1" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
done
|
||
|
||
if [[ -z "${WORK_DIR}" ]]; then
|
||
echo "[ERROR] --work-dir is required" >&2
|
||
exit 1
|
||
fi
|
||
|
||
PAYLOAD_DIR="${WORK_DIR}/payload"
|
||
STATUS_FILE="${WORK_DIR}/status.json"
|
||
LOG_FILE="${WORK_DIR}/restore.log"
|
||
MANIFEST="${PAYLOAD_DIR}/payload-manifest.json"
|
||
# Java 解包阶段会把 payload-manifest.json 转换成下面三份 TSV。
|
||
# shell 阶段只消费 TSV，避免在 bash 中解析复杂 JSON。
|
||
FILES_PLAN="${WORK_DIR}/restore-files.tsv"
|
||
DATABASES_PLAN="${WORK_DIR}/restore-databases.tsv"
|
||
MQ_PLAN="${WORK_DIR}/restore-mq.tsv"
|
||
ENV_FILE="${WORK_DIR}/restore-env.sh"
|
||
STOP_COMMANDS_PLAN="${WORK_DIR}/restore-stop-commands.tsv"
|
||
START_COMMANDS_PLAN="${WORK_DIR}/restore-start-commands.tsv"
|
||
SERVICES_STOPPED_FOR_RESTORE=false
|
||
SERVICES_STARTING_AFTER_FAILURE=false
|
||
|
||
json_escape() {
|
||
local value="$1"
|
||
value="${value//\\/\\\\}"
|
||
value="${value//\"/\\\"}"
|
||
value="${value//$'\n'/ }"
|
||
printf '%s' "${value}"
|
||
}
|
||
|
||
write_status() {
|
||
local status="$1"
|
||
local phase="$2"
|
||
local message="$3"
|
||
# status.json 是 Java 查询接口和运维排障共同读取的轻量状态文件。
|
||
# 即使 TMS 服务被 stop，脚本仍会持续刷新这个文件，服务重启后可继续展示结果。
|
||
printf '{"status":"%s","phase":"%s","message":"%s","updatedAt":"%s"}\n' \
|
||
"$(json_escape "${status}")" \
|
||
"$(json_escape "${phase}")" \
|
||
"$(json_escape "${message}")" \
|
||
"$(date '+%Y-%m-%dT%H:%M:%S%z')" > "${STATUS_FILE}"
|
||
}
|
||
|
||
log() {
|
||
printf '[%s] %s\n' "$(date '+%F %T')" "$1" >> "${LOG_FILE}"
|
||
}
|
||
|
||
fail() {
|
||
local message="$1"
|
||
local phase="${2:-FAILED}"
|
||
log "[ERROR] ${message}"
|
||
write_status "FAILED" "${phase}" "${message}"
|
||
attempt_service_start_after_failure "${phase}"
|
||
exit 1
|
||
}
|
||
|
||
require_env() {
|
||
local name="$1"
|
||
local value="${!name:-}"
|
||
if [[ -z "${value}" ]]; then
|
||
fail "required environment variable ${name} is empty" "${2:-PRECHECK}"
|
||
fi
|
||
}
|
||
|
||
restore_files() {
|
||
[[ -f "${FILES_PLAN}" ]] || return 0
|
||
# restore-files.tsv 每行格式：payload内相对路径<TAB>目标绝对路径。
|
||
# 目标路径白名单已在 Java 解包阶段校验，这里只负责按计划覆盖文件。
|
||
while IFS=$'\t' read -r entry_path restore_path || [[ -n "${entry_path:-}" ]]; do
|
||
[[ -n "${entry_path:-}" ]] || continue
|
||
[[ -n "${restore_path:-}" ]] || fail "restore file target is empty for ${entry_path}" "FILE"
|
||
local source_file="${PAYLOAD_DIR}/${entry_path}"
|
||
[[ -f "${source_file}" ]] || fail "restore source file not found: ${entry_path}" "FILE"
|
||
mkdir -p "$(dirname "${restore_path}")"
|
||
cp -f "${source_file}" "${restore_path}"
|
||
log "restored file: ${restore_path}"
|
||
done < "${FILES_PLAN}"
|
||
}
|
||
|
||
restore_databases() {
|
||
[[ -f "${DATABASES_PLAN}" ]] || return 0
|
||
[[ -s "${DATABASES_PLAN}" ]] || return 0
|
||
local script="${DB_RESTORE_SCRIPT_PATH:-}"
|
||
[[ -n "${script}" && -x "${script}" ]] || fail "DB_RESTORE_SCRIPT_PATH is not executable" "DATABASE"
|
||
# 数据库连接参数来自 Java 生成的 restore-env.sh；实际导入命令下沉到 restore-db.sh，
|
||
# 方便现场根据 MySQL/TiDB 客户端和账号策略调整实现。
|
||
require_env "DB_HOST" "DATABASE"
|
||
require_env "DB_PORT" "DATABASE"
|
||
require_env "DB_USER" "DATABASE"
|
||
local mysql_path="${MYSQL_PATH:-mysql}"
|
||
local db_password="${DB_PASSWORD:-}"
|
||
local timeout_seconds="${RESTORE_DB_TIMEOUT_SECONDS:-300}"
|
||
[[ "${timeout_seconds}" =~ ^[0-9]+$ ]] || timeout_seconds=300
|
||
(( timeout_seconds > 0 )) || timeout_seconds=300
|
||
while IFS=$'\t' read -r entry_path db_name || [[ -n "${entry_path:-}" ]]; do
|
||
[[ -n "${entry_path:-}" ]] || continue
|
||
[[ -n "${db_name:-}" ]] || fail "database name is empty for ${entry_path}" "DATABASE"
|
||
local sql_file="${PAYLOAD_DIR}/${entry_path}"
|
||
[[ -f "${sql_file}" ]] || fail "sql file not found: ${entry_path}" "DATABASE"
|
||
"${script}" \
|
||
--mysql-path "${mysql_path}" \
|
||
--host "${DB_HOST}" \
|
||
--port "${DB_PORT}" \
|
||
--username "${DB_USER}" \
|
||
--password "${db_password}" \
|
||
--db-name "${db_name}" \
|
||
--sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1 &
|
||
local db_pid=$!
|
||
local timeout_marker="${WORK_DIR}/.db-restore-timeout-${db_pid}"
|
||
rm -f "${timeout_marker}" || true
|
||
(
|
||
sleep "${timeout_seconds}"
|
||
if kill -0 "${db_pid}" >/dev/null 2>&1; then
|
||
printf 'timeout' > "${timeout_marker}"
|
||
kill "${db_pid}" >/dev/null 2>&1 || true
|
||
sleep 1
|
||
kill -9 "${db_pid}" >/dev/null 2>&1 || true
|
||
fi
|
||
) &
|
||
local watchdog_pid=$!
|
||
local db_exit=0
|
||
wait "${db_pid}" || db_exit=$?
|
||
kill "${watchdog_pid}" >/dev/null 2>&1 || true
|
||
wait "${watchdog_pid}" >/dev/null 2>&1 || true
|
||
if [[ -f "${timeout_marker}" ]]; then
|
||
rm -f "${timeout_marker}" || true
|
||
fail "database restore timed out after ${timeout_seconds}s: ${db_name}" "DATABASE"
|
||
fi
|
||
rm -f "${timeout_marker}" || true
|
||
if [[ "${db_exit}" -ne 0 ]]; then
|
||
fail "database restore failed: ${db_name}" "DATABASE"
|
||
fi
|
||
log "restored database: ${db_name}"
|
||
done < "${DATABASES_PLAN}"
|
||
}
|
||
|
||
replay_mq() {
|
||
[[ -f "${MQ_PLAN}" ]] || return 0
|
||
[[ -s "${MQ_PLAN}" ]] || return 0
|
||
local script="${MQ_REPLAY_SCRIPT_PATH:-}"
|
||
[[ -n "${script}" && -x "${script}" ]] || fail "MQ_REPLAY_SCRIPT_PATH is not executable" "MQ_REPLAY"
|
||
# MQ replay 当前只接收 Java 生成的上下文文件，具体恢复 RabbitMQ/TLQ/CFMQ 的命令由现场脚本实现。
|
||
while IFS= read -r entry_path || [[ -n "${entry_path:-}" ]]; do
|
||
[[ -n "${entry_path:-}" ]] || continue
|
||
local context_file="${PAYLOAD_DIR}/${entry_path}"
|
||
[[ -f "${context_file}" ]] || fail "mq replay context not found: ${entry_path}" "MQ_REPLAY"
|
||
"${script}" --context-file "${context_file}" --work-dir "${WORK_DIR}" >> "${LOG_FILE}" 2>&1
|
||
log "mq replay applied"
|
||
done < "${MQ_PLAN}"
|
||
}
|
||
|
||
run_service_commands() {
|
||
local plan_file="$1"
|
||
local phase="$2"
|
||
[[ -f "${plan_file}" ]] || return 1
|
||
[[ -s "${plan_file}" ]] || return 1
|
||
while IFS=$'\t' read -r code required command || [[ -n "${code:-}" ]]; do
|
||
[[ -n "${code:-}" ]] || continue
|
||
[[ -n "${command:-}" ]] || fail "service command is empty: ${code}" "${phase}"
|
||
log "${phase} ${code}: ${command}"
|
||
if bash -lc "${command}" >> "${LOG_FILE}" 2>&1; then
|
||
log "${phase} ${code}: success"
|
||
elif [[ "${required}" == "true" ]]; then
|
||
fail "${phase} command failed: ${code}" "${phase}"
|
||
else
|
||
log "[WARN] ${phase} ${code}: failed but ignored"
|
||
fi
|
||
done < "${plan_file}"
|
||
return 0
|
||
}
|
||
|
||
run_service_commands_best_effort() {
|
||
local plan_file="$1"
|
||
local phase="$2"
|
||
[[ -f "${plan_file}" ]] || return 1
|
||
[[ -s "${plan_file}" ]] || return 1
|
||
while IFS=$'\t' read -r code required command || [[ -n "${code:-}" ]]; do
|
||
[[ -n "${code:-}" ]] || continue
|
||
if [[ -z "${command:-}" ]]; then
|
||
log "[WARN] ${phase} ${code}: command is empty"
|
||
continue
|
||
fi
|
||
log "${phase} ${code}: ${command}"
|
||
if bash -lc "${command}" >> "${LOG_FILE}" 2>&1; then
|
||
log "${phase} ${code}: success"
|
||
else
|
||
log "[WARN] ${phase} ${code}: failed while recovering from restore failure"
|
||
fi
|
||
done < "${plan_file}"
|
||
return 0
|
||
}
|
||
|
||
attempt_service_start_after_failure() {
|
||
local failed_phase="${1:-FAILED}"
|
||
if [[ "${SERVICES_STOPPED_FOR_RESTORE}" != "true" ]]; then
|
||
return 0
|
||
fi
|
||
if [[ "${SERVICES_STARTING_AFTER_FAILURE}" == "true" || "${failed_phase}" == "SERVICE_START" ]]; then
|
||
return 0
|
||
fi
|
||
SERVICES_STARTING_AFTER_FAILURE=true
|
||
log "SERVICE_START_AFTER_FAILURE due to ${failed_phase}"
|
||
if run_service_commands_best_effort "${START_COMMANDS_PLAN}" "SERVICE_START_AFTER_FAILURE"; then
|
||
SERVICES_STOPPED_FOR_RESTORE=false
|
||
return 0
|
||
fi
|
||
if [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
|
||
"${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 \
|
||
&& log "SERVICE_START_AFTER_FAILURE TMS fallback: success" \
|
||
|| log "[WARN] SERVICE_START_AFTER_FAILURE TMS fallback: failed"
|
||
SERVICES_STOPPED_FOR_RESTORE=false
|
||
fi
|
||
}
|
||
|
||
health_check() {
|
||
local url="${HEALTH_CHECK_URL:-}"
|
||
[[ -n "${url}" ]] || {
|
||
log "health check skipped: HEALTH_CHECK_URL is empty"
|
||
return 0
|
||
}
|
||
command -v curl >/dev/null 2>&1 || fail "curl command not found for health check" "HEALTH_CHECK"
|
||
local timeout="${HEALTH_CHECK_TIMEOUT_SECONDS:-5}"
|
||
local max_wait="${HEALTH_CHECK_MAX_WAIT_SECONDS:-180}"
|
||
local interval="${HEALTH_CHECK_RETRY_INTERVAL_SECONDS:-5}"
|
||
[[ "${max_wait}" =~ ^[0-9]+$ ]] || max_wait=180
|
||
[[ "${interval}" =~ ^[0-9]+$ ]] || interval=5
|
||
local start_time
|
||
start_time="$(date +%s)"
|
||
local attempt=1
|
||
local response=""
|
||
local curl_output
|
||
while true; do
|
||
curl_output="$(curl -fsS --max-time "${timeout}" "${url}" 2>&1)" && response="${curl_output}" || {
|
||
log "[WARN] health check attempt ${attempt} failed: ${curl_output}"
|
||
response=""
|
||
}
|
||
if [[ "${response}" == *"UP"* ]]; then
|
||
log "health check passed after ${attempt} attempt(s)"
|
||
return 0
|
||
fi
|
||
if [[ -n "${response}" ]]; then
|
||
log "[WARN] health check attempt ${attempt} did not return UP: ${response}"
|
||
fi
|
||
local now
|
||
now="$(date +%s)"
|
||
if (( now - start_time >= max_wait )); then
|
||
fail "health check request failed after ${max_wait}s" "HEALTH_CHECK"
|
||
fi
|
||
if (( interval > 0 )); then
|
||
sleep "${interval}"
|
||
fi
|
||
attempt=$((attempt + 1))
|
||
done
|
||
}
|
||
|
||
mkdir -p "${WORK_DIR}"
|
||
touch "${LOG_FILE}"
|
||
chmod 700 "${WORK_DIR}" || true
|
||
|
||
if [[ -f "${ENV_FILE}" ]]; then
|
||
# shellcheck disable=SC1090
|
||
# restore-env.sh 由 LightweightRestoreApplierLauncher 生成，包含脚本路径、数据库连接和健康检查配置。
|
||
source "${ENV_FILE}"
|
||
fi
|
||
|
||
[[ -f "${MANIFEST}" ]] || fail "payload manifest not found" "PRECHECK"
|
||
|
||
delay_seconds="${RESTORE_APPLY_DELAY_SECONDS:-3}"
|
||
if [[ "${delay_seconds}" =~ ^[0-9]+$ ]] && [[ "${delay_seconds}" -gt 0 ]]; then
|
||
write_status "RUNNING" "APPLY_DELAY" "restore apply will start"
|
||
log "APPLY_DELAY ${delay_seconds}s"
|
||
sleep "${delay_seconds}"
|
||
fi
|
||
|
||
# 从这里开始进入停机恢复窗口。TMS_SCRIPT_PATH stop/start 允许失败后继续，
|
||
# 因为某些部署环境可能由 systemd/外部平台接管服务状态，但文件/数据库/MQ 失败必须中断。
|
||
write_status "RUNNING" "SERVICE_STOP" "stopping services"
|
||
log "SERVICE_STOP"
|
||
SERVICES_STOPPED_FOR_RESTORE=true
|
||
if ! run_service_commands "${STOP_COMMANDS_PLAN}" "SERVICE_STOP" \
|
||
&& [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
|
||
"${TMS_SCRIPT_PATH}" stop >> "${LOG_FILE}" 2>&1 || true
|
||
fi
|
||
|
||
write_status "RUNNING" "FILE" "restoring files"
|
||
log "FILE"
|
||
restore_files
|
||
|
||
write_status "RUNNING" "DATABASE" "restoring databases"
|
||
log "DATABASE"
|
||
restore_databases
|
||
|
||
write_status "RUNNING" "MQ_REPLAY" "replaying mq configuration"
|
||
log "MQ_REPLAY"
|
||
replay_mq
|
||
|
||
write_status "RUNNING" "SERVICE_START" "starting services"
|
||
log "SERVICE_START"
|
||
if ! run_service_commands "${START_COMMANDS_PLAN}" "SERVICE_START" \
|
||
&& [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
|
||
"${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 || true
|
||
fi
|
||
SERVICES_STOPPED_FOR_RESTORE=false
|
||
|
||
write_status "RUNNING" "HEALTH_CHECK" "checking health"
|
||
log "HEALTH_CHECK"
|
||
health_check
|
||
|
||
write_status "SUCCESS" "DONE" "restore applied"
|
||
log "DONE"
|