#!/usr/bin/env bash set -euo pipefail WORK_DIR="" while [[ $# -gt 0 ]]; do case "$1" in --work-dir) WORK_DIR="${2:-}" shift 2 ;; *) echo "[ERROR] unsupported argument: $1" >&2 exit 1 ;; esac done if [[ -z "${WORK_DIR}" ]]; then echo "[ERROR] --work-dir is required" >&2 exit 1 fi PAYLOAD_DIR="${WORK_DIR}/payload" STATUS_FILE="${WORK_DIR}/status.json" LOG_FILE="${WORK_DIR}/restore.log" MANIFEST="${PAYLOAD_DIR}/payload-manifest.json" # Java 解包阶段会把 payload-manifest.json 转换成下面三份 TSV。 # shell 阶段只消费 TSV,避免在 bash 中解析复杂 JSON。 FILES_PLAN="${WORK_DIR}/restore-files.tsv" DATABASES_PLAN="${WORK_DIR}/restore-databases.tsv" MQ_PLAN="${WORK_DIR}/restore-mq.tsv" ENV_FILE="${WORK_DIR}/restore-env.sh" STOP_COMMANDS_PLAN="${WORK_DIR}/restore-stop-commands.tsv" START_COMMANDS_PLAN="${WORK_DIR}/restore-start-commands.tsv" json_escape() { local value="$1" value="${value//\\/\\\\}" value="${value//\"/\\\"}" value="${value//$'\n'/ }" printf '%s' "${value}" } write_status() { local status="$1" local phase="$2" local message="$3" # status.json 是 Java 查询接口和运维排障共同读取的轻量状态文件。 # 即使 TMS 服务被 stop,脚本仍会持续刷新这个文件,服务重启后可继续展示结果。 printf '{"status":"%s","phase":"%s","message":"%s","updatedAt":"%s"}\n' \ "$(json_escape "${status}")" \ "$(json_escape "${phase}")" \ "$(json_escape "${message}")" \ "$(date '+%Y-%m-%dT%H:%M:%S%z')" > "${STATUS_FILE}" } log() { printf '[%s] %s\n' "$(date '+%F %T')" "$1" >> "${LOG_FILE}" } fail() { local message="$1" local phase="${2:-FAILED}" log "[ERROR] ${message}" write_status "FAILED" "${phase}" "${message}" exit 1 } require_env() { local name="$1" local value="${!name:-}" if [[ -z "${value}" ]]; then fail "required environment variable ${name} is empty" "${2:-PRECHECK}" fi } restore_files() { [[ -f "${FILES_PLAN}" ]] || return 0 # restore-files.tsv 每行格式:payload内相对路径目标绝对路径。 # 目标路径白名单已在 Java 解包阶段校验,这里只负责按计划覆盖文件。 while IFS=$'\t' read -r entry_path restore_path || [[ -n "${entry_path:-}" ]]; do [[ -n "${entry_path:-}" ]] || continue [[ -n "${restore_path:-}" ]] || fail "restore file target is empty for ${entry_path}" "FILE" local source_file="${PAYLOAD_DIR}/${entry_path}" [[ -f "${source_file}" ]] || fail "restore source file not found: ${entry_path}" "FILE" mkdir -p "$(dirname "${restore_path}")" cp -f "${source_file}" "${restore_path}" log "restored file: ${restore_path}" done < "${FILES_PLAN}" } restore_databases() { [[ -f "${DATABASES_PLAN}" ]] || return 0 [[ -s "${DATABASES_PLAN}" ]] || return 0 local script="${DB_RESTORE_SCRIPT_PATH:-}" [[ -n "${script}" && -x "${script}" ]] || fail "DB_RESTORE_SCRIPT_PATH is not executable" "DATABASE" # 数据库连接参数来自 Java 生成的 restore-env.sh;实际导入命令下沉到 restore-db.sh, # 方便现场根据 MySQL/TiDB 客户端和账号策略调整实现。 require_env "DB_HOST" "DATABASE" require_env "DB_PORT" "DATABASE" require_env "DB_USER" "DATABASE" local mysql_path="${MYSQL_PATH:-mysql}" local db_password="${DB_PASSWORD:-}" while IFS=$'\t' read -r entry_path db_name || [[ -n "${entry_path:-}" ]]; do [[ -n "${entry_path:-}" ]] || continue [[ -n "${db_name:-}" ]] || fail "database name is empty for ${entry_path}" "DATABASE" local sql_file="${PAYLOAD_DIR}/${entry_path}" [[ -f "${sql_file}" ]] || fail "sql file not found: ${entry_path}" "DATABASE" "${script}" \ --mysql-path "${mysql_path}" \ --host "${DB_HOST}" \ --port "${DB_PORT}" \ --username "${DB_USER}" \ --password "${db_password}" \ --db-name "${db_name}" \ --sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1 log "restored database: ${db_name}" done < "${DATABASES_PLAN}" } replay_mq() { [[ -f "${MQ_PLAN}" ]] || return 0 [[ -s "${MQ_PLAN}" ]] || return 0 local script="${MQ_REPLAY_SCRIPT_PATH:-}" [[ -n "${script}" && -x "${script}" ]] || fail "MQ_REPLAY_SCRIPT_PATH is not executable" "MQ_REPLAY" # MQ replay 当前只接收 Java 生成的上下文文件,具体恢复 RabbitMQ/TLQ/CFMQ 的命令由现场脚本实现。 while IFS= read -r entry_path || [[ -n "${entry_path:-}" ]]; do [[ -n "${entry_path:-}" ]] || continue local context_file="${PAYLOAD_DIR}/${entry_path}" [[ -f "${context_file}" ]] || fail "mq replay context not found: ${entry_path}" "MQ_REPLAY" "${script}" --context-file "${context_file}" --work-dir "${WORK_DIR}" >> "${LOG_FILE}" 2>&1 log "mq replay applied" done < "${MQ_PLAN}" } run_service_commands() { local plan_file="$1" local phase="$2" [[ -f "${plan_file}" ]] || return 1 [[ -s "${plan_file}" ]] || return 1 while IFS=$'\t' read -r code required command || [[ -n "${code:-}" ]]; do [[ -n "${code:-}" ]] || continue [[ -n "${command:-}" ]] || fail "service command is empty: ${code}" "${phase}" log "${phase} ${code}: ${command}" if bash -lc "${command}" >> "${LOG_FILE}" 2>&1; then log "${phase} ${code}: success" elif [[ "${required}" == "true" ]]; then fail "${phase} command failed: ${code}" "${phase}" else log "[WARN] ${phase} ${code}: failed but ignored" fi done < "${plan_file}" return 0 } health_check() { local url="${HEALTH_CHECK_URL:-}" [[ -n "${url}" ]] || { log "health check skipped: HEALTH_CHECK_URL is empty" return 0 } command -v curl >/dev/null 2>&1 || fail "curl command not found for health check" "HEALTH_CHECK" local timeout="${HEALTH_CHECK_TIMEOUT_SECONDS:-5}" local response response="$(curl -fsS --max-time "${timeout}" "${url}")" || fail "health check request failed" "HEALTH_CHECK" if [[ "${response}" != *"UP"* ]]; then fail "health check did not return UP" "HEALTH_CHECK" fi log "health check passed" } mkdir -p "${WORK_DIR}" touch "${LOG_FILE}" chmod 700 "${WORK_DIR}" || true if [[ -f "${ENV_FILE}" ]]; then # shellcheck disable=SC1090 # restore-env.sh 由 LightweightRestoreApplierLauncher 生成,包含脚本路径、数据库连接和健康检查配置。 source "${ENV_FILE}" fi [[ -f "${MANIFEST}" ]] || fail "payload manifest not found" "PRECHECK" delay_seconds="${RESTORE_APPLY_DELAY_SECONDS:-3}" if [[ "${delay_seconds}" =~ ^[0-9]+$ ]] && [[ "${delay_seconds}" -gt 0 ]]; then write_status "RUNNING" "APPLY_DELAY" "restore apply will start" log "APPLY_DELAY ${delay_seconds}s" sleep "${delay_seconds}" fi # 从这里开始进入停机恢复窗口。TMS_SCRIPT_PATH stop/start 允许失败后继续, # 因为某些部署环境可能由 systemd/外部平台接管服务状态,但文件/数据库/MQ 失败必须中断。 write_status "RUNNING" "SERVICE_STOP" "stopping services" log "SERVICE_STOP" if ! run_service_commands "${STOP_COMMANDS_PLAN}" "SERVICE_STOP" \ && [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then "${TMS_SCRIPT_PATH}" stop >> "${LOG_FILE}" 2>&1 || true fi write_status "RUNNING" "FILE" "restoring files" log "FILE" restore_files write_status "RUNNING" "DATABASE" "restoring databases" log "DATABASE" restore_databases write_status "RUNNING" "MQ_REPLAY" "replaying mq configuration" log "MQ_REPLAY" replay_mq write_status "RUNNING" "SERVICE_START" "starting services" log "SERVICE_START" if ! run_service_commands "${START_COMMANDS_PLAN}" "SERVICE_START" \ && [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then "${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 || true fi write_status "RUNNING" "HEALTH_CHECK" "checking health" log "HEALTH_CHECK" health_check write_status "SUCCESS" "DONE" "restore applied" log "DONE"