fix:排除资源备份表
This commit is contained in:
@@ -32,6 +32,8 @@ MQ_PLAN="${WORK_DIR}/restore-mq.tsv"
|
||||
ENV_FILE="${WORK_DIR}/restore-env.sh"
|
||||
STOP_COMMANDS_PLAN="${WORK_DIR}/restore-stop-commands.tsv"
|
||||
START_COMMANDS_PLAN="${WORK_DIR}/restore-start-commands.tsv"
|
||||
SERVICES_STOPPED_FOR_RESTORE=false
|
||||
SERVICES_STARTING_AFTER_FAILURE=false
|
||||
|
||||
json_escape() {
|
||||
local value="$1"
|
||||
@@ -63,6 +65,7 @@ fail() {
|
||||
local phase="${2:-FAILED}"
|
||||
log "[ERROR] ${message}"
|
||||
write_status "FAILED" "${phase}" "${message}"
|
||||
attempt_service_start_after_failure "${phase}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -101,6 +104,9 @@ restore_databases() {
|
||||
require_env "DB_USER" "DATABASE"
|
||||
local mysql_path="${MYSQL_PATH:-mysql}"
|
||||
local db_password="${DB_PASSWORD:-}"
|
||||
local timeout_seconds="${RESTORE_DB_TIMEOUT_SECONDS:-300}"
|
||||
[[ "${timeout_seconds}" =~ ^[0-9]+$ ]] || timeout_seconds=300
|
||||
(( timeout_seconds > 0 )) || timeout_seconds=300
|
||||
while IFS=$'\t' read -r entry_path db_name || [[ -n "${entry_path:-}" ]]; do
|
||||
[[ -n "${entry_path:-}" ]] || continue
|
||||
[[ -n "${db_name:-}" ]] || fail "database name is empty for ${entry_path}" "DATABASE"
|
||||
@@ -113,7 +119,32 @@ restore_databases() {
|
||||
--username "${DB_USER}" \
|
||||
--password "${db_password}" \
|
||||
--db-name "${db_name}" \
|
||||
--sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1
|
||||
--sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1 &
|
||||
local db_pid=$!
|
||||
local timeout_marker="${WORK_DIR}/.db-restore-timeout-${db_pid}"
|
||||
rm -f "${timeout_marker}" || true
|
||||
(
|
||||
sleep "${timeout_seconds}"
|
||||
if kill -0 "${db_pid}" >/dev/null 2>&1; then
|
||||
printf 'timeout' > "${timeout_marker}"
|
||||
kill "${db_pid}" >/dev/null 2>&1 || true
|
||||
sleep 1
|
||||
kill -9 "${db_pid}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
) &
|
||||
local watchdog_pid=$!
|
||||
local db_exit=0
|
||||
wait "${db_pid}" || db_exit=$?
|
||||
kill "${watchdog_pid}" >/dev/null 2>&1 || true
|
||||
wait "${watchdog_pid}" >/dev/null 2>&1 || true
|
||||
if [[ -f "${timeout_marker}" ]]; then
|
||||
rm -f "${timeout_marker}" || true
|
||||
fail "database restore timed out after ${timeout_seconds}s: ${db_name}" "DATABASE"
|
||||
fi
|
||||
rm -f "${timeout_marker}" || true
|
||||
if [[ "${db_exit}" -ne 0 ]]; then
|
||||
fail "database restore failed: ${db_name}" "DATABASE"
|
||||
fi
|
||||
log "restored database: ${db_name}"
|
||||
done < "${DATABASES_PLAN}"
|
||||
}
|
||||
@@ -153,6 +184,49 @@ run_service_commands() {
|
||||
return 0
|
||||
}
|
||||
|
||||
run_service_commands_best_effort() {
|
||||
local plan_file="$1"
|
||||
local phase="$2"
|
||||
[[ -f "${plan_file}" ]] || return 1
|
||||
[[ -s "${plan_file}" ]] || return 1
|
||||
while IFS=$'\t' read -r code required command || [[ -n "${code:-}" ]]; do
|
||||
[[ -n "${code:-}" ]] || continue
|
||||
if [[ -z "${command:-}" ]]; then
|
||||
log "[WARN] ${phase} ${code}: command is empty"
|
||||
continue
|
||||
fi
|
||||
log "${phase} ${code}: ${command}"
|
||||
if bash -lc "${command}" >> "${LOG_FILE}" 2>&1; then
|
||||
log "${phase} ${code}: success"
|
||||
else
|
||||
log "[WARN] ${phase} ${code}: failed while recovering from restore failure"
|
||||
fi
|
||||
done < "${plan_file}"
|
||||
return 0
|
||||
}
|
||||
|
||||
attempt_service_start_after_failure() {
|
||||
local failed_phase="${1:-FAILED}"
|
||||
if [[ "${SERVICES_STOPPED_FOR_RESTORE}" != "true" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "${SERVICES_STARTING_AFTER_FAILURE}" == "true" || "${failed_phase}" == "SERVICE_START" ]]; then
|
||||
return 0
|
||||
fi
|
||||
SERVICES_STARTING_AFTER_FAILURE=true
|
||||
log "SERVICE_START_AFTER_FAILURE due to ${failed_phase}"
|
||||
if run_service_commands_best_effort "${START_COMMANDS_PLAN}" "SERVICE_START_AFTER_FAILURE"; then
|
||||
SERVICES_STOPPED_FOR_RESTORE=false
|
||||
return 0
|
||||
fi
|
||||
if [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
|
||||
"${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 \
|
||||
&& log "SERVICE_START_AFTER_FAILURE TMS fallback: success" \
|
||||
|| log "[WARN] SERVICE_START_AFTER_FAILURE TMS fallback: failed"
|
||||
SERVICES_STOPPED_FOR_RESTORE=false
|
||||
fi
|
||||
}
|
||||
|
||||
health_check() {
|
||||
local url="${HEALTH_CHECK_URL:-}"
|
||||
[[ -n "${url}" ]] || {
|
||||
@@ -217,6 +291,7 @@ fi
|
||||
# 因为某些部署环境可能由 systemd/外部平台接管服务状态,但文件/数据库/MQ 失败必须中断。
|
||||
write_status "RUNNING" "SERVICE_STOP" "stopping services"
|
||||
log "SERVICE_STOP"
|
||||
SERVICES_STOPPED_FOR_RESTORE=true
|
||||
if ! run_service_commands "${STOP_COMMANDS_PLAN}" "SERVICE_STOP" \
|
||||
&& [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
|
||||
"${TMS_SCRIPT_PATH}" stop >> "${LOG_FILE}" 2>&1 || true
|
||||
@@ -240,6 +315,7 @@ if ! run_service_commands "${START_COMMANDS_PLAN}" "SERVICE_START" \
|
||||
&& [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
|
||||
"${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 || true
|
||||
fi
|
||||
SERVICES_STOPPED_FOR_RESTORE=false
|
||||
|
||||
write_status "RUNNING" "HEALTH_CHECK" "checking health"
|
||||
log "HEALTH_CHECK"
|
||||
|
||||
Reference in New Issue
Block a user