diff --git a/README.md b/README.md index eda93b0..8424223 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ Open: - `db/TMS.sql` - `db/CMEP.sql`（包内存在时） - `db/.sql`（标准/间参收发器包内存在时，导回同名机构库） -- `TMS_DB` 备份默认通过 `tms.backup.tms-db-excluded-tables` 排除角色、授权、会话和审计表，恢复时不会覆盖新机器安全状态 +- `TMS_DB` 备份默认通过 `tms.backup.tms-db-excluded-tables` 排除角色、授权、会话、审计和资源备份/恢复任务表，恢复时不会覆盖新机器安全状态，也不会把旧的 `RUNNING` 任务带到新机器 - `MQ_REPLAY` 通过 `replay-mq.sh` 消费 `mq/mq-restore-context.json`，也可用 `MQ_REPLAY_COMMAND` 委派给现场脚本 - Java 启动恢复脚本前会在工作目录生成 `restore-env.sh`，传递数据库连接、恢复脚本路径、MQ replay 脚本路径和健康检查配置；健康检查会按配置重试等待 TMS 真正启动完成 @@ -315,6 +315,7 @@ Open: - `tms.backup.mysqldump-timeout-seconds` - `tms.backup.mysql-path` - `tms.backup.restore-db-script-path` +- `tms.backup.restore-db-timeout-seconds` - `tms.backup.mq-replay-script-path` - `tms.backup.tms-database-name` - `tms.backup.tms-db-excluded-tables` diff --git a/config/application.yml.example b/config/application.yml.example index 1602bc5..e1c6e79 100644 --- a/config/application.yml.example +++ b/config/application.yml.example @@ -115,8 +115,9 @@ tms: product-types: - ENTERPRISE - INDIRECT - # TMS 库备份排除表。默认跳过角色、授权、会话和审计表，避免恢复时覆盖新机器安全状态。 + # TMS 库备份排除表。默认跳过角色、授权、会话、审计和资源任务表，避免恢复时覆盖新机器安全状态或带回旧 RUNNING 任务。 mysqldump-timeout-seconds: 300 + restore-db-timeout-seconds: 300 tms-db-excluded-tables: - tms_role_account - tms_role_ukey_binding @@ -125,6 +126,8 @@ tms: - tms_auth_challenge - tms_auth_audit_log - tms_operation_audit_log + - tms_resource_backup_task + - tms_resource_restore_task allowed-restore-roots: - /home/tms/config - /home/cmep4i diff --git a/scripts/resource-restore/apply-resource-backup.sh b/scripts/resource-restore/apply-resource-backup.sh index a172ffa..b73479e 100755 --- a/scripts/resource-restore/apply-resource-backup.sh +++ b/scripts/resource-restore/apply-resource-backup.sh @@ -32,6 +32,8 @@ MQ_PLAN="${WORK_DIR}/restore-mq.tsv" ENV_FILE="${WORK_DIR}/restore-env.sh" STOP_COMMANDS_PLAN="${WORK_DIR}/restore-stop-commands.tsv" START_COMMANDS_PLAN="${WORK_DIR}/restore-start-commands.tsv" +SERVICES_STOPPED_FOR_RESTORE=false +SERVICES_STARTING_AFTER_FAILURE=false json_escape() { local value="$1" @@ -63,6 +65,7 @@ fail() { local phase="${2:-FAILED}" log "[ERROR] ${message}" write_status "FAILED" "${phase}" "${message}" + attempt_service_start_after_failure "${phase}" exit 1 } @@ -101,6 +104,9 @@ restore_databases() { require_env "DB_USER" "DATABASE" local mysql_path="${MYSQL_PATH:-mysql}" local db_password="${DB_PASSWORD:-}" + local timeout_seconds="${RESTORE_DB_TIMEOUT_SECONDS:-300}" + [[ "${timeout_seconds}" =~ ^[0-9]+$ ]] || timeout_seconds=300 + (( timeout_seconds > 0 )) || timeout_seconds=300 while IFS=$'\t' read -r entry_path db_name || [[ -n "${entry_path:-}" ]]; do [[ -n "${entry_path:-}" ]] || continue [[ -n "${db_name:-}" ]] || fail "database name is empty for ${entry_path}" "DATABASE" @@ -113,7 +119,32 @@ restore_databases() { --username "${DB_USER}" \ --password "${db_password}" \ --db-name "${db_name}" \ - --sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1 + --sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1 & + local db_pid=$! + local timeout_marker="${WORK_DIR}/.db-restore-timeout-${db_pid}" + rm -f "${timeout_marker}" || true + ( + sleep "${timeout_seconds}" + if kill -0 "${db_pid}" >/dev/null 2>&1; then + printf 'timeout' > "${timeout_marker}" + kill "${db_pid}" >/dev/null 2>&1 || true + sleep 1 + kill -9 "${db_pid}" >/dev/null 2>&1 || true + fi + ) & + local watchdog_pid=$! + local db_exit=0 + wait "${db_pid}" || db_exit=$? + kill "${watchdog_pid}" >/dev/null 2>&1 || true + wait "${watchdog_pid}" >/dev/null 2>&1 || true + if [[ -f "${timeout_marker}" ]]; then + rm -f "${timeout_marker}" || true + fail "database restore timed out after ${timeout_seconds}s: ${db_name}" "DATABASE" + fi + rm -f "${timeout_marker}" || true + if [[ "${db_exit}" -ne 0 ]]; then + fail "database restore failed: ${db_name}" "DATABASE" + fi log "restored database: ${db_name}" done < "${DATABASES_PLAN}" } @@ -153,6 +184,49 @@ run_service_commands() { return 0 } +run_service_commands_best_effort() { + local plan_file="$1" + local phase="$2" + [[ -f "${plan_file}" ]] || return 1 + [[ -s "${plan_file}" ]] || return 1 + while IFS=$'\t' read -r code required command || [[ -n "${code:-}" ]]; do + [[ -n "${code:-}" ]] || continue + if [[ -z "${command:-}" ]]; then + log "[WARN] ${phase} ${code}: command is empty" + continue + fi + log "${phase} ${code}: ${command}" + if bash -lc "${command}" >> "${LOG_FILE}" 2>&1; then + log "${phase} ${code}: success" + else + log "[WARN] ${phase} ${code}: failed while recovering from restore failure" + fi + done < "${plan_file}" + return 0 +} + +attempt_service_start_after_failure() { + local failed_phase="${1:-FAILED}" + if [[ "${SERVICES_STOPPED_FOR_RESTORE}" != "true" ]]; then + return 0 + fi + if [[ "${SERVICES_STARTING_AFTER_FAILURE}" == "true" || "${failed_phase}" == "SERVICE_START" ]]; then + return 0 + fi + SERVICES_STARTING_AFTER_FAILURE=true + log "SERVICE_START_AFTER_FAILURE due to ${failed_phase}" + if run_service_commands_best_effort "${START_COMMANDS_PLAN}" "SERVICE_START_AFTER_FAILURE"; then + SERVICES_STOPPED_FOR_RESTORE=false + return 0 + fi + if [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then + "${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 \ + && log "SERVICE_START_AFTER_FAILURE TMS fallback: success" \ + || log "[WARN] SERVICE_START_AFTER_FAILURE TMS fallback: failed" + SERVICES_STOPPED_FOR_RESTORE=false + fi +} + health_check() { local url="${HEALTH_CHECK_URL:-}" [[ -n "${url}" ]] || { @@ -217,6 +291,7 @@ fi # 因为某些部署环境可能由 systemd/外部平台接管服务状态，但文件/数据库/MQ 失败必须中断。 write_status "RUNNING" "SERVICE_STOP" "stopping services" log "SERVICE_STOP" +SERVICES_STOPPED_FOR_RESTORE=true if ! run_service_commands "${STOP_COMMANDS_PLAN}" "SERVICE_STOP" \ && [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then "${TMS_SCRIPT_PATH}" stop >> "${LOG_FILE}" 2>&1 || true @@ -240,6 +315,7 @@ if ! run_service_commands "${START_COMMANDS_PLAN}" "SERVICE_START" \ && [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then "${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 || true fi +SERVICES_STOPPED_FOR_RESTORE=false write_status "RUNNING" "HEALTH_CHECK" "checking health" log "HEALTH_CHECK" diff --git a/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java b/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java index c4ad09d..e2b2875 100644 --- a/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java +++ b/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java @@ -2,12 +2,19 @@ package com.cisd.tms.modules.backup.config; import com.cisd.tms.modules.backup.dto.support.ConfiguredFileBackupResource; import com.cisd.tms.modules.backup.dto.support.ConfiguredRestoreCommand; +import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "tms.backup") public class ResourceBackupProperties { + private static final List MANDATORY_TMS_DB_EXCLUDED_TABLES = List.of( + "tms_resource_backup_task", + "tms_resource_restore_task" + ); + private String outputDir = "/home/tms/tmp/resource-backup-packages"; private String precheckStoreDir = "/home/tms/tmp/resource-restore-precheck"; private String restoreTaskRootDir = "/home/tms/tmp/resource-restore"; @@ -16,6 +23,7 @@ public class ResourceBackupProperties { private int mysqldumpTimeoutSeconds = 300; private String mysqlPath = "mysql"; private String restoreDbScriptPath = "/home/tms/bin/resource-restore/restore-db.sh"; + private int restoreDbTimeoutSeconds = 300; private String mqReplayScriptPath = "/home/tms/bin/resource-restore/replay-mq.sh"; private String tmsDatabaseName = ""; private List tmsDbExcludedTables = List.of( @@ -25,7 +33,9 @@ public class ResourceBackupProperties { "tms_auth_session", "tms_auth_challenge", "tms_auth_audit_log", - "tms_operation_audit_log" + "tms_operation_audit_log", + "tms_resource_backup_task", + "tms_resource_restore_task" ); private String cmepDatabaseName = "CMEP"; private String healthCheckUrl = "http://127.0.0.1:8080/actuator/health"; @@ -107,6 +117,14 @@ public class ResourceBackupProperties { this.restoreDbScriptPath = restoreDbScriptPath; } + public int getRestoreDbTimeoutSeconds() { + return restoreDbTimeoutSeconds; + } + + public void setRestoreDbTimeoutSeconds(int restoreDbTimeoutSeconds) { + this.restoreDbTimeoutSeconds = restoreDbTimeoutSeconds; + } + public String getMqReplayScriptPath() { return mqReplayScriptPath; } @@ -124,7 +142,12 @@ public class ResourceBackupProperties { } public List getTmsDbExcludedTables() { - return tmsDbExcludedTables; + LinkedHashSet values = new LinkedHashSet<>(); + if (tmsDbExcludedTables != null) { + values.addAll(tmsDbExcludedTables); + } + values.addAll(MANDATORY_TMS_DB_EXCLUDED_TABLES); + return new ArrayList<>(values); } public void setTmsDbExcludedTables(List tmsDbExcludedTables) { diff --git a/src/main/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollector.java b/src/main/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollector.java index 6909025..0d51a64 100644 --- a/src/main/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollector.java +++ b/src/main/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollector.java @@ -79,45 +79,40 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector { command.add(target.databaseName()); try { - Process process = new ProcessBuilder(command) - .redirectErrorStream(true) - .start(); - AtomicReference outputRef = new AtomicReference<>(new byte[0]); - AtomicReference outputErrorRef = new AtomicReference<>(); - Thread outputReader = new Thread(() -> { - try { - outputRef.set(readAllBytes(process.getInputStream())); - } catch (IOException ex) { - outputErrorRef.set(ex); - } - }, "resource-backup-mysqldump-output-reader"); - outputReader.setDaemon(true); - outputReader.start(); + Process process = new ProcessBuilder(command).start(); + StreamReadResult stdout = readStreamAsync(process.getInputStream(), "resource-backup-mysqldump-stdout-reader"); + StreamReadResult stderr = readStreamAsync(process.getErrorStream(), "resource-backup-mysqldump-stderr-reader"); int timeoutSeconds = Math.max(1, resourceBackupProperties.getMysqldumpTimeoutSeconds()); boolean finished = process.waitFor(timeoutSeconds, java.util.concurrent.TimeUnit.SECONDS); if (!finished) { process.destroyForcibly(); - joinOutputReader(outputReader); + joinOutputReader(stdout.thread()); + joinOutputReader(stderr.thread()); throw new IllegalStateException("mysqldump执行超时 after " + timeoutSeconds + "s"); } - joinOutputReader(outputReader); - if (outputErrorRef.get() != null) { - throw new IllegalStateException("读取mysqldump输出失败", outputErrorRef.get()); + joinOutputReader(stdout.thread()); + joinOutputReader(stderr.thread()); + if (stdout.error().get() != null) { + throw new IllegalStateException("读取mysqldump标准输出失败", stdout.error().get()); } - byte[] stdout = outputRef.get(); + if (stderr.error().get() != null) { + throw new IllegalStateException("读取mysqldump错误输出失败", stderr.error().get()); + } + byte[] stdoutBytes = stdout.bytes().get(); + byte[] stderrBytes = stderr.bytes().get(); int exitCode = process.exitValue(); if (exitCode != 0) { - String error = new String(stdout, StandardCharsets.UTF_8).trim(); + String error = new String(stderrBytes, StandardCharsets.UTF_8).trim(); throw new IllegalStateException("mysqldump 执行失败 with exitCode=" + exitCode + (error.isEmpty() ? "" : ", error=" + error)); } - if (stdout.length == 0) { + if (stdoutBytes.length == 0) { throw new IllegalStateException("mysqldump输出为空"); } CollectedDatabaseDump dump = new CollectedDatabaseDump(); dump.setDatabaseName(target.databaseName()); dump.setFileName(fileName); - dump.setContent(stdout); + dump.setContent(stdoutBytes); return dump; } catch (InterruptedException ex) { Thread.currentThread().interrupt(); @@ -131,6 +126,21 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector { outputReader.join(1000L); } + private static StreamReadResult readStreamAsync(InputStream inputStream, String threadName) { + AtomicReference bytes = new AtomicReference<>(new byte[0]); + AtomicReference error = new AtomicReference<>(); + Thread thread = new Thread(() -> { + try { + bytes.set(readAllBytes(inputStream)); + } catch (IOException ex) { + error.set(ex); + } + }, threadName); + thread.setDaemon(true); + thread.start(); + return new StreamReadResult(thread, bytes, error); + } + private MysqlConnectionTarget resolveTarget(String configuredDatabaseName, String fallbackDatabaseName) { String url = trim(datasourceUrl); if (url.isEmpty()) { @@ -187,4 +197,11 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector { private record MysqlConnectionTarget(String host, int port, String username, String password, String databaseName) { } + + private record StreamReadResult( + Thread thread, + AtomicReference bytes, + AtomicReference error + ) { + } } diff --git a/src/main/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncher.java b/src/main/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncher.java index 49c35f8..f51b482 100644 --- a/src/main/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncher.java +++ b/src/main/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncher.java @@ -113,6 +113,7 @@ public class LightweightRestoreApplierLauncher { // 脚本读取这些变量后执行停服务、导库、MQ replay、健康检查等现场动作。 values.put("TMS_SCRIPT_PATH", trim(properties.getTmsScriptPath())); values.put("DB_RESTORE_SCRIPT_PATH", trim(properties.getRestoreDbScriptPath())); + values.put("RESTORE_DB_TIMEOUT_SECONDS", String.valueOf(properties.getRestoreDbTimeoutSeconds())); values.put("MQ_REPLAY_SCRIPT_PATH", trim(properties.getMqReplayScriptPath())); values.put("MYSQL_PATH", trim(properties.getMysqlPath()).isEmpty() ? "mysql" : trim(properties.getMysqlPath())); values.put("DB_HOST", target.host()); diff --git a/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java b/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java index 4b9c291..420abf0 100644 --- a/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java @@ -28,7 +28,10 @@ class ResourceBackupPropertiesTest { Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_account")); Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_ukey_binding")); Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_auth_session")); + Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_backup_task")); + Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_restore_task")); Assertions.assertEquals(300, properties.getMysqldumpTimeoutSeconds()); + Assertions.assertEquals(300, properties.getRestoreDbTimeoutSeconds()); Assertions.assertEquals(180, properties.getHealthCheckMaxWaitSeconds()); Assertions.assertEquals(5, properties.getHealthCheckRetryIntervalSeconds()); } @@ -52,6 +55,17 @@ class ResourceBackupPropertiesTest { Assertions.assertFalse(properties.getFileResources().get(0).isRequired()); } + @Test + void shouldAlwaysExcludeResourceTaskTablesEvenWhenConfiguredListOverridesDefaults() { + ResourceBackupProperties properties = new ResourceBackupProperties(); + + properties.setTmsDbExcludedTables(java.util.List.of("custom_table")); + + Assertions.assertEquals("custom_table", properties.getTmsDbExcludedTables().get(0)); + Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_backup_task")); + Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_restore_task")); + } + @Test void shouldAcceptConfiguredRestoreStartAndStopCommands() { ResourceBackupProperties properties = new ResourceBackupProperties(); diff --git a/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java b/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java index bfd0113..e2874ea 100644 --- a/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java @@ -68,6 +68,45 @@ class MysqlDatabaseBackupCollectorTest { Assertions.assertTrue(elapsedMillis < 1800, "mysqldump should be killed on timeout"); } + @Test + void shouldKeepMysqldumpWarningsOutOfSqlDumpContent() throws Exception { + Path mysqldump = noisyMysqldump(); + ResourceBackupProperties properties = new ResourceBackupProperties(); + properties.setMysqldumpPath(mysqldump.toString()); + MysqlDatabaseBackupCollector collector = new MysqlDatabaseBackupCollector( + properties, + "jdbc:mysql://127.0.0.1:4000/TMS?useSSL=false", + "root", + "secret" + ); + + DatabaseBackupCollector.CollectedDatabaseDump dump = collector.collectTmsDatabaseDump(); + + String content = new String(dump.getContent()); + Assertions.assertEquals("-- synthetic dump\n", content); + Assertions.assertFalse(content.contains("mysqldump: [Warning]")); + } + + @Test + void shouldExcludeResourceTaskTablesByDefaultWhenDumpingTmsDatabase() throws Exception { + Path argsLog = tempDir.resolve("default-mysqldump-args.log"); + Path mysqldump = fakeMysqldump(argsLog); + ResourceBackupProperties properties = new ResourceBackupProperties(); + properties.setMysqldumpPath(mysqldump.toString()); + MysqlDatabaseBackupCollector collector = new MysqlDatabaseBackupCollector( + properties, + "jdbc:mysql://127.0.0.1:4000/TMS?useSSL=false", + "root", + "secret" + ); + + collector.collectTmsDatabaseDump(); + + String invocation = Files.readString(argsLog); + Assertions.assertTrue(invocation.contains("--ignore-table=TMS.tms_resource_backup_task")); + Assertions.assertTrue(invocation.contains("--ignore-table=TMS.tms_resource_restore_task")); + } + private Path fakeMysqldump(Path argsLog) throws Exception { Path script = tempDir.resolve("mysqldump"); Files.writeString(script, """ @@ -89,4 +128,15 @@ class MysqlDatabaseBackupCollectorTest { script.toFile().setExecutable(true); return script; } + + private Path noisyMysqldump() throws Exception { + Path script = tempDir.resolve("noisy-mysqldump"); + Files.writeString(script, """ + #!/usr/bin/env bash + printf 'mysqldump: [Warning] Using a password on the command line interface can be insecure.\\n' >&2 + printf -- '-- synthetic dump\\n' + """); + script.toFile().setExecutable(true); + return script; + } } diff --git a/src/test/java/com/cisd/tms/modules/backup/restore/ApplyResourceBackupScriptContractTest.java b/src/test/java/com/cisd/tms/modules/backup/restore/ApplyResourceBackupScriptContractTest.java index 3c72e3b..5981815 100644 --- a/src/test/java/com/cisd/tms/modules/backup/restore/ApplyResourceBackupScriptContractTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/restore/ApplyResourceBackupScriptContractTest.java @@ -193,7 +193,111 @@ class ApplyResourceBackupScriptContractTest { Assertions.assertTrue(log.indexOf("start-tms.sh") < log.indexOf("start-standard-apps.sh")); } + @Test + void shouldAttemptServiceStartWhenFileRestoreFailsAfterStop() throws Exception { + Path workDir = tempDir.resolve("work"); + Path payloadDir = workDir.resolve("payload"); + Files.createDirectories(payloadDir); + Files.writeString(payloadDir.resolve("payload-manifest.json"), "{\"resources\":[]}"); + Files.writeString(workDir.resolve("restore-files.tsv"), + "resources/missing.yml\t" + tempDir.resolve("target/application.yml") + System.lineSeparator()); + Path commandLog = tempDir.resolve("service-recovery.log"); + Path stopScript = fakeScript("stop-tms.sh", commandLog); + Path startScript = fakeScript("start-tms.sh", commandLog); + Files.writeString(workDir.resolve("restore-stop-commands.tsv"), + "TMS\ttrue\t" + stopScript + System.lineSeparator()); + Files.writeString(workDir.resolve("restore-start-commands.tsv"), + "TMS\ttrue\t" + startScript + System.lineSeparator()); + + int exitCode = runScriptForExit(workDir, Map.of()); + + Assertions.assertEquals(1, exitCode); + String log = Files.readString(commandLog); + Assertions.assertTrue(log.indexOf("stop-tms.sh") < log.indexOf("start-tms.sh")); + String status = Files.readString(workDir.resolve("status.json")); + Assertions.assertTrue(status.contains("\"status\":\"FAILED\"")); + Assertions.assertTrue(status.contains("\"phase\":\"FILE\"")); + } + + @Test + void shouldFailAndAttemptServiceStartWhenDatabaseRestoreTimesOut() throws Exception { + Path workDir = tempDir.resolve("work"); + Path payloadDir = workDir.resolve("payload"); + Files.createDirectories(payloadDir.resolve("db")); + Files.writeString(payloadDir.resolve("payload-manifest.json"), "{\"resources\":[]}"); + Files.writeString(payloadDir.resolve("db/TMS.sql"), "select 1;"); + Files.writeString(workDir.resolve("restore-databases.tsv"), "db/TMS.sql\tTMS" + System.lineSeparator()); + Path commandLog = tempDir.resolve("db-timeout-recovery.log"); + Path stopScript = fakeScript("stop-tms.sh", commandLog); + Path startScript = fakeScript("start-tms.sh", commandLog); + Path dbScript = slowScript("db-restore.sh", commandLog); + Files.writeString(workDir.resolve("restore-stop-commands.tsv"), + "TMS\ttrue\t" + stopScript + System.lineSeparator()); + Files.writeString(workDir.resolve("restore-start-commands.tsv"), + "TMS\ttrue\t" + startScript + System.lineSeparator()); + + int exitCode = runScriptForExit(workDir, Map.of( + "DB_RESTORE_SCRIPT_PATH", dbScript.toString(), + "MYSQL_PATH", "mysql-test", + "DB_HOST", "127.0.0.1", + "DB_PORT", "4000", + "DB_USER", "root", + "DB_PASSWORD", "secret", + "RESTORE_DB_TIMEOUT_SECONDS", "1" + )); + + Assertions.assertEquals(1, exitCode); + String log = Files.readString(commandLog); + Assertions.assertTrue(log.indexOf("stop-tms.sh") < log.indexOf("db-restore.sh")); + Assertions.assertTrue(log.indexOf("db-restore.sh") < log.indexOf("start-tms.sh")); + String status = Files.readString(workDir.resolve("status.json")); + Assertions.assertTrue(status.contains("\"status\":\"FAILED\"")); + Assertions.assertTrue(status.contains("\"phase\":\"DATABASE\"")); + Assertions.assertTrue(status.contains("timed out")); + } + + @Test + void shouldFailAndAttemptServiceStartWhenDatabaseRestoreCommandFails() throws Exception { + Path workDir = tempDir.resolve("work"); + Path payloadDir = workDir.resolve("payload"); + Files.createDirectories(payloadDir.resolve("db")); + Files.writeString(payloadDir.resolve("payload-manifest.json"), "{\"resources\":[]}"); + Files.writeString(payloadDir.resolve("db/TMS.sql"), "bad sql"); + Files.writeString(workDir.resolve("restore-databases.tsv"), "db/TMS.sql\tTMS" + System.lineSeparator()); + Path commandLog = tempDir.resolve("db-failure-recovery.log"); + Path stopScript = fakeScript("stop-tms.sh", commandLog); + Path startScript = fakeScript("start-tms.sh", commandLog); + Path dbScript = failingScript("db-restore.sh", commandLog); + Files.writeString(workDir.resolve("restore-stop-commands.tsv"), + "TMS\ttrue\t" + stopScript + System.lineSeparator()); + Files.writeString(workDir.resolve("restore-start-commands.tsv"), + "TMS\ttrue\t" + startScript + System.lineSeparator()); + + int exitCode = runScriptForExit(workDir, Map.of( + "DB_RESTORE_SCRIPT_PATH", dbScript.toString(), + "MYSQL_PATH", "mysql-test", + "DB_HOST", "127.0.0.1", + "DB_PORT", "4000", + "DB_USER", "root", + "DB_PASSWORD", "secret" + )); + + Assertions.assertEquals(1, exitCode); + String log = Files.readString(commandLog); + Assertions.assertTrue(log.indexOf("stop-tms.sh") < log.indexOf("db-restore.sh")); + Assertions.assertTrue(log.indexOf("db-restore.sh") < log.indexOf("start-tms.sh")); + String status = Files.readString(workDir.resolve("status.json")); + Assertions.assertTrue(status.contains("\"status\":\"FAILED\"")); + Assertions.assertTrue(status.contains("\"phase\":\"DATABASE\"")); + Assertions.assertTrue(status.contains("database restore failed")); + } + private static void runScript(Path workDir, Map environment) throws Exception { + int exitCode = runScriptForExit(workDir, environment); + Assertions.assertEquals(0, exitCode); + } + + private static int runScriptForExit(Path workDir, Map environment) throws Exception { ProcessBuilder builder = new ProcessBuilder( "bash", Path.of("scripts/resource-restore/apply-resource-backup.sh").toAbsolutePath().normalize().toString(), @@ -203,8 +307,7 @@ class ApplyResourceBackupScriptContractTest { builder.directory(Path.of(".").toAbsolutePath().normalize().toFile()); builder.environment().putAll(environment); Process process = builder.start(); - int exitCode = process.waitFor(); - Assertions.assertEquals(0, exitCode, new String(process.getErrorStream().readAllBytes())); + return process.waitFor(); } private Path fakeScript(String name, Path commandLog) throws Exception { @@ -216,4 +319,26 @@ class ApplyResourceBackupScriptContractTest { script.toFile().setExecutable(true); return script; } + + private Path slowScript(String name, Path commandLog) throws Exception { + Path script = tempDir.resolve(name); + Files.writeString(script, """ + #!/usr/bin/env bash + printf '%%s %%s\\n' "$0" "$*" >> "%s" + sleep 2 + """.formatted(commandLog)); + script.toFile().setExecutable(true); + return script; + } + + private Path failingScript(String name, Path commandLog) throws Exception { + Path script = tempDir.resolve(name); + Files.writeString(script, """ + #!/usr/bin/env bash + printf '%%s %%s\\n' "$0" "$*" >> "%s" + exit 2 + """.formatted(commandLog)); + script.toFile().setExecutable(true); + return script; + } } diff --git a/src/test/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncherTest.java b/src/test/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncherTest.java index 088163a..8266ee5 100644 --- a/src/test/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncherTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncherTest.java @@ -61,6 +61,7 @@ class LightweightRestoreApplierLauncherTest { String env = Files.readString(workDir.toAbsolutePath().normalize().resolve("restore-env.sh")); Assertions.assertTrue(env.contains("export DB_RESTORE_SCRIPT_PATH='/home/tms/bin/resource-restore/restore-db.sh'")); + Assertions.assertTrue(env.contains("export RESTORE_DB_TIMEOUT_SECONDS='300'")); Assertions.assertTrue(env.contains("export MQ_REPLAY_SCRIPT_PATH='/home/tms/bin/resource-restore/replay-mq.sh'")); Assertions.assertTrue(env.contains("export MYSQL_PATH='mysql'")); Assertions.assertTrue(env.contains("export DB_HOST='db-host'"));