Sunyard/cisd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix：排除资源备份表

Browse Source
This commit is contained in:
waner 2026-05-11 17:45:07 +08:00
parent d1f136f8e9
commit 750c08f202
10 changed files with 340 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -297,7 +297,7 @@ Open:
- `db/TMS.sql`
- `db/CMEP.sql`（包内存在时）
- `db/<orgCode>.sql`（标准/间参收发器包内存在时，导回同名机构库）
- `TMS_DB` 备份默认通过 `tms.backup.tms-db-excluded-tables` 排除角色、授权、会话和审计表，恢复时不会覆盖新机器安全状态
- `TMS_DB` 备份默认通过 `tms.backup.tms-db-excluded-tables` 排除角色、授权、会话、审计和资源备份/恢复任务表，恢复时不会覆盖新机器安全状态，也不会把旧的 `RUNNING` 任务带到新机器
- `MQ_REPLAY` 通过 `replay-mq.sh` 消费 `mq/mq-restore-context.json`，也可用 `MQ_REPLAY_COMMAND` 委派给现场脚本
- Java 启动恢复脚本前会在工作目录生成 `restore-env.sh`，传递数据库连接、恢复脚本路径、MQ replay 脚本路径和健康检查配置；健康检查会按配置重试等待 TMS 真正启动完成
@ -315,6 +315,7 @@ Open:
- `tms.backup.mysqldump-timeout-seconds`
- `tms.backup.mysql-path`
- `tms.backup.restore-db-script-path`
- `tms.backup.restore-db-timeout-seconds`
- `tms.backup.mq-replay-script-path`
- `tms.backup.tms-database-name`
- `tms.backup.tms-db-excluded-tables`

5
config/application.yml.example
View File

@ -115,8 +115,9 @@ tms:
product-types:
- ENTERPRISE
- INDIRECT
# TMS 库备份排除表。默认跳过角色、授权、会话和审计表，避免恢复时覆盖新机器安全状态。
# TMS 库备份排除表。默认跳过角色、授权、会话、审计和资源任务表，避免恢复时覆盖新机器安全状态或带回旧 RUNNING 任务。
mysqldump-timeout-seconds: 300
restore-db-timeout-seconds: 300
tms-db-excluded-tables:
- tms_role_account
- tms_role_ukey_binding
@ -125,6 +126,8 @@ tms:
- tms_auth_challenge
- tms_auth_audit_log
- tms_operation_audit_log
- tms_resource_backup_task
- tms_resource_restore_task
allowed-restore-roots:
- /home/tms/config
- /home/cmep4i

78
scripts/resource-restore/apply-resource-backup.sh
View File

@ -32,6 +32,8 @@ MQ_PLAN="${WORK_DIR}/restore-mq.tsv"
ENV_FILE="${WORK_DIR}/restore-env.sh"
STOP_COMMANDS_PLAN="${WORK_DIR}/restore-stop-commands.tsv"
START_COMMANDS_PLAN="${WORK_DIR}/restore-start-commands.tsv"
SERVICES_STOPPED_FOR_RESTORE=false
SERVICES_STARTING_AFTER_FAILURE=false
json_escape() {
local value="$1"
@ -63,6 +65,7 @@ fail() {
local phase="${2:-FAILED}"
log "[ERROR] ${message}"
write_status "FAILED" "${phase}" "${message}"
attempt_service_start_after_failure "${phase}"
exit 1
}
@ -101,6 +104,9 @@ restore_databases() {
require_env "DB_USER" "DATABASE"
local mysql_path="${MYSQL_PATH:-mysql}"
local db_password="${DB_PASSWORD:-}"
local timeout_seconds="${RESTORE_DB_TIMEOUT_SECONDS:-300}"
[[ "${timeout_seconds}" =~ ^[0-9]+$ ]] || timeout_seconds=300
(( timeout_seconds > 0 )) || timeout_seconds=300
while IFS=$'\t' read -r entry_path db_name || [[ -n "${entry_path:-}" ]]; do
[[ -n "${entry_path:-}" ]] || continue
[[ -n "${db_name:-}" ]] || fail "database name is empty for ${entry_path}" "DATABASE"
@ -113,7 +119,32 @@ restore_databases() {
--username "${DB_USER}" \
--password "${db_password}" \
--db-name "${db_name}" \
--sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1
--sql-file "${sql_file}" >> "${LOG_FILE}" 2>&1 &
local db_pid=$!
local timeout_marker="${WORK_DIR}/.db-restore-timeout-${db_pid}"
rm -f "${timeout_marker}" || true
(
sleep "${timeout_seconds}"
if kill -0 "${db_pid}" >/dev/null 2>&1; then
printf 'timeout' > "${timeout_marker}"
kill "${db_pid}" >/dev/null 2>&1 || true
sleep 1
kill -9 "${db_pid}" >/dev/null 2>&1 || true
fi
) &
local watchdog_pid=$!
local db_exit=0
wait "${db_pid}" || db_exit=$?
kill "${watchdog_pid}" >/dev/null 2>&1 || true
wait "${watchdog_pid}" >/dev/null 2>&1 || true
if [[ -f "${timeout_marker}" ]]; then
rm -f "${timeout_marker}" || true
fail "database restore timed out after ${timeout_seconds}s: ${db_name}" "DATABASE"
fi
rm -f "${timeout_marker}" || true
if [[ "${db_exit}" -ne 0 ]]; then
fail "database restore failed: ${db_name}" "DATABASE"
fi
log "restored database: ${db_name}"
done < "${DATABASES_PLAN}"
}
@ -153,6 +184,49 @@ run_service_commands() {
return 0
}
run_service_commands_best_effort() {
local plan_file="$1"
local phase="$2"
[[ -f "${plan_file}" ]] || return 1
[[ -s "${plan_file}" ]] || return 1
while IFS=$'\t' read -r code required command || [[ -n "${code:-}" ]]; do
[[ -n "${code:-}" ]] || continue
if [[ -z "${command:-}" ]]; then
log "[WARN] ${phase} ${code}: command is empty"
continue
fi
log "${phase} ${code}: ${command}"
if bash -lc "${command}" >> "${LOG_FILE}" 2>&1; then
log "${phase} ${code}: success"
else
log "[WARN] ${phase} ${code}: failed while recovering from restore failure"
fi
done < "${plan_file}"
return 0
}
attempt_service_start_after_failure() {
local failed_phase="${1:-FAILED}"
if [[ "${SERVICES_STOPPED_FOR_RESTORE}" != "true" ]]; then
return 0
fi
if [[ "${SERVICES_STARTING_AFTER_FAILURE}" == "true" || "${failed_phase}" == "SERVICE_START" ]]; then
return 0
fi
SERVICES_STARTING_AFTER_FAILURE=true
log "SERVICE_START_AFTER_FAILURE due to ${failed_phase}"
if run_service_commands_best_effort "${START_COMMANDS_PLAN}" "SERVICE_START_AFTER_FAILURE"; then
SERVICES_STOPPED_FOR_RESTORE=false
return 0
fi
if [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
"${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 \
&& log "SERVICE_START_AFTER_FAILURE TMS fallback: success" \
|| log "[WARN] SERVICE_START_AFTER_FAILURE TMS fallback: failed"
SERVICES_STOPPED_FOR_RESTORE=false
fi
}
health_check() {
local url="${HEALTH_CHECK_URL:-}"
[[ -n "${url}" ]] || {
@ -217,6 +291,7 @@ fi
# 因为某些部署环境可能由 systemd/外部平台接管服务状态，但文件/数据库/MQ 失败必须中断。
write_status "RUNNING" "SERVICE_STOP" "stopping services"
log "SERVICE_STOP"
SERVICES_STOPPED_FOR_RESTORE=true
if ! run_service_commands "${STOP_COMMANDS_PLAN}" "SERVICE_STOP" \
&& [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
"${TMS_SCRIPT_PATH}" stop >> "${LOG_FILE}" 2>&1 || true
@ -240,6 +315,7 @@ if ! run_service_commands "${START_COMMANDS_PLAN}" "SERVICE_START" \
&& [[ -n "${TMS_SCRIPT_PATH:-}" && -x "${TMS_SCRIPT_PATH}" ]]; then
"${TMS_SCRIPT_PATH}" start >> "${LOG_FILE}" 2>&1 || true
fi
SERVICES_STOPPED_FOR_RESTORE=false
write_status "RUNNING" "HEALTH_CHECK" "checking health"
log "HEALTH_CHECK"

27
src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java
View File

@ -2,12 +2,19 @@ package com.cisd.tms.modules.backup.config;
import com.cisd.tms.modules.backup.dto.support.ConfiguredFileBackupResource;
import com.cisd.tms.modules.backup.dto.support.ConfiguredRestoreCommand;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "tms.backup")
public class ResourceBackupProperties {
private static final List<String> MANDATORY_TMS_DB_EXCLUDED_TABLES = List.of(
"tms_resource_backup_task",
"tms_resource_restore_task"
);
private String outputDir = "/home/tms/tmp/resource-backup-packages";
private String precheckStoreDir = "/home/tms/tmp/resource-restore-precheck";
private String restoreTaskRootDir = "/home/tms/tmp/resource-restore";
@ -16,6 +23,7 @@ public class ResourceBackupProperties {
private int mysqldumpTimeoutSeconds = 300;
private String mysqlPath = "mysql";
private String restoreDbScriptPath = "/home/tms/bin/resource-restore/restore-db.sh";
private int restoreDbTimeoutSeconds = 300;
private String mqReplayScriptPath = "/home/tms/bin/resource-restore/replay-mq.sh";
private String tmsDatabaseName = "";
private List<String> tmsDbExcludedTables = List.of(
@ -25,7 +33,9 @@ public class ResourceBackupProperties {
"tms_auth_session",
"tms_auth_challenge",
"tms_auth_audit_log",
"tms_operation_audit_log"
"tms_operation_audit_log",
"tms_resource_backup_task",
"tms_resource_restore_task"
);
private String cmepDatabaseName = "CMEP";
private String healthCheckUrl = "http://127.0.0.1:8080/actuator/health";
@ -107,6 +117,14 @@ public class ResourceBackupProperties {
this.restoreDbScriptPath = restoreDbScriptPath;
}
public int getRestoreDbTimeoutSeconds() {
return restoreDbTimeoutSeconds;
}
public void setRestoreDbTimeoutSeconds(int restoreDbTimeoutSeconds) {
this.restoreDbTimeoutSeconds = restoreDbTimeoutSeconds;
}
public String getMqReplayScriptPath() {
return mqReplayScriptPath;
}
@ -124,7 +142,12 @@ public class ResourceBackupProperties {
}
public List<String> getTmsDbExcludedTables() {
return tmsDbExcludedTables;
LinkedHashSet<String> values = new LinkedHashSet<>();
if (tmsDbExcludedTables != null) {
values.addAll(tmsDbExcludedTables);
}
values.addAll(MANDATORY_TMS_DB_EXCLUDED_TABLES);
return new ArrayList<>(values);
}
public void setTmsDbExcludedTables(List<String> tmsDbExcludedTables) {

61
src/main/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollector.java
View File

@ -79,45 +79,40 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector {
command.add(target.databaseName());
try {
Process process = new ProcessBuilder(command)
.redirectErrorStream(true)
.start();
AtomicReference<byte[]> outputRef = new AtomicReference<>(new byte[0]);
AtomicReference<IOException> outputErrorRef = new AtomicReference<>();
Thread outputReader = new Thread(() -> {
try {
outputRef.set(readAllBytes(process.getInputStream()));
} catch (IOException ex) {
outputErrorRef.set(ex);
}
}, "resource-backup-mysqldump-output-reader");
outputReader.setDaemon(true);
outputReader.start();
Process process = new ProcessBuilder(command).start();
StreamReadResult stdout = readStreamAsync(process.getInputStream(), "resource-backup-mysqldump-stdout-reader");
StreamReadResult stderr = readStreamAsync(process.getErrorStream(), "resource-backup-mysqldump-stderr-reader");
int timeoutSeconds = Math.max(1, resourceBackupProperties.getMysqldumpTimeoutSeconds());
boolean finished = process.waitFor(timeoutSeconds, java.util.concurrent.TimeUnit.SECONDS);
if (!finished) {
process.destroyForcibly();
joinOutputReader(outputReader);
joinOutputReader(stdout.thread());
joinOutputReader(stderr.thread());
throw new IllegalStateException("mysqldump执行超时 after " + timeoutSeconds + "s");
}
joinOutputReader(outputReader);
if (outputErrorRef.get() != null) {
throw new IllegalStateException("读取mysqldump输出失败", outputErrorRef.get());
joinOutputReader(stdout.thread());
joinOutputReader(stderr.thread());
if (stdout.error().get() != null) {
throw new IllegalStateException("读取mysqldump标准输出失败", stdout.error().get());
}
byte[] stdout = outputRef.get();
if (stderr.error().get() != null) {
throw new IllegalStateException("读取mysqldump错误输出失败", stderr.error().get());
}
byte[] stdoutBytes = stdout.bytes().get();
byte[] stderrBytes = stderr.bytes().get();
int exitCode = process.exitValue();
if (exitCode != 0) {
String error = new String(stdout, StandardCharsets.UTF_8).trim();
String error = new String(stderrBytes, StandardCharsets.UTF_8).trim();
throw new IllegalStateException("mysqldump 执行失败 with exitCode=" + exitCode
+ (error.isEmpty() ? "" : ", error=" + error));
}
if (stdout.length == 0) {
if (stdoutBytes.length == 0) {
throw new IllegalStateException("mysqldump输出为空");
}
CollectedDatabaseDump dump = new CollectedDatabaseDump();
dump.setDatabaseName(target.databaseName());
dump.setFileName(fileName);
dump.setContent(stdout);
dump.setContent(stdoutBytes);
return dump;
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
@ -131,6 +126,21 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector {
outputReader.join(1000L);
}
private static StreamReadResult readStreamAsync(InputStream inputStream, String threadName) {
AtomicReference<byte[]> bytes = new AtomicReference<>(new byte[0]);
AtomicReference<IOException> error = new AtomicReference<>();
Thread thread = new Thread(() -> {
try {
bytes.set(readAllBytes(inputStream));
} catch (IOException ex) {
error.set(ex);
}
}, threadName);
thread.setDaemon(true);
thread.start();
return new StreamReadResult(thread, bytes, error);
}
private MysqlConnectionTarget resolveTarget(String configuredDatabaseName, String fallbackDatabaseName) {
String url = trim(datasourceUrl);
if (url.isEmpty()) {
@ -187,4 +197,11 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector {
private record MysqlConnectionTarget(String host, int port, String username, String password, String databaseName) {
}
private record StreamReadResult(
Thread thread,
AtomicReference<byte[]> bytes,
AtomicReference<IOException> error
) {
}
}

1
src/main/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncher.java
View File

@ -113,6 +113,7 @@ public class LightweightRestoreApplierLauncher {
// 脚本读取这些变量后执行停服务、导库、MQ replay、健康检查等现场动作。
values.put("TMS_SCRIPT_PATH", trim(properties.getTmsScriptPath()));
values.put("DB_RESTORE_SCRIPT_PATH", trim(properties.getRestoreDbScriptPath()));
values.put("RESTORE_DB_TIMEOUT_SECONDS", String.valueOf(properties.getRestoreDbTimeoutSeconds()));
values.put("MQ_REPLAY_SCRIPT_PATH", trim(properties.getMqReplayScriptPath()));
values.put("MYSQL_PATH", trim(properties.getMysqlPath()).isEmpty() ? "mysql" : trim(properties.getMysqlPath()));
values.put("DB_HOST", target.host());

14
src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java
View File

@ -28,7 +28,10 @@ class ResourceBackupPropertiesTest {
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_account"));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_ukey_binding"));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_auth_session"));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_backup_task"));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_restore_task"));
Assertions.assertEquals(300, properties.getMysqldumpTimeoutSeconds());
Assertions.assertEquals(300, properties.getRestoreDbTimeoutSeconds());
Assertions.assertEquals(180, properties.getHealthCheckMaxWaitSeconds());
Assertions.assertEquals(5, properties.getHealthCheckRetryIntervalSeconds());
}
@ -52,6 +55,17 @@ class ResourceBackupPropertiesTest {
Assertions.assertFalse(properties.getFileResources().get(0).isRequired());
}
@Test
void shouldAlwaysExcludeResourceTaskTablesEvenWhenConfiguredListOverridesDefaults() {
ResourceBackupProperties properties = new ResourceBackupProperties();
properties.setTmsDbExcludedTables(java.util.List.of("custom_table"));
Assertions.assertEquals("custom_table", properties.getTmsDbExcludedTables().get(0));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_backup_task"));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_resource_restore_task"));
}
@Test
void shouldAcceptConfiguredRestoreStartAndStopCommands() {
ResourceBackupProperties properties = new ResourceBackupProperties();

50
src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java
View File

@ -68,6 +68,45 @@ class MysqlDatabaseBackupCollectorTest {
Assertions.assertTrue(elapsedMillis < 1800, "mysqldump should be killed on timeout");
}
@Test
void shouldKeepMysqldumpWarningsOutOfSqlDumpContent() throws Exception {
Path mysqldump = noisyMysqldump();
ResourceBackupProperties properties = new ResourceBackupProperties();
properties.setMysqldumpPath(mysqldump.toString());
MysqlDatabaseBackupCollector collector = new MysqlDatabaseBackupCollector(
properties,
"jdbc:mysql://127.0.0.1:4000/TMS?useSSL=false",
"root",
"secret"
);
DatabaseBackupCollector.CollectedDatabaseDump dump = collector.collectTmsDatabaseDump();
String content = new String(dump.getContent());
Assertions.assertEquals("-- synthetic dump\n", content);
Assertions.assertFalse(content.contains("mysqldump: [Warning]"));
}
@Test
void shouldExcludeResourceTaskTablesByDefaultWhenDumpingTmsDatabase() throws Exception {
Path argsLog = tempDir.resolve("default-mysqldump-args.log");
Path mysqldump = fakeMysqldump(argsLog);
ResourceBackupProperties properties = new ResourceBackupProperties();
properties.setMysqldumpPath(mysqldump.toString());
MysqlDatabaseBackupCollector collector = new MysqlDatabaseBackupCollector(
properties,
"jdbc:mysql://127.0.0.1:4000/TMS?useSSL=false",
"root",
"secret"
);
collector.collectTmsDatabaseDump();
String invocation = Files.readString(argsLog);
Assertions.assertTrue(invocation.contains("--ignore-table=TMS.tms_resource_backup_task"));
Assertions.assertTrue(invocation.contains("--ignore-table=TMS.tms_resource_restore_task"));
}
private Path fakeMysqldump(Path argsLog) throws Exception {
Path script = tempDir.resolve("mysqldump");
Files.writeString(script, """
@ -89,4 +128,15 @@ class MysqlDatabaseBackupCollectorTest {
script.toFile().setExecutable(true);
return script;
}
private Path noisyMysqldump() throws Exception {
Path script = tempDir.resolve("noisy-mysqldump");
Files.writeString(script, """
#!/usr/bin/env bash
printf 'mysqldump: [Warning] Using a password on the command line interface can be insecure.\\n' >&2
printf -- '-- synthetic dump\\n'
""");
script.toFile().setExecutable(true);
return script;
}
}

129
src/test/java/com/cisd/tms/modules/backup/restore/ApplyResourceBackupScriptContractTest.java
View File

@ -193,7 +193,111 @@ class ApplyResourceBackupScriptContractTest {
Assertions.assertTrue(log.indexOf("start-tms.sh") < log.indexOf("start-standard-apps.sh"));
}
@Test
void shouldAttemptServiceStartWhenFileRestoreFailsAfterStop() throws Exception {
Path workDir = tempDir.resolve("work");
Path payloadDir = workDir.resolve("payload");
Files.createDirectories(payloadDir);
Files.writeString(payloadDir.resolve("payload-manifest.json"), "{\"resources\":[]}");
Files.writeString(workDir.resolve("restore-files.tsv"),
"resources/missing.yml\t" + tempDir.resolve("target/application.yml") + System.lineSeparator());
Path commandLog = tempDir.resolve("service-recovery.log");
Path stopScript = fakeScript("stop-tms.sh", commandLog);
Path startScript = fakeScript("start-tms.sh", commandLog);
Files.writeString(workDir.resolve("restore-stop-commands.tsv"),
"TMS\ttrue\t" + stopScript + System.lineSeparator());
Files.writeString(workDir.resolve("restore-start-commands.tsv"),
"TMS\ttrue\t" + startScript + System.lineSeparator());
int exitCode = runScriptForExit(workDir, Map.of());
Assertions.assertEquals(1, exitCode);
String log = Files.readString(commandLog);
Assertions.assertTrue(log.indexOf("stop-tms.sh") < log.indexOf("start-tms.sh"));
String status = Files.readString(workDir.resolve("status.json"));
Assertions.assertTrue(status.contains("\"status\":\"FAILED\""));
Assertions.assertTrue(status.contains("\"phase\":\"FILE\""));
}
@Test
void shouldFailAndAttemptServiceStartWhenDatabaseRestoreTimesOut() throws Exception {
Path workDir = tempDir.resolve("work");
Path payloadDir = workDir.resolve("payload");
Files.createDirectories(payloadDir.resolve("db"));
Files.writeString(payloadDir.resolve("payload-manifest.json"), "{\"resources\":[]}");
Files.writeString(payloadDir.resolve("db/TMS.sql"), "select 1;");
Files.writeString(workDir.resolve("restore-databases.tsv"), "db/TMS.sql\tTMS" + System.lineSeparator());
Path commandLog = tempDir.resolve("db-timeout-recovery.log");
Path stopScript = fakeScript("stop-tms.sh", commandLog);
Path startScript = fakeScript("start-tms.sh", commandLog);
Path dbScript = slowScript("db-restore.sh", commandLog);
Files.writeString(workDir.resolve("restore-stop-commands.tsv"),
"TMS\ttrue\t" + stopScript + System.lineSeparator());
Files.writeString(workDir.resolve("restore-start-commands.tsv"),
"TMS\ttrue\t" + startScript + System.lineSeparator());
int exitCode = runScriptForExit(workDir, Map.of(
"DB_RESTORE_SCRIPT_PATH", dbScript.toString(),
"MYSQL_PATH", "mysql-test",
"DB_HOST", "127.0.0.1",
"DB_PORT", "4000",
"DB_USER", "root",
"DB_PASSWORD", "secret",
"RESTORE_DB_TIMEOUT_SECONDS", "1"
));
Assertions.assertEquals(1, exitCode);
String log = Files.readString(commandLog);
Assertions.assertTrue(log.indexOf("stop-tms.sh") < log.indexOf("db-restore.sh"));
Assertions.assertTrue(log.indexOf("db-restore.sh") < log.indexOf("start-tms.sh"));
String status = Files.readString(workDir.resolve("status.json"));
Assertions.assertTrue(status.contains("\"status\":\"FAILED\""));
Assertions.assertTrue(status.contains("\"phase\":\"DATABASE\""));
Assertions.assertTrue(status.contains("timed out"));
}
@Test
void shouldFailAndAttemptServiceStartWhenDatabaseRestoreCommandFails() throws Exception {
Path workDir = tempDir.resolve("work");
Path payloadDir = workDir.resolve("payload");
Files.createDirectories(payloadDir.resolve("db"));
Files.writeString(payloadDir.resolve("payload-manifest.json"), "{\"resources\":[]}");
Files.writeString(payloadDir.resolve("db/TMS.sql"), "bad sql");
Files.writeString(workDir.resolve("restore-databases.tsv"), "db/TMS.sql\tTMS" + System.lineSeparator());
Path commandLog = tempDir.resolve("db-failure-recovery.log");
Path stopScript = fakeScript("stop-tms.sh", commandLog);
Path startScript = fakeScript("start-tms.sh", commandLog);
Path dbScript = failingScript("db-restore.sh", commandLog);
Files.writeString(workDir.resolve("restore-stop-commands.tsv"),
"TMS\ttrue\t" + stopScript + System.lineSeparator());
Files.writeString(workDir.resolve("restore-start-commands.tsv"),
"TMS\ttrue\t" + startScript + System.lineSeparator());
int exitCode = runScriptForExit(workDir, Map.of(
"DB_RESTORE_SCRIPT_PATH", dbScript.toString(),
"MYSQL_PATH", "mysql-test",
"DB_HOST", "127.0.0.1",
"DB_PORT", "4000",
"DB_USER", "root",
"DB_PASSWORD", "secret"
));
Assertions.assertEquals(1, exitCode);
String log = Files.readString(commandLog);
Assertions.assertTrue(log.indexOf("stop-tms.sh") < log.indexOf("db-restore.sh"));
Assertions.assertTrue(log.indexOf("db-restore.sh") < log.indexOf("start-tms.sh"));
String status = Files.readString(workDir.resolve("status.json"));
Assertions.assertTrue(status.contains("\"status\":\"FAILED\""));
Assertions.assertTrue(status.contains("\"phase\":\"DATABASE\""));
Assertions.assertTrue(status.contains("database restore failed"));
}
private static void runScript(Path workDir, Map<String, String> environment) throws Exception {
int exitCode = runScriptForExit(workDir, environment);
Assertions.assertEquals(0, exitCode);
}
private static int runScriptForExit(Path workDir, Map<String, String> environment) throws Exception {
ProcessBuilder builder = new ProcessBuilder(
"bash",
Path.of("scripts/resource-restore/apply-resource-backup.sh").toAbsolutePath().normalize().toString(),
@ -203,8 +307,7 @@ class ApplyResourceBackupScriptContractTest {
builder.directory(Path.of(".").toAbsolutePath().normalize().toFile());
builder.environment().putAll(environment);
Process process = builder.start();
int exitCode = process.waitFor();
Assertions.assertEquals(0, exitCode, new String(process.getErrorStream().readAllBytes()));
return process.waitFor();
}
private Path fakeScript(String name, Path commandLog) throws Exception {
@ -216,4 +319,26 @@ class ApplyResourceBackupScriptContractTest {
script.toFile().setExecutable(true);
return script;
}
private Path slowScript(String name, Path commandLog) throws Exception {
Path script = tempDir.resolve(name);
Files.writeString(script, """
#!/usr/bin/env bash
printf '%%s %%s\\n' "$0" "$*" >> "%s"
sleep 2
""".formatted(commandLog));
script.toFile().setExecutable(true);
return script;
}
private Path failingScript(String name, Path commandLog) throws Exception {
Path script = tempDir.resolve(name);
Files.writeString(script, """
#!/usr/bin/env bash
printf '%%s %%s\\n' "$0" "$*" >> "%s"
exit 2
""".formatted(commandLog));
script.toFile().setExecutable(true);
return script;
}
}

1
src/test/java/com/cisd/tms/modules/backup/restore/LightweightRestoreApplierLauncherTest.java
View File

@ -61,6 +61,7 @@ class LightweightRestoreApplierLauncherTest {
String env = Files.readString(workDir.toAbsolutePath().normalize().resolve("restore-env.sh"));
Assertions.assertTrue(env.contains("export DB_RESTORE_SCRIPT_PATH='/home/tms/bin/resource-restore/restore-db.sh'"));
Assertions.assertTrue(env.contains("export RESTORE_DB_TIMEOUT_SECONDS='300'"));
Assertions.assertTrue(env.contains("export MQ_REPLAY_SCRIPT_PATH='/home/tms/bin/resource-restore/replay-mq.sh'"));
Assertions.assertTrue(env.contains("export MYSQL_PATH='mysql'"));
Assertions.assertTrue(env.contains("export DB_HOST='db-host'"));