diff --git a/README.md b/README.md index 1f4291c..eda93b0 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,7 @@ Open: - `tms.backup.allowed-restore-roots` - `tms.backup.file-resources` - `tms.backup.mysqldump-path` +- `tms.backup.mysqldump-timeout-seconds` - `tms.backup.mysql-path` - `tms.backup.restore-db-script-path` - `tms.backup.mq-replay-script-path` diff --git a/config/application.yml.example b/config/application.yml.example index be13e19..1602bc5 100644 --- a/config/application.yml.example +++ b/config/application.yml.example @@ -116,6 +116,7 @@ tms: - ENTERPRISE - INDIRECT # TMS 库备份排除表。默认跳过角色、授权、会话和审计表，避免恢复时覆盖新机器安全状态。 + mysqldump-timeout-seconds: 300 tms-db-excluded-tables: - tms_role_account - tms_role_ukey_binding diff --git a/scripts/standard-init/start_standard_apps.sh b/scripts/standard-init/start_standard_apps.sh index 890fed5..57a23d2 100644 --- a/scripts/standard-init/start_standard_apps.sh +++ b/scripts/standard-init/start_standard_apps.sh @@ -3,6 +3,9 @@ set -euo pipefail STANDARD_HOME_DIR="${STANDARD_HOME_DIR:?STANDARD_HOME_DIR is required}" STANDARD_RUN_USER="${STANDARD_RUN_USER:?STANDARD_RUN_USER is required}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +"${SCRIPT_DIR}/stop_standard_apps.sh" || true su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmsp/start_cmsp.sh" su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmtp/start_cmtp.sh" diff --git a/scripts/standard-init/stop_standard_apps.sh b/scripts/standard-init/stop_standard_apps.sh index 909d25f..1ec9922 100644 --- a/scripts/standard-init/stop_standard_apps.sh +++ b/scripts/standard-init/stop_standard_apps.sh @@ -3,15 +3,40 @@ set -euo pipefail STANDARD_HOME_DIR="${STANDARD_HOME_DIR:?STANDARD_HOME_DIR is required}" STANDARD_RUN_USER="${STANDARD_RUN_USER:?STANDARD_RUN_USER is required}" +STOP_TIMEOUT_SECONDS="${STOP_TIMEOUT_SECONDS:-30}" + +wait_until_stopped() { + local pattern="$1" + local label="$2" + local deadline=$((SECONDS + STOP_TIMEOUT_SECONDS)) + while pgrep -f "${pattern}" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo "[WARN] ${label} still running after ${STOP_TIMEOUT_SECONDS}s, force killing" + pkill -9 -f "${pattern}" >/dev/null 2>&1 || true + break + fi + sleep 1 + done +} + +stop_by_pattern() { + local pattern="$1" + local label="$2" + if pgrep -f "${pattern}" >/dev/null 2>&1; then + echo "[INFO] stopping ${label}: ${pattern}" + pkill -f "${pattern}" >/dev/null 2>&1 || true + wait_until_stopped "${pattern}" "${label}" + fi +} if [ -x "${STANDARD_HOME_DIR}/cmsp/stop_cmsp.sh" ]; then su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmsp/stop_cmsp.sh" || true -else - pkill -f "${STANDARD_HOME_DIR}/cmsp" >/dev/null 2>&1 || true fi +stop_by_pattern "${STANDARD_HOME_DIR}/cmsp" "CMEP-CMSP path" +stop_by_pattern "CMEP-CMSP" "CMEP-CMSP jar" if [ -x "${STANDARD_HOME_DIR}/cmtp/stop_cmtp.sh" ]; then su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmtp/stop_cmtp.sh" || true -else - pkill -f "${STANDARD_HOME_DIR}/cmtp" >/dev/null 2>&1 || true fi +stop_by_pattern "${STANDARD_HOME_DIR}/cmtp" "CMEP-CMTP path" +stop_by_pattern "CMEP-CMTP" "CMEP-CMTP jar" diff --git a/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java b/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java index 5f0ec90..c4ad09d 100644 --- a/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java +++ b/src/main/java/com/cisd/tms/modules/backup/config/ResourceBackupProperties.java @@ -13,6 +13,7 @@ public class ResourceBackupProperties { private String restoreTaskRootDir = "/home/tms/tmp/resource-restore"; private String tmsScriptPath = "/home/tms/scripts/tms.sh"; private String mysqldumpPath = "mysqldump"; + private int mysqldumpTimeoutSeconds = 300; private String mysqlPath = "mysql"; private String restoreDbScriptPath = "/home/tms/bin/resource-restore/restore-db.sh"; private String mqReplayScriptPath = "/home/tms/bin/resource-restore/replay-mq.sh"; @@ -82,6 +83,14 @@ public class ResourceBackupProperties { this.mysqldumpPath = mysqldumpPath; } + public int getMysqldumpTimeoutSeconds() { + return mysqldumpTimeoutSeconds; + } + + public void setMysqldumpTimeoutSeconds(int mysqldumpTimeoutSeconds) { + this.mysqldumpTimeoutSeconds = mysqldumpTimeoutSeconds; + } + public String getMysqlPath() { return mysqlPath; } diff --git a/src/main/java/com/cisd/tms/modules/backup/mapper/ResourceBackupTaskMapper.java b/src/main/java/com/cisd/tms/modules/backup/mapper/ResourceBackupTaskMapper.java index 7775694..81078e4 100644 --- a/src/main/java/com/cisd/tms/modules/backup/mapper/ResourceBackupTaskMapper.java +++ b/src/main/java/com/cisd/tms/modules/backup/mapper/ResourceBackupTaskMapper.java @@ -5,6 +5,7 @@ import com.cisd.tms.modules.backup.entity.ResourceBackupTaskEntity; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; @Mapper public interface ResourceBackupTaskMapper extends BaseMapperX { @@ -17,4 +18,29 @@ public interface ResourceBackupTaskMapper extends BaseMapperX outputRef = new AtomicReference<>(new byte[0]); + AtomicReference outputErrorRef = new AtomicReference<>(); + Thread outputReader = new Thread(() -> { + try { + outputRef.set(readAllBytes(process.getInputStream())); + } catch (IOException ex) { + outputErrorRef.set(ex); + } + }, "resource-backup-mysqldump-output-reader"); + outputReader.setDaemon(true); + outputReader.start(); + int timeoutSeconds = Math.max(1, resourceBackupProperties.getMysqldumpTimeoutSeconds()); + boolean finished = process.waitFor(timeoutSeconds, java.util.concurrent.TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + joinOutputReader(outputReader); + throw new IllegalStateException("mysqldump执行超时 after " + timeoutSeconds + "s"); + } + joinOutputReader(outputReader); + if (outputErrorRef.get() != null) { + throw new IllegalStateException("读取mysqldump输出失败", outputErrorRef.get()); + } + byte[] stdout = outputRef.get(); + int exitCode = process.exitValue(); if (exitCode != 0) { - String error = new String(stderr, StandardCharsets.UTF_8).trim(); + String error = new String(stdout, StandardCharsets.UTF_8).trim(); throw new IllegalStateException("mysqldump 执行失败 with exitCode=" + exitCode + (error.isEmpty() ? "" : ", error=" + error)); } @@ -103,6 +127,10 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector { } } + private static void joinOutputReader(Thread outputReader) throws InterruptedException { + outputReader.join(1000L); + } + private MysqlConnectionTarget resolveTarget(String configuredDatabaseName, String fallbackDatabaseName) { String url = trim(datasourceUrl); if (url.isEmpty()) { diff --git a/src/main/java/com/cisd/tms/modules/backup/repository/impl/ResourceBackupTaskRepositoryImpl.java b/src/main/java/com/cisd/tms/modules/backup/repository/impl/ResourceBackupTaskRepositoryImpl.java index 7904c16..be10080 100644 --- a/src/main/java/com/cisd/tms/modules/backup/repository/impl/ResourceBackupTaskRepositoryImpl.java +++ b/src/main/java/com/cisd/tms/modules/backup/repository/impl/ResourceBackupTaskRepositoryImpl.java @@ -23,7 +23,10 @@ public class ResourceBackupTaskRepositoryImpl implements ResourceBackupTaskRepos @Override public void update(ResourceBackupTaskEntity entity) { - resourceBackupTaskMapper.updateById(entity); + int updated = resourceBackupTaskMapper.updateByTaskId(entity); + if (updated <= 0) { + throw new IllegalStateException("资源备份任务更新失败：" + (entity == null ? "" : entity.getTaskId())); + } } @Override diff --git a/src/main/java/com/cisd/tms/modules/backup/service/impl/ResourceBackupServiceImpl.java b/src/main/java/com/cisd/tms/modules/backup/service/impl/ResourceBackupServiceImpl.java index ae22042..d001a5f 100644 --- a/src/main/java/com/cisd/tms/modules/backup/service/impl/ResourceBackupServiceImpl.java +++ b/src/main/java/com/cisd/tms/modules/backup/service/impl/ResourceBackupServiceImpl.java @@ -88,6 +88,7 @@ public class ResourceBackupServiceImpl implements ResourceBackupService { // 先落 RUNNING，再开始真实采集。这样第二个备份/恢复请求能被互斥守卫拦住。 resourceBackupTaskRepository.save(entity); + LinkageError linkageError = null; try { // 先根据初始化快照收敛资源范围，再交给打包服务生成本地可见的 .tmsbak 文件。 // createPackage 内部会完成：资源采集 -> payload.zip -> 数字信封加密 -> 外层 .tmsbak 写盘。 @@ -115,11 +116,21 @@ public class ResourceBackupServiceImpl implements ResourceBackupService { entity.setStatus("FAILED"); entity.setErrorCode("PACKAGE_CREATE_FAILED"); entity.setErrorMessage(rootCauseAwareMessage(ex)); + } catch (LinkageError ex) { + // JNA/native library failures are Errors, not RuntimeExceptions. Persist a terminal task state + // before rethrowing so the admission guard will not be blocked by a stale RUNNING row. + entity.setStatus("FAILED"); + entity.setErrorCode("PACKAGE_CREATE_FAILED"); + entity.setErrorMessage(rootCauseAwareMessage(ex)); + linkageError = ex; } entity.setFinishTime(LocalDateTime.now()); entity.setUpdateTime(entity.getFinishTime()); // 同一条任务记录从 RUNNING 更新为 SUCCESS/FAILED，列表和详情均以这里的终态为准。 resourceBackupTaskRepository.update(entity); + if (linkageError != null) { + throw linkageError; + } CreateResourceBackupResponse response = new CreateResourceBackupResponse(); response.setTaskId(taskId); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 46b12bb..7dff4d2 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -294,6 +294,8 @@ tms: required: ${TMS_BACKUP_STANDARD_CMTP_APP_CONFIG_REQUIRED:true} # 导出 TMS 库时使用的 mysqldump 命令；现场可按实际安装路径覆盖。 mysqldump-path: ${TMS_BACKUP_MYSQLDUMP_PATH:mysqldump} + # mysqldump 单次导出最大等待时间（秒）；超时会终止进程并将备份任务置为 FAILED。 + mysqldump-timeout-seconds: ${TMS_BACKUP_MYSQLDUMP_TIMEOUT_SECONDS:300} # 恢复 TMS 库时使用的 mysql 客户端命令。 mysql-path: ${TMS_BACKUP_MYSQL_PATH:mysql} # 数据库恢复脚本路径；apply 脚本在 RESTORE_DATABASES 阶段调用。 @@ -330,7 +332,7 @@ tms: internal-token: ${TMS_INTERNAL_TOKEN:change-me-internal-token} internal-auth: # 是否启用 /api/** 的内部登录鉴权；开发联调时可临时关闭。 - enabled: ${TMS_SECURITY_INTERNAL_AUTH_ENABLED:false} + enabled: ${TMS_SECURITY_INTERNAL_AUTH_ENABLED:true} # 关闭内部鉴权后，注入请求上下文的调试角色。 debug-role-code: ${TMS_SECURITY_INTERNAL_AUTH_DEBUG_ROLE_CODE:OPS_ADMIN} # 关闭内部鉴权后，注入请求上下文的调试认证等级。 diff --git a/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java b/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java index 510a4d7..4b9c291 100644 --- a/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/ResourceBackupPropertiesTest.java @@ -28,6 +28,7 @@ class ResourceBackupPropertiesTest { Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_account")); Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_ukey_binding")); Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_auth_session")); + Assertions.assertEquals(300, properties.getMysqldumpTimeoutSeconds()); Assertions.assertEquals(180, properties.getHealthCheckMaxWaitSeconds()); Assertions.assertEquals(5, properties.getHealthCheckRetryIntervalSeconds()); } diff --git a/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java b/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java index 83e3d2e..bfd0113 100644 --- a/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/packagex/MysqlDatabaseBackupCollectorTest.java @@ -44,6 +44,30 @@ class MysqlDatabaseBackupCollectorTest { Assertions.assertFalse(invocations[1].contains("--ignore-table=")); } + @Test + void shouldFailWhenMysqldumpExceedsConfiguredTimeout() throws Exception { + Path mysqldump = slowMysqldump(); + ResourceBackupProperties properties = new ResourceBackupProperties(); + properties.setMysqldumpPath(mysqldump.toString()); + properties.setMysqldumpTimeoutSeconds(1); + MysqlDatabaseBackupCollector collector = new MysqlDatabaseBackupCollector( + properties, + "jdbc:mysql://127.0.0.1:4000/TMS?useSSL=false", + "root", + "secret" + ); + + long startedAt = System.nanoTime(); + IllegalStateException ex = Assertions.assertThrows( + IllegalStateException.class, + collector::collectTmsDatabaseDump + ); + long elapsedMillis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + + Assertions.assertTrue(ex.getMessage().contains("mysqldump执行超时")); + Assertions.assertTrue(elapsedMillis < 1800, "mysqldump should be killed on timeout"); + } + private Path fakeMysqldump(Path argsLog) throws Exception { Path script = tempDir.resolve("mysqldump"); Files.writeString(script, """ @@ -54,4 +78,15 @@ class MysqlDatabaseBackupCollectorTest { script.toFile().setExecutable(true); return script; } + + private Path slowMysqldump() throws Exception { + Path script = tempDir.resolve("slow-mysqldump"); + Files.writeString(script, """ + #!/usr/bin/env bash + sleep 2 + printf -- '-- synthetic dump\\n' + """); + script.toFile().setExecutable(true); + return script; + } } diff --git a/src/test/java/com/cisd/tms/modules/backup/repository/ResourceTaskRepositoryTest.java b/src/test/java/com/cisd/tms/modules/backup/repository/ResourceTaskRepositoryTest.java index afe38bf..2733e09 100644 --- a/src/test/java/com/cisd/tms/modules/backup/repository/ResourceTaskRepositoryTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/repository/ResourceTaskRepositoryTest.java @@ -43,6 +43,22 @@ class ResourceTaskRepositoryTest { Assertions.assertEquals("nightly", found.getRemark()); } + @Test + void shouldUpdateBackupTaskByBusinessTaskId() { + ResourceBackupTaskMapper mapper = Mockito.mock(ResourceBackupTaskMapper.class); + ResourceBackupTaskRepository repository = new ResourceBackupTaskRepositoryImpl(mapper); + ResourceBackupTaskEntity entity = new ResourceBackupTaskEntity(); + entity.setTaskId("RBKP-001"); + entity.setStatus("SUCCESS"); + + Mockito.when(mapper.updateByTaskId(entity)).thenReturn(1); + + repository.update(entity); + + Mockito.verify(mapper).updateByTaskId(entity); + Mockito.verify(mapper, Mockito.never()).updateById(entity); + } + @Test void shouldSaveAndQueryRestoreTask() { ResourceRestoreTaskMapper mapper = Mockito.mock(ResourceRestoreTaskMapper.class); diff --git a/src/test/java/com/cisd/tms/modules/backup/service/ResourceBackupServiceTest.java b/src/test/java/com/cisd/tms/modules/backup/service/ResourceBackupServiceTest.java index 1dd389c..3df3c2c 100644 --- a/src/test/java/com/cisd/tms/modules/backup/service/ResourceBackupServiceTest.java +++ b/src/test/java/com/cisd/tms/modules/backup/service/ResourceBackupServiceTest.java @@ -164,6 +164,47 @@ class ResourceBackupServiceTest { Assertions.assertTrue(saved.getErrorMessage().contains("SDF_Encrypt")); } + @Test + void shouldPersistFailedTaskWhenPackageCreationThrowsLinkageError() { + InitTaskRepository initTaskRepository = Mockito.mock(InitTaskRepository.class); + ResourceDiscoveryService resourceDiscoveryService = Mockito.mock(ResourceDiscoveryService.class); + ResourcePackageService resourcePackageService = Mockito.mock(ResourcePackageService.class); + ResourceBackupTaskRepository resourceBackupTaskRepository = Mockito.mock(ResourceBackupTaskRepository.class); + ResourceTaskAdmissionGuard admissionGuard = Mockito.mock(ResourceTaskAdmissionGuard.class); + + InitTaskEntity initTask = sampleInitTask(); + List discoveredItems = List.of(resource("TMS_CONFIG", "/home/tms/config", true)); + + Mockito.when(initTaskRepository.findLatestByTaskTypeAndStatus("INIT", "SUCCESS")).thenReturn(Optional.of(initTask)); + Mockito.when(resourceDiscoveryService.discover(initTask)).thenReturn(discoveredItems); + Mockito.when(resourcePackageService.createPackage( + Mockito.anyString(), + Mockito.anyString(), + Mockito.same(initTask), + Mockito.eq(discoveredItems), + Mockito.any(CreateResourceBackupRequest.class) + )).thenThrow(new UnsatisfiedLinkError("libsdf.so not found")); + + ResourceBackupService service = new ResourceBackupServiceImpl( + initTaskRepository, + resourceDiscoveryService, + resourcePackageService, + resourceBackupTaskRepository, + new ResourceTaskIdGenerator(), + admissionGuard + ); + + Assertions.assertThrows(UnsatisfiedLinkError.class, () -> service.createBackupTask(sampleRequest())); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ResourceBackupTaskEntity.class); + Mockito.verify(resourceBackupTaskRepository).update(captor.capture()); + ResourceBackupTaskEntity saved = captor.getValue(); + Assertions.assertEquals("FAILED", saved.getStatus()); + Assertions.assertEquals("PACKAGE_CREATE_FAILED", saved.getErrorCode()); + Assertions.assertTrue(saved.getErrorMessage().contains("libsdf.so not found")); + Assertions.assertNotNull(saved.getFinishTime()); + } + @Test void shouldPersistRunningTaskWithConfiguredKeyFingerprintsBeforePackaging() { InitTaskRepository initTaskRepository = Mockito.mock(InitTaskRepository.class); diff --git a/src/test/java/com/cisd/tms/modules/init/executor/ConfigurableInitStepExecutorTest.java b/src/test/java/com/cisd/tms/modules/init/executor/ConfigurableInitStepExecutorTest.java index 46d842e..57061a4 100644 --- a/src/test/java/com/cisd/tms/modules/init/executor/ConfigurableInitStepExecutorTest.java +++ b/src/test/java/com/cisd/tms/modules/init/executor/ConfigurableInitStepExecutorTest.java @@ -1056,6 +1056,7 @@ class ConfigurableInitStepExecutorTest { String appContent = Files.readString(appScript); Assertions.assertTrue(appContent.contains("STANDARD_HOME_DIR")); Assertions.assertTrue(appContent.contains("STANDARD_RUN_USER")); + Assertions.assertTrue(appContent.contains("stop_standard_apps.sh")); Assertions.assertTrue(appContent.contains("start_cmsp.sh")); Assertions.assertTrue(appContent.contains("start_cmtp.sh")); Assertions.assertFalse(appContent.contains("/home/cemp4i/cmsp/start_cmsp.sh")); @@ -1066,6 +1067,9 @@ class ConfigurableInitStepExecutorTest { Assertions.assertTrue(stopAppContent.contains("STANDARD_RUN_USER")); Assertions.assertTrue(stopAppContent.contains("stop_cmsp.sh")); Assertions.assertTrue(stopAppContent.contains("stop_cmtp.sh")); + Assertions.assertTrue(stopAppContent.contains("CMEP-CMSP")); + Assertions.assertTrue(stopAppContent.contains("CMEP-CMTP")); + Assertions.assertTrue(stopAppContent.contains("wait_until_stopped")); String stopRabbitContent = Files.readString(stopRabbitScript); Assertions.assertTrue(stopRabbitContent.contains("systemctl stop"));