fix:备份流程
This commit is contained in:
@@ -13,6 +13,7 @@ public class ResourceBackupProperties {
|
||||
private String restoreTaskRootDir = "/home/tms/tmp/resource-restore";
|
||||
private String tmsScriptPath = "/home/tms/scripts/tms.sh";
|
||||
private String mysqldumpPath = "mysqldump";
|
||||
private int mysqldumpTimeoutSeconds = 300;
|
||||
private String mysqlPath = "mysql";
|
||||
private String restoreDbScriptPath = "/home/tms/bin/resource-restore/restore-db.sh";
|
||||
private String mqReplayScriptPath = "/home/tms/bin/resource-restore/replay-mq.sh";
|
||||
@@ -82,6 +83,14 @@ public class ResourceBackupProperties {
|
||||
this.mysqldumpPath = mysqldumpPath;
|
||||
}
|
||||
|
||||
public int getMysqldumpTimeoutSeconds() {
|
||||
return mysqldumpTimeoutSeconds;
|
||||
}
|
||||
|
||||
public void setMysqldumpTimeoutSeconds(int mysqldumpTimeoutSeconds) {
|
||||
this.mysqldumpTimeoutSeconds = mysqldumpTimeoutSeconds;
|
||||
}
|
||||
|
||||
public String getMysqlPath() {
|
||||
return mysqlPath;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import com.cisd.tms.modules.backup.entity.ResourceBackupTaskEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
@Mapper
|
||||
public interface ResourceBackupTaskMapper extends BaseMapperX<ResourceBackupTaskEntity> {
|
||||
@@ -17,4 +18,29 @@ public interface ResourceBackupTaskMapper extends BaseMapperX<ResourceBackupTask
|
||||
+ "FROM tms_resource_backup_task WHERE task_id = #{taskId}"
|
||||
)
|
||||
ResourceBackupTaskEntity selectByTaskId(@Param("taskId") String taskId);
|
||||
|
||||
@Update(
|
||||
"UPDATE tms_resource_backup_task SET "
|
||||
+ "status = #{status}, "
|
||||
+ "product_type = #{productType}, "
|
||||
+ "mq_type = #{mqType}, "
|
||||
+ "org_code = #{orgCode}, "
|
||||
+ "source_device_id = #{sourceDeviceId}, "
|
||||
+ "package_name = #{packageName}, "
|
||||
+ "package_path = #{packagePath}, "
|
||||
+ "package_size = #{packageSize}, "
|
||||
+ "package_hash = #{packageHash}, "
|
||||
+ "storage_enc_key_fingerprint = #{storageEncKeyFingerprint}, "
|
||||
+ "authenticity_key_fingerprint = #{authenticityKeyFingerprint}, "
|
||||
+ "manifest_summary_json = #{manifestSummaryJson}, "
|
||||
+ "error_code = #{errorCode}, "
|
||||
+ "error_message = #{errorMessage}, "
|
||||
+ "remark = #{remark}, "
|
||||
+ "create_by = #{createBy}, "
|
||||
+ "update_time = #{updateTime}, "
|
||||
+ "start_time = #{startTime}, "
|
||||
+ "finish_time = #{finishTime} "
|
||||
+ "WHERE task_id = #{taskId}"
|
||||
)
|
||||
int updateByTaskId(ResourceBackupTaskEntity entity);
|
||||
}
|
||||
|
||||
+33
-5
@@ -8,6 +8,7 @@ import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -78,12 +79,35 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector {
|
||||
command.add(target.databaseName());
|
||||
|
||||
try {
|
||||
Process process = new ProcessBuilder(command).start();
|
||||
byte[] stdout = readAllBytes(process.getInputStream());
|
||||
byte[] stderr = readAllBytes(process.getErrorStream());
|
||||
int exitCode = process.waitFor();
|
||||
Process process = new ProcessBuilder(command)
|
||||
.redirectErrorStream(true)
|
||||
.start();
|
||||
AtomicReference<byte[]> outputRef = new AtomicReference<>(new byte[0]);
|
||||
AtomicReference<IOException> outputErrorRef = new AtomicReference<>();
|
||||
Thread outputReader = new Thread(() -> {
|
||||
try {
|
||||
outputRef.set(readAllBytes(process.getInputStream()));
|
||||
} catch (IOException ex) {
|
||||
outputErrorRef.set(ex);
|
||||
}
|
||||
}, "resource-backup-mysqldump-output-reader");
|
||||
outputReader.setDaemon(true);
|
||||
outputReader.start();
|
||||
int timeoutSeconds = Math.max(1, resourceBackupProperties.getMysqldumpTimeoutSeconds());
|
||||
boolean finished = process.waitFor(timeoutSeconds, java.util.concurrent.TimeUnit.SECONDS);
|
||||
if (!finished) {
|
||||
process.destroyForcibly();
|
||||
joinOutputReader(outputReader);
|
||||
throw new IllegalStateException("mysqldump执行超时 after " + timeoutSeconds + "s");
|
||||
}
|
||||
joinOutputReader(outputReader);
|
||||
if (outputErrorRef.get() != null) {
|
||||
throw new IllegalStateException("读取mysqldump输出失败", outputErrorRef.get());
|
||||
}
|
||||
byte[] stdout = outputRef.get();
|
||||
int exitCode = process.exitValue();
|
||||
if (exitCode != 0) {
|
||||
String error = new String(stderr, StandardCharsets.UTF_8).trim();
|
||||
String error = new String(stdout, StandardCharsets.UTF_8).trim();
|
||||
throw new IllegalStateException("mysqldump 执行失败 with exitCode=" + exitCode
|
||||
+ (error.isEmpty() ? "" : ", error=" + error));
|
||||
}
|
||||
@@ -103,6 +127,10 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector {
|
||||
}
|
||||
}
|
||||
|
||||
private static void joinOutputReader(Thread outputReader) throws InterruptedException {
|
||||
outputReader.join(1000L);
|
||||
}
|
||||
|
||||
private MysqlConnectionTarget resolveTarget(String configuredDatabaseName, String fallbackDatabaseName) {
|
||||
String url = trim(datasourceUrl);
|
||||
if (url.isEmpty()) {
|
||||
|
||||
+4
-1
@@ -23,7 +23,10 @@ public class ResourceBackupTaskRepositoryImpl implements ResourceBackupTaskRepos
|
||||
|
||||
@Override
|
||||
public void update(ResourceBackupTaskEntity entity) {
|
||||
resourceBackupTaskMapper.updateById(entity);
|
||||
int updated = resourceBackupTaskMapper.updateByTaskId(entity);
|
||||
if (updated <= 0) {
|
||||
throw new IllegalStateException("资源备份任务更新失败:" + (entity == null ? "" : entity.getTaskId()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -88,6 +88,7 @@ public class ResourceBackupServiceImpl implements ResourceBackupService {
|
||||
// 先落 RUNNING,再开始真实采集。这样第二个备份/恢复请求能被互斥守卫拦住。
|
||||
resourceBackupTaskRepository.save(entity);
|
||||
|
||||
LinkageError linkageError = null;
|
||||
try {
|
||||
// 先根据初始化快照收敛资源范围,再交给打包服务生成本地可见的 .tmsbak 文件。
|
||||
// createPackage 内部会完成:资源采集 -> payload.zip -> 数字信封加密 -> 外层 .tmsbak 写盘。
|
||||
@@ -115,11 +116,21 @@ public class ResourceBackupServiceImpl implements ResourceBackupService {
|
||||
entity.setStatus("FAILED");
|
||||
entity.setErrorCode("PACKAGE_CREATE_FAILED");
|
||||
entity.setErrorMessage(rootCauseAwareMessage(ex));
|
||||
} catch (LinkageError ex) {
|
||||
// JNA/native library failures are Errors, not RuntimeExceptions. Persist a terminal task state
|
||||
// before rethrowing so the admission guard will not be blocked by a stale RUNNING row.
|
||||
entity.setStatus("FAILED");
|
||||
entity.setErrorCode("PACKAGE_CREATE_FAILED");
|
||||
entity.setErrorMessage(rootCauseAwareMessage(ex));
|
||||
linkageError = ex;
|
||||
}
|
||||
entity.setFinishTime(LocalDateTime.now());
|
||||
entity.setUpdateTime(entity.getFinishTime());
|
||||
// 同一条任务记录从 RUNNING 更新为 SUCCESS/FAILED,列表和详情均以这里的终态为准。
|
||||
resourceBackupTaskRepository.update(entity);
|
||||
if (linkageError != null) {
|
||||
throw linkageError;
|
||||
}
|
||||
|
||||
CreateResourceBackupResponse response = new CreateResourceBackupResponse();
|
||||
response.setTaskId(taskId);
|
||||
|
||||
@@ -294,6 +294,8 @@ tms:
|
||||
required: ${TMS_BACKUP_STANDARD_CMTP_APP_CONFIG_REQUIRED:true}
|
||||
# 导出 TMS 库时使用的 mysqldump 命令;现场可按实际安装路径覆盖。
|
||||
mysqldump-path: ${TMS_BACKUP_MYSQLDUMP_PATH:mysqldump}
|
||||
# mysqldump 单次导出最大等待时间(秒);超时会终止进程并将备份任务置为 FAILED。
|
||||
mysqldump-timeout-seconds: ${TMS_BACKUP_MYSQLDUMP_TIMEOUT_SECONDS:300}
|
||||
# 恢复 TMS 库时使用的 mysql 客户端命令。
|
||||
mysql-path: ${TMS_BACKUP_MYSQL_PATH:mysql}
|
||||
# 数据库恢复脚本路径;apply 脚本在 RESTORE_DATABASES 阶段调用。
|
||||
@@ -330,7 +332,7 @@ tms:
|
||||
internal-token: ${TMS_INTERNAL_TOKEN:change-me-internal-token}
|
||||
internal-auth:
|
||||
# 是否启用 /api/** 的内部登录鉴权;开发联调时可临时关闭。
|
||||
enabled: ${TMS_SECURITY_INTERNAL_AUTH_ENABLED:false}
|
||||
enabled: ${TMS_SECURITY_INTERNAL_AUTH_ENABLED:true}
|
||||
# 关闭内部鉴权后,注入请求上下文的调试角色。
|
||||
debug-role-code: ${TMS_SECURITY_INTERNAL_AUTH_DEBUG_ROLE_CODE:OPS_ADMIN}
|
||||
# 关闭内部鉴权后,注入请求上下文的调试认证等级。
|
||||
|
||||
@@ -28,6 +28,7 @@ class ResourceBackupPropertiesTest {
|
||||
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_account"));
|
||||
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_ukey_binding"));
|
||||
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_auth_session"));
|
||||
Assertions.assertEquals(300, properties.getMysqldumpTimeoutSeconds());
|
||||
Assertions.assertEquals(180, properties.getHealthCheckMaxWaitSeconds());
|
||||
Assertions.assertEquals(5, properties.getHealthCheckRetryIntervalSeconds());
|
||||
}
|
||||
|
||||
+35
@@ -44,6 +44,30 @@ class MysqlDatabaseBackupCollectorTest {
|
||||
Assertions.assertFalse(invocations[1].contains("--ignore-table="));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFailWhenMysqldumpExceedsConfiguredTimeout() throws Exception {
|
||||
Path mysqldump = slowMysqldump();
|
||||
ResourceBackupProperties properties = new ResourceBackupProperties();
|
||||
properties.setMysqldumpPath(mysqldump.toString());
|
||||
properties.setMysqldumpTimeoutSeconds(1);
|
||||
MysqlDatabaseBackupCollector collector = new MysqlDatabaseBackupCollector(
|
||||
properties,
|
||||
"jdbc:mysql://127.0.0.1:4000/TMS?useSSL=false",
|
||||
"root",
|
||||
"secret"
|
||||
);
|
||||
|
||||
long startedAt = System.nanoTime();
|
||||
IllegalStateException ex = Assertions.assertThrows(
|
||||
IllegalStateException.class,
|
||||
collector::collectTmsDatabaseDump
|
||||
);
|
||||
long elapsedMillis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
|
||||
|
||||
Assertions.assertTrue(ex.getMessage().contains("mysqldump执行超时"));
|
||||
Assertions.assertTrue(elapsedMillis < 1800, "mysqldump should be killed on timeout");
|
||||
}
|
||||
|
||||
private Path fakeMysqldump(Path argsLog) throws Exception {
|
||||
Path script = tempDir.resolve("mysqldump");
|
||||
Files.writeString(script, """
|
||||
@@ -54,4 +78,15 @@ class MysqlDatabaseBackupCollectorTest {
|
||||
script.toFile().setExecutable(true);
|
||||
return script;
|
||||
}
|
||||
|
||||
private Path slowMysqldump() throws Exception {
|
||||
Path script = tempDir.resolve("slow-mysqldump");
|
||||
Files.writeString(script, """
|
||||
#!/usr/bin/env bash
|
||||
sleep 2
|
||||
printf -- '-- synthetic dump\\n'
|
||||
""");
|
||||
script.toFile().setExecutable(true);
|
||||
return script;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,6 +43,22 @@ class ResourceTaskRepositoryTest {
|
||||
Assertions.assertEquals("nightly", found.getRemark());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUpdateBackupTaskByBusinessTaskId() {
|
||||
ResourceBackupTaskMapper mapper = Mockito.mock(ResourceBackupTaskMapper.class);
|
||||
ResourceBackupTaskRepository repository = new ResourceBackupTaskRepositoryImpl(mapper);
|
||||
ResourceBackupTaskEntity entity = new ResourceBackupTaskEntity();
|
||||
entity.setTaskId("RBKP-001");
|
||||
entity.setStatus("SUCCESS");
|
||||
|
||||
Mockito.when(mapper.updateByTaskId(entity)).thenReturn(1);
|
||||
|
||||
repository.update(entity);
|
||||
|
||||
Mockito.verify(mapper).updateByTaskId(entity);
|
||||
Mockito.verify(mapper, Mockito.never()).updateById(entity);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldSaveAndQueryRestoreTask() {
|
||||
ResourceRestoreTaskMapper mapper = Mockito.mock(ResourceRestoreTaskMapper.class);
|
||||
|
||||
@@ -164,6 +164,47 @@ class ResourceBackupServiceTest {
|
||||
Assertions.assertTrue(saved.getErrorMessage().contains("SDF_Encrypt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPersistFailedTaskWhenPackageCreationThrowsLinkageError() {
|
||||
InitTaskRepository initTaskRepository = Mockito.mock(InitTaskRepository.class);
|
||||
ResourceDiscoveryService resourceDiscoveryService = Mockito.mock(ResourceDiscoveryService.class);
|
||||
ResourcePackageService resourcePackageService = Mockito.mock(ResourcePackageService.class);
|
||||
ResourceBackupTaskRepository resourceBackupTaskRepository = Mockito.mock(ResourceBackupTaskRepository.class);
|
||||
ResourceTaskAdmissionGuard admissionGuard = Mockito.mock(ResourceTaskAdmissionGuard.class);
|
||||
|
||||
InitTaskEntity initTask = sampleInitTask();
|
||||
List<DiscoveredResourceItem> discoveredItems = List.of(resource("TMS_CONFIG", "/home/tms/config", true));
|
||||
|
||||
Mockito.when(initTaskRepository.findLatestByTaskTypeAndStatus("INIT", "SUCCESS")).thenReturn(Optional.of(initTask));
|
||||
Mockito.when(resourceDiscoveryService.discover(initTask)).thenReturn(discoveredItems);
|
||||
Mockito.when(resourcePackageService.createPackage(
|
||||
Mockito.anyString(),
|
||||
Mockito.anyString(),
|
||||
Mockito.same(initTask),
|
||||
Mockito.eq(discoveredItems),
|
||||
Mockito.any(CreateResourceBackupRequest.class)
|
||||
)).thenThrow(new UnsatisfiedLinkError("libsdf.so not found"));
|
||||
|
||||
ResourceBackupService service = new ResourceBackupServiceImpl(
|
||||
initTaskRepository,
|
||||
resourceDiscoveryService,
|
||||
resourcePackageService,
|
||||
resourceBackupTaskRepository,
|
||||
new ResourceTaskIdGenerator(),
|
||||
admissionGuard
|
||||
);
|
||||
|
||||
Assertions.assertThrows(UnsatisfiedLinkError.class, () -> service.createBackupTask(sampleRequest()));
|
||||
|
||||
ArgumentCaptor<ResourceBackupTaskEntity> captor = ArgumentCaptor.forClass(ResourceBackupTaskEntity.class);
|
||||
Mockito.verify(resourceBackupTaskRepository).update(captor.capture());
|
||||
ResourceBackupTaskEntity saved = captor.getValue();
|
||||
Assertions.assertEquals("FAILED", saved.getStatus());
|
||||
Assertions.assertEquals("PACKAGE_CREATE_FAILED", saved.getErrorCode());
|
||||
Assertions.assertTrue(saved.getErrorMessage().contains("libsdf.so not found"));
|
||||
Assertions.assertNotNull(saved.getFinishTime());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPersistRunningTaskWithConfiguredKeyFingerprintsBeforePackaging() {
|
||||
InitTaskRepository initTaskRepository = Mockito.mock(InitTaskRepository.class);
|
||||
|
||||
@@ -1056,6 +1056,7 @@ class ConfigurableInitStepExecutorTest {
|
||||
String appContent = Files.readString(appScript);
|
||||
Assertions.assertTrue(appContent.contains("STANDARD_HOME_DIR"));
|
||||
Assertions.assertTrue(appContent.contains("STANDARD_RUN_USER"));
|
||||
Assertions.assertTrue(appContent.contains("stop_standard_apps.sh"));
|
||||
Assertions.assertTrue(appContent.contains("start_cmsp.sh"));
|
||||
Assertions.assertTrue(appContent.contains("start_cmtp.sh"));
|
||||
Assertions.assertFalse(appContent.contains("/home/cemp4i/cmsp/start_cmsp.sh"));
|
||||
@@ -1066,6 +1067,9 @@ class ConfigurableInitStepExecutorTest {
|
||||
Assertions.assertTrue(stopAppContent.contains("STANDARD_RUN_USER"));
|
||||
Assertions.assertTrue(stopAppContent.contains("stop_cmsp.sh"));
|
||||
Assertions.assertTrue(stopAppContent.contains("stop_cmtp.sh"));
|
||||
Assertions.assertTrue(stopAppContent.contains("CMEP-CMSP"));
|
||||
Assertions.assertTrue(stopAppContent.contains("CMEP-CMTP"));
|
||||
Assertions.assertTrue(stopAppContent.contains("wait_until_stopped"));
|
||||
|
||||
String stopRabbitContent = Files.readString(stopRabbitScript);
|
||||
Assertions.assertTrue(stopRabbitContent.contains("systemctl stop"));
|
||||
|
||||
Reference in New Issue
Block a user