fix:备份流程

This commit is contained in:
waner
2026-05-11 15:24:39 +08:00
parent 79d38cc566
commit 54cbef1ef3
15 changed files with 217 additions and 11 deletions
+1
View File
@@ -312,6 +312,7 @@ Open:
- `tms.backup.allowed-restore-roots`
- `tms.backup.file-resources`
- `tms.backup.mysqldump-path`
- `tms.backup.mysqldump-timeout-seconds`
- `tms.backup.mysql-path`
- `tms.backup.restore-db-script-path`
- `tms.backup.mq-replay-script-path`
+1
View File
@@ -116,6 +116,7 @@ tms:
- ENTERPRISE
- INDIRECT
# TMS 库备份排除表。默认跳过角色、授权、会话和审计表,避免恢复时覆盖新机器安全状态。
mysqldump-timeout-seconds: 300
tms-db-excluded-tables:
- tms_role_account
- tms_role_ukey_binding
@@ -3,6 +3,9 @@ set -euo pipefail
STANDARD_HOME_DIR="${STANDARD_HOME_DIR:?STANDARD_HOME_DIR is required}"
STANDARD_RUN_USER="${STANDARD_RUN_USER:?STANDARD_RUN_USER is required}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
"${SCRIPT_DIR}/stop_standard_apps.sh" || true
su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmsp/start_cmsp.sh"
su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmtp/start_cmtp.sh"
+29 -4
View File
@@ -3,15 +3,40 @@ set -euo pipefail
STANDARD_HOME_DIR="${STANDARD_HOME_DIR:?STANDARD_HOME_DIR is required}"
STANDARD_RUN_USER="${STANDARD_RUN_USER:?STANDARD_RUN_USER is required}"
STOP_TIMEOUT_SECONDS="${STOP_TIMEOUT_SECONDS:-30}"
wait_until_stopped() {
local pattern="$1"
local label="$2"
local deadline=$((SECONDS + STOP_TIMEOUT_SECONDS))
while pgrep -f "${pattern}" >/dev/null 2>&1; do
if (( SECONDS >= deadline )); then
echo "[WARN] ${label} still running after ${STOP_TIMEOUT_SECONDS}s, force killing"
pkill -9 -f "${pattern}" >/dev/null 2>&1 || true
break
fi
sleep 1
done
}
stop_by_pattern() {
local pattern="$1"
local label="$2"
if pgrep -f "${pattern}" >/dev/null 2>&1; then
echo "[INFO] stopping ${label}: ${pattern}"
pkill -f "${pattern}" >/dev/null 2>&1 || true
wait_until_stopped "${pattern}" "${label}"
fi
}
if [ -x "${STANDARD_HOME_DIR}/cmsp/stop_cmsp.sh" ]; then
su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmsp/stop_cmsp.sh" || true
else
pkill -f "${STANDARD_HOME_DIR}/cmsp" >/dev/null 2>&1 || true
fi
stop_by_pattern "${STANDARD_HOME_DIR}/cmsp" "CMEP-CMSP path"
stop_by_pattern "CMEP-CMSP" "CMEP-CMSP jar"
if [ -x "${STANDARD_HOME_DIR}/cmtp/stop_cmtp.sh" ]; then
su - "${STANDARD_RUN_USER}" -s /bin/bash "${STANDARD_HOME_DIR}/cmtp/stop_cmtp.sh" || true
else
pkill -f "${STANDARD_HOME_DIR}/cmtp" >/dev/null 2>&1 || true
fi
stop_by_pattern "${STANDARD_HOME_DIR}/cmtp" "CMEP-CMTP path"
stop_by_pattern "CMEP-CMTP" "CMEP-CMTP jar"
@@ -13,6 +13,7 @@ public class ResourceBackupProperties {
private String restoreTaskRootDir = "/home/tms/tmp/resource-restore";
private String tmsScriptPath = "/home/tms/scripts/tms.sh";
private String mysqldumpPath = "mysqldump";
private int mysqldumpTimeoutSeconds = 300;
private String mysqlPath = "mysql";
private String restoreDbScriptPath = "/home/tms/bin/resource-restore/restore-db.sh";
private String mqReplayScriptPath = "/home/tms/bin/resource-restore/replay-mq.sh";
@@ -82,6 +83,14 @@ public class ResourceBackupProperties {
this.mysqldumpPath = mysqldumpPath;
}
public int getMysqldumpTimeoutSeconds() {
return mysqldumpTimeoutSeconds;
}
public void setMysqldumpTimeoutSeconds(int mysqldumpTimeoutSeconds) {
this.mysqldumpTimeoutSeconds = mysqldumpTimeoutSeconds;
}
public String getMysqlPath() {
return mysqlPath;
}
@@ -5,6 +5,7 @@ import com.cisd.tms.modules.backup.entity.ResourceBackupTaskEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
@Mapper
public interface ResourceBackupTaskMapper extends BaseMapperX<ResourceBackupTaskEntity> {
@@ -17,4 +18,29 @@ public interface ResourceBackupTaskMapper extends BaseMapperX<ResourceBackupTask
+ "FROM tms_resource_backup_task WHERE task_id = #{taskId}"
)
ResourceBackupTaskEntity selectByTaskId(@Param("taskId") String taskId);
@Update(
"UPDATE tms_resource_backup_task SET "
+ "status = #{status}, "
+ "product_type = #{productType}, "
+ "mq_type = #{mqType}, "
+ "org_code = #{orgCode}, "
+ "source_device_id = #{sourceDeviceId}, "
+ "package_name = #{packageName}, "
+ "package_path = #{packagePath}, "
+ "package_size = #{packageSize}, "
+ "package_hash = #{packageHash}, "
+ "storage_enc_key_fingerprint = #{storageEncKeyFingerprint}, "
+ "authenticity_key_fingerprint = #{authenticityKeyFingerprint}, "
+ "manifest_summary_json = #{manifestSummaryJson}, "
+ "error_code = #{errorCode}, "
+ "error_message = #{errorMessage}, "
+ "remark = #{remark}, "
+ "create_by = #{createBy}, "
+ "update_time = #{updateTime}, "
+ "start_time = #{startTime}, "
+ "finish_time = #{finishTime} "
+ "WHERE task_id = #{taskId}"
)
int updateByTaskId(ResourceBackupTaskEntity entity);
}
@@ -8,6 +8,7 @@ import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -78,12 +79,35 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector {
command.add(target.databaseName());
try {
Process process = new ProcessBuilder(command).start();
byte[] stdout = readAllBytes(process.getInputStream());
byte[] stderr = readAllBytes(process.getErrorStream());
int exitCode = process.waitFor();
Process process = new ProcessBuilder(command)
.redirectErrorStream(true)
.start();
AtomicReference<byte[]> outputRef = new AtomicReference<>(new byte[0]);
AtomicReference<IOException> outputErrorRef = new AtomicReference<>();
Thread outputReader = new Thread(() -> {
try {
outputRef.set(readAllBytes(process.getInputStream()));
} catch (IOException ex) {
outputErrorRef.set(ex);
}
}, "resource-backup-mysqldump-output-reader");
outputReader.setDaemon(true);
outputReader.start();
int timeoutSeconds = Math.max(1, resourceBackupProperties.getMysqldumpTimeoutSeconds());
boolean finished = process.waitFor(timeoutSeconds, java.util.concurrent.TimeUnit.SECONDS);
if (!finished) {
process.destroyForcibly();
joinOutputReader(outputReader);
throw new IllegalStateException("mysqldump执行超时 after " + timeoutSeconds + "s");
}
joinOutputReader(outputReader);
if (outputErrorRef.get() != null) {
throw new IllegalStateException("读取mysqldump输出失败", outputErrorRef.get());
}
byte[] stdout = outputRef.get();
int exitCode = process.exitValue();
if (exitCode != 0) {
String error = new String(stderr, StandardCharsets.UTF_8).trim();
String error = new String(stdout, StandardCharsets.UTF_8).trim();
throw new IllegalStateException("mysqldump 执行失败 with exitCode=" + exitCode
+ (error.isEmpty() ? "" : ", error=" + error));
}
@@ -103,6 +127,10 @@ public class MysqlDatabaseBackupCollector implements DatabaseBackupCollector {
}
}
private static void joinOutputReader(Thread outputReader) throws InterruptedException {
outputReader.join(1000L);
}
private MysqlConnectionTarget resolveTarget(String configuredDatabaseName, String fallbackDatabaseName) {
String url = trim(datasourceUrl);
if (url.isEmpty()) {
@@ -23,7 +23,10 @@ public class ResourceBackupTaskRepositoryImpl implements ResourceBackupTaskRepos
@Override
public void update(ResourceBackupTaskEntity entity) {
resourceBackupTaskMapper.updateById(entity);
int updated = resourceBackupTaskMapper.updateByTaskId(entity);
if (updated <= 0) {
throw new IllegalStateException("资源备份任务更新失败:" + (entity == null ? "" : entity.getTaskId()));
}
}
@Override
@@ -88,6 +88,7 @@ public class ResourceBackupServiceImpl implements ResourceBackupService {
// 先落 RUNNING,再开始真实采集。这样第二个备份/恢复请求能被互斥守卫拦住。
resourceBackupTaskRepository.save(entity);
LinkageError linkageError = null;
try {
// 先根据初始化快照收敛资源范围,再交给打包服务生成本地可见的 .tmsbak 文件。
// createPackage 内部会完成:资源采集 -> payload.zip -> 数字信封加密 -> 外层 .tmsbak 写盘。
@@ -115,11 +116,21 @@ public class ResourceBackupServiceImpl implements ResourceBackupService {
entity.setStatus("FAILED");
entity.setErrorCode("PACKAGE_CREATE_FAILED");
entity.setErrorMessage(rootCauseAwareMessage(ex));
} catch (LinkageError ex) {
// JNA/native library failures are Errors, not RuntimeExceptions. Persist a terminal task state
// before rethrowing so the admission guard will not be blocked by a stale RUNNING row.
entity.setStatus("FAILED");
entity.setErrorCode("PACKAGE_CREATE_FAILED");
entity.setErrorMessage(rootCauseAwareMessage(ex));
linkageError = ex;
}
entity.setFinishTime(LocalDateTime.now());
entity.setUpdateTime(entity.getFinishTime());
// 同一条任务记录从 RUNNING 更新为 SUCCESS/FAILED,列表和详情均以这里的终态为准。
resourceBackupTaskRepository.update(entity);
if (linkageError != null) {
throw linkageError;
}
CreateResourceBackupResponse response = new CreateResourceBackupResponse();
response.setTaskId(taskId);
+3 -1
View File
@@ -294,6 +294,8 @@ tms:
required: ${TMS_BACKUP_STANDARD_CMTP_APP_CONFIG_REQUIRED:true}
# 导出 TMS 库时使用的 mysqldump 命令;现场可按实际安装路径覆盖。
mysqldump-path: ${TMS_BACKUP_MYSQLDUMP_PATH:mysqldump}
# mysqldump 单次导出最大等待时间(秒);超时会终止进程并将备份任务置为 FAILED。
mysqldump-timeout-seconds: ${TMS_BACKUP_MYSQLDUMP_TIMEOUT_SECONDS:300}
# 恢复 TMS 库时使用的 mysql 客户端命令。
mysql-path: ${TMS_BACKUP_MYSQL_PATH:mysql}
# 数据库恢复脚本路径;apply 脚本在 RESTORE_DATABASES 阶段调用。
@@ -330,7 +332,7 @@ tms:
internal-token: ${TMS_INTERNAL_TOKEN:change-me-internal-token}
internal-auth:
# 是否启用 /api/** 的内部登录鉴权;开发联调时可临时关闭。
enabled: ${TMS_SECURITY_INTERNAL_AUTH_ENABLED:false}
enabled: ${TMS_SECURITY_INTERNAL_AUTH_ENABLED:true}
# 关闭内部鉴权后,注入请求上下文的调试角色。
debug-role-code: ${TMS_SECURITY_INTERNAL_AUTH_DEBUG_ROLE_CODE:OPS_ADMIN}
# 关闭内部鉴权后,注入请求上下文的调试认证等级。
@@ -28,6 +28,7 @@ class ResourceBackupPropertiesTest {
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_account"));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_role_ukey_binding"));
Assertions.assertTrue(properties.getTmsDbExcludedTables().contains("tms_auth_session"));
Assertions.assertEquals(300, properties.getMysqldumpTimeoutSeconds());
Assertions.assertEquals(180, properties.getHealthCheckMaxWaitSeconds());
Assertions.assertEquals(5, properties.getHealthCheckRetryIntervalSeconds());
}
@@ -44,6 +44,30 @@ class MysqlDatabaseBackupCollectorTest {
Assertions.assertFalse(invocations[1].contains("--ignore-table="));
}
@Test
void shouldFailWhenMysqldumpExceedsConfiguredTimeout() throws Exception {
Path mysqldump = slowMysqldump();
ResourceBackupProperties properties = new ResourceBackupProperties();
properties.setMysqldumpPath(mysqldump.toString());
properties.setMysqldumpTimeoutSeconds(1);
MysqlDatabaseBackupCollector collector = new MysqlDatabaseBackupCollector(
properties,
"jdbc:mysql://127.0.0.1:4000/TMS?useSSL=false",
"root",
"secret"
);
long startedAt = System.nanoTime();
IllegalStateException ex = Assertions.assertThrows(
IllegalStateException.class,
collector::collectTmsDatabaseDump
);
long elapsedMillis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
Assertions.assertTrue(ex.getMessage().contains("mysqldump执行超时"));
Assertions.assertTrue(elapsedMillis < 1800, "mysqldump should be killed on timeout");
}
private Path fakeMysqldump(Path argsLog) throws Exception {
Path script = tempDir.resolve("mysqldump");
Files.writeString(script, """
@@ -54,4 +78,15 @@ class MysqlDatabaseBackupCollectorTest {
script.toFile().setExecutable(true);
return script;
}
private Path slowMysqldump() throws Exception {
Path script = tempDir.resolve("slow-mysqldump");
Files.writeString(script, """
#!/usr/bin/env bash
sleep 2
printf -- '-- synthetic dump\\n'
""");
script.toFile().setExecutable(true);
return script;
}
}
@@ -43,6 +43,22 @@ class ResourceTaskRepositoryTest {
Assertions.assertEquals("nightly", found.getRemark());
}
@Test
void shouldUpdateBackupTaskByBusinessTaskId() {
ResourceBackupTaskMapper mapper = Mockito.mock(ResourceBackupTaskMapper.class);
ResourceBackupTaskRepository repository = new ResourceBackupTaskRepositoryImpl(mapper);
ResourceBackupTaskEntity entity = new ResourceBackupTaskEntity();
entity.setTaskId("RBKP-001");
entity.setStatus("SUCCESS");
Mockito.when(mapper.updateByTaskId(entity)).thenReturn(1);
repository.update(entity);
Mockito.verify(mapper).updateByTaskId(entity);
Mockito.verify(mapper, Mockito.never()).updateById(entity);
}
@Test
void shouldSaveAndQueryRestoreTask() {
ResourceRestoreTaskMapper mapper = Mockito.mock(ResourceRestoreTaskMapper.class);
@@ -164,6 +164,47 @@ class ResourceBackupServiceTest {
Assertions.assertTrue(saved.getErrorMessage().contains("SDF_Encrypt"));
}
@Test
void shouldPersistFailedTaskWhenPackageCreationThrowsLinkageError() {
InitTaskRepository initTaskRepository = Mockito.mock(InitTaskRepository.class);
ResourceDiscoveryService resourceDiscoveryService = Mockito.mock(ResourceDiscoveryService.class);
ResourcePackageService resourcePackageService = Mockito.mock(ResourcePackageService.class);
ResourceBackupTaskRepository resourceBackupTaskRepository = Mockito.mock(ResourceBackupTaskRepository.class);
ResourceTaskAdmissionGuard admissionGuard = Mockito.mock(ResourceTaskAdmissionGuard.class);
InitTaskEntity initTask = sampleInitTask();
List<DiscoveredResourceItem> discoveredItems = List.of(resource("TMS_CONFIG", "/home/tms/config", true));
Mockito.when(initTaskRepository.findLatestByTaskTypeAndStatus("INIT", "SUCCESS")).thenReturn(Optional.of(initTask));
Mockito.when(resourceDiscoveryService.discover(initTask)).thenReturn(discoveredItems);
Mockito.when(resourcePackageService.createPackage(
Mockito.anyString(),
Mockito.anyString(),
Mockito.same(initTask),
Mockito.eq(discoveredItems),
Mockito.any(CreateResourceBackupRequest.class)
)).thenThrow(new UnsatisfiedLinkError("libsdf.so not found"));
ResourceBackupService service = new ResourceBackupServiceImpl(
initTaskRepository,
resourceDiscoveryService,
resourcePackageService,
resourceBackupTaskRepository,
new ResourceTaskIdGenerator(),
admissionGuard
);
Assertions.assertThrows(UnsatisfiedLinkError.class, () -> service.createBackupTask(sampleRequest()));
ArgumentCaptor<ResourceBackupTaskEntity> captor = ArgumentCaptor.forClass(ResourceBackupTaskEntity.class);
Mockito.verify(resourceBackupTaskRepository).update(captor.capture());
ResourceBackupTaskEntity saved = captor.getValue();
Assertions.assertEquals("FAILED", saved.getStatus());
Assertions.assertEquals("PACKAGE_CREATE_FAILED", saved.getErrorCode());
Assertions.assertTrue(saved.getErrorMessage().contains("libsdf.so not found"));
Assertions.assertNotNull(saved.getFinishTime());
}
@Test
void shouldPersistRunningTaskWithConfiguredKeyFingerprintsBeforePackaging() {
InitTaskRepository initTaskRepository = Mockito.mock(InitTaskRepository.class);
@@ -1056,6 +1056,7 @@ class ConfigurableInitStepExecutorTest {
String appContent = Files.readString(appScript);
Assertions.assertTrue(appContent.contains("STANDARD_HOME_DIR"));
Assertions.assertTrue(appContent.contains("STANDARD_RUN_USER"));
Assertions.assertTrue(appContent.contains("stop_standard_apps.sh"));
Assertions.assertTrue(appContent.contains("start_cmsp.sh"));
Assertions.assertTrue(appContent.contains("start_cmtp.sh"));
Assertions.assertFalse(appContent.contains("/home/cemp4i/cmsp/start_cmsp.sh"));
@@ -1066,6 +1067,9 @@ class ConfigurableInitStepExecutorTest {
Assertions.assertTrue(stopAppContent.contains("STANDARD_RUN_USER"));
Assertions.assertTrue(stopAppContent.contains("stop_cmsp.sh"));
Assertions.assertTrue(stopAppContent.contains("stop_cmtp.sh"));
Assertions.assertTrue(stopAppContent.contains("CMEP-CMSP"));
Assertions.assertTrue(stopAppContent.contains("CMEP-CMTP"));
Assertions.assertTrue(stopAppContent.contains("wait_until_stopped"));
String stopRabbitContent = Files.readString(stopRabbitScript);
Assertions.assertTrue(stopRabbitContent.contains("systemctl stop"));