From 9b64b754c13a8e9b311c614461591c52f48e7090 Mon Sep 17 00:00:00 2001 From: "haidong.pang" Date: Tue, 11 Aug 2026 14:01:59 +0800 Subject: [PATCH] [storage]: reclaim referenced reimage chains Keep an old root out of reimage trash while downstream snapshot references still depend on it. Persist reference cleanup, protect remaining leaves at execution time, and retry external primary storage deletion from leaf to root using only the undeleted suffix. Sync capacity and remove matching legacy reimage trash after cleanup. Resolves: ZSTAC-81222 Change-Id: I500146d6898405f4df0cc18a1d8a4aaa94fee49d --- .../VolumeSnapshotReferenceTreeVO_.java | 2 +- .../primary/ExternalDeleteVolumeChainGC.java | 70 ++++++++ .../addon/primary/ExternalPrimaryStorage.java | 63 ++++++- .../DeleteVolumeSnapshotReferenceGC.java | 92 ++++++++++ .../VolumeSnapshotReferenceTreeBase.java | 1 + .../VolumeSnapshotReferenceUtils.java | 46 ++--- .../addon/zbs/ZbsPrimaryStorageCase.groovy | 160 ++++++++++++++++++ 7 files changed, 401 insertions(+), 33 deletions(-) create mode 100644 storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java create mode 100644 storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java index 0020210791c..001b1d18d23 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java @@ -12,7 +12,7 @@ public class VolumeSnapshotReferenceTreeVO_ extends ResourceVO_ { public static volatile SingularAttribute rootVolumeSnapshotUuid; public static volatile SingularAttribute rootVolumeUuid; public static volatile SingularAttribute rootVolumeSnapshotTreeUuid; - public static volatile SingularAttribute rootVolumeSnapshotInstallUrl; + public static volatile SingularAttribute rootInstallUrl; public static volatile SingularAttribute primaryStorageUuid; public static volatile SingularAttribute hostUuid; diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java new file mode 100644 index 00000000000..949cbb9b9f4 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java @@ -0,0 +1,70 @@ +package org.zstack.storage.addon.primary; + +import org.apache.commons.collections.CollectionUtils; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.gc.GC; +import org.zstack.core.gc.GCCompletion; +import org.zstack.core.gc.TimeBasedGarbageCollector; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.primary.DeleteVolumeChainOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.DeleteVolumeChainOnPrimaryStorageReply; +import org.zstack.header.storage.primary.PrimaryStorageConstant; +import org.zstack.header.storage.primary.PrimaryStorageVO; +import org.zstack.storage.primary.PrimaryStorageGlobalConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040; + +public class ExternalDeleteVolumeChainGC extends TimeBasedGarbageCollector { + @GC + public String primaryStorageUuid; + @GC + public List installPaths; + @GC + public String chainTop; + + @Override + protected void triggerNow(GCCompletion completion) { + if (!dbf.isExist(primaryStorageUuid, PrimaryStorageVO.class) || CollectionUtils.isEmpty(installPaths)) { + completion.cancel(); + return; + } + + DeleteVolumeChainOnPrimaryStorageMsg msg = new DeleteVolumeChainOnPrimaryStorageMsg(); + msg.setPrimaryStorageUuid(primaryStorageUuid); + msg.setInstallPaths(installPaths); + msg.setChainTop(chainTop); + bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, primaryStorageUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + retry(completion, reply.getError()); + return; + } + + DeleteVolumeChainOnPrimaryStorageReply r = reply.castReply(); + if (CollectionUtils.isEmpty(r.getUndeletedInstallPaths())) { + completion.success(); + return; + } + + installPaths = new ArrayList<>(r.getUndeletedInstallPaths()); + retry(completion, operr(ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040, + "failed to delete volume chain paths %s", installPaths)); + } + }); + } + + private void retry(GCCompletion completion, ErrorCode error) { + NEXT_TIME = PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class); + NEXT_TIME_UNIT = TimeUnit.SECONDS; + updateContext(); + completion.fail(error); + } +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 6e222dd4959..c8ae7c30ba5 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -24,6 +24,8 @@ import org.zstack.header.core.NopeCompletion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.core.trash.InstallPathRecycleVO; +import org.zstack.header.core.trash.InstallPathRecycleVO_; import org.zstack.header.core.workflow.*; import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.errorcode.ErrorCodeList; @@ -56,6 +58,7 @@ import org.zstack.resourceconfig.ResourceConfigFacade; import org.zstack.storage.backup.BackupStorageSystemTags; import org.zstack.storage.primary.*; +import org.zstack.storage.snapshot.reference.VolumeSnapshotReferenceUtils; import org.zstack.storage.volume.VolumeSystemTags; import org.zstack.utils.Utils; import org.zstack.utils.gson.JSONObjectUtil; @@ -63,6 +66,7 @@ import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -1704,11 +1708,18 @@ public void fail(ErrorCode errorCode) { protected void handle(DeleteVolumeChainOnPrimaryStorageMsg msg) { DeleteVolumeChainOnPrimaryStorageReply reply = new DeleteVolumeChainOnPrimaryStorageReply(); + if (CollectionUtils.isEmpty(msg.getInstallPaths())) { + bus.reply(msg, reply); + return; + } + + AtomicInteger deletedCount = new AtomicInteger(); new While<>(msg.getInstallPaths()).each((installPath, compl) -> { // TODO use trash instead, make sure only one snapshot for link clone. controller.deleteVolumeAndSnapshot(installPath, new Completion(compl) { @Override public void success() { + deletedCount.incrementAndGet(); compl.done(); } @@ -1722,11 +1733,54 @@ public void fail(ErrorCode errorCode) { @Override public void done(ErrorCodeList errorCodeList) { if (!errorCodeList.getCauses().isEmpty()) { - reply.setError(errorCodeList.getCauses().get(0)); + List undeletedInstallPaths = new ArrayList<>(msg.getInstallPaths() + .subList(deletedCount.get(), msg.getInstallPaths().size())); + submitDeleteVolumeChainGC(msg, undeletedInstallPaths); + reply.setUndeletedInstallPaths(undeletedInstallPaths); bus.reply(msg, reply); - } else { + return; + } + + syncCapacityAfterDeleteVolumeChain(msg, reply); + } + }); + } + + private void submitDeleteVolumeChainGC(DeleteVolumeChainOnPrimaryStorageMsg msg, + List undeletedInstallPaths) { + ExternalDeleteVolumeChainGC gc = new ExternalDeleteVolumeChainGC(); + gc.primaryStorageUuid = self.getUuid(); + gc.installPaths = undeletedInstallPaths; + gc.chainTop = msg.getChainTop(); + gc.NAME = String.format("gc-external-%s-volume-chain-%s", self.getUuid(), gc.chainTop); + gc.deduplicateSubmit(PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class), + TimeUnit.SECONDS); + } + + private void syncCapacityAfterDeleteVolumeChain(DeleteVolumeChainOnPrimaryStorageMsg msg, + DeleteVolumeChainOnPrimaryStorageReply reply) { + SyncPrimaryStorageCapacityMsg smsg = new SyncPrimaryStorageCapacityMsg(); + smsg.setPrimaryStorageUuid(self.getUuid()); + bus.makeTargetServiceIdByResourceUuid(smsg, PrimaryStorageConstant.SERVICE_ID, self.getUuid()); + bus.send(smsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + submitDeleteVolumeChainGC(msg, new ArrayList<>(msg.getInstallPaths())); + reply.setError(r.getError()); bus.reply(msg, reply); + return; } + + String rootInstallPath = msg.getInstallPaths().get(msg.getInstallPaths().size() - 1); + List trashIds = Q.New(InstallPathRecycleVO.class) + .select(InstallPathRecycleVO_.trashId) + .eq(InstallPathRecycleVO_.storageUuid, self.getUuid()) + .eq(InstallPathRecycleVO_.installPath, rootInstallPath) + .eq(InstallPathRecycleVO_.trashType, TrashType.ReimageVolume.toString()) + .listValues(); + trashIds.forEach(trash::removeFromDb); + bus.reply(msg, reply); } }); } @@ -2104,6 +2158,11 @@ public boolean skip(Map data) { @Override public void run(FlowTrigger trigger, Map data) { + if (VolumeSnapshotReferenceUtils.isVolumeDirectlyReferenceByOthers(msg.getVolume())) { + trigger.next(); + return; + } + boolean hasSnapshot = Q.New(VolumeSnapshotVO.class) .eq(VolumeSnapshotVO_.volumeUuid, msg.getVolume().getUuid()) .like(VolumeSnapshotVO_.primaryStorageInstallPath, String.format("%s%%", msg.getVolume().getInstallPath())) diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java new file mode 100644 index 00000000000..0e34a9550ed --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java @@ -0,0 +1,92 @@ +package org.zstack.storage.snapshot.reference; + +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.Q; +import org.zstack.core.gc.GC; +import org.zstack.core.gc.GCCompletion; +import org.zstack.core.gc.TimeBasedGarbageCollector; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.primary.PrimaryStorageVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotConstant; +import org.zstack.header.storage.snapshot.reference.DeleteVolumeSnapshotReferenceLeafMsg; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceInventory; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeInventory; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO_; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO_; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.storage.primary.PrimaryStorageGlobalConfig; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class DeleteVolumeSnapshotReferenceGC extends TimeBasedGarbageCollector { + @GC + public VolumeSnapshotReferenceInventory leaf; + @GC + public VolumeSnapshotReferenceTreeInventory tree; + @GC + public VolumeInventory deletedVolume; + + @Override + protected void triggerNow(GCCompletion completion) { + if (!dbf.isExist(tree.getPrimaryStorageUuid(), PrimaryStorageVO.class)) { + completion.cancel(); + return; + } + + DeleteVolumeSnapshotReferenceLeafMsg msg = new DeleteVolumeSnapshotReferenceLeafMsg(); + msg.setLeaf(leaf); + msg.setOtherLeafs(getOtherLeafs()); + msg.setTree(tree); + msg.setDeletedVolume(deletedVolume); + bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, tree.getUuid()); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + retry(completion, reply.getError()); + return; + } + + completion.success(); + } + }); + } + + private List getOtherLeafs() { + List refs; + if (leaf.getParentId() == null) { + List treeUuids = Q.New(VolumeSnapshotReferenceTreeVO.class) + .select(VolumeSnapshotReferenceTreeVO_.uuid) + .eq(VolumeSnapshotReferenceTreeVO_.primaryStorageUuid, tree.getPrimaryStorageUuid()) + .eq(VolumeSnapshotReferenceTreeVO_.rootInstallUrl, tree.getRootInstallUrl()) + .listValues(); + if (treeUuids.isEmpty()) { + return Collections.emptyList(); + } + refs = Q.New(VolumeSnapshotReferenceVO.class) + .in(VolumeSnapshotReferenceVO_.treeUuid, treeUuids) + .isNull(VolumeSnapshotReferenceVO_.parentId) + .list(); + } else { + refs = Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.treeUuid, tree.getUuid()) + .eq(VolumeSnapshotReferenceVO_.parentId, leaf.getParentId()) + .list(); + } + + return refs.stream().map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList()); + } + + private void retry(GCCompletion completion, ErrorCode error) { + NEXT_TIME = PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class); + NEXT_TIME_UNIT = TimeUnit.SECONDS; + updateContext(); + completion.fail(error); + } +} diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java index cb1bb6a657e..12205e21323 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java @@ -113,6 +113,7 @@ private void deleteSnapshotRefLeaf(DeleteVolumeSnapshotReferenceLeafMsg msg, Com boolean rootDeleted = msg.getLeaf().getParentId() == null && !Q.New(VolumeVO.class) .eq(VolumeVO_.uuid, msg.getLeaf().getVolumeUuid()) .eq(VolumeVO_.primaryStorageUuid, msg.getTree().getPrimaryStorageUuid()) + .eq(VolumeVO_.installPath, self.getRootInstallUrl()) .isExists(); String endPath = rootDeleted ? self.getRootInstallUrl() : msg.getLeaf().getVolumeSnapshotInstallUrl(); String startPath = msg.getLeaf().getDirectSnapshotInstallUrl(); diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java index 39293324faa..180d362e826 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java @@ -4,7 +4,6 @@ import org.apache.commons.collections.MapUtils; import org.springframework.transaction.support.TransactionSynchronizationManager; import org.zstack.core.Platform; -import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.db.*; import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.exception.CloudRuntimeException; @@ -27,6 +26,7 @@ import javax.persistence.LockModeType; import javax.persistence.Tuple; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.zstack.core.Platform.operr; @@ -258,6 +258,7 @@ protected VolumeSnapshotReferenceTreeVO scripts() { VolumeSnapshotReferenceTreeVO tree = Q.New(VolumeSnapshotReferenceTreeVO.class) .eq(VolumeSnapshotReferenceTreeVO_.rootVolumeUuid, baseSnapshot.getVolumeUuid()) .eq(VolumeSnapshotReferenceTreeVO_.primaryStorageUuid, baseSnapshot.getPrimaryStorageUuid()) + .eq(VolumeSnapshotReferenceTreeVO_.rootInstallUrl, vol.getInstallPath()) .find(); if (tree != null) { return tree; @@ -494,25 +495,13 @@ protected void scripts() { } private static void deleteBitsOnPs(VolumeSnapshotReferenceTreeVO treeVO, VolumeSnapshotReferenceVO ref) { - List treeRefs = Q.New(VolumeSnapshotReferenceVO.class).eq(VolumeSnapshotReferenceVO_.treeUuid, treeVO.getUuid()).list(); - - List otherLeafs; - if (ref.getParentId() == null) { - otherLeafs = treeRefs.stream().filter(it -> it.getParentId() == null).map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList()); - } else { - otherLeafs = treeRefs.stream().filter(it -> ref.getParentId().equals(it.getParentId())).map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList()); - } - otherLeafs.removeIf(it -> it.getId() == ref.getId()); - - DeleteVolumeSnapshotReferenceLeafMsg msg = new DeleteVolumeSnapshotReferenceLeafMsg(); - msg.setLeaf(VolumeSnapshotReferenceInventory.valueOf(ref)); - msg.setOtherLeafs(otherLeafs); - msg.setTree(treeVO.toInventory()); VolumeVO deletedVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, ref.getReferenceVolumeUuid()).find(); - msg.setDeletedVolume(deletedVolume.toInventory()); - CloudBus bus = Platform.getComponentLoader().getComponent(CloudBus.class); - bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, treeVO.getUuid()); - bus.send(msg); + DeleteVolumeSnapshotReferenceGC gc = new DeleteVolumeSnapshotReferenceGC(); + gc.NAME = String.format("gc-volume-snapshot-reference-%s-leaf-%s", treeVO.getUuid(), ref.getId()); + gc.leaf = VolumeSnapshotReferenceInventory.valueOf(ref); + gc.tree = treeVO.toInventory(); + gc.deletedVolume = deletedVolume.toInventory(); + gc.deduplicateSubmit(1L, TimeUnit.SECONDS); } private static void deleteSnapshotRef(VolumeSnapshotReferenceVO ref) { @@ -529,19 +518,16 @@ private static void deleteSnapshotRef(VolumeSnapshotReferenceVO ref) { private static void deleteSnapshotRefLeafInTree(VolumeSnapshotReferenceTreeVO tree, VolumeSnapshotReferenceVO ref) { boolean referenceRedirected = !ref.getDirectSnapshotUuid().equals(ref.getVolumeSnapshotUuid()); - boolean backingVolumeDeletedInDb = SQL.New("select vol.uuid from VolumeVO vol, VolumeSnapshotReferenceTreeVO tree" + - " where vol.uuid = :volUuid" + - " and tree.uuid = :treeUuid" + - " and vol.primaryStorageUuid = tree.primaryStorageUuid", String.class) - .param("volUuid", ref.getVolumeUuid()) - .param("treeUuid", ref.getTreeUuid()) - .find() == null; + if (tree == null) { + tree = Q.New(VolumeSnapshotReferenceTreeVO.class).eq(VolumeSnapshotReferenceTreeVO_.uuid, ref.getTreeUuid()).find(); + } + boolean backingVolumeDeletedInDb = !Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, ref.getVolumeUuid()) + .eq(VolumeVO_.primaryStorageUuid, tree.getPrimaryStorageUuid()) + .eq(VolumeVO_.installPath, tree.getRootInstallUrl()) + .isExists(); if (referenceRedirected || backingVolumeDeletedInDb) { - if (tree == null) { - tree = Q.New(VolumeSnapshotReferenceTreeVO.class).eq(VolumeSnapshotReferenceTreeVO_.uuid, ref.getTreeUuid()).find(); - } - deleteBitsOnPs(tree, ref); } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 589e47918a0..d9147da304d 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -6,12 +6,22 @@ import org.zstack.core.cloudbus.EventCallback import org.zstack.core.cloudbus.EventFacade import org.zstack.core.db.DatabaseFacade import org.zstack.core.db.Q +import org.zstack.core.gc.GCStatus +import org.zstack.core.trash.TrashType import org.zstack.header.errorcode.ErrorCode import org.zstack.header.errorcode.OperationFailureException +import org.zstack.header.core.trash.InstallPathRecycleVO +import org.zstack.header.core.trash.InstallPathRecycleVO_ import org.zstack.header.message.MessageReply +import org.zstack.header.storage.backup.UploadImageToRemoteTargetMsg +import org.zstack.header.storage.backup.UploadImageToRemoteTargetReply import org.zstack.header.storage.primary.GetVolumeBackingChainFromPrimaryStorageMsg import org.zstack.header.storage.primary.GetVolumeBackingChainFromPrimaryStorageReply import org.zstack.header.storage.primary.PrimaryStorageConstant +import org.zstack.header.storage.snapshot.VolumeSnapshotVO +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_ +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO_ import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageMsg import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageReply import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO @@ -40,8 +50,10 @@ import org.zstack.storage.zbs.Config import org.zstack.storage.zbs.ZbsAgentUrl import org.zstack.storage.zbs.ZbsConstants import org.zstack.storage.zbs.ZbsGlobalProperty +import org.zstack.storage.zbs.ZbsHelper import org.zstack.storage.zbs.ZbsPrimaryStorageMdsBase import org.zstack.storage.zbs.ZbsStorageController +import org.zstack.storage.volume.VolumeSystemTags import org.zstack.test.integration.storage.StorageTest import org.zstack.testlib.EnvSpec import org.zstack.testlib.HttpError @@ -194,6 +206,7 @@ class ZbsPrimaryStorageCase extends SubCase { testAttachPrimaryStorageFailsWhenActivatingHeartbeatVolumeFails() testMdsConnectFailed() testLifecycle() + testReimageReferencedRootCleanup() testDataVolumeLifecycle() testMdsPing() testCheckHostStorageConnection() @@ -756,6 +769,153 @@ class ZbsPrimaryStorageCase extends SubCase { } } + void testReimageReferencedRootCleanup() { + attachPrimaryStorageToCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + + env.message(UploadImageToRemoteTargetMsg.class) { UploadImageToRemoteTargetMsg msg, CloudBus bus -> + bus.reply(msg, new UploadImageToRemoteTargetReply()) + } + + def instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory + def image = env.inventoryByName("image") as ImageInventory + def l3 = env.inventoryByName("l3") as L3NetworkInventory + def vm = createVmInstance { + name = "reimage-referenced-root" + imageUuid = image.uuid + l3NetworkUuids = [l3.uuid] + instanceOfferingUuid = instanceOffering.uuid + } as VmInstanceInventory + def oldRoot = queryVolume { + conditions = ["uuid=${vm.rootVolumeUuid}"] + }[0] as VolumeInventory + def snapshot = createVolumeSnapshot { + name = "referenced-root-snapshot" + volumeUuid = oldRoot.uuid + } as VolumeSnapshotInventory + def child = createDataVolumeFromVolumeSnapshot { + name = "referenced-root-child" + volumeSnapshotUuid = snapshot.uuid + systemTags = [VolumeSystemTags.FAST_CREATE.tagFormat] + } as VolumeInventory + + assert Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.referenceVolumeUuid, child.uuid).isExists() : + "FAST_CREATE child must retain the backing reference: childUuid=${child.uuid}" + + List deleteAttempts = Collections.synchronizedList(new ArrayList<>()) + AtomicBoolean cleanupStarted = new AtomicBoolean(false) + AtomicBoolean failSnapshotDeleteOnce = new AtomicBoolean(true) + env.simulator(ZbsStorageController.QUERY_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.QueryVolumeCmd.class) + def rsp = new ZbsStorageController.QueryVolumeRsp() + rsp.size = oldRoot.size + rsp.actualSize = oldRoot.actualSize + if (ZbsHelper.normalizeToZbsPath(cmd.path) == snapshot.primaryStorageInstallPath) { + rsp.parentUri = oldRoot.installPath + } + return rsp + } + env.simulator(ZbsStorageController.DELETE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.DeleteVolumeCmd.class) + String path = ZbsHelper.normalizeToZbsPath(cmd.path) + deleteAttempts.add(path) + def rsp = new ZbsStorageController.DeleteVolumeRsp() + if (!cleanupStarted.get() && path == oldRoot.installPath) { + rsp.success = false + rsp.error = "snapshot is still in used" + } else if (cleanupStarted.get() && path == snapshot.primaryStorageInstallPath && + failSnapshotDeleteOnce.compareAndSet(true, false)) { + rsp.success = false + rsp.error = "transient cleanup failure" + } + return rsp + } + + stopVmInstance { + uuid = vm.uuid + } + reimageVmInstance { + vmInstanceUuid = vm.uuid + } + + assert !deleteAttempts.contains(oldRoot.installPath) : + "reimage must not delete an old root with downstream references: oldRoot=${oldRoot.installPath} attempts=${deleteAttempts}" + assert !Q.New(InstallPathRecycleVO.class) + .eq(InstallPathRecycleVO_.storageUuid, ps.uuid) + .eq(InstallPathRecycleVO_.installPath, oldRoot.installPath) + .eq(InstallPathRecycleVO_.trashType, TrashType.ReimageVolume.toString()) + .isExists() : "reimage must not create trash for referenced old root: oldRoot=${oldRoot.installPath}" + + destroyVmInstance { + uuid = vm.uuid + } + expungeVmInstance { + uuid = vm.uuid + } + assert !Q.New(VolumeSnapshotVO.class).eq(VolumeSnapshotVO_.volumeUuid, oldRoot.uuid).isExists() : + "old root snapshots must be removed after VM expunge: volumeUuid=${oldRoot.uuid}" + assert Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.referenceVolumeUuid, child.uuid).isExists() : + "VM expunge must retain the FAST_CREATE child reference: childUuid=${child.uuid}" + + cleanupStarted.set(true) + int cleanupStartIndex = deleteAttempts.size() + deleteDataVolume { + uuid = child.uuid + } + expungeDataVolume { + uuid = child.uuid + } + + List chainJobs = [] + retryInSecs { + chainJobs = queryGCJob { + conditions = [ + "runnerClass=org.zstack.storage.addon.primary.ExternalDeleteVolumeChainGC", + "context~=%${oldRoot.installPath}%" + ] + } as List + assert chainJobs.size() == 1 : + "failed chain cleanup must leave one GC job: oldRoot=${oldRoot.installPath} jobs=${chainJobs*.uuid}" + assert chainJobs[0].status != GCStatus.Done.toString() : + "chain GC must retain the undeleted suffix after transient failure: job=${chainJobs[0].uuid}" + } + + triggerGCJob { + uuid = chainJobs[0].uuid + } + retryInSecs { + def job = queryGCJob { + conditions = ["uuid=${chainJobs[0].uuid}"] + }[0] as GarbageCollectorInventory + List cleanupAttempts = deleteAttempts.drop(cleanupStartIndex) + int snapshotIndex = cleanupAttempts.lastIndexOf(snapshot.primaryStorageInstallPath) + int oldRootIndex = cleanupAttempts.indexOf(oldRoot.installPath) + assert job.status == GCStatus.Done.toString() : + "chain GC must finish after retry: job=${job.uuid} status=${job.status} context=${job.context}" + assert snapshotIndex >= 0 && oldRootIndex > snapshotIndex : + "cleanup must delete leaf-to-root: snapshot=${snapshot.primaryStorageInstallPath} oldRoot=${oldRoot.installPath} attempts=${cleanupAttempts}" + } + + assert !Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.referenceVolumeUuid, child.uuid).isExists() : + "reference must be removed after child expunge: childUuid=${child.uuid}" + assert !Q.New(InstallPathRecycleVO.class) + .eq(InstallPathRecycleVO_.storageUuid, ps.uuid) + .eq(InstallPathRecycleVO_.installPath, oldRoot.installPath) + .eq(InstallPathRecycleVO_.trashType, TrashType.ReimageVolume.toString()) + .isExists() : "completed chain cleanup must remove legacy reimage trash: oldRoot=${oldRoot.installPath}" + + detachPrimaryStorageFromCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + env.cleanSimulatorHandlers() + } + void testMdsPing() { ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data = null long count = 0