diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java index 0020210791c..001b1d18d23 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java @@ -12,7 +12,7 @@ public class VolumeSnapshotReferenceTreeVO_ extends ResourceVO_ { public static volatile SingularAttribute rootVolumeSnapshotUuid; public static volatile SingularAttribute rootVolumeUuid; public static volatile SingularAttribute rootVolumeSnapshotTreeUuid; - public static volatile SingularAttribute rootVolumeSnapshotInstallUrl; + public static volatile SingularAttribute rootInstallUrl; public static volatile SingularAttribute primaryStorageUuid; public static volatile SingularAttribute hostUuid; diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java new file mode 100644 index 00000000000..949cbb9b9f4 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java @@ -0,0 +1,70 @@ +package org.zstack.storage.addon.primary; + +import org.apache.commons.collections.CollectionUtils; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.gc.GC; +import org.zstack.core.gc.GCCompletion; +import org.zstack.core.gc.TimeBasedGarbageCollector; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.primary.DeleteVolumeChainOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.DeleteVolumeChainOnPrimaryStorageReply; +import org.zstack.header.storage.primary.PrimaryStorageConstant; +import org.zstack.header.storage.primary.PrimaryStorageVO; +import org.zstack.storage.primary.PrimaryStorageGlobalConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040; + +public class ExternalDeleteVolumeChainGC extends TimeBasedGarbageCollector { + @GC + public String primaryStorageUuid; + @GC + public List installPaths; + @GC + public String chainTop; + + @Override + protected void triggerNow(GCCompletion completion) { + if (!dbf.isExist(primaryStorageUuid, PrimaryStorageVO.class) || CollectionUtils.isEmpty(installPaths)) { + completion.cancel(); + return; + } + + DeleteVolumeChainOnPrimaryStorageMsg msg = new DeleteVolumeChainOnPrimaryStorageMsg(); + msg.setPrimaryStorageUuid(primaryStorageUuid); + msg.setInstallPaths(installPaths); + msg.setChainTop(chainTop); + bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, primaryStorageUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + retry(completion, reply.getError()); + return; + } + + DeleteVolumeChainOnPrimaryStorageReply r = reply.castReply(); + if (CollectionUtils.isEmpty(r.getUndeletedInstallPaths())) { + completion.success(); + return; + } + + installPaths = new ArrayList<>(r.getUndeletedInstallPaths()); + retry(completion, operr(ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040, + "failed to delete volume chain paths %s", installPaths)); + } + }); + } + + private void retry(GCCompletion completion, ErrorCode error) { + NEXT_TIME = PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class); + NEXT_TIME_UNIT = TimeUnit.SECONDS; + updateContext(); + completion.fail(error); + } +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 6e222dd4959..c8ae7c30ba5 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -24,6 +24,8 @@ import org.zstack.header.core.NopeCompletion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.core.trash.InstallPathRecycleVO; +import org.zstack.header.core.trash.InstallPathRecycleVO_; import org.zstack.header.core.workflow.*; import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.errorcode.ErrorCodeList; @@ -56,6 +58,7 @@ import org.zstack.resourceconfig.ResourceConfigFacade; import org.zstack.storage.backup.BackupStorageSystemTags; import org.zstack.storage.primary.*; +import org.zstack.storage.snapshot.reference.VolumeSnapshotReferenceUtils; import org.zstack.storage.volume.VolumeSystemTags; import org.zstack.utils.Utils; import org.zstack.utils.gson.JSONObjectUtil; @@ -63,6 +66,7 @@ import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -1704,11 +1708,18 @@ public void fail(ErrorCode errorCode) { protected void handle(DeleteVolumeChainOnPrimaryStorageMsg msg) { DeleteVolumeChainOnPrimaryStorageReply reply = new DeleteVolumeChainOnPrimaryStorageReply(); + if (CollectionUtils.isEmpty(msg.getInstallPaths())) { + bus.reply(msg, reply); + return; + } + + AtomicInteger deletedCount = new AtomicInteger(); new While<>(msg.getInstallPaths()).each((installPath, compl) -> { // TODO use trash instead, make sure only one snapshot for link clone. controller.deleteVolumeAndSnapshot(installPath, new Completion(compl) { @Override public void success() { + deletedCount.incrementAndGet(); compl.done(); } @@ -1722,11 +1733,54 @@ public void fail(ErrorCode errorCode) { @Override public void done(ErrorCodeList errorCodeList) { if (!errorCodeList.getCauses().isEmpty()) { - reply.setError(errorCodeList.getCauses().get(0)); + List undeletedInstallPaths = new ArrayList<>(msg.getInstallPaths() + .subList(deletedCount.get(), msg.getInstallPaths().size())); + submitDeleteVolumeChainGC(msg, undeletedInstallPaths); + reply.setUndeletedInstallPaths(undeletedInstallPaths); bus.reply(msg, reply); - } else { + return; + } + + syncCapacityAfterDeleteVolumeChain(msg, reply); + } + }); + } + + private void submitDeleteVolumeChainGC(DeleteVolumeChainOnPrimaryStorageMsg msg, + List undeletedInstallPaths) { + ExternalDeleteVolumeChainGC gc = new ExternalDeleteVolumeChainGC(); + gc.primaryStorageUuid = self.getUuid(); + gc.installPaths = undeletedInstallPaths; + gc.chainTop = msg.getChainTop(); + gc.NAME = String.format("gc-external-%s-volume-chain-%s", self.getUuid(), gc.chainTop); + gc.deduplicateSubmit(PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class), + TimeUnit.SECONDS); + } + + private void syncCapacityAfterDeleteVolumeChain(DeleteVolumeChainOnPrimaryStorageMsg msg, + DeleteVolumeChainOnPrimaryStorageReply reply) { + SyncPrimaryStorageCapacityMsg smsg = new SyncPrimaryStorageCapacityMsg(); + smsg.setPrimaryStorageUuid(self.getUuid()); + bus.makeTargetServiceIdByResourceUuid(smsg, PrimaryStorageConstant.SERVICE_ID, self.getUuid()); + bus.send(smsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + submitDeleteVolumeChainGC(msg, new ArrayList<>(msg.getInstallPaths())); + reply.setError(r.getError()); bus.reply(msg, reply); + return; } + + String rootInstallPath = msg.getInstallPaths().get(msg.getInstallPaths().size() - 1); + List trashIds = Q.New(InstallPathRecycleVO.class) + .select(InstallPathRecycleVO_.trashId) + .eq(InstallPathRecycleVO_.storageUuid, self.getUuid()) + .eq(InstallPathRecycleVO_.installPath, rootInstallPath) + .eq(InstallPathRecycleVO_.trashType, TrashType.ReimageVolume.toString()) + .listValues(); + trashIds.forEach(trash::removeFromDb); + bus.reply(msg, reply); } }); } @@ -2104,6 +2158,11 @@ public boolean skip(Map data) { @Override public void run(FlowTrigger trigger, Map data) { + if (VolumeSnapshotReferenceUtils.isVolumeDirectlyReferenceByOthers(msg.getVolume())) { + trigger.next(); + return; + } + boolean hasSnapshot = Q.New(VolumeSnapshotVO.class) .eq(VolumeSnapshotVO_.volumeUuid, msg.getVolume().getUuid()) .like(VolumeSnapshotVO_.primaryStorageInstallPath, String.format("%s%%", msg.getVolume().getInstallPath())) diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java new file mode 100644 index 00000000000..0e34a9550ed --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java @@ -0,0 +1,92 @@ +package org.zstack.storage.snapshot.reference; + +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.Q; +import org.zstack.core.gc.GC; +import org.zstack.core.gc.GCCompletion; +import org.zstack.core.gc.TimeBasedGarbageCollector; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.primary.PrimaryStorageVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotConstant; +import org.zstack.header.storage.snapshot.reference.DeleteVolumeSnapshotReferenceLeafMsg; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceInventory; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeInventory; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO_; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO; +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO_; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.storage.primary.PrimaryStorageGlobalConfig; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class DeleteVolumeSnapshotReferenceGC extends TimeBasedGarbageCollector { + @GC + public VolumeSnapshotReferenceInventory leaf; + @GC + public VolumeSnapshotReferenceTreeInventory tree; + @GC + public VolumeInventory deletedVolume; + + @Override + protected void triggerNow(GCCompletion completion) { + if (!dbf.isExist(tree.getPrimaryStorageUuid(), PrimaryStorageVO.class)) { + completion.cancel(); + return; + } + + DeleteVolumeSnapshotReferenceLeafMsg msg = new DeleteVolumeSnapshotReferenceLeafMsg(); + msg.setLeaf(leaf); + msg.setOtherLeafs(getOtherLeafs()); + msg.setTree(tree); + msg.setDeletedVolume(deletedVolume); + bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, tree.getUuid()); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + retry(completion, reply.getError()); + return; + } + + completion.success(); + } + }); + } + + private List getOtherLeafs() { + List refs; + if (leaf.getParentId() == null) { + List treeUuids = Q.New(VolumeSnapshotReferenceTreeVO.class) + .select(VolumeSnapshotReferenceTreeVO_.uuid) + .eq(VolumeSnapshotReferenceTreeVO_.primaryStorageUuid, tree.getPrimaryStorageUuid()) + .eq(VolumeSnapshotReferenceTreeVO_.rootInstallUrl, tree.getRootInstallUrl()) + .listValues(); + if (treeUuids.isEmpty()) { + return Collections.emptyList(); + } + refs = Q.New(VolumeSnapshotReferenceVO.class) + .in(VolumeSnapshotReferenceVO_.treeUuid, treeUuids) + .isNull(VolumeSnapshotReferenceVO_.parentId) + .list(); + } else { + refs = Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.treeUuid, tree.getUuid()) + .eq(VolumeSnapshotReferenceVO_.parentId, leaf.getParentId()) + .list(); + } + + return refs.stream().map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList()); + } + + private void retry(GCCompletion completion, ErrorCode error) { + NEXT_TIME = PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class); + NEXT_TIME_UNIT = TimeUnit.SECONDS; + updateContext(); + completion.fail(error); + } +} diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java index cb1bb6a657e..12205e21323 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java @@ -113,6 +113,7 @@ private void deleteSnapshotRefLeaf(DeleteVolumeSnapshotReferenceLeafMsg msg, Com boolean rootDeleted = msg.getLeaf().getParentId() == null && !Q.New(VolumeVO.class) .eq(VolumeVO_.uuid, msg.getLeaf().getVolumeUuid()) .eq(VolumeVO_.primaryStorageUuid, msg.getTree().getPrimaryStorageUuid()) + .eq(VolumeVO_.installPath, self.getRootInstallUrl()) .isExists(); String endPath = rootDeleted ? self.getRootInstallUrl() : msg.getLeaf().getVolumeSnapshotInstallUrl(); String startPath = msg.getLeaf().getDirectSnapshotInstallUrl(); diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java index 39293324faa..180d362e826 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java @@ -4,7 +4,6 @@ import org.apache.commons.collections.MapUtils; import org.springframework.transaction.support.TransactionSynchronizationManager; import org.zstack.core.Platform; -import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.db.*; import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.exception.CloudRuntimeException; @@ -27,6 +26,7 @@ import javax.persistence.LockModeType; import javax.persistence.Tuple; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.zstack.core.Platform.operr; @@ -258,6 +258,7 @@ protected VolumeSnapshotReferenceTreeVO scripts() { VolumeSnapshotReferenceTreeVO tree = Q.New(VolumeSnapshotReferenceTreeVO.class) .eq(VolumeSnapshotReferenceTreeVO_.rootVolumeUuid, baseSnapshot.getVolumeUuid()) .eq(VolumeSnapshotReferenceTreeVO_.primaryStorageUuid, baseSnapshot.getPrimaryStorageUuid()) + .eq(VolumeSnapshotReferenceTreeVO_.rootInstallUrl, vol.getInstallPath()) .find(); if (tree != null) { return tree; @@ -494,25 +495,13 @@ protected void scripts() { } private static void deleteBitsOnPs(VolumeSnapshotReferenceTreeVO treeVO, VolumeSnapshotReferenceVO ref) { - List treeRefs = Q.New(VolumeSnapshotReferenceVO.class).eq(VolumeSnapshotReferenceVO_.treeUuid, treeVO.getUuid()).list(); - - List otherLeafs; - if (ref.getParentId() == null) { - otherLeafs = treeRefs.stream().filter(it -> it.getParentId() == null).map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList()); - } else { - otherLeafs = treeRefs.stream().filter(it -> ref.getParentId().equals(it.getParentId())).map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList()); - } - otherLeafs.removeIf(it -> it.getId() == ref.getId()); - - DeleteVolumeSnapshotReferenceLeafMsg msg = new DeleteVolumeSnapshotReferenceLeafMsg(); - msg.setLeaf(VolumeSnapshotReferenceInventory.valueOf(ref)); - msg.setOtherLeafs(otherLeafs); - msg.setTree(treeVO.toInventory()); VolumeVO deletedVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, ref.getReferenceVolumeUuid()).find(); - msg.setDeletedVolume(deletedVolume.toInventory()); - CloudBus bus = Platform.getComponentLoader().getComponent(CloudBus.class); - bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, treeVO.getUuid()); - bus.send(msg); + DeleteVolumeSnapshotReferenceGC gc = new DeleteVolumeSnapshotReferenceGC(); + gc.NAME = String.format("gc-volume-snapshot-reference-%s-leaf-%s", treeVO.getUuid(), ref.getId()); + gc.leaf = VolumeSnapshotReferenceInventory.valueOf(ref); + gc.tree = treeVO.toInventory(); + gc.deletedVolume = deletedVolume.toInventory(); + gc.deduplicateSubmit(1L, TimeUnit.SECONDS); } private static void deleteSnapshotRef(VolumeSnapshotReferenceVO ref) { @@ -529,19 +518,16 @@ private static void deleteSnapshotRef(VolumeSnapshotReferenceVO ref) { private static void deleteSnapshotRefLeafInTree(VolumeSnapshotReferenceTreeVO tree, VolumeSnapshotReferenceVO ref) { boolean referenceRedirected = !ref.getDirectSnapshotUuid().equals(ref.getVolumeSnapshotUuid()); - boolean backingVolumeDeletedInDb = SQL.New("select vol.uuid from VolumeVO vol, VolumeSnapshotReferenceTreeVO tree" + - " where vol.uuid = :volUuid" + - " and tree.uuid = :treeUuid" + - " and vol.primaryStorageUuid = tree.primaryStorageUuid", String.class) - .param("volUuid", ref.getVolumeUuid()) - .param("treeUuid", ref.getTreeUuid()) - .find() == null; + if (tree == null) { + tree = Q.New(VolumeSnapshotReferenceTreeVO.class).eq(VolumeSnapshotReferenceTreeVO_.uuid, ref.getTreeUuid()).find(); + } + boolean backingVolumeDeletedInDb = !Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, ref.getVolumeUuid()) + .eq(VolumeVO_.primaryStorageUuid, tree.getPrimaryStorageUuid()) + .eq(VolumeVO_.installPath, tree.getRootInstallUrl()) + .isExists(); if (referenceRedirected || backingVolumeDeletedInDb) { - if (tree == null) { - tree = Q.New(VolumeSnapshotReferenceTreeVO.class).eq(VolumeSnapshotReferenceTreeVO_.uuid, ref.getTreeUuid()).find(); - } - deleteBitsOnPs(tree, ref); } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 589e47918a0..d9147da304d 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -6,12 +6,22 @@ import org.zstack.core.cloudbus.EventCallback import org.zstack.core.cloudbus.EventFacade import org.zstack.core.db.DatabaseFacade import org.zstack.core.db.Q +import org.zstack.core.gc.GCStatus +import org.zstack.core.trash.TrashType import org.zstack.header.errorcode.ErrorCode import org.zstack.header.errorcode.OperationFailureException +import org.zstack.header.core.trash.InstallPathRecycleVO +import org.zstack.header.core.trash.InstallPathRecycleVO_ import org.zstack.header.message.MessageReply +import org.zstack.header.storage.backup.UploadImageToRemoteTargetMsg +import org.zstack.header.storage.backup.UploadImageToRemoteTargetReply import org.zstack.header.storage.primary.GetVolumeBackingChainFromPrimaryStorageMsg import org.zstack.header.storage.primary.GetVolumeBackingChainFromPrimaryStorageReply import org.zstack.header.storage.primary.PrimaryStorageConstant +import org.zstack.header.storage.snapshot.VolumeSnapshotVO +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_ +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO_ import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageMsg import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageReply import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO @@ -40,8 +50,10 @@ import org.zstack.storage.zbs.Config import org.zstack.storage.zbs.ZbsAgentUrl import org.zstack.storage.zbs.ZbsConstants import org.zstack.storage.zbs.ZbsGlobalProperty +import org.zstack.storage.zbs.ZbsHelper import org.zstack.storage.zbs.ZbsPrimaryStorageMdsBase import org.zstack.storage.zbs.ZbsStorageController +import org.zstack.storage.volume.VolumeSystemTags import org.zstack.test.integration.storage.StorageTest import org.zstack.testlib.EnvSpec import org.zstack.testlib.HttpError @@ -194,6 +206,7 @@ class ZbsPrimaryStorageCase extends SubCase { testAttachPrimaryStorageFailsWhenActivatingHeartbeatVolumeFails() testMdsConnectFailed() testLifecycle() + testReimageReferencedRootCleanup() testDataVolumeLifecycle() testMdsPing() testCheckHostStorageConnection() @@ -756,6 +769,153 @@ class ZbsPrimaryStorageCase extends SubCase { } } + void testReimageReferencedRootCleanup() { + attachPrimaryStorageToCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + + env.message(UploadImageToRemoteTargetMsg.class) { UploadImageToRemoteTargetMsg msg, CloudBus bus -> + bus.reply(msg, new UploadImageToRemoteTargetReply()) + } + + def instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory + def image = env.inventoryByName("image") as ImageInventory + def l3 = env.inventoryByName("l3") as L3NetworkInventory + def vm = createVmInstance { + name = "reimage-referenced-root" + imageUuid = image.uuid + l3NetworkUuids = [l3.uuid] + instanceOfferingUuid = instanceOffering.uuid + } as VmInstanceInventory + def oldRoot = queryVolume { + conditions = ["uuid=${vm.rootVolumeUuid}"] + }[0] as VolumeInventory + def snapshot = createVolumeSnapshot { + name = "referenced-root-snapshot" + volumeUuid = oldRoot.uuid + } as VolumeSnapshotInventory + def child = createDataVolumeFromVolumeSnapshot { + name = "referenced-root-child" + volumeSnapshotUuid = snapshot.uuid + systemTags = [VolumeSystemTags.FAST_CREATE.tagFormat] + } as VolumeInventory + + assert Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.referenceVolumeUuid, child.uuid).isExists() : + "FAST_CREATE child must retain the backing reference: childUuid=${child.uuid}" + + List deleteAttempts = Collections.synchronizedList(new ArrayList<>()) + AtomicBoolean cleanupStarted = new AtomicBoolean(false) + AtomicBoolean failSnapshotDeleteOnce = new AtomicBoolean(true) + env.simulator(ZbsStorageController.QUERY_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.QueryVolumeCmd.class) + def rsp = new ZbsStorageController.QueryVolumeRsp() + rsp.size = oldRoot.size + rsp.actualSize = oldRoot.actualSize + if (ZbsHelper.normalizeToZbsPath(cmd.path) == snapshot.primaryStorageInstallPath) { + rsp.parentUri = oldRoot.installPath + } + return rsp + } + env.simulator(ZbsStorageController.DELETE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.DeleteVolumeCmd.class) + String path = ZbsHelper.normalizeToZbsPath(cmd.path) + deleteAttempts.add(path) + def rsp = new ZbsStorageController.DeleteVolumeRsp() + if (!cleanupStarted.get() && path == oldRoot.installPath) { + rsp.success = false + rsp.error = "snapshot is still in used" + } else if (cleanupStarted.get() && path == snapshot.primaryStorageInstallPath && + failSnapshotDeleteOnce.compareAndSet(true, false)) { + rsp.success = false + rsp.error = "transient cleanup failure" + } + return rsp + } + + stopVmInstance { + uuid = vm.uuid + } + reimageVmInstance { + vmInstanceUuid = vm.uuid + } + + assert !deleteAttempts.contains(oldRoot.installPath) : + "reimage must not delete an old root with downstream references: oldRoot=${oldRoot.installPath} attempts=${deleteAttempts}" + assert !Q.New(InstallPathRecycleVO.class) + .eq(InstallPathRecycleVO_.storageUuid, ps.uuid) + .eq(InstallPathRecycleVO_.installPath, oldRoot.installPath) + .eq(InstallPathRecycleVO_.trashType, TrashType.ReimageVolume.toString()) + .isExists() : "reimage must not create trash for referenced old root: oldRoot=${oldRoot.installPath}" + + destroyVmInstance { + uuid = vm.uuid + } + expungeVmInstance { + uuid = vm.uuid + } + assert !Q.New(VolumeSnapshotVO.class).eq(VolumeSnapshotVO_.volumeUuid, oldRoot.uuid).isExists() : + "old root snapshots must be removed after VM expunge: volumeUuid=${oldRoot.uuid}" + assert Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.referenceVolumeUuid, child.uuid).isExists() : + "VM expunge must retain the FAST_CREATE child reference: childUuid=${child.uuid}" + + cleanupStarted.set(true) + int cleanupStartIndex = deleteAttempts.size() + deleteDataVolume { + uuid = child.uuid + } + expungeDataVolume { + uuid = child.uuid + } + + List chainJobs = [] + retryInSecs { + chainJobs = queryGCJob { + conditions = [ + "runnerClass=org.zstack.storage.addon.primary.ExternalDeleteVolumeChainGC", + "context~=%${oldRoot.installPath}%" + ] + } as List + assert chainJobs.size() == 1 : + "failed chain cleanup must leave one GC job: oldRoot=${oldRoot.installPath} jobs=${chainJobs*.uuid}" + assert chainJobs[0].status != GCStatus.Done.toString() : + "chain GC must retain the undeleted suffix after transient failure: job=${chainJobs[0].uuid}" + } + + triggerGCJob { + uuid = chainJobs[0].uuid + } + retryInSecs { + def job = queryGCJob { + conditions = ["uuid=${chainJobs[0].uuid}"] + }[0] as GarbageCollectorInventory + List cleanupAttempts = deleteAttempts.drop(cleanupStartIndex) + int snapshotIndex = cleanupAttempts.lastIndexOf(snapshot.primaryStorageInstallPath) + int oldRootIndex = cleanupAttempts.indexOf(oldRoot.installPath) + assert job.status == GCStatus.Done.toString() : + "chain GC must finish after retry: job=${job.uuid} status=${job.status} context=${job.context}" + assert snapshotIndex >= 0 && oldRootIndex > snapshotIndex : + "cleanup must delete leaf-to-root: snapshot=${snapshot.primaryStorageInstallPath} oldRoot=${oldRoot.installPath} attempts=${cleanupAttempts}" + } + + assert !Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.referenceVolumeUuid, child.uuid).isExists() : + "reference must be removed after child expunge: childUuid=${child.uuid}" + assert !Q.New(InstallPathRecycleVO.class) + .eq(InstallPathRecycleVO_.storageUuid, ps.uuid) + .eq(InstallPathRecycleVO_.installPath, oldRoot.installPath) + .eq(InstallPathRecycleVO_.trashType, TrashType.ReimageVolume.toString()) + .isExists() : "completed chain cleanup must remove legacy reimage trash: oldRoot=${oldRoot.installPath}" + + detachPrimaryStorageFromCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + env.cleanSimulatorHandlers() + } + void testMdsPing() { ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data = null long count = 0