<fix>[storage]: ZSTAC-81222 defer referenced reimage trash

header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceTreeVO_.java

@@ -12,7 +12,7 @@ public class VolumeSnapshotReferenceTreeVO_ extends ResourceVO_ {
     public static volatile SingularAttribute<VolumeSnapshotReferenceTreeVO, String> rootVolumeSnapshotUuid;
     public static volatile SingularAttribute<VolumeSnapshotReferenceTreeVO, String> rootVolumeUuid;
     public static volatile SingularAttribute<VolumeSnapshotReferenceTreeVO, String> rootVolumeSnapshotTreeUuid;
-    public static volatile SingularAttribute<VolumeSnapshotReferenceTreeVO, String> rootVolumeSnapshotInstallUrl;
+    public static volatile SingularAttribute<VolumeSnapshotReferenceTreeVO, String> rootInstallUrl;
 
     public static volatile SingularAttribute<VolumeSnapshotReferenceTreeVO, String> primaryStorageUuid;
     public static volatile SingularAttribute<VolumeSnapshotReferenceTreeVO, String> hostUuid;

storage/src/main/java/org/zstack/storage/addon/primary/ExternalDeleteVolumeChainGC.java

@@ -0,0 +1,70 @@
+package org.zstack.storage.addon.primary;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.zstack.core.cloudbus.CloudBusCallBack;
+import org.zstack.core.gc.GC;
+import org.zstack.core.gc.GCCompletion;
+import org.zstack.core.gc.TimeBasedGarbageCollector;
+import org.zstack.header.errorcode.ErrorCode;
+import org.zstack.header.message.MessageReply;
+import org.zstack.header.storage.primary.DeleteVolumeChainOnPrimaryStorageMsg;
+import org.zstack.header.storage.primary.DeleteVolumeChainOnPrimaryStorageReply;
+import org.zstack.header.storage.primary.PrimaryStorageConstant;
+import org.zstack.header.storage.primary.PrimaryStorageVO;
+import org.zstack.storage.primary.PrimaryStorageGlobalConfig;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.zstack.core.Platform.operr;
+import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040;
+
+public class ExternalDeleteVolumeChainGC extends TimeBasedGarbageCollector {
+    @GC
+    public String primaryStorageUuid;
+    @GC
+    public List<String> installPaths;
+    @GC
+    public String chainTop;
+
+    @Override
+    protected void triggerNow(GCCompletion completion) {
+        if (!dbf.isExist(primaryStorageUuid, PrimaryStorageVO.class) || CollectionUtils.isEmpty(installPaths)) {
+            completion.cancel();
+            return;
+        }
+
+        DeleteVolumeChainOnPrimaryStorageMsg msg = new DeleteVolumeChainOnPrimaryStorageMsg();
+        msg.setPrimaryStorageUuid(primaryStorageUuid);
+        msg.setInstallPaths(installPaths);
+        msg.setChainTop(chainTop);
+        bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, primaryStorageUuid);
+        bus.send(msg, new CloudBusCallBack(completion) {
+            @Override
+            public void run(MessageReply reply) {
+                if (!reply.isSuccess()) {
+                    retry(completion, reply.getError());
+                    return;
+                }
+
+                DeleteVolumeChainOnPrimaryStorageReply r = reply.castReply();
+                if (CollectionUtils.isEmpty(r.getUndeletedInstallPaths())) {
+                    completion.success();
+                    return;
+                }
+
+                installPaths = new ArrayList<>(r.getUndeletedInstallPaths());
+                retry(completion, operr(ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040,
+                        "failed to delete volume chain paths %s", installPaths));
+            }
+        });
+    }
+
+    private void retry(GCCompletion completion, ErrorCode error) {
+        NEXT_TIME = PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class);
+        NEXT_TIME_UNIT = TimeUnit.SECONDS;
+        updateContext();
+        completion.fail(error);
+    }
+}

storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java

@@ -24,6 +24,8 @@
 import org.zstack.header.core.NopeCompletion;
 import org.zstack.header.core.ReturnValueCompletion;
 import org.zstack.header.core.WhileDoneCompletion;
+import org.zstack.header.core.trash.InstallPathRecycleVO;
+import org.zstack.header.core.trash.InstallPathRecycleVO_;
 import org.zstack.header.core.workflow.*;
 import org.zstack.header.errorcode.ErrorCode;
 import org.zstack.header.errorcode.ErrorCodeList;
@@ -56,13 +58,15 @@
 import org.zstack.resourceconfig.ResourceConfigFacade;
 import org.zstack.storage.backup.BackupStorageSystemTags;
 import org.zstack.storage.primary.*;
+import org.zstack.storage.snapshot.reference.VolumeSnapshotReferenceUtils;
 import org.zstack.storage.volume.VolumeSystemTags;
 import org.zstack.utils.Utils;
 import org.zstack.utils.gson.JSONObjectUtil;
 import org.zstack.utils.logging.CLogger;
 
 import java.util.*;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -1704,11 +1708,18 @@ public void fail(ErrorCode errorCode) {
 
     protected void handle(DeleteVolumeChainOnPrimaryStorageMsg msg) {
         DeleteVolumeChainOnPrimaryStorageReply reply = new DeleteVolumeChainOnPrimaryStorageReply();
+        if (CollectionUtils.isEmpty(msg.getInstallPaths())) {
+            bus.reply(msg, reply);
+            return;
+        }
+
+        AtomicInteger deletedCount = new AtomicInteger();
         new While<>(msg.getInstallPaths()).each((installPath, compl) -> {
             // TODO use trash instead, make sure only one snapshot for link clone.
             controller.deleteVolumeAndSnapshot(installPath, new Completion(compl) {
                 @Override
                 public void success() {
+                    deletedCount.incrementAndGet();
                     compl.done();
                 }
 
@@ -1722,11 +1733,54 @@ public void fail(ErrorCode errorCode) {
             @Override
             public void done(ErrorCodeList errorCodeList) {
                 if (!errorCodeList.getCauses().isEmpty()) {
-                    reply.setError(errorCodeList.getCauses().get(0));
+                    List<String> undeletedInstallPaths = new ArrayList<>(msg.getInstallPaths()
+                            .subList(deletedCount.get(), msg.getInstallPaths().size()));
+                    submitDeleteVolumeChainGC(msg, undeletedInstallPaths);
+                    reply.setUndeletedInstallPaths(undeletedInstallPaths);
                     bus.reply(msg, reply);
-                } else {
+                    return;
+                }
+
+                syncCapacityAfterDeleteVolumeChain(msg, reply);
+            }
+        });
+    }
+
+    private void submitDeleteVolumeChainGC(DeleteVolumeChainOnPrimaryStorageMsg msg,
+                                           List<String> undeletedInstallPaths) {
+        ExternalDeleteVolumeChainGC gc = new ExternalDeleteVolumeChainGC();
+        gc.primaryStorageUuid = self.getUuid();
+        gc.installPaths = undeletedInstallPaths;
+        gc.chainTop = msg.getChainTop();
+        gc.NAME = String.format("gc-external-%s-volume-chain-%s", self.getUuid(), gc.chainTop);
+        gc.deduplicateSubmit(PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class),
+                TimeUnit.SECONDS);
+    }
+
+    private void syncCapacityAfterDeleteVolumeChain(DeleteVolumeChainOnPrimaryStorageMsg msg,
+                                                     DeleteVolumeChainOnPrimaryStorageReply reply) {
+        SyncPrimaryStorageCapacityMsg smsg = new SyncPrimaryStorageCapacityMsg();
+        smsg.setPrimaryStorageUuid(self.getUuid());
+        bus.makeTargetServiceIdByResourceUuid(smsg, PrimaryStorageConstant.SERVICE_ID, self.getUuid());
+        bus.send(smsg, new CloudBusCallBack(msg) {
+            @Override
+            public void run(MessageReply r) {
+                if (!r.isSuccess()) {
+                    submitDeleteVolumeChainGC(msg, new ArrayList<>(msg.getInstallPaths()));
+                    reply.setError(r.getError());
                     bus.reply(msg, reply);
+                    return;
                 }
+
+                String rootInstallPath = msg.getInstallPaths().get(msg.getInstallPaths().size() - 1);
+                List<Long> trashIds = Q.New(InstallPathRecycleVO.class)
+                        .select(InstallPathRecycleVO_.trashId)
+                        .eq(InstallPathRecycleVO_.storageUuid, self.getUuid())
+                        .eq(InstallPathRecycleVO_.installPath, rootInstallPath)
+                        .eq(InstallPathRecycleVO_.trashType, TrashType.ReimageVolume.toString())
+                        .listValues();
+                trashIds.forEach(trash::removeFromDb);
+                bus.reply(msg, reply);
             }
         });
     }
@@ -2104,6 +2158,11 @@ public boolean skip(Map data) {
 
                     @Override
                     public void run(FlowTrigger trigger, Map data) {
+                        if (VolumeSnapshotReferenceUtils.isVolumeDirectlyReferenceByOthers(msg.getVolume())) {
+                            trigger.next();
+                            return;
+                        }
+
                         boolean hasSnapshot = Q.New(VolumeSnapshotVO.class)
                                 .eq(VolumeSnapshotVO_.volumeUuid, msg.getVolume().getUuid())
                                 .like(VolumeSnapshotVO_.primaryStorageInstallPath, String.format("%s%%", msg.getVolume().getInstallPath()))

storage/src/main/java/org/zstack/storage/snapshot/reference/DeleteVolumeSnapshotReferenceGC.java

@@ -0,0 +1,92 @@
+package org.zstack.storage.snapshot.reference;
+
+import org.zstack.core.cloudbus.CloudBusCallBack;
+import org.zstack.core.db.Q;
+import org.zstack.core.gc.GC;
+import org.zstack.core.gc.GCCompletion;
+import org.zstack.core.gc.TimeBasedGarbageCollector;
+import org.zstack.header.errorcode.ErrorCode;
+import org.zstack.header.message.MessageReply;
+import org.zstack.header.storage.primary.PrimaryStorageVO;
+import org.zstack.header.storage.snapshot.VolumeSnapshotConstant;
+import org.zstack.header.storage.snapshot.reference.DeleteVolumeSnapshotReferenceLeafMsg;
+import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceInventory;
+import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeInventory;
+import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO;
+import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO_;
+import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO;
+import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO_;
+import org.zstack.header.volume.VolumeInventory;
+import org.zstack.storage.primary.PrimaryStorageGlobalConfig;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class DeleteVolumeSnapshotReferenceGC extends TimeBasedGarbageCollector {
+    @GC
+    public VolumeSnapshotReferenceInventory leaf;
+    @GC
+    public VolumeSnapshotReferenceTreeInventory tree;
+    @GC
+    public VolumeInventory deletedVolume;
+
+    @Override
+    protected void triggerNow(GCCompletion completion) {
+        if (!dbf.isExist(tree.getPrimaryStorageUuid(), PrimaryStorageVO.class)) {
+            completion.cancel();
+            return;
+        }
+
+        DeleteVolumeSnapshotReferenceLeafMsg msg = new DeleteVolumeSnapshotReferenceLeafMsg();
+        msg.setLeaf(leaf);
+        msg.setOtherLeafs(getOtherLeafs());
+        msg.setTree(tree);
+        msg.setDeletedVolume(deletedVolume);
+        bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, tree.getUuid());
+        bus.send(msg, new CloudBusCallBack(completion) {
+            @Override
+            public void run(MessageReply reply) {
+                if (!reply.isSuccess()) {
+                    retry(completion, reply.getError());
+                    return;
+                }
+
+                completion.success();
+            }
+        });
+    }
+
+    private List<VolumeSnapshotReferenceInventory> getOtherLeafs() {
+        List<VolumeSnapshotReferenceVO> refs;
+        if (leaf.getParentId() == null) {
+            List<String> treeUuids = Q.New(VolumeSnapshotReferenceTreeVO.class)
+                    .select(VolumeSnapshotReferenceTreeVO_.uuid)
+                    .eq(VolumeSnapshotReferenceTreeVO_.primaryStorageUuid, tree.getPrimaryStorageUuid())
+                    .eq(VolumeSnapshotReferenceTreeVO_.rootInstallUrl, tree.getRootInstallUrl())
+                    .listValues();
+            if (treeUuids.isEmpty()) {
+                return Collections.emptyList();
+            }
+            refs = Q.New(VolumeSnapshotReferenceVO.class)
+                    .in(VolumeSnapshotReferenceVO_.treeUuid, treeUuids)
+                    .isNull(VolumeSnapshotReferenceVO_.parentId)
+                    .list();
+        } else {
+            refs = Q.New(VolumeSnapshotReferenceVO.class)
+                    .eq(VolumeSnapshotReferenceVO_.treeUuid, tree.getUuid())
+                    .eq(VolumeSnapshotReferenceVO_.parentId, leaf.getParentId())
+                    .list();
+        }
+
+        return refs.stream().map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList());
+    }
+
+    private void retry(GCCompletion completion, ErrorCode error) {
+        NEXT_TIME = PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class);
+        NEXT_TIME_UNIT = TimeUnit.SECONDS;
+        updateContext();
+        completion.fail(error);
+    }
+}

storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceTreeBase.java

@@ -113,6 +113,7 @@ private void deleteSnapshotRefLeaf(DeleteVolumeSnapshotReferenceLeafMsg msg, Com
         boolean rootDeleted = msg.getLeaf().getParentId() == null && !Q.New(VolumeVO.class)
                 .eq(VolumeVO_.uuid, msg.getLeaf().getVolumeUuid())
                 .eq(VolumeVO_.primaryStorageUuid, msg.getTree().getPrimaryStorageUuid())
+                .eq(VolumeVO_.installPath, self.getRootInstallUrl())
                 .isExists();
         String endPath = rootDeleted ? self.getRootInstallUrl() : msg.getLeaf().getVolumeSnapshotInstallUrl();
         String startPath = msg.getLeaf().getDirectSnapshotInstallUrl();

storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java

@@ -4,7 +4,6 @@
 import org.apache.commons.collections.MapUtils;
 import org.springframework.transaction.support.TransactionSynchronizationManager;
 import org.zstack.core.Platform;
-import org.zstack.core.cloudbus.CloudBus;
 import org.zstack.core.db.*;
 import org.zstack.header.errorcode.OperationFailureException;
 import org.zstack.header.exception.CloudRuntimeException;
@@ -27,6 +26,7 @@
 import javax.persistence.LockModeType;
 import javax.persistence.Tuple;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.zstack.core.Platform.operr;
@@ -258,6 +258,7 @@ protected VolumeSnapshotReferenceTreeVO scripts() {
                 VolumeSnapshotReferenceTreeVO tree = Q.New(VolumeSnapshotReferenceTreeVO.class)
                         .eq(VolumeSnapshotReferenceTreeVO_.rootVolumeUuid, baseSnapshot.getVolumeUuid())
                         .eq(VolumeSnapshotReferenceTreeVO_.primaryStorageUuid, baseSnapshot.getPrimaryStorageUuid())
+                        .eq(VolumeSnapshotReferenceTreeVO_.rootInstallUrl, vol.getInstallPath())
                         .find();
                 if (tree != null) {
                     return tree;
@@ -494,25 +495,13 @@ protected void scripts() {
     }
 
     private static void deleteBitsOnPs(VolumeSnapshotReferenceTreeVO treeVO, VolumeSnapshotReferenceVO ref) {
-        List<VolumeSnapshotReferenceVO> treeRefs = Q.New(VolumeSnapshotReferenceVO.class).eq(VolumeSnapshotReferenceVO_.treeUuid, treeVO.getUuid()).list();
-
-        List<VolumeSnapshotReferenceInventory> otherLeafs;
-        if (ref.getParentId() == null) {
-            otherLeafs = treeRefs.stream().filter(it -> it.getParentId() == null).map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList());
-        } else {
-            otherLeafs = treeRefs.stream().filter(it -> ref.getParentId().equals(it.getParentId())).map(VolumeSnapshotReferenceInventory::valueOf).collect(Collectors.toList());
-        }
-        otherLeafs.removeIf(it -> it.getId() == ref.getId());
-
-        DeleteVolumeSnapshotReferenceLeafMsg msg = new DeleteVolumeSnapshotReferenceLeafMsg();
-        msg.setLeaf(VolumeSnapshotReferenceInventory.valueOf(ref));
-        msg.setOtherLeafs(otherLeafs);
-        msg.setTree(treeVO.toInventory());
         VolumeVO deletedVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, ref.getReferenceVolumeUuid()).find();
-        msg.setDeletedVolume(deletedVolume.toInventory());
-        CloudBus bus = Platform.getComponentLoader().getComponent(CloudBus.class);
-        bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, treeVO.getUuid());
-        bus.send(msg);
+        DeleteVolumeSnapshotReferenceGC gc = new DeleteVolumeSnapshotReferenceGC();
+        gc.NAME = String.format("gc-volume-snapshot-reference-%s-leaf-%s", treeVO.getUuid(), ref.getId());
+        gc.leaf = VolumeSnapshotReferenceInventory.valueOf(ref);
+        gc.tree = treeVO.toInventory();
+        gc.deletedVolume = deletedVolume.toInventory();
+        gc.deduplicateSubmit(1L, TimeUnit.SECONDS);
     }
 
     private static void deleteSnapshotRef(VolumeSnapshotReferenceVO ref) {
@@ -529,19 +518,16 @@ private static void deleteSnapshotRef(VolumeSnapshotReferenceVO ref) {
 
     private static void deleteSnapshotRefLeafInTree(VolumeSnapshotReferenceTreeVO tree, VolumeSnapshotReferenceVO ref) {
         boolean referenceRedirected = !ref.getDirectSnapshotUuid().equals(ref.getVolumeSnapshotUuid());
-        boolean backingVolumeDeletedInDb = SQL.New("select vol.uuid from VolumeVO vol, VolumeSnapshotReferenceTreeVO tree" +
-                        " where vol.uuid = :volUuid" +
-                        " and tree.uuid = :treeUuid" +
-                        " and vol.primaryStorageUuid = tree.primaryStorageUuid", String.class)
-                .param("volUuid", ref.getVolumeUuid())
-                .param("treeUuid", ref.getTreeUuid())
-                .find() == null;
+        if (tree == null) {
+            tree = Q.New(VolumeSnapshotReferenceTreeVO.class).eq(VolumeSnapshotReferenceTreeVO_.uuid, ref.getTreeUuid()).find();
+        }
+        boolean backingVolumeDeletedInDb = !Q.New(VolumeVO.class)
+                .eq(VolumeVO_.uuid, ref.getVolumeUuid())
+                .eq(VolumeVO_.primaryStorageUuid, tree.getPrimaryStorageUuid())
+                .eq(VolumeVO_.installPath, tree.getRootInstallUrl())
+                .isExists();
 
         if (referenceRedirected || backingVolumeDeletedInDb) {
-            if (tree == null) {
-                tree = Q.New(VolumeSnapshotReferenceTreeVO.class).eq(VolumeSnapshotReferenceTreeVO_.uuid, ref.getTreeUuid()).find();
-            }
-
             deleteBitsOnPs(tree, ref);
         }