diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapDiffDependencyEntry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapDiffDependencyEntry.java new file mode 100644 index 000000000000..df3b5e70ef2b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapDiffDependencyEntry.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; + +/** + * Metadata for a classified snapshot diff entry used to build the dependency + * graph for dependency-ordered report emission. + */ +public final class SnapDiffDependencyEntry { + + private final long objectId; + private final long parentObjectId; + private final DiffReportEntry reportEntry; + + public SnapDiffDependencyEntry(long objectId, long parentObjectId, + DiffReportEntry reportEntry) { + this.objectId = objectId; + this.parentObjectId = parentObjectId; + this.reportEntry = Objects.requireNonNull(reportEntry, "reportEntry"); + } + + public long getObjectId() { + return objectId; + } + + public long getParentObjectId() { + return parentObjectId; + } + + public DiffReportEntry getReportEntry() { + return reportEntry; + } + + public DiffType getDiffType() { + return reportEntry.getType(); + } + + public String getSourcePath() { + return new String(reportEntry.getSourcePath(), StandardCharsets.UTF_8); + } + + public String getTargetPath() { + if (reportEntry.getTargetPath() == null) { + return null; + } + return new String(reportEntry.getTargetPath(), StandardCharsets.UTF_8); + } + + public boolean isDelete() { + return getDiffType() == DiffType.DELETE; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof SnapDiffDependencyEntry)) { + return false; + } + SnapDiffDependencyEntry that = (SnapDiffDependencyEntry) other; + return objectId == that.objectId + && parentObjectId == that.parentObjectId + && reportEntry.equals(that.reportEntry); + } + + @Override + public int hashCode() { + return Objects.hash(objectId, parentObjectId, reportEntry); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapDiffDependencyGraph.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapDiffDependencyGraph.java new file mode 100644 index 000000000000..a1b8c44751de --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapDiffDependencyGraph.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; + +/** + * Directed graph of snapshot diff entries and Kahn topological sort for + * dependency-ordered report emission. + * + *

Dependency rules encoded by edges (edge {@code u -> v} means {@code u} + * must appear before {@code v}): + *

+ * + *

A RENAME target path cannot match a CREATE path in the same diff report. + */ +public final class SnapDiffDependencyGraph { + + private final List nodes = new ArrayList<>(); + private final Map> adjacencyList = new HashMap<>(); + private final Map inDegree = new HashMap<>(); + + /** + * @throws IllegalStateException if entries contain a RENAME target path that + * matches a CREATE path, or if dependency edges form a cycle + */ + public SnapDiffDependencyGraph(List entries) { + for (SnapDiffDependencyEntry entry : entries) { + addNode(entry); + } + buildDependencyEdges(); + } + + /** + * Returns entries in dependency order using Kahn's algorithm. + * + * @return topologically sorted dependency entries + * @throws IllegalStateException if the graph contains a cycle + */ + public List getOrderedEntries() { + Queue zeroInDegree = new ArrayDeque<>(); + for (int nodeId = 0; nodeId < nodes.size(); nodeId++) { + if (inDegree.get(nodeId) == 0) { + zeroInDegree.add(nodeId); + } + } + + List orderedEntries = new ArrayList<>(nodes.size()); + while (!zeroInDegree.isEmpty()) { + int nodeId = zeroInDegree.remove(); + orderedEntries.add(nodes.get(nodeId)); + for (int dependentNodeId : adjacencyList.get(nodeId)) { + int updatedInDegree = inDegree.get(dependentNodeId) - 1; + inDegree.put(dependentNodeId, updatedInDegree); + if (updatedInDegree == 0) { + zeroInDegree.add(dependentNodeId); + } + } + } + + if (orderedEntries.size() != nodes.size()) { + throw new IllegalStateException( + "Cycle detected in snapshot diff dependency graph"); + } + return orderedEntries; + } + + /** + * Converts dependency-ordered entries to report entries. + */ + public static List toOrderedReportEntries( + List orderedEntries) { + List reportEntries = + new ArrayList<>(orderedEntries.size()); + for (SnapDiffDependencyEntry entry : orderedEntries) { + reportEntries.add(entry.getReportEntry()); + } + return reportEntries; + } + + private int addNode(SnapDiffDependencyEntry entry) { + int nodeId = nodes.size(); + nodes.add(entry); + adjacencyList.put(nodeId, new HashSet<>()); + inDegree.put(nodeId, 0); + return nodeId; + } + + private void addEdge(int fromNodeId, int toNodeId) { + if (fromNodeId == toNodeId) { + return; + } + if (adjacencyList.get(fromNodeId).add(toNodeId)) { + inDegree.put(toNodeId, inDegree.get(toNodeId) + 1); + } + } + + private void buildDependencyEdges() { + // One objectId can map to multiple non-delete nodes, for example RENAME + MODIFY. + Map> nonDeleteNodesByObjectId = new HashMap<>(); + // Each objectId has at most one DELETE entry after top-level delete retention. + Map deleteNodesByObjectId = new HashMap<>(); + Map> createNodesByPath = new HashMap<>(); + Map> deleteNodesByPath = new HashMap<>(); + Map> renameNodesBySourcePath = new HashMap<>(); + Map> renameNodesByTargetPath = new HashMap<>(); + + for (int nodeId = 0; nodeId < nodes.size(); nodeId++) { + SnapDiffDependencyEntry entry = nodes.get(nodeId); + if (entry.isDelete()) { + deleteNodesByObjectId.put(entry.getObjectId(), nodeId); + addToPathIndex(deleteNodesByPath, entry.getSourcePath(), nodeId); + } else { + nonDeleteNodesByObjectId + .computeIfAbsent(entry.getObjectId(), ignored -> new ArrayList<>()) + .add(nodeId); + DiffType diffType = entry.getDiffType(); + if (diffType == DiffType.CREATE) { + addToPathIndex(createNodesByPath, entry.getSourcePath(), nodeId); + } else if (diffType == DiffType.RENAME) { + addToPathIndex(renameNodesBySourcePath, entry.getSourcePath(), nodeId); + addToPathIndex(renameNodesByTargetPath, entry.getTargetPath(), nodeId); + } + } + } + + validateRenameTargetDoesNotMatchCreatePath(renameNodesByTargetPath, + createNodesByPath); + + for (int nodeId = 0; nodeId < nodes.size(); nodeId++) { + SnapDiffDependencyEntry entry = nodes.get(nodeId); + long parentObjectId = entry.getParentObjectId(); + if (parentObjectId <= 0L) { + continue; + } + + Integer parentDeleteNodeId = deleteNodesByObjectId.get(parentObjectId); + if (entry.isDelete()) { + if (parentDeleteNodeId != null) { + addEdge(nodeId, parentDeleteNodeId); + } + } else { + addEdgesFromNodes( + nonDeleteNodesByObjectId.getOrDefault(parentObjectId, + Collections.emptyList()), + nodeId); + if (parentDeleteNodeId != null) { + addEdge(nodeId, parentDeleteNodeId); + } + } + } + + addPathConflictEdges(deleteNodesByPath, createNodesByPath, + renameNodesByTargetPath); + addRenameBeforeCreateEdges(renameNodesBySourcePath, createNodesByPath); + } + + private static void addToPathIndex(Map> pathIndex, + String path, int nodeId) { + if (path == null) { + return; + } + pathIndex.computeIfAbsent(path, ignored -> new ArrayList<>()).add(nodeId); + } + + private void addEdgesToNodes(int fromNodeId, List toNodeIds) { + for (int toNodeId : toNodeIds) { + addEdge(fromNodeId, toNodeId); + } + } + + private void addEdgesFromNodes(List fromNodeIds, int toNodeId) { + for (int fromNodeId : fromNodeIds) { + addEdge(fromNodeId, toNodeId); + } + } + + private void addPathConflictEdges(Map> deleteNodesByPath, + Map> createNodesByPath, + Map> renameNodesByTargetPath) { + for (Map.Entry> deleteEntry : + deleteNodesByPath.entrySet()) { + String path = deleteEntry.getKey(); + for (int deleteNodeId : deleteEntry.getValue()) { + addEdgesToNodes(deleteNodeId, + createNodesByPath.getOrDefault(path, Collections.emptyList())); + addEdgesToNodes(deleteNodeId, + renameNodesByTargetPath.getOrDefault(path, + Collections.emptyList())); + } + } + } + + private void addRenameBeforeCreateEdges( + Map> renameNodesBySourcePath, + Map> createNodesByPath) { + for (Map.Entry> createEntry : + createNodesByPath.entrySet()) { + String path = createEntry.getKey(); + for (int renameNodeId : renameNodesBySourcePath.getOrDefault(path, + Collections.emptyList())) { + addEdgesToNodes(renameNodeId, createEntry.getValue()); + } + } + } + + private static void validateRenameTargetDoesNotMatchCreatePath( + Map> renameNodesByTargetPath, + Map> createNodesByPath) { + for (Map.Entry> createEntry : + createNodesByPath.entrySet()) { + String path = createEntry.getKey(); + if (!renameNodesByTargetPath.containsKey(path)) { + continue; + } + throw new IllegalStateException(String.format( + "Invalid snapshot diff report: RENAME target path '%s' cannot match " + + "a CREATE path in the same diff", path)); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapDiffDependencyGraph.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapDiffDependencyGraph.java new file mode 100644 index 000000000000..cdd91f07ad2a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapDiffDependencyGraph.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType.CREATE; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType.DELETE; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType.MODIFY; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType.RENAME; +import static org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone.getDiffReportEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; +import org.junit.jupiter.api.Test; + +class TestSnapDiffDependencyGraph { + + @Test + void testParentCreateBeforeChildCreate() { + List entries = Arrays.asList( + entry(2L, 1L, CREATE, "parent/child"), + entry(1L, 0L, CREATE, "parent")); + + List orderedTypes = toDiffTypes(sort(entries)); + assertEquals(Arrays.asList(CREATE, CREATE), orderedTypes); + assertPathOrder(entries, "parent", "parent/child"); + } + + @Test + void testChildDeleteBeforeParentDelete() { + List entries = Arrays.asList( + entry(1L, 0L, DELETE, "parent"), + entry(2L, 1L, DELETE, "parent/child")); + + List orderedTypes = toDiffTypes(sort(entries)); + assertEquals(Arrays.asList(DELETE, DELETE), orderedTypes); + assertPathOrder(entries, "parent/child", "parent"); + } + + @Test + void testDeleteBeforeCreateOnSamePath() { + List entries = Arrays.asList( + entry(2L, 0L, CREATE, "dir/key"), + entry(1L, 0L, DELETE, "dir/key")); + + List orderedTypes = toDiffTypes(sort(entries)); + assertEquals(Arrays.asList(DELETE, CREATE), orderedTypes); + } + + @Test + void testDeleteBeforeRenameOnSameTargetPath() { + List entries = Arrays.asList( + entry(2L, 0L, RENAME, "old/key", "dir/key"), + entry(1L, 0L, DELETE, "dir/key")); + + List orderedTypes = toDiffTypes(sort(entries)); + assertEquals(Arrays.asList(DELETE, RENAME), orderedTypes); + } + + @Test + void testRenameBeforeCreateWhenRenameFreesSourcePath() { + List entries = Arrays.asList( + entry(3L, 0L, CREATE, "dir/key"), + entry(2L, 0L, RENAME, "dir/key", "dir/renamed-key")); + + List orderedTypes = toDiffTypes(sort(entries)); + assertEquals(Arrays.asList(RENAME, CREATE), orderedTypes); + } + + @Test + void testRenameTargetPathMatchingCreateThrowsException() { + List entries = Arrays.asList( + entry(3L, 0L, CREATE, "dir/key"), + entry(2L, 0L, RENAME, "old/key", "dir/key")); + + assertThrows(IllegalStateException.class, + () -> new SnapDiffDependencyGraph(entries)); + } + + @Test + void testRenameBeforeDeleteParentDirectory() { + List entries = Arrays.asList( + entry(1L, 0L, DELETE, "A"), + entry(2L, 1L, RENAME, "A/B", "C/B")); + + List orderedTypes = toDiffTypes(sort(entries)); + assertEquals(Arrays.asList(RENAME, DELETE), orderedTypes); + } + + @Test + void testModifyAndRenameForSameObjectKeepDependencyOrder() { + List entries = Arrays.asList( + entry(2L, 1L, MODIFY, "parent/child"), + entry(2L, 1L, RENAME, "parent/old-child", "parent/child"), + entry(1L, 0L, CREATE, "parent")); + + List ordered = sort(entries); + assertEquals(CREATE, ordered.get(0).getDiffType()); + assertEquals("parent", ordered.get(0).getSourcePath()); + assertTrue(ordered.subList(1, 3).stream() + .noneMatch(SnapDiffDependencyEntry::isDelete)); + assertTrue(ordered.subList(1, 3).stream() + .map(SnapDiffDependencyEntry::getObjectId) + .allMatch(objectId -> objectId == 2L)); + } + + @Test + void testTopologicalSortDetectsCycle() { + List entries = Arrays.asList( + entry(1L, 2L, CREATE, "a"), + entry(2L, 1L, CREATE, "b")); + + assertThrows(IllegalStateException.class, + () -> new SnapDiffDependencyGraph(entries).getOrderedEntries()); + } + + @Test + void testToOrderedReportEntries() { + List entries = Arrays.asList( + entry(2L, 1L, CREATE, "parent/child"), + entry(1L, 0L, CREATE, "parent")); + + List orderedEntries = SnapDiffDependencyGraph + .toOrderedReportEntries(new SnapDiffDependencyGraph(entries).getOrderedEntries()); + + assertEquals(2, orderedEntries.size()); + assertEquals(CREATE, orderedEntries.get(0).getType()); + assertEquals("parent", + new String(orderedEntries.get(0).getSourcePath(), StandardCharsets.UTF_8)); + assertEquals("parent/child", + new String(orderedEntries.get(1).getSourcePath(), StandardCharsets.UTF_8)); + } + + private static List sort( + List entries) { + return new SnapDiffDependencyGraph(entries).getOrderedEntries(); + } + + private static SnapDiffDependencyEntry entry(long objectId, long parentObjectId, + DiffType diffType, String sourcePath) { + return new SnapDiffDependencyEntry(objectId, parentObjectId, + getDiffReportEntry(diffType, sourcePath)); + } + + private static SnapDiffDependencyEntry entry(long objectId, long parentObjectId, + DiffType diffType, String sourcePath, String targetPath) { + return new SnapDiffDependencyEntry(objectId, parentObjectId, + getDiffReportEntry(diffType, sourcePath, targetPath)); + } + + private static List toDiffTypes( + List entries) { + return entries.stream() + .map(SnapDiffDependencyEntry::getDiffType) + .collect(Collectors.toList()); + } + + private static void assertPathOrder(List entries, + String firstPath, String secondPath) { + List ordered = sort(entries); + assertEquals(firstPath, ordered.get(0).getSourcePath()); + assertEquals(secondPath, ordered.get(1).getSourcePath()); + } +}