diff --git a/fe/fe-benchmark/pom.xml b/fe/fe-benchmark/pom.xml new file mode 100644 index 00000000000000..2de619284a9f69 --- /dev/null +++ b/fe/fe-benchmark/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + + org.apache.doris + fe + ${revision} + ../pom.xml + + + fe-benchmark + Doris FE Benchmarks + + + + ${project.groupId} + fe-core + ${project.version} + + + diff --git a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh new file mode 100755 index 00000000000000..7ce2e0be8e3d4f --- /dev/null +++ b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +BENCHMARK_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +FE_DIR=$(cd -- "${BENCHMARK_DIR}/.." && pwd) +CLASSPATH_FILE=$(mktemp) +trap 'rm -f "${CLASSPATH_FILE}"' EXIT + +( + cd "${FE_DIR}" + mvn -Pbenchmark -pl fe-benchmark -am compile -DskipTests -Dskip.clean=true + mvn -Pbenchmark -pl fe-benchmark -am dependency:build-classpath \ + -Dskip.clean=true \ + -DincludeScope=test \ + -Dmdep.outputFile="${CLASSPATH_FILE}" +) + +REACTOR_CLASSES= +while IFS= read -r -d '' CLASSES_DIR; do + REACTOR_CLASSES+="${CLASSES_DIR}:" +done < <(find "${FE_DIR}" -type d -path '*/target/classes' -print0) +DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}") +BENCHMARK_FILTER=${BENCHMARK_FILTER:-'HivePartitionValuesSizeBenchmark|IcebergCacheSizeBenchmark|PaimonCacheSizeBenchmark|MetaCacheSoftValueBenchmark'} + +BENCHMARK_CLASSES=( + org.apache.doris.datasource.hive.HivePartitionValuesSizeBenchmark + org.apache.doris.datasource.iceberg.IcebergCacheSizeBenchmark + org.apache.doris.datasource.paimon.PaimonCacheSizeBenchmark + org.apache.doris.datasource.metacache.MetaCacheSoftValueBenchmark +) + +for BENCHMARK_CLASS in "${BENCHMARK_CLASSES[@]}"; do + if [[ "${BENCHMARK_CLASS##*.}" =~ ${BENCHMARK_FILTER} ]]; then + java \ + -Xms1g \ + -Xmx4g \ + -classpath "${REACTOR_CLASSES}${DEPENDENCY_CLASSES}" \ + "${BENCHMARK_CLASS}" \ + "$@" + fi +done diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java new file mode 100644 index 00000000000000..c1e9234c70fa67 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.benchmark; + +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** Small dependency-free harness for opt-in FE microbenchmarks. */ +public final class BenchmarkHarness { + private static final long WARMUP_MILLIS = Long.getLong("benchmark.warmup.millis", 500L); + private static final long MEASUREMENT_MILLIS = Long.getLong("benchmark.measurement.millis", 500L); + private static final int MEASUREMENT_ITERATIONS = Integer.getInteger("benchmark.iterations", 3); + private static final boolean PRINT_RESULT = Boolean.getBoolean("benchmark.print.result"); + private static volatile Object sink; + + private BenchmarkHarness() { + } + + @FunctionalInterface + public interface Operation { + Object run() throws Exception; + } + + public static void measure(String name, TimeUnit outputUnit, Operation operation) throws Exception { + runWindow(operation, WARMUP_MILLIS); + double totalNanosPerOperation = 0.0D; + long totalOperations = 0L; + for (int iteration = 0; iteration < MEASUREMENT_ITERATIONS; iteration++) { + Window result = runWindow(operation, MEASUREMENT_MILLIS); + totalNanosPerOperation += result.nanosPerOperation; + totalOperations += result.operations; + } + double averageNanos = totalNanosPerOperation / MEASUREMENT_ITERATIONS; + String result = PRINT_RESULT ? ", result=" + sink : ""; + System.out.printf(Locale.ROOT, "%-72s %12.3f %s/op (%d ops%s)%n", + name, convertFromNanos(averageNanos, outputUnit), unitName(outputUnit), totalOperations, result); + } + + private static Window runWindow(Operation operation, long minimumMillis) throws Exception { + long start = System.nanoTime(); + long deadline = start + TimeUnit.MILLISECONDS.toNanos(minimumMillis); + long operations = 0L; + do { + sink = operation.run(); + operations++; + } while (System.nanoTime() < deadline); + long elapsed = System.nanoTime() - start; + return new Window(operations, (double) elapsed / operations); + } + + private static double convertFromNanos(double nanos, TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return nanos; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return nanos / 1_000.0D; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return nanos / 1_000_000.0D; + } else if (outputUnit == TimeUnit.SECONDS) { + return nanos / 1_000_000_000.0D; + } + throw new IllegalArgumentException("unsupported benchmark time unit: " + outputUnit); + } + + private static String unitName(TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return "ns"; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return "us"; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return "ms"; + } else if (outputUnit == TimeUnit.SECONDS) { + return "s"; + } + return outputUnit.name().toLowerCase(Locale.ROOT); + } + + private static final class Window { + private final long operations; + private final double nanosPerOperation; + + private Window(long operations, double nanosPerOperation) { + this.operations = operations; + this.nanosPerOperation = nanosPerOperation; + } + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java new file mode 100644 index 00000000000000..7e3acd9e2f97ac --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; +import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; + +import com.google.common.collect.HashBiMap; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.MoreExecutors; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** Measures count publication, weighted preparation, and prepared-value admission separately. */ +public class HivePartitionValuesSizeBenchmark { + private static final int TAIL_PAYLOAD_BYTES = 1024 * 1024; + private static final long MAX_WEIGHT_BYTES = 4L * 1024L * 1024L * 1024L; + + public int countPublicationBaseline(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + return state.partitionValues.getSortedPartitionRanges() + .map(ranges -> ranges.sortedPartitions.size() + ranges.defaultPartitions.size()) + .orElse(0); + } + + public int sealPublicationWithoutEstimate(UnsealedState state) { + state.partitionValues.sealForPublication(); + return state.partitionValues.getIdToPartitionItem().size(); + } + + public long weightedPublication(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + state.partitionValues.prepareForCachePublication(state.key); + return requireComplete(state.partitionValues); + } + + public long eventCopySealAndEstimate(PreparedState state) { + HivePartitionValues copy = state.partitionValues.mutableCopy(); + copy.rebuildSortedPartitionRangesForPublication(); + copy.prepareForCachePublication(state.key); + return requireComplete(copy); + } + + public long preparedSizeProvider(PreparedState state) { + return state.partitionValues.getSizeEstimate().getBytes(); + } + + public long estimateFormula(PreparedState state) { + state.partitionValues.prepareSizeEstimate(state.key); + return requireComplete(state.partitionValues); + } + + public void replacementAdmission(PreparedState state) { + MetaCacheEntry.ReplaceResult result = state.cacheEntry.tryReplace( + state.key, state.currentPartitionValues, state.nextPartitionValues); + if (result != MetaCacheEntry.ReplaceResult.REPLACED) { + throw new IllegalStateException("replacement failed: " + result); + } + state.currentPartitionValues = state.nextPartitionValues; + state.nextPartitionValues = state.nextPartitionValues == state.partitionValues + ? state.replacementPartitionValues : state.partitionValues; + } + + public HivePartitionValues countStrongCacheHit(PreparedState state) { + return state.countCacheEntry.getIfPresent(state.key); + } + + public HivePartitionValues weightedSoftCacheHit(PreparedState state) { + return state.cacheEntry.getIfPresent(state.key); + } + + public static void main(String[] args) throws Exception { + HivePartitionValuesSizeBenchmark benchmark = new HivePartitionValuesSizeBenchmark(); + for (int partitionCount : new int[] {1000, 10000, 100000}) { + for (String distribution : new String[] {"uniform", "tail_skew"}) { + String suffix = "[partitions=" + partitionCount + ",distribution=" + distribution + "]"; + BenchmarkHarness.measure("hive.countPublicationBaseline" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.countPublicationBaseline(state); + }); + BenchmarkHarness.measure("hive.sealPublicationWithoutEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.sealPublicationWithoutEstimate(state); + }); + BenchmarkHarness.measure("hive.weightedPublication" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.weightedPublication(state); + }); + + PreparedState prepared = new PreparedState(); + prepared.partitionCount = partitionCount; + prepared.distribution = distribution; + prepared.setup(); + try { + BenchmarkHarness.measure("hive.eventCopySealAndEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> benchmark.eventCopySealAndEstimate(prepared)); + BenchmarkHarness.measure("hive.preparedSizeProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedSizeProvider(prepared)); + BenchmarkHarness.measure("hive.estimateFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.estimateFormula(prepared)); + BenchmarkHarness.measure("hive.countStrongCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.countStrongCacheHit(prepared)); + BenchmarkHarness.measure("hive.weightedSoftCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.weightedSoftCacheHit(prepared)); + BenchmarkHarness.measure("hive.replacementAdmission" + suffix, + TimeUnit.NANOSECONDS, () -> { + benchmark.replacementAdmission(prepared); + return null; + }); + } finally { + prepared.tearDown(); + } + } + } + } + + private static UnsealedState unsealedState(int partitionCount, String distribution) throws Exception { + UnsealedState state = new UnsealedState(); + state.partitionCount = partitionCount; + state.distribution = distribution; + state.setupInvocation(); + return state; + } + + /** Fresh graph per invocation so all publication work stays inside the measured method. */ + public static class UnsealedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + + public void setupInvocation() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + } + } + + public static class PreparedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + private HivePartitionValues replacementPartitionValues; + private HivePartitionValues currentPartitionValues; + private HivePartitionValues nextPartitionValues; + private MetaCacheEntry cacheEntry; + private MetaCacheEntry countCacheEntry; + private ExecutorService cacheExecutor; + + public void setup() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + partitionValues.prepareForCachePublication(key); + requireComplete(partitionValues); + + // A distinct value root makes Caffeine perform a real replacement while sharing + // immutable payload objects to keep the fixture's resident heap bounded. + replacementPartitionValues = new HivePartitionValues( + partitionValues.getIdToPartitionItem(), + partitionValues.getPartitionNameToIdMap(), + partitionValues.getPartitionValuesMap()); + replacementPartitionValues.prepareForCachePublication(key); + requireComplete(replacementPartitionValues); + + cacheExecutor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "hive", "partition_values_benchmark", OptionalLong.empty(), OptionalLong.empty()); + cacheEntry = new MetaCacheEntry<>( + "partition_values_benchmark", + ignored -> partitionValues, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 1L, MAX_WEIGHT_BYTES), + cacheExecutor, + false, + false, + (entryKey, value) -> value.prepareForCachePublication(entryKey), + entryBudget); + cacheEntry.put(key, partitionValues); + countCacheEntry = new MetaCacheEntry<>( + "partition_values_count_benchmark", + ignored -> partitionValues, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1L), + cacheExecutor, + false, + false); + countCacheEntry.put(key, partitionValues); + currentPartitionValues = partitionValues; + nextPartitionValues = replacementPartitionValues; + } + + public void tearDown() { + if (cacheEntry != null) { + cacheEntry.close(); + } + if (countCacheEntry != null) { + countCacheEntry.close(); + } + if (cacheExecutor != null) { + cacheExecutor.shutdownNow(); + } + } + } + + private static List benchmarkTypes() { + return Collections.singletonList(Type.STRING); + } + + private static PartitionValueCacheKey benchmarkKey(List types) { + return new PartitionValueCacheKey( + NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"), types); + } + + private static long requireComplete(HivePartitionValues value) { + MetaCacheSizeEstimate estimate = value.getSizeEstimate(); + if (!estimate.isComplete()) { + throw new IllegalStateException("benchmark graph is not fully measurable: " + + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private static HivePartitionValues createPartitionValues( + int count, String distribution, List types) throws Exception { + HashBiMap nameToId = HashBiMap.create(count); + Map idToItem = Maps.newHashMapWithExpectedSize(count); + Map> idToValues = Maps.newHashMapWithExpectedSize(count); + String tailPayload = "tail_skew".equals(distribution) ? repeat('x', TAIL_PAYLOAD_BYTES) : null; + long partitionNameCharacterCount = 0L; + + for (int i = 0; i < count; i++) { + long id = i; + String value = i == count - 1 && tailPayload != null ? tailPayload : "value-" + i; + String name = "p=" + value; + partitionNameCharacterCount += name.length(); + List rawValues = Collections.singletonList(new PartitionValue(value)); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes(rawValues, types, true); + List keys = new ArrayList<>(1); + keys.add(partitionKey); + + nameToId.put(name, id); + idToItem.put(id, new ListPartitionItem(keys)); + idToValues.put(id, new ArrayList<>(Collections.singletonList(value))); + } + return new HivePartitionValues( + idToItem, nameToId, idToValues, partitionNameCharacterCount, types.size()); + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, value); + return new String(chars); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java new file mode 100644 index 00000000000000..2cca17a879b625 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java @@ -0,0 +1,411 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** Measures Iceberg table, long-history table, snapshot and manifest publication. */ +public class IcebergCacheSizeBenchmark { + public int tableValueConstruction(TablePublicationState state) { + return new IcebergTableCacheValue(state.table).getIcebergTable().schema().schemaId(); + } + + public long tablePublication(TablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(TablePublicationState state) { + return IcebergCacheSizeEstimator.retainedTablePayloadBytes(state.table); + } + + public long longHistoryTablePublication(LongHistoryTablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long snapshotPublication(SnapshotPublicationState state) { + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table); + value.prepareForCachePublication(state.snapshotKey); + return requireComplete(value.getSizeEstimate()); + } + + public int snapshotValueConstruction(SnapshotPublicationState state) { + return new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table) + .getPartitionInfo().getNameToIcebergPartition().size(); + } + + public long preparedSnapshotCacheHit(SnapshotPublicationState state) { + return state.preparedSnapshotValue.getIcebergTable().get() + .currentSnapshot().snapshotId(); + } + + public long manifestPublication(ManifestState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateManifestEntry(state.key, state.value)); + } + + public int manifestValueConstruction(ManifestState state) { + return ManifestCacheValue.forDataFiles(state.files).getDataFiles().size(); + } + + public int denseManifestReaderBaseline(DenseManifestState state) { + List collected = new ArrayList<>(); + for (DataFile file : state.files) { + collected.add(file.copy()); + } + return ImmutableList.copyOf(collected).size(); + } + + public int denseManifestValueConstruction(DenseManifestState state) { + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); + for (DataFile file : state.files) { + builder.addDataFile(file.copy()); + } + return builder.build().getDataFiles().size(); + } + + public long preparedWeightProvider(PreparedState state) { + return state.preparedTableValue.getSizeEstimate().getBytes(); + } + + public long tableFormula(PreparedState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateTableEntry( + state.mapping, state.preparedTableValue)); + } + + public int preparedTableCacheHit(PreparedState state) { + return state.preparedTableValue.getIcebergTable().schema().schemaId(); + } + + public static void main(String[] args) throws Exception { + IcebergCacheSizeBenchmark benchmark = new IcebergCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + String suffix = "[fields=" + fieldCount + "]"; + TablePublicationState tableState = new TablePublicationState(); + tableState.fieldCount = fieldCount; + tableState.setup(); + BenchmarkHarness.measure("iceberg.tableValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableValueConstruction(tableState)); + BenchmarkHarness.measure("iceberg.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(tableState)); + BenchmarkHarness.measure("iceberg.tablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(tableState)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.setup(); + BenchmarkHarness.measure("iceberg.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("iceberg.tableFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableFormula(prepared)); + BenchmarkHarness.measure("iceberg.preparedTableCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedTableCacheHit(prepared)); + } + for (int snapshotCount : new int[] {1000, 10000}) { + String suffix = "[snapshots=" + snapshotCount + "]"; + LongHistoryTablePublicationState state = new LongHistoryTablePublicationState(); + state.snapshotCount = snapshotCount; + state.setup(); + BenchmarkHarness.measure("iceberg.longHistoryTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.longHistoryTablePublication(state)); + } + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + SnapshotPublicationState state = new SnapshotPublicationState(); + state.fieldCount = fieldCount; + state.partitionCount = partitionCount; + state.setup(); + BenchmarkHarness.measure("iceberg.snapshotValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotValueConstruction(state)); + BenchmarkHarness.measure("iceberg.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(state)); + BenchmarkHarness.measure("iceberg.preparedSnapshotCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedSnapshotCacheHit(state)); + } + } + for (int fileCount : new int[] {100, 10000}) { + ManifestState state = new ManifestState(); + state.fileCount = fileCount; + state.setup(); + BenchmarkHarness.measure("iceberg.manifestPublication[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestPublication(state)); + BenchmarkHarness.measure("iceberg.manifestValueConstruction[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestValueConstruction(state)); + } + for (int metricColumns : new int[] {100, 1000}) { + for (int fileCount : new int[] {100, 10000}) { + DenseManifestState state = new DenseManifestState(); + state.metricColumns = metricColumns; + state.fileCount = fileCount; + state.setup(); + String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + "]"; + BenchmarkHarness.measure("iceberg.denseManifestReaderBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestReaderBaseline(state)); + BenchmarkHarness.measure("iceberg.denseManifestValueConstruction" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestValueConstruction(state)); + } + } + } + + public static class TablePublicationState { + public int fieldCount; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount); + } + } + + public static class SnapshotPublicationState { + public int fieldCount; + + public int partitionCount; + + private Table table; + private IcebergSnapshotEntryKey snapshotKey; + private IcebergPartitionInfo partitionInfo; + private IcebergSnapshotCacheValue preparedSnapshotValue; + + public void setup() { + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount); + partitionInfo = newPartitionInfo(partitionCount); + snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table) + .orElseThrow(() -> new IllegalStateException("benchmark table has no generation key")); + preparedSnapshotValue = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), table); + preparedSnapshotValue.prepareForCachePublication(snapshotKey); + requireComplete(preparedSnapshotValue.getSizeEstimate()); + } + } + + public static class LongHistoryTablePublicationState { + public int snapshotCount; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newLongHistoryTable(snapshotCount); + } + } + + public static class PreparedState { + public int fieldCount; + + private IcebergTableCacheValue preparedTableValue; + private NameMapping mapping; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + Table table = newTable(fieldCount); + preparedTableValue = new IcebergTableCacheValue(table); + preparedTableValue.prepareForCachePublication(mapping); + requireComplete(preparedTableValue.getSizeEstimate()); + } + } + + public static class ManifestState { + public int fileCount; + + private IcebergManifestEntryKey key; + private ManifestCacheValue value; + private List files; + + public void setup() { + key = new IcebergManifestEntryKey("/benchmark/manifest.avro", ManifestContent.DATA); + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/file-" + index + ".parquet") + .withFileSizeInBytes(1024L + index) + .withRecordCount(10L + index) + .build()); + } + value = ManifestCacheValue.forDataFiles(files); + } + } + + public static class DenseManifestState { + public int fileCount; + + public int metricColumns; + + private List files; + + public void setup() { + int poolSize = Math.min(fileCount, 256); + List filePool = new ArrayList<>(poolSize); + for (int fileIndex = 0; fileIndex < poolSize; fileIndex++) { + HashMap lowerBounds = new HashMap<>(metricColumns); + HashMap upperBounds = new HashMap<>(metricColumns); + for (int columnIndex = 0; columnIndex < metricColumns; columnIndex++) { + lowerBounds.put(columnIndex, ByteBuffer.allocate(16)); + upperBounds.put(columnIndex, ByteBuffer.allocate(32)); + } + Metrics metrics = new Metrics( + 10L, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + lowerBounds, + upperBounds); + filePool.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/dense-" + fileIndex + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build()); + } + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(filePool.get(index % poolSize)); + } + } + } + + private static Table newTable(int fieldCount) { + List fields = new ArrayList<>(fieldCount); + for (int index = 0; index < fieldCount; index++) { + fields.add(Types.NestedField.optional(index + 1, "field_" + index, Types.StringType.get())); + } + Schema schema = new Schema(fields); + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:/benchmark/table", Collections.emptyMap()); + InMemoryFileIO fileIO = new InMemoryFileIO(); + StringBuilder snapshotJson = new StringBuilder("{\"snapshot-id\":7,\"timestamp-ms\":1,") + .append("\"summary\":{\"operation\":\"append\"},\"manifests\":["); + for (int index = 0; index < 10; index++) { + if (index > 0) { + snapshotJson.append(','); + } + String manifestPath = "/benchmark/manifest-" + index + ".avro"; + snapshotJson.append('"').append(manifestPath).append('"'); + fileIO.addFile(manifestPath, new byte[0]); + } + Snapshot snapshot = SnapshotParser.fromJson(snapshotJson.append("],\"schema-id\":0}").toString()); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges() + .withMetadataLocation("file:/benchmark/table/metadata/v1.json").build(); + return new BaseTable(new StaticTableOperations(metadata, fileIO), "benchmark.table"); + } + + private static Table newLongHistoryTable(int snapshotCount) { + long currentSnapshotId = 1000L + snapshotCount - 1L; + StringBuilder json = new StringBuilder() + .append("{\"format-version\":2,\"table-uuid\":\"benchmark-table\",") + .append("\"location\":\"file:/benchmark/table\",\"last-sequence-number\":") + .append(snapshotCount).append(",\"last-updated-ms\":").append(snapshotCount) + .append(",\"last-column-id\":1,\"current-schema-id\":0,") + .append("\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[") + .append("{\"id\":1,\"name\":\"field\",\"required\":false,\"type\":\"string\"}]}],") + .append("\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}],") + .append("\"last-partition-id\":999,\"default-sort-order-id\":0,") + .append("\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{},") + .append("\"current-snapshot-id\":").append(currentSnapshotId) + .append(",\"refs\":{\"main\":{\"snapshot-id\":").append(currentSnapshotId) + .append(",\"type\":\"branch\"}},\"snapshots\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"sequence-number\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index) + .append(",\"timestamp-ms\":").append(index + 1L) + .append(",\"summary\":{\"operation\":\"append\"},") + .append("\"manifest-list\":\"/benchmark/history/list-").append(index) + .append(".avro\",\"schema-id\":0}"); + } + json.append("],\"statistics\":[],\"partition-statistics\":[],\"snapshot-log\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"timestamp-ms\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index).append('}'); + } + json.append("],\"metadata-log\":[]}"); + TableMetadata metadata = TableMetadataParser.fromJson( + "file:/benchmark/table/metadata/v1.json", json.toString()); + return new BaseTable( + new StaticTableOperations(metadata, new InMemoryFileIO()), "benchmark.table"); + } + + private static IcebergPartitionInfo newPartitionInfo(int partitionCount) { + HashMap partitions = new HashMap<>(partitionCount); + long retainedPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=value_" + index; + IcebergPartition partition = new IcebergPartition(name, 0, 10L + index, 1024L + index, + 1L, 1_700_000_000_000L + index, 7L, + Collections.singletonList("value_" + index), Collections.singletonList("identity")); + partitions.put(name, partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); + } + return new IcebergPartitionInfo( + Collections.emptyMap(), partitions, Collections.emptyMap(), retainedPayloadBytes); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java new file mode 100644 index 00000000000000..b488f428e8b9a9 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.util.concurrent.MoreExecutors; + +import java.lang.ref.Reference; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +/** Measures reservation cleanup after Caffeine reports soft values as COLLECTED. */ +public final class MetaCacheSoftValueBenchmark { + private static final int SAMPLE_COUNT = 3; + private static final int VALUE_COUNT = 10_000; + private static final long MAX_WEIGHT_BYTES = 16L * 1024L * 1024L; + + private MetaCacheSoftValueBenchmark() { + } + + public static void main(String[] args) throws Exception { + long totalNanos = 0L; + for (int sample = 0; sample < SAMPLE_COUNT; sample++) { + CollectedState state = CollectedState.create(VALUE_COUNT); + try { + state.enqueueAll(); + long start = System.nanoTime(); + state.cleanUp(); + totalNanos += System.nanoTime() - start; + } finally { + state.close(); + } + } + double averageNanos = (double) totalNanos / SAMPLE_COUNT; + System.out.printf(Locale.ROOT, + "%-72s %12.3f us/batch (%.3f ns/value, %d samples)%n", + "metacache.collectedCleanup[values=" + VALUE_COUNT + "]", + averageNanos / TimeUnit.MICROSECONDS.toNanos(1L), + averageNanos / VALUE_COUNT, + SAMPLE_COUNT); + } + + private static final class CollectedState implements AutoCloseable { + private final ExecutorService executor; + private final ExternalMetaCacheBudgetManager budgetManager; + private final MetaCacheEntry entry; + private final LoadingCache loadingCache; + private final List> references; + + private CollectedState(ExecutorService executor, + ExternalMetaCacheBudgetManager budgetManager, + MetaCacheEntry entry, + LoadingCache loadingCache, + List> references) { + this.executor = executor; + this.budgetManager = budgetManager; + this.entry = entry; + this.loadingCache = loadingCache; + this.references = references; + } + + private static CollectedState create(int valueCount) throws Exception { + ExecutorService executor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget budget = budgetManager.createEntryBudget( + 1L, "benchmark", "soft_cleanup", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft_cleanup", + key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, valueCount, MAX_WEIGHT_BYTES), + executor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), + budget); + for (int index = 0; index < valueCount; index++) { + entry.put("key-" + index, new byte[1]); + } + + LoadingCache loadingCache = (LoadingCache) readField(entry, "loadingData"); + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + List> references = new ArrayList<>(nodes.size()); + for (Object node : nodes.values()) { + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + references.add((Reference) valueReferenceMethod.invoke(node)); + } + if (references.size() != valueCount) { + entry.close(); + executor.shutdownNow(); + throw new IllegalStateException( + "benchmark admission retained " + references.size() + " of " + valueCount + " values"); + } + return new CollectedState(executor, budgetManager, entry, loadingCache, references); + } + + private void enqueueAll() { + for (Reference reference : references) { + reference.clear(); + reference.enqueue(); + } + } + + private void cleanUp() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L); + while (budgetManager.getGlobalUsedWeight() != 0L + && System.nanoTime() < deadline) { + loadingCache.cleanUp(); + LockSupport.parkNanos(TimeUnit.MICROSECONDS.toNanos(100L)); + } + if (budgetManager.getGlobalUsedWeight() != 0L) { + throw new IllegalStateException( + "COLLECTED cleanup retained " + budgetManager.getGlobalUsedWeight() + " bytes"); + } + } + + @Override + public void close() { + entry.close(); + executor.shutdownNow(); + } + } + + private static Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private static Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java new file mode 100644 index 00000000000000..f73eca412f3c44 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.AppendOnlyFileStoreTable; +import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Measures Paimon 1.4.2 nested-schema/non-empty snapshot publication and prepared weight lookup. */ +public class PaimonCacheSizeBenchmark { + public long snapshotPublication(PublicationState state) { + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue(state.partitionInfo, state.snapshot); + value.prepareForCachePublication(state.key); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(PublicationState state) { + return PaimonCacheSizeEstimator.retainedTablePayloadBytes(state.snapshot.getTable()); + } + + public long preparedWeightProvider(PreparedState state) { + return state.value.getSizeEstimate().getBytes(); + } + + public long snapshotFormula(PreparedState state) { + return requireComplete(PaimonCacheSizeEstimator.estimateSnapshotEntry(state.key, state.value)); + } + + public long partitionMapBaseline(PartitionPayloadState state) { + return buildPartitionInfo(state, false); + } + + public long partitionMapWithRetainedCounter(PartitionPayloadState state) { + return buildPartitionInfo(state, true); + } + + public static void main(String[] args) throws Exception { + PaimonCacheSizeBenchmark benchmark = new PaimonCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + PublicationState publication = new PublicationState(); + publication.fieldCount = fieldCount; + publication.partitionCount = partitionCount; + publication.setup(); + BenchmarkHarness.measure("paimon.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(publication)); + BenchmarkHarness.measure("paimon.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(publication)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.partitionCount = partitionCount; + prepared.setup(); + BenchmarkHarness.measure("paimon.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("paimon.snapshotFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotFormula(prepared)); + } + } + for (int partitionCount : new int[] {1000, 10000}) { + for (boolean tailSkew : new boolean[] {false, true}) { + PartitionPayloadState state = new PartitionPayloadState(); + state.partitionCount = partitionCount; + state.tailSkew = tailSkew; + state.setup(); + String suffix = "[partitions=" + partitionCount + + ",distribution=" + (tailSkew ? "tail-skew" : "uniform") + "]"; + BenchmarkHarness.measure("paimon.partitionMapBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapBaseline(state)); + BenchmarkHarness.measure("paimon.partitionMapWithRetainedCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapWithRetainedCounter(state)); + } + } + } + + public static class PublicationState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonPartitionInfo partitionInfo; + private PaimonSnapshot snapshot; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + partitionInfo = fixture.value.getPartitionInfo(); + snapshot = fixture.value.getSnapshot(); + } + } + + public static class PreparedState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonSnapshotCacheValue value; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + value = fixture.value; + value.prepareForCachePublication(fixture.key); + requireComplete(value.getSizeEstimate()); + } + } + + public static class PartitionPayloadState { + public int partitionCount; + + public boolean tailSkew; + + private List partitions; + + public void setup() { + partitions = new ArrayList<>(partitionCount); + String longTail = String.join("", Collections.nCopies(64 * 1024, "x")); + for (int index = 0; index < partitionCount; index++) { + String value = tailSkew && index % 997 == 0 ? longTail : String.valueOf(index); + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < 4; field++) { + typedSpec.put("partition_key_" + field, value + '_' + field); + } + String displayName = "partition_key_0=" + value; + partitions.add(new PartitionPayload( + displayName, new ArrayList<>(typedSpec.values()), index)); + } + } + } + + private static Fixture newFixture(int fieldCount, int partitionCount) throws Exception { + List fields = new ArrayList<>(fieldCount + 2); + fields.add(new DataField(0, "partition_key", new IntType())); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(index + 1, "field_" + index, new VarCharType())); + } + List nestedFields = new ArrayList<>(); + for (int index = 0; index < 8; index++) { + nestedFields.add(DataTypes.FIELD(fieldCount + index + 1, + "nested_field_" + index, DataTypes.STRING())); + } + fields.add(new DataField(fieldCount + 9, "nested_payload", new RowType(nestedFields))); + TableSchema schema = new TableSchema( + 0L, fields, fieldCount + 9, Collections.singletonList("partition_key"), + Collections.emptyList(), Collections.emptyMap(), null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path("file:/tmp/doris-paimon-cache-size-benchmark"), + schema, CatalogEnvironment.empty()); + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 7L, schema.id(), 1L); + HashMap partitions = new HashMap<>(partitionCount); + long retainedPartitionPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=" + index; + String value = String.valueOf(index); + partitions.put(name, new Partition(Collections.singletonMap("partition_key", value), + 10L + index, 1024L + index, 1L, 1_700_000_000_000L + index, 1, true, + 1_700_000_000_000L, "benchmark", 1_700_000_000_000L + index, "benchmark", + Collections.singletonMap("source", "benchmark"))); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(name)); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes("partition_key")); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + new PaimonPartitionInfo(Collections.emptyMap(), partitions, retainedPartitionPayloadBytes), + new PaimonSnapshot(7L, schema.id(), table)); + return new Fixture(key, value); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private long buildPartitionInfo(PartitionPayloadState state, boolean countPayload) { + HashMap partitions = new HashMap<>(state.partitionCount); + long retainedPayloadBytes = 0L; + for (PartitionPayload payload : state.partitions) { + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < payload.values.size(); field++) { + String fieldName = "partition_key_" + field; + String fieldValue = payload.values.get(field); + typedSpec.put(fieldName, fieldValue); + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldName); + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldValue); + } + } + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, payload.displayName); + } + int index = payload.index; + partitions.put(payload.displayName, new Partition( + typedSpec, 10L + index, 1024L + index, 1L, + 1_700_000_000_000L + index, 1, false)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo( + Collections.emptyMap(), partitions, retainedPayloadBytes); + return MetaCacheWeightUtils.saturatedAdd( + partitionInfo.getNameToPartition().size(), partitionInfo.getRetainedPayloadBytes()); + } + + private static class PartitionPayload { + private final String displayName; + private final List values; + private final int index; + + private PartitionPayload( + String displayName, List values, int index) { + this.displayName = displayName; + this.values = values; + this.index = index; + } + } + + private static class Fixture { + private final PaimonSnapshotEntryKey key; + private final PaimonSnapshotCacheValue value; + + private Fixture(PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + this.key = key; + this.value = value; + } + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4e535950aae0e3..c3e95e2d4ff24a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2462,6 +2462,11 @@ public class Config extends ConfigBase { }) public static long external_cache_refresh_time_minutes = 10; // 10 mins + @ConfField(mutable = false, masterOnly = false, + description = {"FE-wide maximum weight for managed external metadata caches. Supports byte units " + + "or a percentage of the JVM max heap; 0 disables the global quota."}) + public static String external_meta_cache_max_weight = "0"; + // Enable manual miss load for external meta cache to avoid blocking replayer on slow loaders. @ConfField(mutable = true, masterOnly = false, description = {"Whether external meta cache uses manual miss load instead of Caffeine sync load."}) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index a880c5ffad4db4..93ceeb49b58144 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -851,6 +851,11 @@ under the License. mockito-inline test + + org.openjdk.jol + jol-core + test + diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java index 674bf0aa39cd5b..7ad55174e303e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import com.github.benmanes.caffeine.cache.Weigher; import org.jetbrains.annotations.NotNull; import java.time.Duration; @@ -49,7 +50,10 @@ public class CacheFactory { private OptionalLong expireAfterAccessSec; private OptionalLong refreshAfterWriteSec; private long maxSize; + private OptionalLong maxWeight; + private Weigher weigher; private boolean enableStats; + private boolean softValues; // Ticker is used to provide a time source for the cache. // Only used for test, to provide a fake time source. // If not provided, the system time is used. @@ -61,11 +65,34 @@ public CacheFactory( long maxSize, boolean enableStats, Ticker ticker) { + this(expireAfterAccessSec, refreshAfterWriteSec, maxSize, OptionalLong.empty(), null, enableStats, ticker); + } + + @SuppressWarnings("unchecked") + public CacheFactory( + OptionalLong expireAfterAccessSec, + OptionalLong refreshAfterWriteSec, + long maxSize, + OptionalLong maxWeight, + Weigher weigher, + boolean enableStats, + Ticker ticker) { this.expireAfterAccessSec = expireAfterAccessSec; this.refreshAfterWriteSec = refreshAfterWriteSec; this.maxSize = maxSize; + this.maxWeight = maxWeight; + this.weigher = (Weigher) weigher; this.enableStats = enableStats; this.ticker = ticker; + if (maxWeight.isPresent() && this.weigher == null) { + throw new IllegalArgumentException("maximumWeight requires a weigher"); + } + } + + /** Configure values as soft references so unused cache entries may be reclaimed under heap pressure. */ + public CacheFactory withSoftValues() { + softValues = true; + return this; } // Build a loading cache, without executor, it will use fork-join pool for refresh @@ -116,7 +143,11 @@ public AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cac @NotNull private Caffeine buildWithParams() { Caffeine builder = Caffeine.newBuilder(); - builder.maximumSize(maxSize); + if (maxWeight.isPresent()) { + builder.maximumWeight(maxWeight.getAsLong()).weigher(weigher); + } else { + builder.maximumSize(maxSize); + } if (expireAfterAccessSec.isPresent()) { builder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSec.getAsLong())); @@ -129,6 +160,10 @@ private Caffeine buildWithParams() { builder.recordStats(); } + if (softValues) { + builder.softValues(); + } + if (ticker != null) { builder.ticker(ticker); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index b44cb735f2ac61..3e8a8f813571ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -648,7 +648,15 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope // Only legacy validators publish a tentative candidate. Detached validators // leave the live CatalogProperty untouched while concurrent initialization runs. if (oldProperties != null && tentativelyMutated) { - ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null + ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + } else { + cacheMgr.rollbackCatalogProperties( + (ExternalCatalog) catalog, oldProperties); + } } if (validationException instanceof DdlException) { throw (DdlException) validationException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 481c1fcb3aca5f..f32c77a7015818 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -49,6 +49,7 @@ import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase; import org.apache.doris.datasource.lance.LanceExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; @@ -446,6 +447,19 @@ protected void checkProperties(CatalogProperty property) throws DdlException { } } + try { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr extMetaCacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (extMetaCacheMgr == null) { + // This fallback is only for isolated construction tests before Env is initialized. + ExternalMetaCacheBudgetManager.fromConfig().validateCatalogMaxWeight(properties); + } else { + extMetaCacheMgr.validateCatalogCacheProperties(this, properties); + } + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + // check schema.cache.ttl-second parameter String schemaCacheTtlSecond = property.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); if (java.util.Objects.nonNull(schemaCacheTtlSecond) && NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL) @@ -1367,9 +1381,21 @@ public int hashCode() { public void notifyPropertiesUpdated(Map updatedProps) { CatalogIf.super.notifyPropertiesUpdated(updatedProps); String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); - if (java.util.Objects.nonNull(schemaCacheTtl)) { - ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + if (java.util.Objects.nonNull(schemaCacheTtl) + || updatedProps.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { extMetaCacheMgr.removeCatalog(id); + return; + } + for (String key : updatedProps.keySet()) { + if (key == null || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + if (separator > 0) { + extMetaCacheMgr.removeCatalogByEngine(id, remainder.substring(0, separator)); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 007e850e54e24e..e58a984eacc462 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.maxcompute.MaxComputeExternalMetaCache; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.ExternalMetaCacheRegistry; import org.apache.doris.datasource.metacache.ExternalMetaCacheRouteResolver; import org.apache.doris.datasource.metacache.LegacyMetaCacheFactory; @@ -38,6 +39,7 @@ import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Striped; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,8 +49,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.Lock; import java.util.function.Consumer; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -95,6 +100,10 @@ public class ExternalMetaCacheMgr { private final ExternalMetaCacheRegistry cacheRegistry; private final ExternalMetaCacheRouteResolver routeResolver; private final LegacyMetaCacheFactory legacyMetaCacheFactory; + private final ExternalMetaCacheBudgetManager budgetManager; + // Catalog property publication and cache-group replacement share this striped lifecycle fence. + // Initialized lookups retain the lock-free fast path above it. + private final Striped catalogLifecycleLocks = Striped.lock(64); // all catalogs could share the same fsCache. private FileSystemCache fsCache; @@ -102,6 +111,7 @@ public class ExternalMetaCacheMgr { private ExternalRowCountCache rowCountCache; public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { + budgetManager = ExternalMetaCacheBudgetManager.fromConfig(); rowCountRefreshExecutor = newThreadPool(isCheckpointCatalog, Config.max_external_cache_loader_thread_pool_size, Config.max_external_cache_loader_thread_pool_size * 1000, @@ -191,28 +201,123 @@ public DorisExternalMetaCache doris(long catalogId) { } public void prepareCatalog(long catalogId) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalog"); - return; + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalog"); + return; + } + Map runtimeProperties = sanitizeCatalogCachePropertiesForRuntime( + catalogId, catalogProperties); + routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, runtimeProperties)); + } finally { + lifecycleLock.unlock(); } - routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); } public void prepareCatalogByEngine(long catalogId, String engine) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + ExternalMetaCache targetCache = this.engine(engine); + if (targetCache.isCatalogInitialized(catalogId)) { return; } - prepareCatalogByEngine(catalogId, engine, catalogProperties); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + return; + } + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } } public void prepareCatalogByEngine(long catalogId, String engine, Map catalogProperties) { + ExternalMetaCache targetCache = this.engine(engine); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } + } + + private void prepareCatalogByEngineLocked( + long catalogId, ExternalMetaCache targetCache, Map catalogProperties) { Map safeCatalogProperties = catalogProperties == null ? Maps.newHashMap() : Maps.newHashMap(catalogProperties); - routeSpecifiedEngine(engine, cache -> cache.initCatalog(catalogId, safeCatalogProperties)); + safeCatalogProperties = sanitizeCatalogCachePropertiesForRuntime(catalogId, safeCatalogProperties); + targetCache.initCatalog(catalogId, safeCatalogProperties); + } + + public void validateCatalogCacheProperties(Map catalogProperties) { + budgetManager.validateCatalogMaxWeight(catalogProperties); + validateCatalogCachePropertyNamespaces(catalogProperties); + cacheRegistry.allCaches().forEach(cache -> cache.validateCatalogProperties(catalogProperties)); + } + + private Map sanitizeCatalogCachePropertiesForRuntime( + long catalogId, Map catalogProperties) { + Map sanitized = Maps.newHashMap(catalogProperties); + try { + budgetManager.parseCatalogMaxWeight(sanitized); + } catch (IllegalArgumentException e) { + sanitized.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' for catalog {}: {}", + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, catalogId, e.getMessage()); + } + return sanitized; + } + + private void validateCatalogCachePropertyNamespaces(Map catalogProperties) { + String globalKey = ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY; + String prefix = "meta.cache."; + for (String key : catalogProperties.keySet()) { + if (key == null || globalKey.equals(key) || !key.startsWith(prefix)) { + continue; + } + String remainder = key.substring(prefix.length()); + int separator = remainder.indexOf('.'); + if (separator <= 0) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String configuredEngine = remainder.substring(0, separator); + ExternalMetaCache resolved = cacheRegistry.resolve(configuredEngine); + if (!resolved.engine().equals(configuredEngine)) { + throw new IllegalArgumentException("External meta cache properties must use canonical engine '" + + resolved.engine() + "' instead of alias '" + configuredEngine + "': " + key); + } + } + } + + /** Strict DDL validation also rejects a valid engine namespace not routed by the catalog type. */ + public void validateCatalogCacheProperties(CatalogIf catalog, Map catalogProperties) { + validateCatalogCacheProperties(catalogProperties); + Set routedEngines = routeResolver.resolveCatalogCaches(catalog.getId(), catalog).stream() + .map(ExternalMetaCache::engine) + .collect(Collectors.toSet()); + for (String key : catalogProperties.keySet()) { + if (key == null || ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY.equals(key) + || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + String configuredEngine = separator < 0 ? remainder : remainder.substring(0, separator); + if (!routedEngines.contains(configuredEngine)) { + throw new IllegalArgumentException("External meta cache engine '" + configuredEngine + + "' is not supported by catalog type " + catalog.getClass().getSimpleName() + ": " + key); + } + } } public void invalidateCatalog(long catalogId) { @@ -228,15 +333,42 @@ public void invalidateCatalogByEngine(long catalogId, String engine) { } public void removeCatalog(long catalogId) { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "removeCatalog", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "removeCatalog", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + } + + /** Restore catalog properties and retire any group initialized from the rejected candidate atomically. */ + public void rollbackCatalogProperties(ExternalCatalog catalog, Map oldProperties) { + long catalogId = catalog.getId(); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + catalog.rollBackCatalogProps(oldProperties); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "rollbackCatalogProperties", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void removeCatalogByEngine(long catalogId, String engine) { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "removeCatalogByEngine", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "removeCatalogByEngine", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void invalidateDb(long catalogId, String dbName) { @@ -302,13 +434,13 @@ private void initEngineCaches() { } private void registerBuiltinEngineCaches() { - cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor)); - cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor)); - cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor)); + cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor, budgetManager)); + cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor, budgetManager)); } private void routeCatalogEngines(long catalogId, Consumer action) { @@ -428,8 +560,9 @@ void replaceEngineCachesForTest(List caches) { * loading/invalidation. No engine-specific metadata (partitions/files/snapshots) is cached. */ private static class DefaultExternalMetaCache extends AbstractExternalMetaCache { - DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor) { - super(engine, refreshExecutor); + DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(engine, refreshExecutor, budgetManager); registerEntry(MetaCacheEntryDef.of( ENTRY_SCHEMA, SchemaCacheKey.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java index d14ba5645bf269..e7487c065b5ce3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java @@ -26,6 +26,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -72,7 +73,11 @@ public class DorisExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public DorisExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public DorisExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); backendsEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_BACKENDS, String.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index e2d73fd7a16edf..883a38780149d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -28,10 +28,8 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.hudi.HudiExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.iceberg.IcebergUtils; -import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractHiveProperties; import org.apache.doris.fs.FileSystemProvider; @@ -218,10 +216,6 @@ public void notifyPropertiesUpdated(Map updatedProps) { if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) { Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); } - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); - } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java new file mode 100644 index 00000000000000..c26a021890e880 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +/** Constant-time retained-weight formula for Hive partition-value cache entries. */ +final class HiveCacheSizeEstimator { + // Calibrated against complete 4.1 object graphs. The per-character reserve covers the + // partition name plus derived value/literal strings and therefore remains skew-sensitive. + private static final long ENTRY_BASE_BYTES = 2L * 1024L; + private static final long PARTITION_BASE_BYTES = 4L * 1024L; + private static final long PARTITION_COLUMN_BYTES = 384L; + private static final long PARTITION_NAME_CHARACTER_BYTES = 8L; + + private HiveCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimatePartitionValuesEntry( + PartitionValueCacheKey key, HivePartitionValues value) { + long partitionCount = value.getIdToPartitionItem() == null + ? 0L : value.getIdToPartitionItem().size(); + long perPartitionBytes = MetaCacheWeightUtils.saturatedAdd( + PARTITION_BASE_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionColumnCount(), PARTITION_COLUMN_BYTES)); + long bytes = MetaCacheWeightUtils.saturatedAdd( + ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionNameCharacterCount(), PARTITION_NAME_CHARACTER_BYTES)); + return MetaCacheSizeEstimate.complete(bytes); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 73986138c51cb0..dfa5b00289feb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -42,8 +42,12 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.fs.FileSystemCache; import org.apache.doris.fs.FileSystemDirectoryLister; @@ -59,10 +63,12 @@ import com.google.common.base.Strings; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import lombok.Data; +import lombok.Getter; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Partition; @@ -108,6 +114,7 @@ */ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private static final Logger LOG = LogManager.getLogger(HiveExternalMetaCache.class); + private static final int PARTITION_EVENT_REPLACE_MAX_RETRIES = 8; public static final String ENGINE = "hive"; public static final String ENTRY_SCHEMA = "schema"; @@ -127,7 +134,12 @@ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private final PartitionCacheCoordinator partitionCacheCoordinator = new PartitionCacheCoordinator(); public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, fileListingExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); this.fileListingExecutor = fileListingExecutor; schemaEntry = registerEntry(MetaCacheEntryDef.of( @@ -144,7 +156,8 @@ public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fi CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, - Config.max_hive_partition_table_cache_num))); + Config.max_hive_partition_table_cache_num)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); partitionEntry = registerEntry(MetaCacheEntryDef.of( ENTRY_PARTITION, PartitionCacheKey.class, @@ -292,9 +305,12 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); + long partitionNameCharacterCount = 0L; String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); for (String partitionName : partitionNames) { + partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( + partitionNameCharacterCount, partitionName.length()); long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); idToPartitionItem.put(partitionId, listPartitionItem); @@ -302,7 +318,15 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap); + HivePartitionValues partitionValues = + new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + partitionNameCharacterCount, key.types == null ? 0 : key.types.size()); + preparePartitionValuesForPublication(partitionValues); + return partitionValues; + } + + private void preparePartitionValuesForPublication(HivePartitionValues partitionValues) { + partitionValues.rebuildSortedPartitionRangesForPublication(); } private ListPartitionItem toListPartitionItem(String partitionName, List types, String catalogName) { @@ -635,7 +659,7 @@ private void invalidatePartitionCache(NameMapping nameMapping, String partitionN List values = HiveUtil.toPartitionValues(partitionName); PartitionCacheKey partKey = new PartitionCacheKey(nameMapping, values); - HivePartition partition = partitionEntry.getIfPresent(partKey); + HivePartition partition = partitionEntry.peekIfPresent(partKey); if (partition == null) { // Partition metadata cache miss: the exact FileCacheKey cannot be rebuilt here because it // needs the partition path and input format carried by HivePartition. Invalidate this @@ -715,41 +739,64 @@ private void addPartitionsCache(NameMapping nameMapping, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, partitionColumnTypes); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; - } - - HivePartitionValues copy = partitionValues.copy(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItem = new HashMap<>(); - HMSExternalCatalog catalog = hmsCatalog(catalogId); String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); - for (String partitionName : partitionNames) { - if (partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", - partitionName, localTblName); - continue; + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; } - long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); - ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); - idToPartitionItemBefore.put(partitionId, listPartitionItem); - idToPartitionItem.put(partitionId, listPartitionItem); - partitionNameToIdMapBefore.put(partitionName, partitionId); - } - Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); - Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - partitionValuesMapBefore.putAll(partitionValuesMap); - copy.rebuildSortedPartitionRanges(); - - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); + HivePartitionValues copy = current.mutableCopy(); + Map allItems = copy.getIdToPartitionItem(); + Map allNames = copy.getPartitionNameToIdMap(); + Map addedItems = new HashMap<>(); + for (String partitionName : partitionNames) { + if (allNames.containsKey(partitionName)) { + if (attempt == 0) { + LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", + partitionName, localTblName); + } + continue; + } + long partitionId = Util.genIdByName( + catalog.getName(), localDbName, localTblName, partitionName); + ListPartitionItem item = toListPartitionItem(partitionName, key.types, catalog.getName()); + allItems.put(partitionId, item); + addedItems.put(partitionId, item); + allNames.put(partitionName, partitionId); + copy.addPartitionNameCharacters(partitionName.length()); + } + if (addedItems.isEmpty()) { + // Even a replay/no-op event must fence a refresh that started before the event. + // Otherwise that refresh could replace this already-correct graph with stale HMS data. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } + continue; + } + copy.getPartitionValuesMap().putAll( + ListPartitionPrunerV2.getPartitionValuesMap(addedItems)); + preparePartitionValuesForPublication(copy); + + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after add event was rejected: {}", key); + return; + } } + // Repeated conflicts mean we cannot prove the cached graph contains this event. Force + // the next reader to rebuild it from HMS rather than retaining a possibly stale value. + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated add-event conflicts: {}", key); } private void dropPartitionsCache(ExternalTable dorisTable, @@ -765,41 +812,63 @@ private void dropPartitionsCache(ExternalTable dorisTable, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, null); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; + if (invalidPartitionCache) { + for (String partitionName : partitionNames) { + invalidatePartitionCache(nameMapping, partitionName); + } } - HivePartitionValues copy = partitionValues.copy(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map> partitionValuesMap = copy.getPartitionValuesMap(); - - for (String partitionName : partitionNames) { - if (!partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", - partitionName, nameMapping.getFullLocalName()); + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; + } + HivePartitionValues copy = current.mutableCopy(); + Map allNames = copy.getPartitionNameToIdMap(); + Map allItems = copy.getIdToPartitionItem(); + Map> allValues = copy.getPartitionValuesMap(); + boolean changed = false; + for (String partitionName : partitionNames) { + Long partitionId = allNames.remove(partitionName); + if (partitionId == null) { + LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", + partitionName, nameMapping.getFullLocalName()); + continue; + } + allItems.remove(partitionId); + allValues.remove(partitionId); + copy.removePartitionNameCharacters(partitionName.length()); + changed = true; + } + if (!changed) { + // See the add-event no-op path: event ordering still has to win over an older refresh. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } continue; } - Long partitionId = partitionNameToIdMapBefore.remove(partitionName); - idToPartitionItemBefore.remove(partitionId); - partitionValuesMap.remove(partitionId); - - if (invalidPartitionCache) { - invalidatePartitionCache(nameMapping, partitionName); + preparePartitionValuesForPublication(copy); + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after drop event was rejected: {}", key); + return; } } - - copy.rebuildSortedPartitionRanges(); - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); - } + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated drop-event conflicts: {}", key); } } @VisibleForTesting public void putPartitionValuesCacheForTest(PartitionValueCacheKey key, HivePartitionValues values) { + preparePartitionValuesForPublication(values); partitionValuesEntry.get(key.getNameMapping().getCtlId()).put(key, values); } @@ -842,15 +911,15 @@ public List getFilesByTransaction(List partitions /** * The key of hive partition values cache. */ - @Data + @Getter public static class PartitionValueCacheKey { - private NameMapping nameMapping; + private final NameMapping nameMapping; // Not part of cache identity. - private List types; + private final List types; public PartitionValueCacheKey(NameMapping nameMapping, List types) { this.nameMapping = nameMapping; - this.types = types; + this.types = types == null ? null : ImmutableList.copyOf(types); } @Override @@ -1037,7 +1106,7 @@ public static class HiveFileStatus { AcidInfo acidInfo; } - @Data + @Getter public static class HivePartitionValues { private BiMap partitionNameToIdMap; private Map idToPartitionItem; @@ -1045,6 +1114,12 @@ public static class HivePartitionValues { // Sorted partition ranges for binary search filtering. private SortedPartitionRanges sortedPartitionRanges; + // Prepared once after construction/update; the cache weigher only reads this value. + private transient volatile MetaCacheSizeEstimate sizeEstimate; + // Maintained while the metadata is already being loaded or updated. Admission only reads it. + private long partitionNameCharacterCount; + private int partitionColumnCount; + private transient boolean sortedPartitionRangesPrepared; public HivePartitionValues() { } @@ -1052,22 +1127,98 @@ public HivePartitionValues() { public HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap) { + this(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + countPartitionNameCharacters(partitionNameToIdMap), + inferPartitionColumnCount(partitionValuesMap)); + } + + HivePartitionValues(Map idToPartitionItem, + BiMap partitionNameToIdMap, + Map> partitionValuesMap, + long partitionNameCharacterCount, + int partitionColumnCount) { this.idToPartitionItem = idToPartitionItem; this.partitionNameToIdMap = partitionNameToIdMap; this.partitionValuesMap = partitionValuesMap; - this.sortedPartitionRanges = buildSortedPartitionRanges(); + this.partitionNameCharacterCount = partitionNameCharacterCount; + this.partitionColumnCount = partitionColumnCount; } - public HivePartitionValues copy() { + HivePartitionValues mutableCopy() { HivePartitionValues copy = new HivePartitionValues(); - copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap)); - copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); - copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap)); + copy.partitionNameToIdMap = partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap); + copy.idToPartitionItem = idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem); + copy.partitionValuesMap = partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap); + copy.partitionNameCharacterCount = partitionNameCharacterCount; + copy.partitionColumnCount = partitionColumnCount; return copy; } - public void rebuildSortedPartitionRanges() { - this.sortedPartitionRanges = buildSortedPartitionRanges(); + /** Compatibility hook for tests and benchmarks; publication uses copy-on-write updates. */ + void sealForPublication() { + if (!sortedPartitionRangesPrepared) { + rebuildSortedPartitionRangesForPublication(); + } + } + + void rebuildSortedPartitionRangesForPublication() { + sortedPartitionRanges = buildSortedPartitionRanges(); + sortedPartitionRangesPrepared = true; + } + + MetaCacheSizeEstimate prepareForCachePublication(PartitionValueCacheKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely( + "hive_partition_values_preparation_failed", () -> { + prepareSizeEstimate(key); + return getSizeEstimate(); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + MetaCacheSizeEstimate result = sizeEstimate; + return result == null ? MetaCacheSizeEstimate.incomplete("estimate_not_prepared") : result; + } + + void prepareSizeEstimate(PartitionValueCacheKey key) { + sizeEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, this); + } + + long getPartitionNameCharacterCount() { + return partitionNameCharacterCount; + } + + int getPartitionColumnCount() { + return partitionColumnCount; + } + + private void addPartitionNameCharacters(int characters) { + partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( + partitionNameCharacterCount, characters); + } + + private void removePartitionNameCharacters(int characters) { + partitionNameCharacterCount = Math.max(0L, partitionNameCharacterCount - characters); + } + + private static long countPartitionNameCharacters(BiMap names) { + long characters = 0L; + if (names != null) { + for (String name : names.keySet()) { + characters = MetaCacheWeightUtils.saturatedAdd(characters, name.length()); + } + } + return characters; + } + + private static int inferPartitionColumnCount(Map> values) { + if (values == null || values.isEmpty()) { + return 0; + } + List first = values.values().iterator().next(); + return first == null ? 0 : first.size(); } public java.util.Optional> getSortedPartitionRanges() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index 74d2aa99900340..a83777b1e53e68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -83,7 +84,11 @@ public class HudiExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public HudiExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_PARTITION, HudiPartitionCacheKey.class, TablePartitionValues.class, this::loadPartitionValuesCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiPartitionCacheKey::getNameMapping))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java new file mode 100644 index 00000000000000..8d483ecfb5153f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.Map; + +/** Publication-time retained-weight formulas for Iceberg cache entries. */ +final class IcebergCacheSizeEstimator { + private static final long KEY_BASE_BYTES = 128L; + private static final long TABLE_BASE_BYTES = 16L * 1024L; + private static final long SCHEMA_VERSION_BYTES = 512L; + private static final long SCHEMA_FIELD_BYTES = 512L; + private static final long NESTED_SCHEMA_FIELD_BYTES = 512L; + private static final long PARTITION_SPEC_BYTES = 256L; + private static final long PARTITION_SPEC_FIELD_BYTES = 384L; + private static final long SORT_ORDER_BYTES = 256L; + private static final long SORT_FIELD_BYTES = 256L; + private static final long TABLE_PROPERTY_BYTES = 256L; + private static final long CURRENT_SNAPSHOT_BYTES = 512L; + private static final long HISTORICAL_SNAPSHOT_BYTES = 1024L; + private static final long SNAPSHOT_LOG_ENTRY_BYTES = 64L; + private static final long METADATA_LOG_ENTRY_BYTES = 128L; + private static final long SNAPSHOT_REF_BYTES = 128L; + private static final long STATISTICS_FILE_BYTES = 512L; + private static final long PARTITION_STATISTICS_FILE_BYTES = 256L; + private static final long ENCRYPTED_KEY_BYTES = 256L; + private static final long PARTITION_BYTES = 512L; + private static final long PARTITION_ALIAS_BYTES = 256L; + private static final long NAME_MAPPING_ENTRY_BYTES = 256L; + private static final long MANIFEST_ENTRY_BASE_BYTES = 256L; + private static final long DATA_FILE_BYTES = 16L * 1024L; + private static final long DELETE_FILE_BYTES = 18L * 1024L; + private static final long FILE_METRIC_ENTRY_BYTES = 160L; + + private IcebergCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { + Table table = value.getRetainedIcebergTable(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { + long bytes = KEY_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation())); + + IcebergPartitionInfo partitionInfo = value.getPartitionInfo(); + bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); + bytes = addCount(bytes, value.getNameMapping().map(java.util.Map::size).orElse(0), + NAME_MAPPING_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedNameMappingPayloadBytes()); + + if (value.getRetainedIcebergTable().isPresent()) { + Table table = value.getRetainedIcebergTable().get(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + } + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateManifestEntry( + IcebergManifestEntryKey key, ManifestCacheValue value) { + if (!value.isAccountingComplete()) { + return MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete"); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + MANIFEST_ENTRY_BASE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath())); + bytes = addCount(bytes, value.getDataFiles().size(), DATA_FILE_BYTES); + bytes = addCount(bytes, value.getDeleteFiles().size(), DELETE_FILE_BYTES); + bytes = addCount(bytes, value.getDataFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = addCount(bytes, value.getDeleteFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + private static MetaCacheSizeEstimate checkSupportedTable(Table table) { + if (table == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table"); + } + if (!(table instanceof HasTableOperations)) { + return MetaCacheSizeEstimate.incomplete( + "unsupported_iceberg_table:" + table.getClass().getName()); + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata"); + } + if (metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_metadata_location"); + } + return MetaCacheSizeEstimate.complete(1L); + } + + /** Reads only metadata collection sizes and a constant number of strings; no FileIO is used. */ + private static long estimateTable(Table table) { + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + long bytes = MetaCacheWeightUtils.saturatedAdd( + TABLE_BASE_BYTES, MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.location())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.metadataFileLocation())); + + bytes = addCount(bytes, metadata.properties().size(), TABLE_PROPERTY_BYTES); + if (metadata.currentSnapshot() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CURRENT_SNAPSHOT_BYTES); + } + return bytes; + } + + /** Captures exact historical cardinalities and skew-sensitive payload once before admission. */ + static long retainedTablePayloadBytes(Table table) { + if (!(table instanceof HasTableOperations)) { + return 0L; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return 0L; + } + + long bytes = 0L; + for (Schema schema : metadata.schemas()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SCHEMA_VERSION_BYTES); + for (Types.NestedField field : schema.columns()) { + bytes = addFieldPayload(bytes, field, false); + } + } + for (PartitionSpec spec : metadata.specs()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_BYTES); + for (org.apache.iceberg.PartitionField field : spec.fields()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_FIELD_BYTES); + bytes = addString(bytes, field.name()); + } + } + for (SortOrder sortOrder : metadata.sortOrders()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_ORDER_BYTES); + bytes = addCount(bytes, sortOrder.fields().size(), SORT_FIELD_BYTES); + } + for (Map.Entry property : metadata.properties().entrySet()) { + bytes = addString(bytes, property.getKey()); + bytes = addString(bytes, property.getValue()); + } + for (Snapshot snapshot : metadata.snapshots()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HISTORICAL_SNAPSHOT_BYTES); + bytes = addString(bytes, snapshot.operation()); + bytes = addString(bytes, snapshot.manifestListLocation()); + bytes = addStringMap(bytes, snapshot.summary(), TABLE_PROPERTY_BYTES); + } + bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_ENTRY_BYTES); + for (TableMetadata.MetadataLogEntry previousFile : metadata.previousFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_ENTRY_BYTES); + bytes = addString(bytes, previousFile.file()); + } + bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES); + for (String refName : metadata.refs().keySet()) { + bytes = addString(bytes, refName); + } + for (StatisticsFile statisticsFile : metadata.statisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + bytes = addCount(bytes, statisticsFile.blobMetadata().size(), TABLE_PROPERTY_BYTES); + } + for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + } + for (EncryptedKey encryptedKey : metadata.encryptionKeys()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_BYTES); + bytes = addString(bytes, encryptedKey.keyId()); + bytes = addString(bytes, encryptedKey.encryptedById()); + bytes = addBufferPayload(bytes, encryptedKey.encryptedKeyMetadata()); + bytes = addStringMap(bytes, encryptedKey.properties(), TABLE_PROPERTY_BYTES); + } + bytes = addString(bytes, metadata.uuid()); + return bytes; + } + + private static long addBufferPayload(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + + private static long addFieldPayload(long bytes, Types.NestedField field, boolean nested) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + nested ? NESTED_SCHEMA_FIELD_BYTES : SCHEMA_FIELD_BYTES); + bytes = addString(bytes, field.name()); + bytes = addString(bytes, field.doc()); + bytes = addDefaultPayload(bytes, field.initialDefault()); + bytes = addDefaultPayload(bytes, field.writeDefault()); + return addTypePayload(bytes, field.type()); + } + + private static long addTypePayload(long bytes, Type type) { + if (type.isStructType()) { + for (Types.NestedField field : type.asStructType().fields()) { + bytes = addFieldPayload(bytes, field, true); + } + } else if (type.isListType()) { + bytes = addTypePayload(bytes, type.asListType().elementType()); + } else if (type.isMapType()) { + bytes = addTypePayload(bytes, type.asMapType().keyType()); + bytes = addTypePayload(bytes, type.asMapType().valueType()); + } + return bytes; + } + + private static long addDefaultPayload(long bytes, Object value) { + if (value instanceof CharSequence) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + return MetaCacheWeightUtils.saturatedAdd(bytes, ((ByteBuffer) value).capacity()); + } else if (value instanceof byte[]) { + return MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addStringMap(long bytes, Map values, long entryBytes) { + if (values == null) { + return bytes; + } + bytes = addCount(bytes, values.size(), entryBytes); + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 8407a29d8908a9..e8b24299d53df2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; @@ -29,9 +28,12 @@ import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -47,16 +49,19 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; +import javax.annotation.Nullable; /** * Iceberg engine implementation of {@link AbstractExternalMetaCache}. * *

Registered entries: *

    - *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping, each - * memoizing its latest snapshot runtime projection
    • + *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping
    • + *
  • {@code snapshot}: immutable snapshot projections keyed by a stable metadata generation
    • *
  • {@code view}: loaded Iceberg {@link View} instances
    • *
  • {@code manifest}: parsed manifest payload ({@link ManifestCacheValue}) keyed by * manifest path and content type
    • @@ -69,7 +74,7 @@ *

    Invalidation behavior: *

      *
    • catalog invalidation clears all entries and drops Iceberg {@link ManifestFiles} IO cache
      • - *
    • db/table invalidation clears table/view/schema entries, while keeping manifest entries
      • + *
    • db/table invalidation clears table/snapshot/view/schema entries, while keeping manifest entries
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ @@ -78,26 +83,41 @@ public class IcebergExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "iceberg"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_VIEW = "view"; public static final String ENTRY_MANIFEST = "manifest"; public static final String ENTRY_SCHEMA = "schema"; private static final long DEFAULT_MANIFEST_CACHE_CAPACITY = 100_000L; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle viewEntry; private final EntryHandle manifestEntry; private final EntryHandle schemaEntry; public IcebergExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withSizeEstimator(this::prepareTableForCachePublication) + .withReplacementListener(this::retireTableGeneration)); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); viewEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_VIEW, NameMapping.class, View.class, this::loadView, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); manifestEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_MANIFEST, IcebergManifestEntryKey.class, ManifestCacheValue.class, - CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY))); + CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimator.estimateSafely( + "iceberg_manifest_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateManifestEntry(key, value)))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSchemaCacheKey::getNameMapping))); @@ -108,13 +128,87 @@ public Table getIcebergTable(ExternalTable dorisTable) { return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getIcebergTable(); } + public Table getWritableIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + + " when loading a writable Iceberg table"); + } + IcebergMetadataOps ops = resolveMetadataOps(catalog); + // DDL/actions must start from the live catalog generation. DML that was planned against a + // retained read generation wraps this live table separately in IcebergTransaction. + return executeAuthenticated(catalog, () -> ops.loadTable( + nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); + } + + Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + MetaCacheEntry entry = + tableEntry.get(nameMapping.getCtlId()); + IcebergTableCacheValue tableValue = + entry.get(nameMapping); + return createQueryTable(nameMapping, tableValue); + } + + private Table createQueryTable( + NameMapping nameMapping, IcebergTableCacheValue tableValue) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || snapshotEntry.get(nameMapping.getCtlId()).isWeightBounded(); + if (!isolateForQueries) { + return tableValue.getIcebergTable(); + } + Table queryTable = tableValue.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(queryTable); + return queryTable; + } + public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + IcebergTableCacheValue tableValue = + tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + Table retainedTable = tableValue.getRetainedIcebergTable(); + java.util.Optional optionalKey = + IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!optionalKey.isPresent()) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared(); + return executeAuthenticated(nameMapping.getCtlId(), + () -> loadSnapshotProjection( + dorisTable, + isolateForQueries ? tableValue.newQueryScopedTable() + : tableValue.getIcebergTable(), + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries)); + } + IcebergSnapshotEntryKey key = optionalKey.get(); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || entry.isWeightBounded(); + IcebergSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { + Table projectionTable = isolateForQueries + ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); + IcebergSnapshotCacheValue value = loadSnapshotProjection( + dorisTable, projectionTable, + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); + if (entry.isWeightBounded()) { + value.prepareForCachePublication(key); + } + return value; + })); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && !tableValue.isSamePhysicalGeneration(currentTable)) { + // A query may have captured the previous table immediately before refresh publication. + // It can use that immutable value, but must not republish an unreachable old projection. + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public List getSnapshotList(ExternalTable dorisTable) { - Table icebergTable = getIcebergTable(dorisTable); + Table icebergTable = getQueryScopedIcebergTable(dorisTable); List snapshots = com.google.common.collect.Lists.newArrayList(); com.google.common.collect.Iterables.addAll(snapshots, icebergTable.snapshots()); return snapshots; @@ -126,8 +220,31 @@ public View getIcebergView(ExternalTable dorisTable) { } public IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId) { - IcebergSchemaCacheKey key = new IcebergSchemaCacheKey(nameMapping, schemaId); - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()).get(key); + IcebergTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable()); + } + + IcebergSchemaCacheValue getIcebergSchemaCacheValue( + NameMapping nameMapping, long schemaId, Table retainedTable) { + Optional generation = IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!generation.isPresent()) { + return (IcebergSchemaCacheValue) loadSchemaCacheValue( + new IcebergSchemaCacheKey(nameMapping, schemaId), retainedTable); + } + IcebergSchemaCacheKey key = new IcebergSchemaCacheKey( + nameMapping, generation.get().getTableUuid(), schemaId); + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry + .get(key, ignored -> loadSchemaCacheValue(key, retainedTable)); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null) { + Optional currentGeneration = IcebergSnapshotEntryKey.tryCreate( + nameMapping, currentTable.getRetainedIcebergTable()); + if (!currentGeneration.isPresent() + || !currentGeneration.get().getTableUuid().equals(generation.get().getTableUuid())) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } + } return (IcebergSchemaCacheValue) schemaCacheValue; } @@ -139,11 +256,13 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, MetaCacheEntry manifestEntry = this.manifestEntry.get(nameMapping.getCtlId()); IcebergManifestEntryKey key = IcebergManifestEntryKey.of(manifest); - boolean hit = manifestEntry.getIfPresent(key) != null; + boolean hit = manifestEntry.peekIfPresent(key) != null; if (cacheHitRecorder != null) { cacheHitRecorder.accept(hit); } - return manifestEntry.get(key, ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); + return manifestEntry.get(key, + ignored -> loadManifestCacheValue( + manifest, icebergTable, key.getContent(), manifestEntry.isWeightBounded())); } @Override @@ -159,25 +278,32 @@ public void invalidateCatalogEntries(long catalogId) { } private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { throw new RuntimeException(String.format("Cannot find catalog %d when loading table %s/%s.", nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); } IcebergMetadataOps ops = resolveMetadataOps(catalog); - try { - Table table = ((ExternalCatalog) catalog).getExecutionAuthenticator() - .execute(() -> ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - ExternalTable dorisTable = findExternalTable(nameMapping, ENGINE); - return new IcebergTableCacheValue(table, () -> loadSnapshotProjection(dorisTable, table)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } + return executeAuthenticated(catalog, () -> { + Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + IcebergTableCacheValue value = new IcebergTableCacheValue(table); + MetaCacheEntry currentEntry = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + if (currentEntry != null && currentEntry.isWeightBounded()) { + prepareTableForCachePublication(nameMapping, value); + } + return value; + }); + } + + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + return value.prepareForCachePublication(nameMapping); } private View loadView(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (!(catalog instanceof IcebergExternalCatalog)) { return null; } @@ -191,7 +317,7 @@ private View loadView(NameMapping nameMapping) { } private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFile manifest, Table icebergTable, - ManifestContent content) { + ManifestContent content, boolean accountRetainedSize) { if (manifest == null || icebergTable == null) { String manifestPath = manifest == null ? "null" : manifest.path(); throw new CacheException("Manifest cache loader context is missing for %s", @@ -199,10 +325,9 @@ private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFil } try { if (content == ManifestContent.DELETES) { - return ManifestCacheValue.forDeleteFiles( - loadDeleteFiles(manifest, icebergTable)); + return loadDeleteFiles(manifest, icebergTable, accountRetainedSize); } - return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, icebergTable)); + return loadDataFiles(manifest, icebergTable, accountRetainedSize); } catch (IOException e) { throw new CacheException("Failed to read manifest %s", e, manifest.path()); } @@ -216,27 +341,64 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } - private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table icebergTable) { + private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + dorisTable.setUpdateTime(System.currentTimeMillis()); + boolean isView = dorisTable instanceof IcebergExternalTable + && ((IcebergExternalTable) dorisTable).isView(); + return IcebergUtils.loadSchemaCacheValue( + dorisTable, key.getSchemaId(), isView, retainedTable).orElseThrow(() -> + new CacheException("failed to load iceberg schema cache value for: %s.%s.%s, schemaId: %s", + null, key.getNameMapping().getCtlId(), key.getNameMapping().getLocalDbName(), + key.getNameMapping().getLocalTblName(), key.getSchemaId())); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable IcebergTableCacheValue previousValue, IcebergTableCacheValue currentValue) { + if (previousValue != null && previousValue.isSamePhysicalGeneration(currentValue)) { + return; + } + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.belongsTo(currentValue)); + } + Optional currentUuid = currentValue.getTableUuid(); + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.getTableUuid().equals(currentUuid)); + } + } + + private IcebergSnapshotCacheValue loadSnapshotProjection( + ExternalTable dorisTable, Table projectionTable, Table retainedTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); } try { - // Freeze before deriving snapshot, partitions, and aliases; BaseTable accessors share - // refreshable operations and otherwise could mix two concurrent metadata generations. - Table retainedTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); MTMVRelatedTableIf table = (MTMVRelatedTableIf) dorisTable; - IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(retainedTable); + IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(projectionTable); IcebergPartitionInfo icebergPartitionInfo; if (!table.isValidRelatedTable()) { icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { - icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, retainedTable, + icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, projectionTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue( - icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(retainedTable), - retainedTable); + Optional>> nameMapping = + IcebergUtils.getNameMapping(projectionTable); + return isolateForQueries + ? new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable, retainedCurrentSnapshotJson) + : new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } @@ -251,32 +413,58 @@ private IcebergMetadataOps resolveMetadataOps(CatalogIf catalog) { throw new RuntimeException("Only support 'hms' and 'iceberg' type for iceberg table"); } + private T executeAuthenticated(long catalogId, Callable task) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + catalogId + " when loading Iceberg metadata."); + } + return executeAuthenticated(catalog, task); + } + + private T executeAuthenticated(CatalogIf catalog, Callable task) { + if (!(catalog instanceof ExternalCatalog)) { + throw new RuntimeException("Iceberg metadata cache requires an external catalog"); + } + try { + return ((ExternalCatalog) catalog).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.iceberg.table.enable", "meta.cache.iceberg.snapshot.enable"); + compatibility.put("meta.cache.iceberg.table.ttl-second", "meta.cache.iceberg.snapshot.ttl-second"); + compatibility.put("meta.cache.iceberg.table.capacity", "meta.cache.iceberg.snapshot.capacity"); + return compatibility; } - private List loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDataFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - List dataFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.read(manifest, table.io())) { for (org.apache.iceberg.DataFile dataFile : reader) { - dataFiles.add(dataFile.copy()); + builder.addDataFile(dataFile.copy()); } } - return dataFiles; + return builder.build(); } - private List loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDeleteFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - List deleteFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { for (org.apache.iceberg.DeleteFile deleteFile : reader) { - deleteFiles.add(deleteFile.copy()); + builder.addDeleteFile(deleteFile.copy()); } } - return deleteFiles; + return builder.build(); } private void dropManifestFileIoCacheForCatalog(long catalogId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 5d59440a5a62b1..9c9ee6d53b6416 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -148,6 +148,10 @@ public Table getIcebergTable() { return IcebergUtils.getIcebergTable(this); } + public Table getWritableIcebergTable() { + return IcebergUtils.getWritableIcebergTable(this); + } + @Override public String getComment() { return properties().getOrDefault(TABLE_COMMENT_PROP, ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index bcbe03de6e6d01..d996f80754a37f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -483,7 +483,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) @Override public void createOrReplaceBranchImpl(ExternalTable dorisTable, CreateOrReplaceBranchInfo branchInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); BranchOptions branchOptions = branchInfo.getBranchOptions(); Long snapshotId = branchOptions.getSnapshotId() @@ -571,7 +571,7 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { @Override public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagInfo tagInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); TagOptions tagOptions = tagInfo.getTagOptions(); Long snapshotId = tagOptions.getSnapshotId() .orElse( @@ -623,7 +623,7 @@ public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagI public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws UserException { String tagName = tagInfo.getTagName(); boolean ifExists = tagInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(tagName); if (snapshotRef != null || !ifExists) { @@ -644,7 +644,7 @@ public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws Us public void dropBranchImpl(ExternalTable dorisTable, DropBranchInfo branchInfo) throws UserException { String branchName = branchInfo.getBranchName(); boolean ifExists = branchInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(branchName); if (snapshotRef != null || !ifExists) { @@ -747,7 +747,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateAddColumnMetadata(column, true); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); @@ -778,7 +778,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); if (!parentPath.getType().isStructType()) { @@ -808,7 +808,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); for (Column column : columns) { validateAddColumnMetadata(column, true); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); @@ -831,7 +831,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnName, "drop"); ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -851,7 +851,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -868,7 +868,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, oldName, "rename"); validateRowLineageColumnMutation(icebergTable, newName, "rename to"); Schema schema = icebergTable.schema(); @@ -893,7 +893,7 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), @@ -955,7 +955,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); NestedField currentCol = icebergTable.schema().asStruct() .caseInsensitiveField(columnPath.getTopLevelName()); @@ -1024,7 +1024,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); validateCollectionPseudoFieldComment( @@ -1075,7 +1075,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column @Override public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); if (!columnPath.isNested()) { validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); } @@ -1642,7 +1642,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); List canonicalOrder = new ArrayList<>(newOrder.size()); Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (String columnName : newOrder) { @@ -1709,7 +1709,7 @@ private Term getTransform(String transformName, String columnName, Integer trans */ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); String transformName = clause.getTransformName(); @@ -1738,7 +1738,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause */ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); if (clause.getPartitionFieldName() != null) { @@ -1765,7 +1765,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus */ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); // remove old partition field diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java index cccc6244a0d0cc..96ed0a0dcc1d8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + import java.util.List; public class IcebergPartition { @@ -29,10 +31,19 @@ public class IcebergPartition { private final long lastUpdateTime; private final long lastSnapshotId; private final List transforms; + private final long retainedPayloadBytes; public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, long lastUpdateTime, long lastSnapshotId, List partitionValues, List transforms) { + this(partitionName, specId, recordCount, fileSizeInBytes, fileCount, lastUpdateTime, + lastSnapshotId, partitionValues, transforms, + estimateRetainedPayloadBytes(partitionName, partitionValues, transforms)); + } + + public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, + long lastUpdateTime, long lastSnapshotId, List partitionValues, + List transforms, long retainedPayloadBytes) { this.partitionName = partitionName; this.specId = specId; this.recordCount = recordCount; @@ -42,6 +53,7 @@ public IcebergPartition(String partitionName, int specId, long recordCount, long this.lastSnapshotId = lastSnapshotId; this.partitionValues = partitionValues; this.transforms = transforms; + this.retainedPayloadBytes = retainedPayloadBytes; } public String getPartitionName() { @@ -79,4 +91,26 @@ public List getPartitionValues() { public List getTransforms() { return transforms; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long estimateRetainedPayloadBytes( + String partitionName, List partitionValues, List transforms) { + long bytes = MetaCacheWeightUtils.estimatedStringBytes(partitionName); + bytes = addStrings(bytes, partitionValues); + return addStrings(bytes, transforms); + } + + private static long addStrings(long bytes, List values) { + if (values == null) { + return bytes; + } + for (String value : values) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(value)); + } + return bytes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index de36f0855ddd14..5c43cc56f8bbf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -18,9 +18,9 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; -import com.google.common.collect.Maps; - +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -28,21 +28,32 @@ public class IcebergPartitionInfo { private final Map nameToPartitionItem; private final Map nameToIcebergPartition; private final Map> nameToIcebergPartitionNames; + private final long retainedPayloadBytes; private static final IcebergPartitionInfo EMPTY = new IcebergPartitionInfo(); private IcebergPartitionInfo() { - this.nameToPartitionItem = Maps.newHashMap(); - this.nameToIcebergPartition = Maps.newHashMap(); - this.nameToIcebergPartitionNames = Maps.newHashMap(); + this.nameToPartitionItem = Collections.emptyMap(); + this.nameToIcebergPartition = Collections.emptyMap(); + this.nameToIcebergPartitionNames = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public IcebergPartitionInfo(Map nameToPartitionItem, Map nameToIcebergPartition, Map> nameToIcebergPartitionNames) { + this(nameToPartitionItem, nameToIcebergPartition, nameToIcebergPartitionNames, + retainedPayloadBytes(nameToIcebergPartition)); + } + + public IcebergPartitionInfo(Map nameToPartitionItem, + Map nameToIcebergPartition, + Map> nameToIcebergPartitionNames, + long retainedPayloadBytes) { this.nameToPartitionItem = nameToPartitionItem; this.nameToIcebergPartition = nameToIcebergPartition; this.nameToIcebergPartitionNames = nameToIcebergPartitionNames; + this.retainedPayloadBytes = retainedPayloadBytes; } static IcebergPartitionInfo empty() { @@ -57,6 +68,28 @@ public Map getNameToIcebergPartition() { return nameToIcebergPartition; } + Map> getNameToIcebergPartitionNames() { + return nameToIcebergPartitionNames; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (IcebergPartition partition : partitions.values()) { + if (partition != null) { + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, partition.getRetainedPayloadBytes()); + } + } + return bytes; + } + public long getLatestSnapshotId(String partitionName) { Set icebergPartitionNames = nameToIcebergPartitionNames.get(partitionName); if (icebergPartitionNames == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java index 7c2d09511a2c93..9916d0afbb17e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java @@ -22,14 +22,26 @@ import com.google.common.base.Objects; +import java.util.Optional; + public class IcebergSchemaCacheKey extends SchemaCacheKey { + private final String tableUuid; private final long schemaId; public IcebergSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, "", schemaId); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId) { super(nameMapping); + this.tableUuid = java.util.Objects.requireNonNull(tableUuid, "tableUuid can not be null"); this.schemaId = schemaId; } + public Optional getTableUuid() { + return tableUuid.isEmpty() ? Optional.empty() : Optional.of(tableUuid); + } + public long getSchemaId() { return schemaId; } @@ -46,11 +58,11 @@ public boolean equals(Object o) { return false; } IcebergSchemaCacheKey that = (IcebergSchemaCacheKey) o; - return schemaId == that.schemaId; + return schemaId == that.schemaId && tableUuid.equals(that.tableUuid); } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableUuid, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 30cf64fcfc6bfa..7cfc5bcc2bcd31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,8 +17,17 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; import org.apache.iceberg.BaseTable; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.HistoryEntry; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; @@ -27,7 +36,6 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -40,36 +48,65 @@ public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; private final Optional>> nameMapping; - private final Optional icebergTable; + private Optional
    icebergTable; + private final long retainedNameMappingPayloadBytes; + private String retainedCurrentSnapshotJson; + private boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { - this(partitionInfo, snapshot, Optional.empty(), Optional.empty()); + this(partitionInfo, snapshot, Optional.empty(), Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping) { - this(partitionInfo, snapshot, nameMapping, Optional.empty()); + this(partitionInfo, snapshot, nameMapping, Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping, Table icebergTable) { - this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable)); + this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable), null, false); + } + + IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping, Table retainedTable, + String retainedCurrentSnapshotJson) { + this(partitionInfo, snapshot, nameMapping, Optional.of(retainedTable), + retainedCurrentSnapshotJson, true); } private IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, - Optional>> nameMapping, Optional
    icebergTable) { + Optional>> nameMapping, Optional
    icebergTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; // A cached BaseTable shares live TableOperations; retain a metadata-only generation so a // later commit through that same Table cannot move an already bound statement forward. this.icebergTable = icebergTable.map(IcebergSnapshotCacheValue::retainTableGeneration); - this.nameMapping = nameMapping.map(mapping -> { + this.retainedCurrentSnapshotJson = retainedCurrentSnapshotJson; + if (isolateForQueries) { + this.icebergTable = this.icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + this.queryIsolationPrepared = true; + } + if (nameMapping.isPresent()) { Map> copy = new HashMap<>(); - // Preserve the immutable snapshot contract while remaining compatible with branch-4.1's Java target. - mapping.forEach((id, names) -> copy.put(id, - Collections.unmodifiableList(new ArrayList<>(names)))); - return Collections.unmodifiableMap(copy); - }); + long payloadBytes = 0L; + for (Map.Entry> entry : nameMapping.get().entrySet()) { + List names = ImmutableList.copyOf(entry.getValue()); + copy.put(entry.getKey(), names); + for (String name : names) { + payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(name)); + } + } + this.nameMapping = Optional.of(Collections.unmodifiableMap(copy)); + this.retainedNameMappingPayloadBytes = payloadBytes; + } else { + this.nameMapping = Optional.empty(); + this.retainedNameMappingPayloadBytes = 0L; + } } public IcebergPartitionInfo getPartitionInfo() { @@ -85,6 +122,51 @@ public Optional>> getNameMapping() { } public Optional
    getIcebergTable() { + return queryIsolationPrepared + ? icebergTable.map(table -> createQueryScopedTable( + table, retainedCurrentSnapshotJson)) + : icebergTable; + } + + MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", + () -> { + if (retainedCurrentSnapshotJson == null) { + retainedCurrentSnapshotJson = icebergTable + .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); + } + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); + return IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); + if (sizeEstimate.isComplete()) { + icebergTable = icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + long getRetainedNameMappingPayloadBytes() { + return retainedNameMappingPayloadBytes; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return retainedSnapshotJsonBytes(retainedCurrentSnapshotJson); + } + + Optional
    getRetainedIcebergTable() { return icebergTable; } @@ -95,15 +177,69 @@ static Table retainTableGeneration(Table table) { TableOperations operations = ((HasTableOperations) table).operations(); // Capture current() exactly once so every projection derived from the returned table sees // one metadata generation even when the shared catalog handle refreshes concurrently. - TableOperations frozenOperations = new FrozenTableOperations(operations, operations.current()); + TableOperations frozenOperations = new FrozenTableOperations( + operations, operations.current(), false); return tableWithOperations(table, frozenOperations); } + static Table retainNonGrowingGeneration(Table table) { + if (!isFrozenGeneration(table) || isNonGrowingGeneration(table)) { + return table; + } + TableOperations retainedOperations = ((HasTableOperations) table).operations(); + // Do not rebuild parsed metadata with Iceberg's write-side Builder. Builder validation and + // ID reuse rules are intentionally stricter than metadata parsing and can reject legal + // upgraded tables or renumber sparse/equivalent schema histories. The frozen metadata is + // never exposed after query isolation; each caller receives exact query-local operations. + return tableWithOperations(table, new FrozenTableOperations( + retainedOperations, retainedOperations.current(), true)); + } + + static String retainCurrentSnapshotJson(Table table) { + if (!(table instanceof HasTableOperations)) { + return null; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + Snapshot snapshot = metadata == null ? null : metadata.currentSnapshot(); + return snapshot == null ? null : SnapshotParser.toJson(snapshot, false); + } + + static long retainedSnapshotJsonBytes(String snapshotJson) { + return MetaCacheWeightUtils.estimatedStringBytes(snapshotJson); + } + + static Table createQueryScopedTable(Table retainedTable, String currentSnapshotJson) { + if (!isFrozenGeneration(retainedTable)) { + return retainedTable; + } + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + if (retainedTable instanceof BaseTable) { + return new QueryScopedTable(retainedOperations, retainedTable.name(), + ((BaseTable) retainedTable).reporter(), currentSnapshotJson); + } + return new QueryScopedTable(retainedOperations, retainedTable.name(), null, + currentSnapshotJson); + } + + static void loadQueryMetadataForStatement(Table table) { + if (table instanceof QueryScopedTable) { + ((QueryScopedTable) table).queryMetadata(); + } + } + static boolean isFrozenGeneration(Table table) { return table instanceof HasTableOperations && ((HasTableOperations) table).operations() instanceof FrozenTableOperations; } + static TableOperations unwrapRetainedTableOperations(TableOperations operations) { + TableOperations current = Objects.requireNonNull(operations, "operations can not be null"); + while (current instanceof RetainedTableOperations) { + current = ((RetainedTableOperations) current).delegate; + } + return current; + } + static Table createWritableTable(Table retainedTable, Table liveTable) { if (!isFrozenGeneration(retainedTable)) { return retainedTable; @@ -113,12 +249,19 @@ static Table createWritableTable(Table retainedTable, Table liveTable) { throw new IllegalArgumentException( "Iceberg commit table must provide writable table operations"); } - TableMetadata retainedMetadata = ((HasTableOperations) retainedTable).operations().current(); - TableOperations liveOperations = ((HasTableOperations) liveTable).operations(); + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + TableMetadata retainedMetadata = retainedOperations.current(); + TableOperations liveOperations = unwrapRetainedTableOperations( + ((HasTableOperations) liveTable).operations()); return tableWithOperations(retainedTable, new WritableTableOperations(liveOperations, retainedMetadata)); } + static boolean isNonGrowingGeneration(Table table) { + return isFrozenGeneration(table) + && ((FrozenTableOperations) ((HasTableOperations) table).operations()).nonGrowing; + } + private static Table tableWithOperations(Table table, TableOperations operations) { if (table instanceof BaseTable) { return new BaseTable(operations, table.name(), ((BaseTable) table).reporter()); @@ -166,15 +309,63 @@ public LocationProvider locationProvider() { } } - private static class FrozenTableOperations extends RetainedTableOperations { - private FrozenTableOperations(TableOperations delegate, TableMetadata metadata) { - super(delegate, metadata); + private static class FrozenTableOperations implements TableOperations { + private final TableMetadata metadata; + private final FileIO fileIO; + private final EncryptionManager encryptionManager; + private final LocationProvider locationProvider; + private final boolean nonGrowing; + + private FrozenTableOperations(TableOperations source, TableMetadata metadata, + boolean nonGrowing) { + this.metadata = metadata; + this.fileIO = source.io(); + this.encryptionManager = source.encryption(); + this.locationProvider = source.locationProvider(); + this.nonGrowing = nonGrowing; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; } @Override public void commit(TableMetadata base, TableMetadata newMetadata) { throw new UnsupportedOperationException("Frozen Iceberg table generation is read-only"); } + + @Override + public FileIO io() { + return fileIO; + } + + @Override + public EncryptionManager encryption() { + return encryptionManager; + } + + @Override + public String metadataFileLocation(String fileName) { + String metadataLocation = metadata.metadataFileLocation(); + if (metadataLocation == null) { + throw new UnsupportedOperationException( + "Frozen Iceberg table has no metadata directory"); + } + int separator = metadataLocation.lastIndexOf('/'); + return separator < 0 ? fileName + : metadataLocation.substring(0, separator + 1) + fileName; + } + + @Override + public LocationProvider locationProvider() { + return locationProvider; + } } private static class WritableTableOperations extends RetainedTableOperations { @@ -208,8 +399,9 @@ public TableMetadata refresh() { @Override public void commit(TableMetadata base, TableMetadata newMetadata) { - delegate.commit(base, newMetadata); - currentMetadata = newMetadata; + TableMetadata delegateBase = prepareDelegateCommit(delegate, base, currentMetadata); + delegate.commit(delegateBase, newMetadata); + currentMetadata = delegate.current(); } private boolean isWriterCompatible(TableMetadata refreshedMetadata) { @@ -221,4 +413,128 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { && Objects.equals(retainedMetadata.properties(), refreshedMetadata.properties()); } } + + /** Query-local operations expose exact retained metadata without shared lazy table state. */ + private static final class QueryScopedTableOperations extends RetainedTableOperations { + private QueryScopedTableOperations(TableOperations retainedOperations) { + super(retainedOperations, retainedOperations.current()); + } + + @Override + public void commit(TableMetadata base, TableMetadata metadata) { + throw new UnsupportedOperationException("Query-scoped Iceberg table is read-only"); + } + } + + /** A per-caller view whose Iceberg lazy snapshot state is never written into the cache value. */ + private static final class QueryScopedTable extends BaseTable { + private final QueryScopedTableOperations queryOperations; + private final Snapshot currentSnapshot; + private final Map querySnapshots = new HashMap<>(); + + private QueryScopedTable(TableOperations retainedOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + this(new QueryScopedTableOperations(retainedOperations), name, reporter, currentSnapshotJson); + } + + private QueryScopedTable(QueryScopedTableOperations queryOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + super(queryOperations, name, reporter == null + ? org.apache.iceberg.metrics.LoggingMetricsReporter.instance() : reporter); + this.queryOperations = queryOperations; + this.currentSnapshot = currentSnapshotJson == null + ? null : SnapshotParser.fromJson(currentSnapshotJson); + if (currentSnapshot != null) { + querySnapshots.put(currentSnapshot.snapshotId(), currentSnapshot); + } + } + + @Override + public Snapshot currentSnapshot() { + return currentSnapshot; + } + + @Override + public Snapshot snapshot(long snapshotId) { + if (currentSnapshot != null && currentSnapshot.snapshotId() == snapshotId) { + return currentSnapshot; + } + return copyForQuery(queryMetadata().snapshot(snapshotId)); + } + + @Override + public Iterable snapshots() { + ImmutableList.Builder snapshots = ImmutableList.builder(); + for (Snapshot snapshot : queryMetadata().snapshots()) { + snapshots.add(copyForQuery(snapshot)); + } + return snapshots.build(); + } + + @Override + public List history() { + return queryMetadata().snapshotLog(); + } + + @Override + public Map refs() { + return queryMetadata().refs(); + } + + @Override + public List statisticsFiles() { + return queryMetadata().statisticsFiles(); + } + + @Override + public List partitionStatisticsFiles() { + return queryMetadata().partitionStatisticsFiles(); + } + + private synchronized TableMetadata queryMetadata() { + return queryOperations.current(); + } + + private synchronized Snapshot copyForQuery(Snapshot snapshot) { + if (snapshot == null) { + return null; + } + return querySnapshots.computeIfAbsent(snapshot.snapshotId(), ignored -> + SnapshotParser.fromJson(SnapshotParser.toJson(snapshot, false))); + } + } + + private static TableMetadata prepareDelegateCommit(TableOperations delegate, + TableMetadata base, TableMetadata wrapperCurrent) { + if (base != wrapperCurrent) { + throw new CommitFailedException("Cannot commit from a stale Iceberg table view"); + } + TableMetadata delegateCurrent = delegate.current(); + if (!isSameGeneration(base, delegateCurrent)) { + throw new CommitFailedException("Cannot commit from a stale Iceberg metadata generation"); + } + return delegateCurrent; + } + + private static boolean isSameGeneration(TableMetadata retained, TableMetadata live) { + if (retained == live) { + return true; + } + if (retained == null || live == null) { + return false; + } + if (!Objects.equals(retained.uuid(), live.uuid())) { + return false; + } + if (retained.metadataFileLocation() != null || live.metadataFileLocation() != null) { + return Objects.equals(retained.metadataFileLocation(), live.metadataFileLocation()); + } + return retained.lastUpdatedMillis() == live.lastUpdatedMillis() + && retained.lastSequenceNumber() == live.lastSequenceNumber() + && retained.currentSchemaId() == live.currentSchemaId() + && retained.defaultSpecId() == live.defaultSpecId() + && retained.defaultSortOrderId() == live.defaultSortOrderId() + && Objects.equals(retained.location(), live.location()) + && Objects.equals(retained.properties(), live.properties()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java new file mode 100644 index 00000000000000..13db520e2d755d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; + +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; + +import java.util.Objects; +import java.util.Optional; + +/** Stable identity for an Iceberg snapshot projection built from one frozen metadata generation. */ +public final class IcebergSnapshotEntryKey { + private final NameMapping nameMapping; + private final String tableUuid; + private final String metadataFileLocation; + private final long snapshotId; + private final int schemaId; + private final int defaultSpecId; + + private IcebergSnapshotEntryKey(NameMapping nameMapping, String tableUuid, String metadataFileLocation, + long snapshotId, int schemaId, int defaultSpecId) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.tableUuid = Objects.requireNonNull(tableUuid, "tableUuid can not be null"); + this.metadataFileLocation = Objects.requireNonNull( + metadataFileLocation, "metadataFileLocation can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.defaultSpecId = defaultSpecId; + } + + /** + * Build a key from the same retained table generation that will be used by the value loader. + * Tables without a stable metadata location intentionally bypass the snapshot cache. + */ + public static Optional tryCreate(NameMapping nameMapping, Table retainedTable) { + if (!(retainedTable instanceof HasTableOperations)) { + return Optional.empty(); + } + TableMetadata metadata = ((HasTableOperations) retainedTable).operations().current(); + if (metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + || metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return Optional.empty(); + } + Snapshot snapshot = metadata.currentSnapshot(); + long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); + return Optional.of(new IcebergSnapshotEntryKey(nameMapping, metadata.uuid(), metadata.metadataFileLocation(), + snapshotId, metadata.currentSchemaId(), metadata.defaultSpecId())); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public String getMetadataFileLocation() { + return metadataFileLocation; + } + + public String getTableUuid() { + return tableUuid; + } + + public long getSnapshotId() { + return snapshotId; + } + + public int getSchemaId() { + return schemaId; + } + + public int getDefaultSpecId() { + return defaultSpecId; + } + + boolean belongsTo(IcebergTableCacheValue tableValue) { + Optional generation = tryCreate( + nameMapping, tableValue.getRetainedIcebergTable()); + return generation.isPresent() + && tableUuid.equals(generation.get().tableUuid) + && metadataFileLocation.equals(generation.get().metadataFileLocation); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof IcebergSnapshotEntryKey)) { + return false; + } + IcebergSnapshotEntryKey that = (IcebergSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && defaultSpecId == that.defaultSpecId + && nameMapping.equals(that.nameMapping) + && tableUuid.equals(that.tableUuid) + && metadataFileLocation.equals(that.metadataFileLocation); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, tableUuid, metadataFileLocation, snapshotId, schemaId, defaultSpecId); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index 28e45b47acd250..5bb537340ba6e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -44,9 +44,6 @@ public class IcebergSysExternalTable extends ExternalTable { private final IcebergExternalTable sourceTable; private final String sysTableType; - private volatile Table sysIcebergTable; - private volatile List fullSchema; - private volatile SchemaCacheValue schemaCacheValue; public IcebergSysExternalTable(IcebergExternalTable sourceTable, String sysTableType) { super(generateSysTableId(sourceTable.getId(), sysTableType), @@ -100,24 +97,19 @@ public boolean supportsSnapshotSelection() { } public Table getSysIcebergTable() { - if (sysIcebergTable == null) { - synchronized (this) { - if (sysIcebergTable == null) { - Table baseTable = sourceTable.getIcebergTable(); - MetadataTableType tableType = MetadataTableType.from(sysTableType); - if (tableType == null) { - throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); - } - sysIcebergTable = MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); - } - } + Table baseTable = IcebergUtils.getQueryScopedIcebergTable(sourceTable); + MetadataTableType tableType = MetadataTableType.from(sysTableType); + if (tableType == null) { + throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); } - return sysIcebergTable; + // Metadata tables capture their base operations. Keep them statement-local so exact + // previousFiles/history state and stale-generation retry never leak into this table object. + return MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); } @Override public List getFullSchema() { - return getOrCreateSchemaCacheValue().getSchema(); + return loadSchemaCacheValue().getSchema(); } @Override @@ -156,12 +148,12 @@ public long fetchRowCount() { @Override public Optional initSchema(SchemaCacheKey key) { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override public Optional getSchemaCacheValue() { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override @@ -178,19 +170,12 @@ private static long generateSysTableId(long sourceTableId, String sysTableType) return sourceTableId ^ (sysTableType.hashCode() * 31L); } - private SchemaCacheValue getOrCreateSchemaCacheValue() { - if (schemaCacheValue == null) { - synchronized (this) { - if (schemaCacheValue == null) { - if (fullSchema == null) { - fullSchema = IcebergUtils.parseSchema(getSysIcebergTable().schema(), - getCatalog().getEnableMappingVarbinary(), - getCatalog().getEnableMappingTimestampTz()); - } - schemaCacheValue = new SchemaCacheValue(fullSchema); - } - } - } - return schemaCacheValue; + private SchemaCacheValue loadSchemaCacheValue() { + // Metadata-table schemas may change after source schema or partition-spec evolution. + // Resolve the schema from the same latest-generation path instead of permanently pairing + // this long-lived system-table object with its first observed generation. + return new SchemaCacheValue(IcebergUtils.parseSchema(getSysIcebergTable().schema(), + getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index cdef77346ade27..a4d4f600f66d6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -17,25 +17,122 @@ package org.apache.doris.datasource.iceberg; -import com.google.common.base.Suppliers; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; -import java.util.function.Supplier; +import java.util.Objects; +import java.util.Optional; public class IcebergTableCacheValue { - private final Table icebergTable; - private final Supplier latestSnapshotCacheValue; + private volatile Table icebergTable; + private String retainedCurrentSnapshotJson; + private volatile boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; - public IcebergTableCacheValue(Table icebergTable, Supplier latestSnapshotCacheValue) { - this.icebergTable = icebergTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + public IcebergTableCacheValue(Table icebergTable) { + this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); } public Table getIcebergTable() { + Table retainedTable = icebergTable; + return queryIsolationPrepared || IcebergSnapshotCacheValue.isNonGrowingGeneration(retainedTable) + ? IcebergSnapshotCacheValue.createQueryScopedTable( + retainedTable, retainedCurrentSnapshotJson) + : retainedTable; + } + + public Table getWritableIcebergTable(Table liveTable) { + Table retainedTable = icebergTable; + return IcebergSnapshotCacheValue.createWritableTable(retainedTable, liveTable); + } + + synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", + () -> { + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + retainedTablePayloadBytes = + IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + return IcebergCacheSizeEstimator.estimateTableEntry(key, this); + }); + if (sizeEstimate.isComplete()) { + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + Table getRetainedIcebergTable() { return icebergTable; } - public IcebergSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + synchronized Table newQueryScopedTable() { + if (!queryIsolationPrepared) { + // A failed optional size preparation must only reject weighted cache admission. Do not + // repeat the same unsupported metadata access on the query path and turn it into a + // table-load failure; this value is not retained by the weighted cache in that case. + if (sizeEstimate != null && !sizeEstimate.isComplete()) { + return icebergTable; + } + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + return IcebergSnapshotCacheValue.createQueryScopedTable( + icebergTable, retainedCurrentSnapshotJson); + } + + String getRetainedCurrentSnapshotJson() { + return retainedCurrentSnapshotJson; + } + + boolean isQueryIsolationPrepared() { + return queryIsolationPrepared; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return IcebergSnapshotCacheValue.retainedSnapshotJsonBytes( + retainedCurrentSnapshotJson); + } + + Optional getTableUuid() { + TableMetadata metadata = retainedMetadata(); + return metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + ? Optional.empty() : Optional.of(metadata.uuid()); + } + + boolean isSamePhysicalGeneration(IcebergTableCacheValue other) { + if (other == null) { + return false; + } + TableMetadata left = retainedMetadata(); + TableMetadata right = other.retainedMetadata(); + return left != null && right != null + && Objects.equals(left.uuid(), right.uuid()) + && Objects.equals(left.metadataFileLocation(), right.metadataFileLocation()); + } + + private TableMetadata retainedMetadata() { + Table retainedTable = icebergTable; + return retainedTable instanceof HasTableOperations + ? ((HasTableOperations) retainedTable).operations().current() : null; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 71935cbd88157b..138e4fc4b1289c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -299,7 +299,7 @@ private Table createTransactionTable(ExternalTable dorisTable, Table retainedTab // Reads stay on the retained generation; commit refreshes may follow data-only snapshots, // while writer-contract changes still invalidate files produced for the retained metadata. return IcebergSnapshotCacheValue.createWritableTable( - retainedTable, IcebergUtils.getIcebergTable(dorisTable)); + retainedTable, IcebergUtils.getWritableIcebergTable(dorisTable)); } /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 0b375c70d6791e..bee37c0c37cea4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -56,6 +56,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.metastore.HMSBaseProperties; @@ -1057,6 +1058,14 @@ public static Table getIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTable(dorisTable); } + public static Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getQueryScopedIcebergTable(dorisTable); + } + + public static Table getWritableIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); + } + private static IcebergExternalMetaCache icebergExternalMetaCache(ExternalCatalog catalog) { Preconditions.checkNotNull(catalog, "catalog can not be null"); return Env.getCurrentEnv().getExtMetaCacheMgr().iceberg(catalog.getId()); @@ -1716,6 +1725,12 @@ public static IcebergSchemaCacheValue getSchemaCacheValue(ExternalTable dorisTab .getIcebergSchemaCacheValue(dorisTable.getOrBuildNameMapping(), schemaId); } + static IcebergSchemaCacheValue getSchemaCacheValue( + ExternalTable dorisTable, long schemaId, Table retainedTable) { + return icebergExternalMetaCache(dorisTable).getIcebergSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), schemaId, retainedTable); + } + public static IcebergSnapshot getLatestIcebergSnapshot(Table table) { Snapshot snapshot = table.currentSnapshot(); long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); @@ -1751,10 +1766,14 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T } Map nameToPartition = Maps.newHashMap(); Map nameToPartitionItem = Maps.newHashMap(); + long retainedPayloadBytes = 0L; - List partitionColumns = IcebergUtils.getSchemaCacheValue(dorisTable, schemaId).getPartitionColumns(); + List partitionColumns = IcebergUtils.getSchemaCacheValue( + dorisTable, schemaId, table).getPartitionColumns(); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString(); Range partitionRange = getPartitionRange( partition.getPartitionValues().get(0), transform, partitionColumns); @@ -1762,7 +1781,8 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T nameToPartitionItem.put(partition.getPartitionName(), item); } Map> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem); - return new IcebergPartitionInfo(nameToPartitionItem, nameToPartition, partitionNameMap); + return new IcebergPartitionInfo( + nameToPartitionItem, nameToPartition, partitionNameMap, retainedPayloadBytes); } private static List loadIcebergPartition(Table table, long snapshotId) { @@ -1802,6 +1822,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike StringBuilder sb = new StringBuilder(); List partitionValues = Lists.newArrayList(); List transforms = Lists.newArrayList(); + long retainedPayloadBytes = 0L; for (int i = 0; i < partitionSpec.fields().size(); ++i) { PartitionField partitionField = partitionSpec.fields().get(i); Class fieldClass = partitionSpec.javaClasses()[i]; @@ -1817,12 +1838,19 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike sb.append(fieldValue); sb.append("/"); partitionValues.add(fieldValue); - transforms.add(partitionField.transform().toString()); + String transform = partitionField.transform().toString(); + transforms.add(transform); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(fieldValue)); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(transform)); } if (sb.length() > 0) { sb.delete(sb.length() - 1, sb.length()); } String partitionName = sb.toString(); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(partitionName)); long recordCount = row.get(2, Long.class); long fileCount = row.get(3, Integer.class); long fileSizeInBytes = row.get(4, Long.class); @@ -1841,7 +1869,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike lastUpdateSnapShotId = UNKNOWN_SNAPSHOT_ID; } return new IcebergPartition(partitionName, specId, recordCount, fileSizeInBytes, fileCount, - lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms); + lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms, retainedPayloadBytes); } @VisibleForTesting @@ -1979,7 +2007,10 @@ public int compare(Map.Entry p1, Map.Entry retainedTable = sv.getRetainedIcebergTable(); + return retainedTable.isPresent() + ? getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId(), retainedTable.get()) + : getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId()); } public static IcebergSnapshotCacheValue getLatestSnapshotCacheValue(ExternalTable dorisTable) { @@ -2000,7 +2031,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( Optional scanParams) { if (tableSnapshot.isPresent() || IcebergUtils.isIcebergBranchOrTag(scanParams)) { // If a snapshot is specified, use the specified snapshot and the corresponding schema (not latest). - Table icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(getIcebergTable(dorisTable)); + IcebergExternalMetaCache metaCache = icebergExternalMetaCache(dorisTable); + Table icebergTable = metaCache.getQueryScopedIcebergTable(dorisTable); IcebergTableQueryInfo info; try { info = getQuerySpecSnapshot(icebergTable, tableSnapshot, scanParams); @@ -2010,8 +2042,7 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), - getNameMapping(icebergTable), - icebergTable); + getNameMapping(icebergTable), icebergTable); } return getLatestSnapshotCacheValue(dorisTable); } @@ -2027,11 +2058,12 @@ public static List getIcebergSchema(ExternalTable dorisTable, Optional getIcebergPartitionColumns(Optional snapshot, ExternalTable dorisTable) { IcebergSnapshotCacheValue snapshotValue = getSnapshotCacheValue(snapshot, dorisTable); - if (snapshotValue.getIcebergTable().isPresent()) { + Optional
    snapshotTable = snapshotValue.getIcebergTable(); + if (snapshotTable.isPresent()) { // Schema ID alone cannot identify the partition spec; metadata-only evolution may keep // the same schema and snapshot IDs while changing spec(), so derive both from T0. return buildTableSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId(), - snapshotValue.getIcebergTable().get()).getPartitionColumns(); + snapshotTable.get()).getPartitionColumns(); } return getSchemaCacheValue(dorisTable, snapshotValue).getPartitionColumns(); } @@ -2047,9 +2079,16 @@ public static View getIcebergView(ExternalTable dorisTable) { public static Optional loadSchemaCacheValue( ExternalTable dorisTable, long schemaId, boolean isView) { + return loadSchemaCacheValue(dorisTable, schemaId, isView, null); + } + + public static Optional loadSchemaCacheValue( + ExternalTable dorisTable, long schemaId, boolean isView, Table retainedTable) { return isView ? loadViewSchemaCacheValue(dorisTable, schemaId) - : loadTableSchemaCacheValue(dorisTable, schemaId); + : retainedTable == null + ? loadTableSchemaCacheValue(dorisTable, schemaId) + : Optional.of(buildTableSchemaCacheValue(dorisTable, schemaId, retainedTable)); } private static Optional loadViewSchemaCacheValue(ExternalTable dorisTable, long schemaId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java index e303f0e9111486..94924514a58a48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long sourceSnapshotId = namedArguments.getLong(SNAPSHOT_ID); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java index 0937af8ba4cac4..82a93022354067 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java @@ -149,7 +149,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); // Parse parameters String olderThan = namedArguments.getString(OLDER_THAN); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java index a5560db65520e4..cd746a7dbe6959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String sourceBranch = namedArguments.getString(BRANCH); String desBranch = namedArguments.getString(TO); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java index e1bf8cbdad4472..bf3f116d1cba81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java @@ -66,7 +66,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String targetWapId = namedArguments.getString(WAP_ID); // Find the target WAP snapshot diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java index 430e9fe9d5e22d..dce45c2729693b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { try { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot current = icebergTable.currentSnapshot(); if (current == null) { // No current snapshot means the table is empty, no manifests to rewrite diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java index 8d6b3842a9dc80..a5609f83439d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long targetSnapshotId = namedArguments.getLong(SNAPSHOT_ID); Snapshot targetSnapshot = icebergTable.snapshot(targetSnapshotId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java index 6957c563512657..de7e2a680791c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java @@ -96,7 +96,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String timestampStr = namedArguments.getString(TIMESTAMP); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java index 44df40f8f492b9..5b2c5bd220eb7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java @@ -87,7 +87,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot previousSnapshot = icebergTable.currentSnapshot(); Long previousSnapshotId = previousSnapshot != null ? previousSnapshot.snapshotId() : null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index e98ca6b2fb2808..4d63af348324e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -17,30 +17,73 @@ package org.apache.doris.datasource.iceberg.cache; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.StructLike; -import java.util.Collections; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Cached manifest payload containing parsed files. */ public class ManifestCacheValue { + private static final long AUXILIARY_LIST_ENTRY_BYTES = 32L; + private final List dataFiles; private final List deleteFiles; + private final long dataFileMetricEntryCount; + private final long deleteFileMetricEntryCount; + private final long retainedPayloadBytes; + private final boolean accountingComplete; - private ManifestCacheValue(List dataFiles, List deleteFiles) { - this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles; - this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles; + private ManifestCacheValue(List dataFiles, List deleteFiles, + long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes, + boolean accountingComplete) { + this.dataFiles = ImmutableList.copyOf(dataFiles); + this.deleteFiles = ImmutableList.copyOf(deleteFiles); + this.dataFileMetricEntryCount = dataFileMetricEntryCount; + this.deleteFileMetricEntryCount = deleteFileMetricEntryCount; + this.retainedPayloadBytes = retainedPayloadBytes; + this.accountingComplete = accountingComplete; } public static ManifestCacheValue forDataFiles(List dataFiles) { - return new ManifestCacheValue(dataFiles, Collections.emptyList()); + Builder builder = dataFilesBuilder(); + if (dataFiles != null) { + dataFiles.forEach(builder::addDataFile); + } + return builder.build(); } public static ManifestCacheValue forDeleteFiles(List deleteFiles) { - return new ManifestCacheValue(Collections.emptyList(), deleteFiles); + Builder builder = deleteFilesBuilder(); + if (deleteFiles != null) { + deleteFiles.forEach(builder::addDeleteFile); + } + return builder.build(); + } + + public static Builder dataFilesBuilder() { + return dataFilesBuilder(true); + } + + public static Builder dataFilesBuilder(boolean accountRetainedSize) { + return new Builder(true, accountRetainedSize); + } + + public static Builder deleteFilesBuilder() { + return deleteFilesBuilder(true); + } + + public static Builder deleteFilesBuilder(boolean accountRetainedSize) { + return new Builder(false, accountRetainedSize); } public List getDataFiles() { @@ -50,4 +93,158 @@ public List getDataFiles() { public List getDeleteFiles() { return deleteFiles; } + + public long getDataFileMetricEntryCount() { + return dataFileMetricEntryCount; + } + + public long getDeleteFileMetricEntryCount() { + return deleteFileMetricEntryCount; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + public boolean isAccountingComplete() { + return accountingComplete; + } + + /** Accumulates retained-size counters in the manifest reader's existing file loop. */ + public static final class Builder { + private final boolean dataContent; + private final boolean accountRetainedSize; + private final List dataFiles = new ArrayList<>(); + private final List deleteFiles = new ArrayList<>(); + private long metricEntryCount; + private long retainedPayloadBytes; + private boolean accountingComplete; + + private Builder(boolean dataContent, boolean accountRetainedSize) { + this.dataContent = dataContent; + this.accountRetainedSize = accountRetainedSize; + this.accountingComplete = accountRetainedSize; + } + + public void addDataFile(DataFile file) { + if (!dataContent) { + throw new IllegalStateException("delete manifest builder cannot accept a data file"); + } + dataFiles.add(file); + accountSafely(file); + } + + public void addDeleteFile(DeleteFile file) { + if (dataContent) { + throw new IllegalStateException("data manifest builder cannot accept a delete file"); + } + deleteFiles.add(file); + accountSafely(file); + } + + public ManifestCacheValue build() { + return new ManifestCacheValue(dataFiles, deleteFiles, + dataContent ? metricEntryCount : 0L, + dataContent ? 0L : metricEntryCount, + retainedPayloadBytes, accountingComplete); + } + + private void accountSafely(ContentFile file) { + if (!accountRetainedSize || !accountingComplete) { + return; + } + try { + account(file); + } catch (RuntimeException e) { + // A new or third-party ContentFile implementation must not turn optional cache + // accounting into a manifest-read failure. Keep the files for the current query + // and mark the value incomplete so weighted admission rejects it. + metricEntryCount = 0L; + retainedPayloadBytes = 0L; + accountingComplete = false; + } + } + + private void account(ContentFile file) { + metricEntryCount = MetaCacheWeightUtils.saturatedAdd( + metricEntryCount, metricEntryCount(file)); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, retainedPayloadBytes(file)); + } + } + + private static long metricEntryCount(ContentFile file) { + long count = mapSize(file.columnSizes()); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.valueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nullValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nanValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.lowerBounds())); + return MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.upperBounds())); + } + + private static long retainedPayloadBytes(ContentFile file) { + long bytes = MetaCacheWeightUtils.estimatedCharSequenceBytes(file.path()); + bytes = addBuffer(bytes, file.keyMetadata()); + bytes = addBuffers(bytes, file.lowerBounds()); + bytes = addBuffers(bytes, file.upperBounds()); + bytes = addListEntries(bytes, file.splitOffsets()); + bytes = addListEntries(bytes, file.equalityFieldIds()); + if (file instanceof DeleteFile) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes( + ((DeleteFile) file).referencedDataFile())); + } + return addPartitionPayload(bytes, file.partition()); + } + + private static long addBuffers(long bytes, Map buffers) { + if (buffers == null) { + return bytes; + } + for (ByteBuffer buffer : buffers.values()) { + bytes = addBuffer(bytes, buffer); + } + return bytes; + } + + private static long addBuffer(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + + private static long addListEntries(long bytes, List values) { + if (values == null) { + return bytes; + } + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + values.size(), AUXILIARY_LIST_ENTRY_BYTES)); + } + + private static long addPartitionPayload(long bytes, StructLike partition) { + if (partition == null) { + return bytes; + } + try { + for (int index = 0; index < partition.size(); index++) { + Object value = partition.get(index, Object.class); + if (value instanceof CharSequence) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + bytes = addBuffer(bytes, (ByteBuffer) value); + } else if (value instanceof byte[]) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + } + } + } catch (RuntimeException ignored) { + // A third-party StructLike may reject Object.class. The fixed per-file allowance + // remains conservative, and cache accounting must never fail manifest loading. + } + return bytes; + } + + private static int mapSize(Map map) { + return map == null ? 0 : map.size(); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 957ab6ed55e193..570652f245ff71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -781,8 +781,9 @@ private Table useFrozenTableGeneration(Table currentTable) { if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { IcebergSnapshotCacheValue cacheValue = ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); - if (cacheValue.getIcebergTable().isPresent()) { - Table frozenBaseTable = cacheValue.getIcebergTable().get(); + Optional
    frozenTable = cacheValue.getIcebergTable(); + if (frozenTable.isPresent()) { + Table frozenBaseTable = frozenTable.get(); if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); if (systemTable.supportsSnapshotSelection()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java index 46e58f1e380081..da2980f7185ca7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -52,7 +53,12 @@ public class MaxComputeExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public MaxComputeExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public MaxComputeExternalMetaCache(ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionValuesEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_PARTITION_VALUES, NameMapping.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index a3a44151e45e2f..906804e5307c21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -27,12 +27,15 @@ import org.apache.doris.datasource.SchemaCacheValue; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; import java.util.function.Function; import java.util.function.Predicate; @@ -46,6 +49,8 @@ * to initialize a catalog explicitly before accessing entries. */ public abstract class AbstractExternalMetaCache implements ExternalMetaCache { + private static final Logger LOG = LogManager.getLogger(AbstractExternalMetaCache.class); + protected static CacheSpec defaultEntryCacheSpec() { return CacheSpec.of( true, @@ -62,12 +67,19 @@ protected static CacheSpec defaultSchemaCacheSpec() { private final String engine; private final ExecutorService refreshExecutor; + private final ExternalMetaCacheBudgetManager budgetManager; private final Map catalogEntries = Maps.newConcurrentMap(); private final Map> metaCacheEntryDefs = Maps.newConcurrentMap(); protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor) { + this(engine, refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.empty())); + } + + protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { this.engine = engine; this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); } @Override @@ -81,10 +93,77 @@ public Collection aliases() { } @Override - public void initCatalog(long catalogId, Map catalogProperties) { + public void validateCatalogProperties(Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, id -> buildCatalogEntryGroup(safeCatalogProperties)); + validateMappedCatalogProperties(safeCatalogProperties, true); + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + synchronized (this) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( + safeCatalogProperties, engine, metaCacheEntryDefs, + warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); + try { + budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " + + "for engine {}, catalog {}: {}", + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, + engine, catalogId, e.getMessage()); + } + OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { + if (entryDef.getSizeEstimator() == null) { + continue; + } + String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) + + entryDef.getName() + ".max-weight"; + if (!safeCatalogProperties.containsKey(maxWeightKey)) { + continue; + } + CacheSpec cacheSpec = CacheSpec.fromProperties( + safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); + try { + budgetManager.validateCatalogEntryHierarchy( + runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(maxWeightKey); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " + + "for engine {}, catalog {}: {}", + maxWeightKey, engine, catalogId, e.getMessage()); + } + } + validateMappedCatalogProperties(safeCatalogProperties, false); + catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); + } + } + + private void validateMappedCatalogProperties( + Map catalogProperties, boolean validateAgainstLocalGlobalLimit) { + CacheSpec.validateEngineProperties(catalogProperties, engine, metaCacheEntryDefs); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + metaCacheEntryDefs.values().stream() + .filter(entryDef -> entryDef.getSizeEstimator() != null) + .map(entryDef -> CacheSpec.fromProperties( + catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec())) + .forEach(cacheSpec -> { + if (validateAgainstLocalGlobalLimit) { + budgetManager.validateHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } else { + budgetManager.validateCatalogEntryHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } + }); } @Override @@ -114,6 +193,7 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class MetaCacheEntryDef def = requireMetaCacheEntryDef(entryName); ensureTypeCompatible(def, keyType, valueType); + beforeCatalogEntryLookupForTest(catalogId, entryName); MetaCacheEntry cacheEntry = group.get(entryName); if (cacheEntry == null) { throw new IllegalStateException(String.format( @@ -124,10 +204,10 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class } @Override - public void invalidateCatalog(long catalogId) { + public synchronized void invalidateCatalog(long catalogId) { CatalogEntryGroup removed = catalogEntries.remove(catalogId); if (removed != null) { - removed.invalidateAll(); + removed.close(); } } @@ -162,8 +242,8 @@ public Map stats(long catalogId) { } @Override - public void close() { - catalogEntries.values().forEach(CatalogEntryGroup::invalidateAll); + public synchronized void close() { + catalogEntries.values().forEach(CatalogEntryGroup::close); catalogEntries.clear(); } @@ -191,6 +271,10 @@ protected final MetaCacheEntry entry(long catalogId, MetaCacheEntry return entry(catalogId, entryDef.getName(), entryDef.getKeyType(), entryDef.getValueType()); } + // Let tests pause after capturing a group and before looking up its entry. + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + } + protected final String metaCacheTtlKey(String entryName) { return "meta.cache." + engine + "." + entryName + ".ttl-second"; } @@ -283,23 +367,52 @@ private void invalidateEntryIfMatched(CatalogEntryGroup group, MetaCacheE } } - private CatalogEntryGroup buildCatalogEntryGroup(Map catalogProperties) { + private CatalogEntryGroup buildCatalogEntryGroup(long catalogId, Map catalogProperties) { CatalogEntryGroup group = new CatalogEntryGroup(); - metaCacheEntryDefs.values() - .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(entryDef, catalogProperties))); - return group; + try { + metaCacheEntryDefs.values().forEach(entryDef -> group.put( + entryDef.getName(), newMetaCacheEntry(catalogId, entryDef, catalogProperties))); + return group; + } catch (RuntimeException | Error e) { + group.close(); + throw e; + } } @SuppressWarnings("unchecked") private MetaCacheEntry newMetaCacheEntry( - MetaCacheEntryDef rawEntryDef, Map catalogProperties) { + long catalogId, MetaCacheEntryDef rawEntryDef, Map catalogProperties) { MetaCacheEntryDef entryDef = (MetaCacheEntryDef) rawEntryDef; CacheSpec cacheSpec = CacheSpec.fromProperties( catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); - return new MetaCacheEntry<>(entryDef.getName(), - wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), - cacheSpec, - refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly()); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + if (cacheSpec.isWeightBounded() && entryDef.getSizeEstimator() == null) { + throw new IllegalArgumentException(String.format( + "Entry '%s' for engine '%s' configures max-weight but has no estimator.", + entryDef.getName(), engine)); + } + boolean enableWeight = entryDef.getSizeEstimator() != null + && (cacheSpec.isWeightBounded() + || catalogMaxWeight.isPresent() + || budgetManager.getGlobalMaxWeight().isPresent()); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = null; + if (enableWeight) { + entryBudget = budgetManager.createEntryBudget( + catalogId, engine, entryDef.getName(), catalogMaxWeight, cacheSpec.getMaxWeight()); + cacheSpec = cacheSpec.withMaxWeight(entryBudget.getEffectiveMaxWeight()); + } + try { + return new MetaCacheEntry<>(entryDef.getName(), + wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), + cacheSpec, + refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), + entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener()); + } catch (RuntimeException | Error e) { + if (entryBudget != null) { + entryBudget.close(); + } + throw e; + } } private Function wrapSchemaValidator(Function loader, Class valueType) { @@ -327,8 +440,12 @@ public MetaCacheEntry get(long catalogId) { return entry(catalogId, entryDef); } + @SuppressWarnings("unchecked") public MetaCacheEntry getIfInitialized(long catalogId) { - return isCatalogInitialized(catalogId) ? get(catalogId) : null; + // Read the group once. A concurrent invalidation may close that captured entry, which + // is safe; looking the group up a second time could instead throw after the first check. + CatalogEntryGroup group = catalogEntries.get(catalogId); + return group == null ? null : (MetaCacheEntry) group.get(entryDef.getName()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 0bb640ad0d753c..34ccf41c718b1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -21,10 +21,18 @@ import org.apache.commons.lang3.math.NumberUtils; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.Set; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Common cache specification for external metadata caches. @@ -33,7 +41,8 @@ *
      *
    • enable=false disables cache
      • *
    • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
      • - *
    • capacity=0 disables cache; capacity is count-based
      • + *
    • capacity=0 disables cache; otherwise capacity is the count limit only when max-weight is absent
      • + *
    • when max-weight is present, Caffeine uses the weight limit instead of the positive capacity
      • *
    */ public final class CacheSpec { @@ -43,19 +52,36 @@ public final class CacheSpec { private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; + private static final Pattern DATA_VOLUME_PATTERN = Pattern.compile("^([0-9]+)\\s*(B|KB|MB|GB|TB|PB)?$", + Pattern.CASE_INSENSITIVE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + if (maxWeight < 0) { + throw new IllegalArgumentException("maxWeight can not be negative: " + maxWeight); + } + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); + } + + public CacheSpec withMaxWeight(long effectiveMaxWeight) { + return ofWeight(enable, ttlSecond, capacity, effectiveMaxWeight); } public static PropertySpec.Builder propertySpecBuilder() { @@ -77,7 +103,8 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getWeightProperty(properties, propertySpec.getMaxWeightKey()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** @@ -95,6 +122,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT) .build(); } @@ -151,6 +179,68 @@ public static boolean isCacheEnabled(boolean enable, long ttlSecond, long capaci return enable && ttlSecond != 0 && capacity != 0; } + /** + * Parse an exact byte value with an optional binary unit. Percentages are accepted only + * when {@code allowPercent} is true and are resolved against {@code maxHeapBytes}. + */ + public static long parseWeight(String value, String key, boolean allowPercent, long maxHeapBytes) { + String normalized = Objects.requireNonNull(value, "value").trim(); + if (normalized.isEmpty()) { + throw invalidWeight(key, value); + } + if (normalized.endsWith("%")) { + if (!allowPercent || maxHeapBytes <= 0) { + throw invalidWeight(key, value); + } + String percentageText = normalized.substring(0, normalized.length() - 1).trim(); + try { + BigDecimal percentage = new BigDecimal(percentageText); + if (percentage.signum() < 0 || percentage.compareTo(BigDecimal.valueOf(100L)) > 0) { + throw invalidWeight(key, value); + } + BigInteger bytes = BigDecimal.valueOf(maxHeapBytes) + .multiply(percentage) + .divide(BigDecimal.valueOf(100L)) + .toBigInteger(); + return checkedLong(bytes, key, value); + } catch (NumberFormatException e) { + throw invalidWeight(key, value); + } + } + + Matcher matcher = DATA_VOLUME_PATTERN.matcher(normalized); + if (!matcher.matches()) { + throw invalidWeight(key, value); + } + BigInteger amount = new BigInteger(matcher.group(1)); + String rawUnit = matcher.group(2); + String unit = rawUnit == null ? "B" : rawUnit.toUpperCase(Locale.ROOT); + int power; + switch (unit) { + case "B": + power = 0; + break; + case "KB": + power = 1; + break; + case "MB": + power = 2; + break; + case "GB": + power = 3; + break; + case "TB": + power = 4; + break; + case "PB": + power = 5; + break; + default: + throw invalidWeight(key, value); + } + return checkedLong(amount.multiply(BigInteger.valueOf(1024L).pow(power)), key, value); + } + /** * Build standard external meta cache key prefix for one engine. * Example: {@code meta.cache.iceberg.} @@ -166,6 +256,110 @@ public static boolean isMetaCacheKeyForEngine(String key, String engine) { return key != null && engine != null && key.startsWith(metaCacheKeyPrefix(engine)); } + /** + * Strictly validate one engine namespace so misspelled entries/options cannot be silently ignored. + * The catalog-wide {@code meta.cache.max-weight} key is validated by the budget manager. + */ + static void validateEngineProperties(Map properties, String engine, + Map> entryDefs) { + Set weightedEntries = new java.util.HashSet<>(); + for (MetaCacheEntryDef entryDef : entryDefs.values()) { + if (entryDef.getSizeEstimator() != null) { + weightedEntries.add(entryDef.getName()); + } + } + validateEngineProperties(properties, engine, entryDefs.keySet(), weightedEntries); + } + + /** + * Ignore invalid persisted cache options during image/replay initialization. + * New CREATE/ALTER statements still use {@link #validateEngineProperties} and fail strictly. + */ + static Map sanitizeEnginePropertiesForRuntime( + Map properties, String engine, + Map> entryDefs, Consumer warningConsumer) { + Map sanitized = new HashMap<>(properties); + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + try { + validateEngineProperties(Collections.singletonMap(key, property.getValue()), engine, entryDefs); + } catch (IllegalArgumentException e) { + sanitized.remove(key); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + key + "': " + e.getMessage()); + } + } + return sanitized; + } + + public static void validateEngineProperties(Map properties, String engine, + Set entryNames, Set weightedEntryNames) { + if (properties == null || properties.isEmpty()) { + return; + } + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + String remainder = key.substring(enginePrefix.length()); + int optionSeparator = remainder.lastIndexOf('.'); + if (optionSeparator <= 0 || optionSeparator == remainder.length() - 1) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String entryName = remainder.substring(0, optionSeparator); + String option = remainder.substring(optionSeparator + 1); + if (!entryNames.contains(entryName)) { + throw new IllegalArgumentException("Unknown external meta cache entry property: " + key); + } + String value = property.getValue(); + switch (option) { + case "enable": + requireStrictBoolean(key, value); + break; + case "ttl-second": + requireLongAtLeast(key, value, CACHE_NO_TTL); + break; + case "capacity": + requireLongAtLeast(key, value, 0L); + break; + case "max-weight": + if (!weightedEntryNames.contains(entryName)) { + throw new IllegalArgumentException( + "External meta cache entry does not support max-weight: " + key); + } + parseWeight(value, key, false, 0L); + break; + default: + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + } + } + + private static void requireStrictBoolean(String key, String value) { + if (value == null || (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value))) { + throw new IllegalArgumentException("Invalid boolean cache property '" + key + "': " + value); + } + } + + private static void requireLongAtLeast(String key, String value, long minimum) { + final long parsed; + try { + parsed = Long.parseLong(value); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid integer cache property '" + key + "': " + value, e); + } + if (parsed < minimum) { + throw new IllegalArgumentException("Cache property '" + key + "' must be >= " + minimum + + ", but was " + value); + } + } + /** * Convert ttlSecond to OptionalLong for CacheFactory. * ttlSecond=-1 means no expiration; ttlSecond=0 disables cache. @@ -193,6 +387,27 @@ private static long getLongProperty(Map properties, String key, return NumberUtils.toLong(value, defaultValue); } + private static OptionalLong getWeightProperty(Map properties, String key) { + if (key == null) { + return OptionalLong.empty(); + } + String value = properties.get(key); + return value == null + ? OptionalLong.empty() + : OptionalLong.of(parseWeight(value, key, false, 0L)); + } + + private static long checkedLong(BigInteger value, String key, String rawValue) { + if (value.signum() < 0 || value.compareTo(LONG_MAX) > 0) { + throw invalidWeight(key, rawValue); + } + return value.longValue(); + } + + private static IllegalArgumentException invalidWeight(String key, String value) { + return new IllegalArgumentException("Invalid cache weight for '" + key + "': " + value); + } + public boolean isEnable() { return enable; } @@ -205,6 +420,19 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return isCacheEnabled(enable, ttlSecond, capacity) + && (!maxWeight.isPresent() || maxWeight.getAsLong() != 0L); + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -212,15 +440,17 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, String maxWeightKey) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; } public String getEnableKey() { @@ -247,6 +477,10 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -254,6 +488,7 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -273,6 +508,11 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key) { + this.maxWeightKey = Objects.requireNonNull(key, "key"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -280,7 +520,8 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java index c195087f415bfc..d37e91a8922019 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.metacache; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; import java.util.Objects; @@ -27,6 +29,8 @@ * Catalog scoped entry container. */ public class CatalogEntryGroup { + private static final Logger LOG = LogManager.getLogger(CatalogEntryGroup.class); + private final Map> entries = new ConcurrentHashMap<>(); public MetaCacheEntry get(String entryName) { @@ -46,4 +50,19 @@ public Map stats() { public void invalidateAll() { entries.values().forEach(MetaCacheEntry::invalidateAll); } + + public void close() { + entries.forEach((name, entry) -> { + try { + entry.close(); + } catch (RuntimeException e) { + LOG.error("Failed to close external metadata cache entry {}; continuing group retirement", + name, e); + } + }); + // Keep the closed entries reachable from this retired group. A query may have captured the + // group immediately before its catalog is removed; returning a closed entry lets that query + // serve an uncached load instead of spuriously observing an uninitialized entry. The group + // is already absent from the owner map and is reclaimed with the last concurrent reader. + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java index 1a067726ec9136..8b874fac8f659e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java @@ -41,6 +41,10 @@ public interface ExternalMetaCache { */ Collection aliases(); + /** Validate cache properties in this engine's canonical namespace. */ + default void validateCatalogProperties(Map catalogProperties) { + } + /** * Initialize all registered entries for one catalog under current engine. * Entry instances are created eagerly at this stage. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java new file mode 100644 index 00000000000000..904a3fb0ad8f75 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -0,0 +1,587 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongUnaryOperator; +import java.util.stream.Collectors; + +/** + * FE-wide admission accounting for managed external metadata caches. + * + *

    All changes are serialized by one short critical section. Cache loads and + * estimators run outside it, so the lock only protects a few arithmetic and map + * operations while making global/catalog/entry reservation atomic. + */ +public final class ExternalMetaCacheBudgetManager { + private static final Logger LOG = LogManager.getLogger(ExternalMetaCacheBudgetManager.class); + private static final ExecutorService PEER_RECLAIM_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-peer-reclaim"); + thread.setDaemon(true); + return thread; + }); + + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; + + private final Object lock = new Object(); + private final OptionalLong globalMaxWeight; + private final Map catalogBuckets = new HashMap<>(); + private final Map entryBuckets = new HashMap<>(); + private final Map entryBudgets = new HashMap<>(); + private long globalUsedWeight; + private final AtomicLong globalRejectedCount = new AtomicLong(); + + public ExternalMetaCacheBudgetManager(OptionalLong globalMaxWeight) { + this.globalMaxWeight = Objects.requireNonNull(globalMaxWeight, "globalMaxWeight"); + if (globalMaxWeight.isPresent() && globalMaxWeight.getAsLong() <= 0) { + throw new IllegalArgumentException("global max weight must be positive when enabled"); + } + } + + public static ExternalMetaCacheBudgetManager fromConfig() { + String configured = Config.external_meta_cache_max_weight; + long parsed = CacheSpec.parseWeight( + configured, + "external_meta_cache_max_weight", + true, + Runtime.getRuntime().maxMemory()); + if (configured.trim().endsWith("%") && parsed == 0L) { + throw new IllegalArgumentException( + "external_meta_cache_max_weight percentage must be greater than 0%"); + } + return new ExternalMetaCacheBudgetManager(parsed == 0L ? OptionalLong.empty() : OptionalLong.of(parsed)); + } + + public OptionalLong parseCatalogMaxWeight(Map catalogProperties) { + String configured = catalogProperties.get(CATALOG_MAX_WEIGHT_PROPERTY); + if (configured == null) { + return OptionalLong.empty(); + } + long parsed = CacheSpec.parseWeight(configured, CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } + + /** Validate a catalog limit at DDL time against this FE's configured global bound. */ + public OptionalLong validateCatalogMaxWeight(Map catalogProperties) { + OptionalLong catalogMaxWeight = parseCatalogMaxWeight(catalogProperties); + validateHierarchy(catalogMaxWeight, OptionalLong.empty()); + return catalogMaxWeight; + } + + /** + * Create the budget handle used by one physical per-catalog cache entry. + */ + public EntryBudget createEntryBudget(long catalogId, String engine, String entryName, + OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + Objects.requireNonNull(engine, "engine"); + Objects.requireNonNull(entryName, "entryName"); + Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight"); + Objects.requireNonNull(entryMaxWeight, "entryMaxWeight"); + validateCatalogEntryHierarchy(catalogMaxWeight, entryMaxWeight); + + OptionalLong effectiveMax = minimumPresent(globalMaxWeight, catalogMaxWeight, entryMaxWeight); + if (!effectiveMax.isPresent()) { + throw new IllegalArgumentException("entry budget requires at least one configured weight bound"); + } + + EntryScope scope = new EntryScope(catalogId, engine, entryName); + synchronized (lock) { + Bucket catalogBucket = catalogBuckets.get(catalogId); + long catalogLimit = minimumLimit(globalMaxWeight, catalogMaxWeight); + if (catalogBucket == null) { + catalogBucket = new Bucket(catalogLimit); + catalogBuckets.put(catalogId, catalogBucket); + } else if (catalogBucket.maxWeight != catalogLimit) { + throw new IllegalStateException("Conflicting catalog cache max weight for catalog " + catalogId); + } + + if (entryBuckets.containsKey(scope)) { + throw new IllegalStateException("Duplicated external meta cache budget: " + scope); + } + Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + EntryBudget entryBudget = new EntryBudget( + this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); + entryBuckets.put(scope, entryBucket); + entryBudgets.put(scope, entryBudget); + return entryBudget; + } + } + + public OptionalLong getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalUsedWeight() { + synchronized (lock) { + return globalUsedWeight; + } + } + + public long getGlobalRejectedCount() { + return globalRejectedCount.get(); + } + + public void validateHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + if (globalMaxWeight.isPresent() && catalogMaxWeight.isPresent() + && catalogMaxWeight.getAsLong() > globalMaxWeight.getAsLong()) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " can not exceed FE global max weight"); + } + OptionalLong parent = catalogMaxWeight.isPresent() ? catalogMaxWeight : globalMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + /** + * Validate persisted catalog-to-entry hierarchy without comparing it with this FE's local + * global bound. Catalog properties are validated on the master, while the global percentage + * is resolved independently from each FE's heap. Runtime admission therefore clamps to the + * local global limit instead of rejecting a catalog accepted on a larger master. + */ + public void validateCatalogEntryHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + OptionalLong parent = catalogMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + private Optional tryReserve(EntryBudget entryBudget, long bytes) { + checkWeight(bytes); + synchronized (lock) { + if (entryBudget.closed) { + return Optional.empty(); + } + if (!fits(limitOf(globalMaxWeight), globalUsedWeight, bytes) + || !fits(entryBudget.catalogBucket.maxWeight, entryBudget.catalogBucket.usedWeight, bytes) + || !fits(entryBudget.entryBucket.maxWeight, entryBudget.entryBucket.usedWeight, bytes)) { + entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return Optional.empty(); + } + addUsed(entryBudget, bytes); + return Optional.of(new AdmissionReservation(this, entryBudget, bytes)); + } + } + + private boolean resize(AdmissionReservation reservation, long newBytes) { + checkWeight(newBytes); + synchronized (lock) { + if (!reservation.active || reservation.entryBudget.closed) { + return false; + } + long delta = newBytes - reservation.bytes; + if (delta > 0 && (!fits(limitOf(globalMaxWeight), globalUsedWeight, delta) + || !fits(reservation.entryBudget.catalogBucket.maxWeight, + reservation.entryBudget.catalogBucket.usedWeight, delta) + || !fits(reservation.entryBudget.entryBucket.maxWeight, + reservation.entryBudget.entryBucket.usedWeight, delta))) { + reservation.entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return false; + } + if (delta >= 0) { + addUsed(reservation.entryBudget, delta); + } else { + subtractUsed(reservation.entryBudget, -delta); + } + reservation.bytes = newBytes; + return true; + } + } + + private void release(AdmissionReservation reservation) { + synchronized (lock) { + if (!reservation.active) { + return; + } + if (reservation.entryBudget.closed) { + reservation.bytes = 0L; + reservation.active = false; + return; + } + subtractUsed(reservation.entryBudget, reservation.bytes); + reservation.bytes = 0L; + reservation.active = false; + } + } + + private void close(EntryBudget entryBudget) { + synchronized (lock) { + if (entryBudget.closed) { + return; + } + if (entryBudget.entryBucket.usedWeight != 0L) { + long leakedWeight = entryBudget.entryBucket.usedWeight; + LOG.error("Force-closing external metadata cache budget {} with {} bytes still reserved", + entryBudget.scope, leakedWeight); + if (leakedWeight <= globalUsedWeight + && leakedWeight <= entryBudget.catalogBucket.usedWeight) { + globalUsedWeight -= leakedWeight; + entryBudget.catalogBucket.usedWeight -= leakedWeight; + entryBudget.entryBucket.usedWeight = 0L; + } else { + LOG.error("External metadata cache accounting is inconsistent while closing {}; " + + "globalUsed={}, catalogUsed={}, entryUsed={}", + entryBudget.scope, globalUsedWeight, + entryBudget.catalogBucket.usedWeight, leakedWeight); + globalUsedWeight = Math.max(0L, globalUsedWeight - leakedWeight); + entryBudget.catalogBucket.usedWeight = Math.max( + 0L, entryBudget.catalogBucket.usedWeight - leakedWeight); + entryBudget.entryBucket.usedWeight = 0L; + } + } + entryBudget.closed = true; + entryBudget.reclaimer = null; + entryBuckets.remove(entryBudget.scope, entryBudget.entryBucket); + entryBudgets.remove(entryBudget.scope, entryBudget); + Bucket catalogBucket = entryBudget.catalogBucket; + boolean catalogStillReferenced = entryBuckets.keySet().stream() + .anyMatch(scope -> scope.catalogId == entryBudget.scope.catalogId); + if (!catalogStillReferenced && catalogBucket.usedWeight == 0L) { + catalogBuckets.remove(entryBudget.scope.catalogId, catalogBucket); + } + } + } + + private void addUsed(EntryBudget entryBudget, long bytes) { + globalUsedWeight += bytes; + entryBudget.catalogBucket.usedWeight += bytes; + entryBudget.entryBucket.usedWeight += bytes; + } + + private void subtractUsed(EntryBudget entryBudget, long bytes) { + if (bytes > globalUsedWeight + || bytes > entryBudget.catalogBucket.usedWeight + || bytes > entryBudget.entryBucket.usedWeight) { + throw new IllegalStateException("external meta cache budget accounting underflow"); + } + globalUsedWeight -= bytes; + entryBudget.catalogBucket.usedWeight -= bytes; + entryBudget.entryBucket.usedWeight -= bytes; + } + + private void requestPeerReclaim(EntryBudget requester, long additionalBytes) { + if (additionalBytes <= 0L || requester.closed) { + return; + } + long reclaimBytes; + synchronized (lock) { + if (requester.closed) { + return; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + reclaimBytes = Math.max(globalDeficit, catalogDeficit); + } + if (reclaimBytes <= 0L) { + return; + } + // Rejected values are returned uncached; there is no queue of pending admissions to fund. + // Coalesce concurrent misses to the largest single admission instead of summing identical + // deficits and evicting an entire peer cache during a miss burst. + requester.requestedAdmissionBytes.accumulateAndGet(additionalBytes, Math::max); + schedulePeerReclaim(requester); + } + + private void schedulePeerReclaim(EntryBudget requester) { + if (!requester.reclaimScheduled.compareAndSet(false, true)) { + return; + } + try { + PEER_RECLAIM_EXECUTOR.execute(() -> drainPeerReclaim(requester)); + } catch (RejectedExecutionException e) { + requester.reclaimScheduled.set(false); + LOG.warn("Failed to schedule peer reclamation for external metadata cache budget {}", + requester.scope, e); + } + } + + private void drainPeerReclaim(EntryBudget requester) { + try { + long requestedAdmissionBytes = requester.requestedAdmissionBytes.getAndSet(0L); + if (requestedAdmissionBytes <= 0L || requester.closed) { + return; + } + List candidates; + synchronized (lock) { + candidates = entryBudgets.values().stream() + .filter(candidate -> candidate != requester && !candidate.closed) + .filter(candidate -> candidate.reclaimer != null) + .filter(candidate -> candidate.entryBucket.usedWeight > 0L) + .sorted((left, right) -> { + boolean leftSibling = left.scope.catalogId == requester.scope.catalogId; + boolean rightSibling = right.scope.catalogId == requester.scope.catalogId; + if (leftSibling != rightSibling) { + return leftSibling ? -1 : 1; + } + return Long.compare( + right.entryBucket.usedWeight, left.entryBucket.usedWeight); + }) + .collect(Collectors.toList()); + } + long remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + for (EntryBudget candidate : candidates) { + boolean sibling = candidate.scope.catalogId == requester.scope.catalogId; + if (!sibling && currentCatalogDeficit(requester, requestedAdmissionBytes) > 0L) { + // Another catalog cannot create headroom under the requester's catalog limit. + continue; + } + LongUnaryOperator reclaimer = candidate.reclaimer; + if (reclaimer == null || candidate.closed) { + continue; + } + try { + reclaimer.applyAsLong(remaining); + remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + } catch (RuntimeException e) { + LOG.warn("Failed to reclaim external metadata cache budget from peer {}", + candidate.scope, e); + } + if (remaining == 0L) { + break; + } + } + } finally { + requester.reclaimScheduled.set(false); + if (!requester.closed && requester.requestedAdmissionBytes.get() > 0L) { + schedulePeerReclaim(requester); + } + } + } + + private long currentReclaimDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + if (requester.closed) { + return 0L; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + return Math.max(globalDeficit, catalogDeficit); + } + } + + private long currentCatalogDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit( + requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes); + } + } + + private static long deficit(long maxWeight, long usedWeight, long additionalBytes) { + if (maxWeight == Long.MAX_VALUE || additionalBytes <= maxWeight - Math.min(usedWeight, maxWeight)) { + return 0L; + } + return MetaCacheWeightUtils.saturatedAdd(usedWeight, additionalBytes) - maxWeight; + } + + private static boolean fits(long maxWeight, long usedWeight, long delta) { + return delta >= 0 && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; + } + + private static long limitOf(OptionalLong configured) { + return configured.isPresent() ? configured.getAsLong() : Long.MAX_VALUE; + } + + private static long minimumLimit(OptionalLong first, OptionalLong second) { + return Math.min(limitOf(first), limitOf(second)); + } + + private static OptionalLong minimumPresent(OptionalLong first, OptionalLong second, OptionalLong third) { + if (!first.isPresent() && !second.isPresent() && !third.isPresent()) { + return OptionalLong.empty(); + } + long minimum = Math.min(limitOf(first), Math.min(limitOf(second), limitOf(third))); + return OptionalLong.of(minimum); + } + + private static void checkWeight(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache reservation can not be negative: " + bytes); + } + } + + private static final class Bucket { + private final long maxWeight; + private long usedWeight; + + private Bucket(long maxWeight) { + this.maxWeight = maxWeight; + } + } + + private static final class EntryScope { + private final long catalogId; + private final String engine; + private final String entryName; + + private EntryScope(long catalogId, String engine, String entryName) { + this.catalogId = catalogId; + this.engine = engine; + this.entryName = entryName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryScope)) { + return false; + } + EntryScope that = (EntryScope) other; + return catalogId == that.catalogId && engine.equals(that.engine) && entryName.equals(that.entryName); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, engine, entryName); + } + + @Override + public String toString() { + return catalogId + "/" + engine + "/" + entryName; + } + } + + public static final class EntryBudget { + private final ExternalMetaCacheBudgetManager manager; + private final EntryScope scope; + private final Bucket catalogBucket; + private final Bucket entryBucket; + private final long effectiveMaxWeight; + private final AtomicLong rejectedCount = new AtomicLong(); + private final AtomicLong requestedAdmissionBytes = new AtomicLong(); + private final AtomicBoolean reclaimScheduled = new AtomicBoolean(); + private volatile LongUnaryOperator reclaimer; + // Mutated under manager.lock and read by asynchronous reclamation workers. + private volatile boolean closed; + + private EntryBudget(ExternalMetaCacheBudgetManager manager, EntryScope scope, + Bucket catalogBucket, Bucket entryBucket, long effectiveMaxWeight) { + this.manager = manager; + this.scope = scope; + this.catalogBucket = catalogBucket; + this.entryBucket = entryBucket; + this.effectiveMaxWeight = effectiveMaxWeight; + } + + public Optional tryReserve(long bytes) { + return manager.tryReserve(this, bytes); + } + + void setReclaimer(LongUnaryOperator reclaimer) { + this.reclaimer = Objects.requireNonNull(reclaimer, "reclaimer"); + } + + void requestPeerReclaim(long additionalBytes) { + manager.requestPeerReclaim(this, additionalBytes); + } + + public long getEffectiveMaxWeight() { + return effectiveMaxWeight; + } + + public long getUsedWeight() { + synchronized (manager.lock) { + return entryBucket.usedWeight; + } + } + + public long getCatalogUsedWeight() { + synchronized (manager.lock) { + return catalogBucket.usedWeight; + } + } + + public long getCatalogMaxWeight() { + return catalogBucket.maxWeight == Long.MAX_VALUE ? -1L : catalogBucket.maxWeight; + } + + public long getRejectedCount() { + return rejectedCount.get(); + } + + public long getGlobalUsedWeight() { + return manager.getGlobalUsedWeight(); + } + + public long getGlobalMaxWeight() { + return manager.globalMaxWeight.isPresent() ? manager.globalMaxWeight.getAsLong() : -1L; + } + + public void close() { + manager.close(this); + } + } + + public static final class AdmissionReservation { + private final ExternalMetaCacheBudgetManager manager; + private final EntryBudget entryBudget; + private long bytes; + private boolean active = true; + + private AdmissionReservation(ExternalMetaCacheBudgetManager manager, EntryBudget entryBudget, long bytes) { + this.manager = manager; + this.entryBudget = entryBudget; + this.bytes = bytes; + } + + public boolean tryResize(long newBytes) { + return manager.resize(this, newBytes); + } + + public void release() { + manager.release(this); + } + + public long getBytes() { + synchronized (manager.lock) { + return bytes; + } + } + + public boolean isActive() { + synchronized (manager.lock) { + return active; + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 30668163539d3b..de89c7b9d8d6aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -19,14 +19,29 @@ import org.apache.doris.common.CacheFactory; import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.Weigher; import com.github.benmanes.caffeine.cache.stats.CacheStats; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.HashSet; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -40,8 +55,23 @@ * key/predicate/full invalidation, and lightweight runtime stats. */ public class MetaCacheEntry { + private static final Logger LOG = LogManager.getLogger(MetaCacheEntry.class); // Use striped locks to deduplicate slow external loads without managing per-key lock lifecycle. private static final int LOAD_LOCK_STRIPES = 128; + private static final int LOCAL_EVICTION_BATCH_SIZE = 16; + private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; + private static final long WEIGHT_REJECT_LOG_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1L); + // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced + // generation map per physical entry after callbacks return; cleanup tasks never capture values. + private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-removal-cleanup"); + thread.setDaemon(true); + return thread; + }); + // Conservative retained cost outside the estimator-owned key/value graph: Caffeine's data + // node and policy links plus the reservation ConcurrentHashMap node, record and token. This + // deliberately overestimates common compressed-oops layouts; calibrate downward only with JOL. + static final long FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES = 512L; private final String name; @Nullable @@ -49,15 +79,42 @@ public class MetaCacheEntry { private final CacheSpec cacheSpec; private final boolean effectiveEnabled; private final boolean autoRefresh; + private final ExecutorService refreshExecutor; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final EntryBudget entryBudget; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; + private final boolean weightBounded; + // Entries with publication-time work use the same generation-fenced refresh protocol even + // before a max-weight is configured. This keeps estimation and dependency retirement on every + // load/refresh path instead of letting Caffeine publish values behind those hooks. + private final boolean generationFencedRefresh; // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. private final LoadingCache loadingData; // Use the plain cache view for manual miss load so slow I/O does not happen in Caffeine's sync load path. private final Cache data; // Protect one key stripe at a time to deduplicate concurrent miss loads with bounded lock count. private final Object[] loadLocks = new Object[LOAD_LOCK_STRIPES]; + // Serialize weighted cache mutation with reservation ownership changes. + private final Object admissionLock = new Object(); + // Ownership records deliberately contain no V reference. A weighted cache's Caffeine soft + // reference must be the only cache-owned path to its value, while generation fencing keeps + // delayed removal callbacks from releasing a replacement reservation. + private final Map reservations = new ConcurrentHashMap<>(); + private final Map refreshRecords = new ConcurrentHashMap<>(); + private final Map pendingRemovalGenerations = new ConcurrentHashMap<>(); + private final AtomicBoolean removalCleanupScheduled = new AtomicBoolean(false); + private final Map refreshesInFlight = new ConcurrentHashMap<>(); + // A state exists only while a miss/refresh for the key is in flight. Mutations advance that + // state's epoch, fencing stale publication without retaining every key ever observed. + private final Map keyMutationStates = new ConcurrentHashMap<>(); private final AtomicLong invalidateCount = new AtomicLong(0); - // Bump generation before invalidation so in-flight manual loads do not repopulate stale values. - private final AtomicLong invalidateGeneration = new AtomicLong(0); + // Full invalidation is the only cross-key fence. Ordinary mutations use the per-key state. + private final AtomicLong fullInvalidationGeneration = new AtomicLong(0); + // Primitive owner id lets queued refresh work fence a reservation without retaining its value. + private final AtomicLong reservationGeneration = new AtomicLong(0); // Track load statistics outside Caffeine because manual miss loads bypass the built-in load counters. private final AtomicLong loadSuccessCount = new AtomicLong(0); private final AtomicLong loadFailureCount = new AtomicLong(0); @@ -65,6 +122,12 @@ public class MetaCacheEntry { private final AtomicLong lastLoadSuccessTimeMs = new AtomicLong(-1L); private final AtomicLong lastLoadFailureTimeMs = new AtomicLong(-1L); private final AtomicReference lastError = new AtomicReference<>(""); + private final AtomicLong weightAdmissionRejectedCount = new AtomicLong(0L); + private final AtomicLong localEvictionCount = new AtomicLong(0L); + private final AtomicLong localEvictionWeight = new AtomicLong(0L); + private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final AtomicLong lastWeightRejectLogTimeMs = new AtomicLong(0L); + private final AtomicBoolean closed = new AtomicBoolean(false); public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { this(name, loader, cacheSpec, refreshExecutor, true, false); @@ -77,6 +140,20 @@ public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, E public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, null, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + sizeEstimator, entryBudget, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -91,23 +168,48 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); this.autoRefresh = autoRefresh; - Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); + this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.sizeEstimator = sizeEstimator; + this.entryBudget = entryBudget; + this.replacementListener = replacementListener; + this.weightBounded = this.cacheSpec.isWeightBounded(); + this.generationFencedRefresh = autoRefresh + && (sizeEstimator != null || replacementListener != null); + if (weightBounded && (sizeEstimator == null || entryBudget == null)) { + throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); + } + if (weightBounded) { + entryBudget.setReclaimer(this::reclaimForPeer); + } + this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); OptionalLong refreshAfterWriteSec = - effectiveEnabled && autoRefresh + effectiveEnabled && autoRefresh && !weightBounded && !generationFencedRefresh ? OptionalLong.of(Config.external_cache_refresh_time_minutes * 60) : OptionalLong.empty(); long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; + Weigher cacheWeigher = weightBounded ? this::weigh : null; CacheFactory cacheFactory = new CacheFactory( expireAfterAccessSec, refreshAfterWriteSec, maxSize, + weightBounded ? OptionalLong.of(effectiveEnabled ? this.cacheSpec.getMaxWeight().getAsLong() : 0L) + : OptionalLong.empty(), + cacheWeigher, true, null); - this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + if (weightBounded) { + cacheFactory.withSoftValues(); + } + if (weightBounded || generationFencedRefresh) { + // Direct notification avoids queuing REPLACED values. The listener itself is lock-free + // and delegates only current-owner cleanup, so it is safe under Caffeine's eviction lock. + this.loadingData = cacheFactory.buildCacheWithSyncRemovalListener( + this::loadFromDefaultLoader, this::onRemoval); + } else { + this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + } this.data = loadingData; // Initialize striped locks eagerly to keep the hot path allocation-free. for (int i = 0; i < loadLocks.length; i++) { @@ -120,6 +222,9 @@ public String name() { } public V get(K key) { + if (closed.get()) { + return loadAndTrack(key, this::applyDefaultLoader); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key); } @@ -128,6 +233,9 @@ public V get(K key) { public V get(K key, Function missLoader) { Function loadFunction = Objects.requireNonNull(missLoader, "missLoader can not be null"); + if (closed.get()) { + return loadAndTrack(key, loadFunction); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key, typedKey -> loadAndTrack(typedKey, loadFunction)); } @@ -135,42 +243,235 @@ public V get(K key, Function missLoader) { } public V getIfPresent(K key) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return null; } - return data.getIfPresent(key); + V value = data.getIfPresent(key); + if (value != null) { + maybeRefreshManagedValue(key, value); + } + return value; + } + + /** Return the current value without recording a user-visible cache request. */ + public V peekIfPresent(K key) { + if (!effectiveEnabled || closed.get()) { + return null; + } + return data.asMap().get(key); + } + + /** + * Fence loads and refreshes that started before an event, but only while the expected value is + * still current. Publication-managed entries retain the known-good value and advance the key's + * mutation epoch. Other count-based entries must invalidate because their legacy + * Caffeine-managed refresh path does not participate in that generation protocol. + */ + public boolean fenceInFlightLoadIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + synchronized (admissionLock) { + if (!effectiveEnabled || closed.get() || data.asMap().get(key) != expectedCurrent) { + return false; + } + advanceKeyMutation(key); + if (!weightBounded && !generationFencedRefresh) { + if (!data.asMap().remove(key, expectedCurrent)) { + return false; + } + invalidateCount.incrementAndGet(); + } + return true; + } } public void put(K key, V value) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return; } - data.put(key, value); + if (weightBounded) { + admitWeightedValue(key, value, null, false, null, -1L, true); + } else { + synchronized (admissionLock) { + if (!closed.get()) { + advanceKeyMutation(key); + putNonWeightedValue(key, value); + } + } + } } - public void invalidateKey(K key) { - invalidateGeneration.incrementAndGet(); - if (data.asMap().remove(key) != null) { + /** Result of an atomic compare-and-replace operation. */ + public enum ReplaceResult { + REPLACED, + NOT_CURRENT, + REJECTED, + DISABLED + } + + /** + * Replace one cached value only when it is still the expected identity. + * + *

    Weighted entries perform the identity check, budget resize and Caffeine write under the + * same admission lock. Callers can therefore distinguish a concurrent update from admission + * rejection and avoid retaining a value they already know is stale. + */ + public ReplaceResult tryReplace(K key, V expectedCurrent, V newValue) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + Objects.requireNonNull(newValue, "newValue can not be null"); + if (!effectiveEnabled || closed.get()) { + return ReplaceResult.DISABLED; + } + if (weightBounded) { + return toReplaceResult(admitWeightedValue( + key, newValue, expectedCurrent, true, null, -1L, true)); + } + synchronized (admissionLock) { + AtomicReference result = new AtomicReference<>(ReplaceResult.NOT_CURRENT); + AtomicReference published = new AtomicReference<>(); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (closed.get()) { + result.set(ReplaceResult.DISABLED); + return current; + } + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + published.set(publishRefreshRecord(key)); + result.set(ReplaceResult.REPLACED); + return newValue; + }); + RefreshRecord record = published.get(); + if (record != null && refreshRecords.get(key) == record + && data.asMap().get(key) == newValue) { + record.published = true; + } + if (result.get() == ReplaceResult.REPLACED && data.asMap().get(key) == newValue) { + notifyReplacement(key, expectedCurrent, newValue); + } + return result.get(); + } + } + + /** Remove a key only if it still maps to the expected value identity. */ + public boolean invalidateKeyIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + if (!weightBounded) { + synchronized (admissionLock) { + AtomicBoolean removed = new AtomicBoolean(false); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + invalidateCount.incrementAndGet(); + refreshRecords.remove(key); + removed.set(true); + return null; + }); + return removed.get(); + } + } + synchronized (admissionLock) { + V current = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (current != expectedCurrent || record == null || !record.published) { + return false; + } + advanceKeyMutation(key); + if (!data.asMap().remove(key, current)) { + return false; + } + releaseReservation(key, record.generation); invalidateCount.incrementAndGet(); + return true; + } + } + + public void invalidateKey(K key) { + synchronized (admissionLock) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + if (removed != null) { + refreshRecords.remove(key); + invalidateCount.incrementAndGet(); + } + } } } public void invalidateIf(Predicate predicate) { - invalidateGeneration.incrementAndGet(); - data.asMap().keySet().removeIf(key -> { - if (predicate.test(key)) { - invalidateCount.incrementAndGet(); - return true; + synchronized (admissionLock) { + Set candidates = new HashSet<>(data.asMap().keySet()); + candidates.addAll(keyMutationStates.keySet()); + if (weightBounded) { + candidates.addAll(reservations.keySet()); + } else if (generationFencedRefresh) { + candidates.addAll(refreshRecords.keySet()); } - return false; - }); + for (K key : candidates) { + if (predicate.test(key)) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + refreshRecords.remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + } + } + } + } } public void invalidateAll() { - invalidateGeneration.incrementAndGet(); - long size = data.estimatedSize(); - data.invalidateAll(); - invalidateCount.addAndGet(size); + synchronized (admissionLock) { + fullInvalidationGeneration.incrementAndGet(); + if (weightBounded) { + long size = data.estimatedSize(); + beforeWeightedInvalidateAllForTest(); + data.invalidateAll(); + reservations.values().forEach(record -> record.reservation.release()); + reservations.clear(); + pendingRemovalGenerations.clear(); + invalidateCount.addAndGet(size); + } else { + long size = data.estimatedSize(); + data.invalidateAll(); + refreshRecords.clear(); + pendingRemovalGenerations.clear(); + invalidateCount.addAndGet(size); + } + } + } + + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + invalidateAll(); + if (entryBudget != null) { + entryBudget.close(); + } } public void forEach(BiConsumer consumer) { @@ -198,62 +499,621 @@ public MetaCacheEntryStats stats() { failureCount, totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - cacheStats.evictionCount(), + MetaCacheWeightUtils.saturatedAdd( + cacheStats.evictionCount(), localEvictionCount.get()), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), - lastError.get()); + lastError.get(), + weightBounded, + weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, + weightBounded ? entryBudget.getUsedWeight() : -1L, + weightBounded ? MetaCacheWeightUtils.saturatedAdd( + cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, + weightBounded ? weightAdmissionRejectedCount.get() : -1L, + weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, + weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, + weightBounded ? entryBudget.getGlobalMaxWeight() : -1L, + weightBounded ? entryBudget.getGlobalUsedWeight() : -1L, + weightBounded ? lastWeightRejectReason.get() : ""); + } + + public boolean isWeightBounded() { + return weightBounded; + } + + private AdmissionResult admitWeightedValue( + K key, V value, @Nullable V expectedCurrent, boolean requireExpected, + @Nullable KeyMutationToken expectedMutation, long expectedReservationGeneration, + boolean advanceMutationOnAdmission) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + MetaCacheSizeEstimate estimate; + try { + estimate = Objects.requireNonNull(sizeEstimator.estimate(key, value), "size estimate"); + } catch (IllegalArgumentException e) { + throw e; + } catch (RuntimeException e) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + if (!estimate.isComplete()) { + rejectWeight(estimate.getIncompleteReason()); + return AdmissionResult.REJECTED; + } + + long estimatedPayloadBytes = estimate.getBytes(); + // A retained non-null key/value plus Caffeine node can never consume zero bytes. Treat a + // complete zero as an estimator contract violation so an omitted formula cannot bypass + // every quota and admit an unbounded number of zero-weight entries. + if (estimatedPayloadBytes == 0L) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + long newWeight = MetaCacheWeightUtils.saturatedAdd( + estimatedPayloadBytes, FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES); + synchronized (admissionLock) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + if (expectedMutation != null && !isKeyMutationCurrent(key, expectedMutation)) { + return AdmissionResult.NOT_CURRENT; + } + V oldValue = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (expectedReservationGeneration >= 0L + && (record == null || record.generation != expectedReservationGeneration)) { + return AdmissionResult.NOT_CURRENT; + } + if (requireExpected && oldValue != expectedCurrent) { + return AdmissionResult.NOT_CURRENT; + } + if (oldValue == null && record != null) { + reservations.remove(key, record); + record.reservation.release(); + record = null; + } + if (oldValue != null && (record == null || !record.published)) { + rejectWeight("missing_reservation"); + return AdmissionResult.REJECTED; + } + + if (record == null) { + Optional reservation = reserveWithLocalEviction(key, newWeight); + if (!reservation.isPresent()) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, reservation.get(), nextReservationGeneration()); + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + if (reservations.get(key) == newRecord && data.asMap().get(key) == value) { + newRecord.published = true; + notifyReplacement(key, null, value); + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + reservations.remove(key, newRecord); + newRecord.reservation.release(); + throw e; + } + } + + ReservationRecord previousRecord = record; + long reservedWeight = Math.max(previousRecord.weight, newWeight); + if (!resizeWithLocalEviction(key, previousRecord.reservation, reservedWeight)) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, previousRecord.reservation, nextReservationGeneration()); + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + boolean retained = reservations.get(key) == newRecord && data.asMap().get(key) == value; + if (retained) { + newRecord.published = true; + } + if (retained && reservedWeight != newWeight && !newRecord.reservation.tryResize(newWeight)) { + throw new IllegalStateException("failed to release cache replacement reservation delta"); + } + if (retained) { + notifyReplacement(key, oldValue, value); + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + if (reservations.replace(key, newRecord, previousRecord)) { + if (data.asMap().get(key) == null) { + reservations.remove(key, previousRecord); + previousRecord.reservation.release(); + } else if (!previousRecord.reservation.tryResize(previousRecord.weight)) { + throw new IllegalStateException("failed to roll back cache replacement reservation", e); + } + } + throw e; + } + } + } + + private Optional reserveWithLocalEviction(K incomingKey, long bytes) { + if (bytes > entryBudget.getEffectiveMaxWeight()) { + return Optional.empty(); + } + Optional reservation = entryBudget.tryReserve(bytes); + while (!reservation.isPresent()) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + entryBudget.requestPeerReclaim(bytes); + break; + } + reservation = entryBudget.tryReserve(bytes); + } + return reservation; + } + + private boolean resizeWithLocalEviction(K incomingKey, AdmissionReservation reservation, long newBytes) { + if (newBytes > entryBudget.getEffectiveMaxWeight()) { + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + while (true) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + entryBudget.requestPeerReclaim(Math.max(0L, newBytes - reservation.getBytes())); + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + } + } + + private int evictLocalColdest(K incomingKey, int limit) { + if (!data.policy().eviction().isPresent()) { + return 0; + } + Map coldest = data.policy().eviction().get().coldest(limit); + int evicted = 0; + for (Map.Entry candidate : coldest.entrySet()) { + if (Objects.equals(candidate.getKey(), incomingKey)) { + continue; + } + V current = data.asMap().get(candidate.getKey()); + ReservationRecord record = reservations.get(candidate.getKey()); + long evictedWeight = record != null && record.published && current != null ? record.weight : 0L; + if (current == candidate.getValue() && data.asMap().remove(candidate.getKey(), current)) { + if (record != null) { + releaseReservation(candidate.getKey(), record.generation); + } + localEvictionCount.incrementAndGet(); + localEvictionWeight.accumulateAndGet(evictedWeight, MetaCacheWeightUtils::saturatedAdd); + evicted++; + } + } + return evicted; + } + + private long reclaimForPeer(long targetBytes) { + if (targetBytes <= 0L || closed.get()) { + return 0L; + } + synchronized (admissionLock) { + long before = entryBudget.getUsedWeight(); + long reclaimed = 0L; + while (reclaimed < targetBytes + && evictLocalColdest(null, LOCAL_EVICTION_BATCH_SIZE) > 0) { + reclaimed = Math.max(0L, before - entryBudget.getUsedWeight()); + } + return reclaimed; + } + } + + private int weigh(K key, V value) { + ReservationRecord record = reservations.get(key); + // Every supported write path installs the reservation record before calling data.put. + // Missing ownership is an invariant violation, so fail closed without invoking an O(n) + // estimator from Caffeine's hot weigher callback. + long weight = record == null ? Integer.MAX_VALUE : record.weight; + return weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight; + } + + private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { + if (key == null) { + return; + } + if (!weightBounded && !generationFencedRefresh) { + return; + } + if (closed.get()) { + return; + } + // Replacement transfers the existing reservation to the newly published generation. A + // soft-value collection instead reports a null value with COLLECTED and must release it. + if (cause == RemovalCause.REPLACED) { + return; + } + if (Thread.holdsLock(admissionLock)) { + // Other removals have already removed the Caffeine mapping and can release their owner + // inline. A stale callback cannot release a replacement while its mapping is visible. + if (data.asMap().get(key) == null) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null) { + releaseReservation(key, record.generation); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null) { + releaseRefreshRecord(key, record.generation); + } + } + } + return; + } + beforeRemovalOwnerSnapshotForTest(key); + long ownerGeneration = currentOwnerGeneration(key); + if (ownerGeneration >= 0L) { + beforeRemovalReleaseForTest(key); + if (closed.get()) { + return; + } + pendingRemovalGenerations.merge(key, ownerGeneration, Math::max); + if (closed.get()) { + pendingRemovalGenerations.remove(key, ownerGeneration); + return; + } + scheduleRemovalCleanup(); + } + } + + private void scheduleRemovalCleanup() { + if (closed.get()) { + return; + } + if (removalCleanupScheduled.compareAndSet(false, true)) { + try { + REMOVAL_CLEANUP_EXECUTOR.execute(this::drainRemovalCleanups); + } catch (RejectedExecutionException e) { + removalCleanupScheduled.set(false); + LOG.warn("Failed to schedule removal cleanup for external metadata cache entry {}", name, e); + } + } + } + + private void drainRemovalCleanups() { + try { + int processed = 0; + for (Map.Entry cleanup : pendingRemovalGenerations.entrySet()) { + if (processed++ >= REMOVAL_CLEANUP_BATCH_SIZE) { + break; + } + K key = cleanup.getKey(); + long generation = cleanup.getValue(); + // Claim before cleanup. If the same generation is reported again while cleanup + // runs, its notification creates a new pending item instead of being lost when + // this worker finishes. + if (!pendingRemovalGenerations.remove(key, generation)) { + continue; + } + try { + cleanupRemovedReservation(key, generation); + } catch (RuntimeException e) { + // Restore the generation for retry. The finally block requeues one bounded + // drain instead of permanently wedging this entry's scheduled flag. + pendingRemovalGenerations.merge(key, generation, Math::max); + LOG.warn("Failed to clean a removal reservation for external metadata cache entry {}", + name, e); + } + } + } finally { + removalCleanupScheduled.set(false); + if (!closed.get() && !pendingRemovalGenerations.isEmpty()) { + // One bounded task per turn prevents a hot entry from monopolizing the process-wide + // cleanup executor; a later task is queued behind already scheduled catalogs. + scheduleRemovalCleanup(); + } + } + } + + private void cleanupRemovedReservation(K key, long expectedReservationGeneration) { + beforeRemovalCleanupLockForTest(key); + synchronized (admissionLock) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null + && reservations.remove(key, record)) { + record.reservation.release(); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null) { + refreshRecords.remove(key, record); + } + } + } + afterRemovalCleanupForTest(key); + } + + private void releaseReservation(K key, long expectedGeneration) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedGeneration && reservations.remove(key, record)) { + record.reservation.release(); + } + } + + private void releaseRefreshRecord(K key, long expectedGeneration) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedGeneration) { + refreshRecords.remove(key, record); + } + } + + private long currentOwnerGeneration(K key) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + return record == null ? -1L : record.generation; + } + RefreshRecord record = refreshRecords.get(key); + return record == null ? -1L : record.generation; + } + + private void putNonWeightedValue(K key, V value) { + V previousValue = data.asMap().get(key); + RefreshRecord previous = refreshRecords.get(key); + RefreshRecord next = publishRefreshRecord(key); + try { + beforeNonWeightedCachePutForTest(key, value); + data.put(key, value); + if (next != null && refreshRecords.get(key) == next && data.asMap().get(key) == value) { + next.published = true; + } + if (data.asMap().get(key) == value) { + notifyReplacement(key, previousValue, value); + } + } catch (RuntimeException | Error e) { + if (next != null) { + if (previous == null) { + refreshRecords.remove(key, next); + } else { + refreshRecords.replace(key, next, previous); + } + } + throw e; + } + } + + @Nullable + private RefreshRecord publishRefreshRecord(K key) { + if (!generationFencedRefresh) { + return null; + } + RefreshRecord record = new RefreshRecord(nextReservationGeneration()); + refreshRecords.put(key, record); + return record; + } + + private void notifyReplacement(K key, @Nullable V previousValue, V currentValue) { + if (replacementListener == null || previousValue == currentValue) { + return; + } + try { + replacementListener.onReplacement(key, previousValue, currentValue); + } catch (RuntimeException e) { + LOG.warn("Failed to retire dependencies after replacing external metadata cache entry {}", name, e); + } + } + + private long nextReservationGeneration() { + return reservationGeneration.incrementAndGet(); + } + + private void rejectWeight(String reason) { + weightAdmissionRejectedCount.incrementAndGet(); + String normalizedReason = reason == null || reason.isEmpty() ? "unknown" : reason; + lastWeightRejectReason.set(normalizedReason); + long now = System.currentTimeMillis(); + long previous = lastWeightRejectLogTimeMs.get(); + if (now - previous >= WEIGHT_REJECT_LOG_INTERVAL_MS + && lastWeightRejectLogTimeMs.compareAndSet(previous, now)) { + LOG.warn("Rejected external metadata cache admission for entry {}: reason={}, entryUsed={}, " + + "entryMax={}, catalogUsed={}, catalogMax={}, globalUsed={}, globalMax={}", + name, normalizedReason, entryBudget.getUsedWeight(), entryBudget.getEffectiveMaxWeight(), + entryBudget.getCatalogUsedWeight(), entryBudget.getCatalogMaxWeight(), + entryBudget.getGlobalUsedWeight(), entryBudget.getGlobalMaxWeight()); + } + } + + private void maybeRefreshManagedValue(K key, V currentValue) { + if (closed.get() || !generationFencedRefresh || loader == null) { + return; + } + if (!weightBounded) { + maybeRefreshNonWeightedValue(key, currentValue); + return; + } + ReservationRecord record = reservations.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void maybeRefreshNonWeightedValue(K key, V currentValue) { + RefreshRecord record = refreshRecords.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void submitNonWeightedRefresh( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed == null) { + return; + } + synchronized (admissionLock) { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + advanceKeyMutation(key); + putNonWeightedValue(key, refreshed); + } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isRefreshRecordCurrent( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + RefreshRecord record = refreshRecords.get(key); + return record != null && record.generation == expectedRefreshGeneration + && record.published && data.asMap().get(key) != null; + } + + private void submitWeightedRefresh( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isReservationCurrent(key, expectedReservationGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed != null && isKeyMutationCurrent(key, expectedMutation)) { + admitWeightedValue( + key, refreshed, null, false, expectedMutation, + expectedReservationGeneration, true); + // Admission rejection leaves the already reserved, known-good generation + // in place. A larger refresh must not turn a transient quota shortage into + // a forced cache miss for every subsequent reader. + } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isReservationCurrent( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + ReservationRecord record = reservations.get(key); + return record != null && record.generation == expectedReservationGeneration + && record.published && data.asMap().get(key) != null; } // Read the config dynamically so existing cache entries follow runtime config updates. private boolean isManualMissLoadEnabled() { - return Config.enable_external_meta_cache_manual_miss_load; + return weightBounded || generationFencedRefresh || Config.enable_external_meta_cache_manual_miss_load; } // Execute slow miss loads outside Caffeine's sync load path and suppress stale write-back after invalidation. private V getWithManualLoad(K key, Function loadFunction) { - if (!effectiveEnabled) { - // Bypass cache entirely when the entry is disabled so manual miss load does not relax disable semantics. + if (!effectiveEnabled || closed.get()) { + // Disabled and closed entries may still serve the caller, but can not retain the loaded value. return loadAndTrack(key, loadFunction); } V value = data.getIfPresent(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } synchronized (loadLock(key)) { + if (!effectiveEnabled || closed.get()) { + return loadAndTrack(key, loadFunction); + } value = data.asMap().get(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } - long generation = invalidateGeneration.get(); - V loaded = loadAndTrack(key, loadFunction); - if (generation != invalidateGeneration.get()) { - return loaded; - } + KeyMutationToken mutation = beginKeyMutation(key); + try { + V loaded = loadAndTrack(key, loadFunction); + if (!isKeyMutationCurrent(key, mutation)) { + return loaded; + } - // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. - if (loaded == null) { - return null; - } + // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. + if (loaded == null) { + return null; + } - // Leave a narrow hook for tests to pause exactly before the cache put race window. - beforeManualCachePutForTest(key, loaded); - data.put(key, loaded); - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, loaded); + // Leave a narrow hook for tests to pause exactly before the cache put race window. + beforeManualCachePutForTest(key, loaded); + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + if (weightBounded) { + admitWeightedValue(key, loaded, null, false, mutation, -1L, false); + } else { + synchronized (admissionLock) { + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + beforeNonWeightedManualCachePutForTest(key, loaded); + putNonWeightedValue(key, loaded); + } + } + return loaded; + } finally { + endKeyMutation(key, mutation); } - return loaded; } } - // Remove only the value loaded by the current request and keep newer replacements intact. - private void removeLoadedValue(K key, V loaded) { - data.asMap().computeIfPresent(key, (ignored, currentValue) -> currentValue == loaded ? null : currentValue); - } - // Map keys to a fixed lock stripe set to bound memory usage while keeping same-key deduplication. private Object loadLock(K key) { int hash = key == null ? 0 : key.hashCode(); @@ -264,6 +1124,67 @@ private Object loadLock(K key) { void beforeManualCachePutForTest(K key, V loaded) { } + // Let tests pause after the final generation check while holding the admission lock. + void beforeNonWeightedManualCachePutForTest(K key, V loaded) { + } + + // Called inside Caffeine's direct removal callback; tests use it to force the eviction-lock race. + void beforeRemovalReleaseForTest(K key) { + } + + // Let tests pause a callback after Caffeine removal but before reservation-owner lookup. + void beforeRemovalOwnerSnapshotForTest(K key) { + } + + // Called after reservation ownership is published and before Caffeine receives the value. + void beforeWeightedCachePutForTest(K key, V value) { + } + + // Called after refresh ownership is published and before Caffeine receives a count-bounded value. + void beforeNonWeightedCachePutForTest(K key, V value) { + } + + // Called after one asynchronous generation-conditional removal cleanup has finished. + void afterRemovalCleanupForTest(K key) { + } + + // Called immediately before the asynchronous drain tries to acquire admissionLock. + void beforeRemovalCleanupLockForTest(K key) { + } + + // Let tests establish admissionLock -> Caffeine eviction-lock ordering deterministically. + void beforeWeightedInvalidateAllForTest() { + } + + // Invoke the direct-listener branch while holding the mutation lock. + void notifyRemovalUnderAdmissionLockForTest(K key, V value, RemovalCause cause) { + synchronized (admissionLock) { + onRemoval(key, value, cause); + } + } + + // Enqueue the same generation-only refresh task without waiting for the production interval. + void triggerRefreshForTest(K key) { + V current = data.asMap().get(key); + if (current == null || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.published) { + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } else if (generationFencedRefresh) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.published) { + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } + refreshesInFlight.remove(key); + } + private V loadFromDefaultLoader(K key) { return loadAndTrack(key, this::applyDefaultLoader); } @@ -294,4 +1215,100 @@ private V loadAndTrack(K key, Function loadFunction) { throw e; } } + + private KeyMutationToken beginKeyMutation(K key) { + synchronized (admissionLock) { + KeyMutationState state = keyMutationStates.computeIfAbsent(key, ignored -> new KeyMutationState()); + state.inFlight++; + return new KeyMutationToken(state, state.generation, fullInvalidationGeneration.get()); + } + } + + private void advanceKeyMutation(K key) { + // Callers already serialize cache mutation with admissionLock. Keeping the helper lock-free + // prevents accidental deadlock if it is used by a listener reached under that same lock. + KeyMutationState state = keyMutationStates.get(key); + if (state != null) { + state.generation++; + } + } + + private boolean isKeyMutationCurrent(K key, KeyMutationToken token) { + return token.fullInvalidationGeneration == fullInvalidationGeneration.get() + && token.state == keyMutationStates.get(key) + && token.generation == token.state.generation; + } + + private void endKeyMutation(K key, KeyMutationToken token) { + synchronized (admissionLock) { + if (--token.state.inFlight == 0) { + keyMutationStates.remove(key, token.state); + } + } + } + + private static final class ReservationRecord { + private final long weight; + private final long writeNanos; + private final AdmissionReservation reservation; + private final long generation; + private volatile boolean published; + + private ReservationRecord(long weight, AdmissionReservation reservation, long generation) { + this.weight = weight; + this.reservation = reservation; + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class RefreshRecord { + private final long writeNanos; + private final long generation; + private volatile boolean published; + + private RefreshRecord(long generation) { + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class KeyMutationState { + private volatile long generation; + private int inFlight; + } + + private static final class KeyMutationToken { + private final KeyMutationState state; + private final long generation; + private final long fullInvalidationGeneration; + + private KeyMutationToken( + KeyMutationState state, long generation, long fullInvalidationGeneration) { + this.state = state; + this.generation = generation; + this.fullInvalidationGeneration = fullInvalidationGeneration; + } + } + + private static ReplaceResult toReplaceResult(AdmissionResult result) { + switch (result) { + case ADMITTED: + return ReplaceResult.REPLACED; + case NOT_CURRENT: + return ReplaceResult.NOT_CURRENT; + case REJECTED: + return ReplaceResult.REJECTED; + case DISABLED: + default: + return ReplaceResult.DISABLED; + } + } + + private enum AdmissionResult { + ADMITTED, + NOT_CURRENT, + REJECTED, + DISABLED + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 1f48057a44fc40..689d3b6dc7ca99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -101,10 +101,15 @@ public final class MetaCacheEntryDef { private final boolean autoRefresh; private final boolean contextualOnly; private final MetaCacheEntryInvalidation invalidation; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, - MetaCacheEntryInvalidation invalidation) { + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -123,6 +128,8 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.autoRefresh = autoRefresh; this.contextualOnly = contextualOnly; this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); + this.sizeEstimator = sizeEstimator; + this.replacementListener = replacementListener; } /** @@ -142,7 +149,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C public static MetaCacheEntryDef of(String name, Class keyType, Class valueType, Function loader, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, true, false, - invalidation); + invalidation, null, null); } /** @@ -164,7 +171,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, false, - invalidation); + invalidation, null, null); } /** @@ -179,7 +186,22 @@ public static MetaCacheEntryDef contextualOnly( String name, Class keyType, Class valueType, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, null, defaultCacheSpec, false, true, - invalidation); + invalidation, null, null); + } + + /** Return a definition with a publication-time size estimator. */ + public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, + Objects.requireNonNull(estimator, "estimator"), replacementListener); + } + + /** Return a definition that synchronously retires dependencies after a value replacement. */ + public MetaCacheEntryDef withReplacementListener( + MetaCacheEntryReplacementListener listener) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, sizeEstimator, + Objects.requireNonNull(listener, "listener")); } /** @@ -232,4 +254,14 @@ public boolean isContextualOnly() { public MetaCacheEntryInvalidation getInvalidation() { return invalidation; } + + @Nullable + public MetaCacheSizeEstimator getSizeEstimator() { + return sizeEstimator; + } + + @Nullable + public MetaCacheEntryReplacementListener getReplacementListener() { + return replacementListener; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java new file mode 100644 index 00000000000000..1bfb0cf3990962 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import javax.annotation.Nullable; + +/** Receives a successfully published value while the entry mutation is still serialized. */ +@FunctionalInterface +public interface MetaCacheEntryReplacementListener { + void onReplacement(K key, @Nullable V previousValue, V currentValue); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java index 495fd011083bb0..433cc027515873 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java @@ -51,6 +51,16 @@ public final class MetaCacheEntryStats { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final boolean weightBounded; + private final long maxWeight; + private final long estimatedWeight; + private final long evictionWeight; + private final long weightAdmissionRejectedCount; + private final long catalogMaxWeight; + private final long catalogEstimatedWeight; + private final long globalMaxWeight; + private final long globalEstimatedWeight; + private final String lastWeightRejectReason; /** * Build an immutable stats snapshot. @@ -74,7 +84,17 @@ public MetaCacheEntryStats( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + boolean weightBounded, + long maxWeight, + long estimatedWeight, + long evictionWeight, + long weightAdmissionRejectedCount, + long catalogMaxWeight, + long catalogEstimatedWeight, + long globalMaxWeight, + long globalEstimatedWeight, + String lastWeightRejectReason) { this.configEnabled = configEnabled; this.effectiveEnabled = effectiveEnabled; this.autoRefresh = autoRefresh; @@ -94,6 +114,16 @@ public MetaCacheEntryStats( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = Objects.requireNonNull(lastError, "lastError"); + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.evictionWeight = evictionWeight; + this.weightAdmissionRejectedCount = weightAdmissionRejectedCount; + this.catalogMaxWeight = catalogMaxWeight; + this.catalogEstimatedWeight = catalogEstimatedWeight; + this.globalMaxWeight = globalMaxWeight; + this.globalEstimatedWeight = globalEstimatedWeight; + this.lastWeightRejectReason = Objects.requireNonNull(lastWeightRejectReason, "lastWeightRejectReason"); } public boolean isConfigEnabled() { @@ -186,4 +216,44 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public boolean isWeightBounded() { + return weightBounded; + } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getEvictionWeight() { + return evictionWeight; + } + + public long getWeightAdmissionRejectedCount() { + return weightAdmissionRejectedCount; + } + + public long getCatalogMaxWeight() { + return catalogMaxWeight; + } + + public long getCatalogEstimatedWeight() { + return catalogEstimatedWeight; + } + + public long getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalEstimatedWeight() { + return globalEstimatedWeight; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java new file mode 100644 index 00000000000000..d44702ce559212 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; + +/** + * Immutable result value returned by {@link MetaCacheSizeEstimator}; this class is not an + * estimator implementation. An incomplete result carries no usable byte count and must fail + * cache admission closed. + */ +public final class MetaCacheSizeEstimate { + private final long bytes; + private final boolean complete; + private final String incompleteReason; + + private MetaCacheSizeEstimate(long bytes, boolean complete, String incompleteReason) { + this.bytes = bytes; + this.complete = complete; + this.incompleteReason = Objects.requireNonNull(incompleteReason, "incompleteReason"); + } + + public static MetaCacheSizeEstimate complete(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache size estimate can not be negative: " + bytes); + } + return new MetaCacheSizeEstimate(bytes, true, ""); + } + + public static MetaCacheSizeEstimate incomplete(String reason) { + String safeReason = Objects.requireNonNull(reason, "reason").trim(); + if (safeReason.isEmpty()) { + throw new IllegalArgumentException("incomplete cache size estimate requires a reason"); + } + return new MetaCacheSizeEstimate(0L, false, safeReason); + } + + public long getBytes() { + return bytes; + } + + public boolean isComplete() { + return complete; + } + + public String getIncompleteReason() { + return incompleteReason; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..5aff6a2a1a279a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Supplies the admission weight of one key/value pair. + * + *

    The callback runs once after load and before admission. Implementations may linearly count + * loader-owned collections needed to cover skewed payloads, but must not recursively reflect over + * arbitrary object graphs, perform additional IO, materialize lazy SDK state, or copy payloads + * solely to estimate weight. Caffeine's weigher reads only the admitted reservation record, so + * cache hits and eviction remain O(1). + */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + MetaCacheSizeEstimate estimate(K key, V value); + + /** Convert preparation failures into fail-closed incomplete estimates. */ + static MetaCacheSizeEstimate estimateSafely( + String failureReason, Supplier estimation) { + Objects.requireNonNull(failureReason, "failureReason"); + Objects.requireNonNull(estimation, "estimation"); + try { + return Objects.requireNonNull(estimation.get(), "size estimate"); + } catch (RuntimeException e) { + return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java new file mode 100644 index 00000000000000..7869ac5993f48e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.NameMapping; + +/** Constant-time helpers for conservative external metadata cache weights. */ +public final class MetaCacheWeightUtils { + private static final long STRING_BASE_BYTES = 40L; + private static final long STRING_BYTES_PER_CHARACTER = 2L; + private static final long NAME_MAPPING_BASE_BYTES = 64L; + + private MetaCacheWeightUtils() { + } + + /** + * Estimate a String without inspecting its contents. Two bytes per character deliberately + * avoids depending on CompactStrings or VM-private layout details. + */ + public static long estimatedStringBytes(String value) { + return estimatedCharSequenceBytes(value); + } + + /** Estimate retained character data without materializing a String copy. */ + public static long estimatedCharSequenceBytes(CharSequence value) { + return value == null ? 0L : saturatedAdd( + STRING_BASE_BYTES, saturatedMultiply(value.length(), STRING_BYTES_PER_CHARACTER)); + } + + /** Estimate the fixed set of names retained by a cache key. */ + public static long estimatedNameMappingBytes(NameMapping nameMapping) { + if (nameMapping == null) { + return 0L; + } + long bytes = NAME_MAPPING_BASE_BYTES; + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalDbName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalTblName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteDbName())); + return saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteTblName())); + } + + public static long saturatedAdd(long left, long right) { + if (left < 0L || right < 0L || Long.MAX_VALUE - left < right) { + return Long.MAX_VALUE; + } + return left + right; + } + + public static long saturatedMultiply(long left, long right) { + if (left < 0L || right < 0L || (left != 0L && right > Long.MAX_VALUE / left)) { + return Long.MAX_VALUE; + } + return left * right; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java index b6f36b803b24cc..d3088942ace261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java @@ -44,7 +44,8 @@ public final class PaimonLatestSnapshotProjectionLoader { @FunctionalInterface public interface SchemaValueLoader { - PaimonSchemaCacheValue load(NameMapping nameMapping, long schemaId); + PaimonSchemaCacheValue load( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable); } private final PaimonPartitionInfoLoader partitionInfoLoader; @@ -59,7 +60,8 @@ public PaimonLatestSnapshotProjectionLoader(PaimonPartitionInfoLoader partitionI public PaimonSnapshotCacheValue load(NameMapping nameMapping, Table paimonTable) { try { PaimonSnapshot latestSnapshot = resolveLatestSnapshot(paimonTable, true); - List partitionColumns = schemaValueLoader.load(nameMapping, latestSnapshot.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, latestSnapshot.getSchemaId(), 0L, latestSnapshot.getTable()) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, latestSnapshot.getTable(), partitionColumns); @@ -85,11 +87,21 @@ public PaimonSnapshotCacheValue loadFence(NameMapping nameMapping, Table paimonT } public PaimonSnapshotCacheValue loadAtFence(NameMapping nameMapping, PaimonSnapshot fence) { - return loadEffectiveAtFence(nameMapping, fence.getTable(), fence); + return loadAtFence(nameMapping, fence, 0L); + } + + public PaimonSnapshotCacheValue loadAtFence( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return loadEffectiveAtFence(nameMapping, fence.getTable(), fence, tableGeneration); } public PaimonSnapshotCacheValue loadEffectiveAtFence( NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence) { + return loadEffectiveAtFence(nameMapping, effectiveTable, fence, 0L); + } + + public PaimonSnapshotCacheValue loadEffectiveAtFence( + NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence, long tableGeneration) { try { // The fence owns both version and table generation. Reopening the catalog here can pair // the old snapshot id with a newer schema or branch after invalidation. @@ -102,12 +114,14 @@ public PaimonSnapshotCacheValue loadEffectiveAtFence( latestSchemaTable.copyWithoutTimeTravel( PaimonScanParams.isolateSnapshotRead(fence.getSnapshotId()))); } - List partitionColumns = schemaValueLoader.load(nameMapping, fence.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, fence.getSchemaId(), tableGeneration, effectiveTable) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, snapshotTable, partitionColumns); return new PaimonSnapshotCacheValue(partitionInfo, - new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable)); + new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable), + false, tableGeneration); } catch (Exception e) { throw new CacheException("failed to load paimon snapshot at fence %s.%s.%s: %s", e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java index 0a134cfd7d7d32..fc9fbeaa755752 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java @@ -25,6 +25,7 @@ import org.apache.paimon.table.Table; import java.io.IOException; +import java.util.concurrent.Callable; /** * Loads the base Paimon table handle used by cache entries and runtime projections. @@ -45,4 +46,14 @@ public PaimonExternalCatalog catalog(NameMapping nameMapping) throws IOException return (PaimonExternalCatalog) Env.getCurrentEnv().getCatalogMgr() .getCatalogOrException(nameMapping.getCtlId(), id -> new IOException("Catalog not found: " + id)); } + + public T executeAuthenticated(NameMapping nameMapping, Callable task) { + try { + return catalog(nameMapping).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new CacheException("failed to load authenticated paimon metadata %s.%s.%s: %s", + e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), + e.getMessage()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java new file mode 100644 index 00000000000000..b5a6178538a9c4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.privilege.PrivilegedFileStoreTable; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; + +import java.util.List; +import java.util.Map; + +/** Constant-time retained-weight formula for Paimon snapshot projections. */ +final class PaimonCacheSizeEstimator { + private static final long KEY_BASE_BYTES = 128L; + private static final long SNAPSHOT_BASE_BYTES = 4L * 1024L; + private static final long TABLE_BASE_BYTES = 16L * 1024L; + private static final long TABLE_FIELD_BYTES = 3584L; + private static final long TABLE_OPTION_BYTES = 256L; + private static final long TABLE_KEY_BYTES = 128L; + private static final long NESTED_FIELD_BYTES = 512L; + private static final long PARTITION_BYTES = 1280L; + private static final long PARTITION_ITEM_BYTES = 1024L; + private static final long WRAPPER_BYTES = 512L; + + private PaimonCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + Table table = value.getSnapshot().getTable(); + if (!isSupportedTable(table)) { + return MetaCacheSizeEstimate.incomplete("unsupported_paimon_table:" + + (table == null ? "null" : table.getClass().getName())); + } + + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getPartitionInfo().getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); + } + + private static boolean isSupportedTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return isSupportedTable(((PrivilegedFileStoreTable) table).wrapped()); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return isSupportedTable(fallback.wrapped()) && isSupportedTable(fallback.other()); + } + if (!(table instanceof FileStoreTable)) { + return false; + } + String className = table.getClass().getName(); + return "org.apache.paimon.table.AppendOnlyFileStoreTable".equals(className) + || "org.apache.paimon.table.PrimaryKeyFileStoreTable".equals(className); + } + + /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ + private static long estimateTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, + estimateTable(((PrivilegedFileStoreTable) table).wrapped())); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + long bytes = MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, estimateTable(fallback.wrapped())); + return MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(fallback.other())); + } + + FileStoreTable fileStoreTable = (FileStoreTable) table; + TableSchema schema = fileStoreTable.schema(); + long bytes = TABLE_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(fileStoreTable.location().toString())); + bytes = addCount(bytes, schema.fields().size(), TABLE_FIELD_BYTES); + bytes = addCount(bytes, schema.options().size(), TABLE_OPTION_BYTES); + bytes = addCount(bytes, schema.partitionKeys().size(), TABLE_KEY_BYTES); + bytes = addCount(bytes, schema.primaryKeys().size(), TABLE_KEY_BYTES); + return addCount(bytes, schema.bucketKeys().size(), TABLE_KEY_BYTES); + } + + /** + * Captures skew-sensitive schema text once when the snapshot cache value is constructed. + * All collections are already materialized in TableSchema; this never opens the table store. + */ + static long retainedTablePayloadBytes(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return retainedTablePayloadBytes(((PrivilegedFileStoreTable) table).wrapped()); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return MetaCacheWeightUtils.saturatedAdd( + retainedTablePayloadBytes(fallback.wrapped()), + retainedTablePayloadBytes(fallback.other())); + } + if (!(table instanceof FileStoreTable)) { + return 0L; + } + + TableSchema schema = ((FileStoreTable) table).schema(); + if (schema == null) { + return 0L; + } + long bytes = addString(0L, schema.comment()); + for (DataField field : schema.fields()) { + bytes = addFieldPayload(bytes, field, false); + } + for (Map.Entry option : schema.options().entrySet()) { + bytes = addString(bytes, option.getKey()); + bytes = addString(bytes, option.getValue()); + } + bytes = addStrings(bytes, schema.partitionKeys()); + bytes = addStrings(bytes, schema.primaryKeys()); + return addStrings(bytes, schema.bucketKeys()); + } + + private static long addStrings(long bytes, List values) { + for (String value : values) { + bytes = addString(bytes, value); + } + return bytes; + } + + private static long addFieldPayload(long bytes, DataField field, boolean nested) { + if (nested) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + } + bytes = addString(bytes, field.name()); + bytes = addString(bytes, field.description()); + bytes = addString(bytes, field.defaultValue()); + return addTypePayload(bytes, field.type()); + } + + private static long addTypePayload(long bytes, DataType type) { + if (type instanceof RowType) { + for (DataField field : ((RowType) type).getFields()) { + bytes = addFieldPayload(bytes, field, true); + } + } else if (type instanceof ArrayType) { + bytes = addTypePayload(bytes, ((ArrayType) type).getElementType()); + } else if (type instanceof MapType) { + bytes = addTypePayload(bytes, ((MapType) type).getKeyType()); + bytes = addTypePayload(bytes, ((MapType) type).getValueType()); + } else if (type instanceof MultisetType) { + bytes = addTypePayload(bytes, ((MultisetType) type).getElementType()); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index cde2bbb31efd18..a308896bb299e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -23,6 +23,8 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; @@ -33,6 +35,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; +import javax.annotation.Nullable; /** * Paimon engine implementation of {@link AbstractExternalMetaCache}. @@ -40,36 +43,49 @@ *

    Registered entries: *

      *
    • {@code table}: loaded Paimon table handle per table mapping
      • + *
    • {@code snapshot}: immutable partition projection keyed by a captured snapshot/schema fence
      • *
    • {@code schema}: schema cache keyed by table identity + schema id
      • *
    * - *

    Latest snapshot metadata is modeled as a runtime projection memoized inside the table cache - * value instead of as an independent cache entry. + *

    The latest main-branch snapshot is captured once as a fence and loaded through an independent + * contextual entry. Branch/tag/options projections remain statement-local and are not aliased to + * this main-snapshot key. * *

    Invalidation behavior: *

      - *
    • db/table invalidation clears table and schema entries by matching local names
      • + *
    • db/table invalidation clears table, snapshot and schema entries by matching local names
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ public class PaimonExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "paimon"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_SCHEMA = "schema"; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle schemaEntry; private final PaimonTableLoader tableLoader; private final PaimonLatestSnapshotProjectionLoader latestSnapshotProjectionLoader; public PaimonExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableLoader = new PaimonTableLoader(); latestSnapshotProjectionLoader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withReplacementListener(this::retireTableGeneration)); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, PaimonSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSchemaCacheKey::getNameMapping))); @@ -86,41 +102,76 @@ public Table getPaimonTable(NameMapping nameMapping) { public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()).getSnapshot(); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + nameMapping, fence, tableValue.getGeneration()); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + PaimonSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence( + nameMapping, fence, tableValue.getGeneration()))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableValue.getGeneration()) { + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { - return latestSnapshotProjectionLoader.load(dorisTable.getOrBuildNameMapping(), effectiveTable); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.load(nameMapping, effectiveTable)); } public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - Table table = tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getPaimonTable(); - return latestSnapshotProjectionLoader.loadFence(nameMapping, table); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, PaimonSnapshot fence) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, Table effectiveTable, PaimonSnapshot fence) { - return latestSnapshotProjectionLoader.loadEffectiveAtFence( - dorisTable.getOrBuildNameMapping(), effectiveTable, fence); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadEffectiveAtFence( + nameMapping, effectiveTable, fence)); } public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, long schemaId) { - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()) - .get(new PaimonSchemaCacheKey(nameMapping, schemaId)); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getPaimonSchemaCacheValue( + nameMapping, schemaId, tableValue.getGeneration(), tableValue.getPaimonTable()); + } + + PaimonSchemaCacheValue getPaimonSchemaCacheValue( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable) { + PaimonSchemaCacheKey key = new PaimonSchemaCacheKey(nameMapping, tableGeneration, schemaId); + if (tableGeneration <= 0L) { + return (PaimonSchemaCacheValue) executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable)); + } + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableGeneration) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } return (PaimonSchemaCacheValue) schemaCacheValue; } private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - Table paimonTable = tableLoader.load(nameMapping); - return new PaimonTableCacheValue(paimonTable, - () -> latestSnapshotProjectionLoader.load(nameMapping, paimonTable)); + return new PaimonTableCacheValue(tableLoader.load(nameMapping)); } private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @@ -131,8 +182,47 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } + private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + if (!(dorisTable instanceof PaimonExternalTable)) { + return loadSchemaCacheValue(key); + } + dorisTable.setUpdateTime(System.currentTimeMillis()); + return ((PaimonExternalTable) dorisTable).loadSchemaForCache(retainedTable, key.getSchemaId()); + } + + private PaimonSnapshotCacheValue loadLatestSnapshotFence(NameMapping nameMapping, Table retainedTable) { + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadFence(nameMapping, retainedTable)); + } + + private T executeAuthenticated(NameMapping nameMapping, java.util.concurrent.Callable task) { + return tableLoader.executeAuthenticated(nameMapping, task); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable PaimonTableCacheValue previousValue, PaimonTableCacheValue currentValue) { + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.paimon.table.enable", "meta.cache.paimon.snapshot.enable"); + compatibility.put("meta.cache.paimon.table.ttl-second", "meta.cache.paimon.snapshot.ttl-second"); + compatibility.put("meta.cache.paimon.table.capacity", "meta.cache.paimon.snapshot.capacity"); + return compatibility; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index d0a3c858f4b847..defa3dd4d00ec5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -490,6 +490,14 @@ private PaimonSchemaCacheValue loadSchema(DataTable table, long schemaId) { return loadSchema(table.schemaManager().schema(schemaId)); } + PaimonSchemaCacheValue loadSchemaForCache(Table retainedTable, long schemaId) { + if (!(retainedTable instanceof DataTable)) { + throw new CacheException("retained paimon table does not expose schema history: %s", + null, retainedTable == null ? "null" : retainedTable.getClass().getName()); + } + return loadSchema((DataTable) retainedTable, schemaId); + } + private PaimonSchemaCacheValue loadSchema(TableSchema tableSchema) { List columns = tableSchema.fields(); List dorisColumns = Lists.newArrayListWithCapacity(columns.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 207810b66f5a68..004adc197061b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.paimon.partition.Partition; @@ -49,18 +50,26 @@ public enum PruningStatus { private final PruningStatus pruningStatus; private final Map nameToPartitionItem; private final Map nameToPartition; + private final long retainedPayloadBytes; private PaimonPartitionInfo(PruningStatus pruningStatus) { this.pruningStatus = pruningStatus; this.nameToPartitionItem = Collections.emptyMap(); this.nameToPartition = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public PaimonPartitionInfo(Map nameToPartitionItem, Map nameToPartition) { + this(nameToPartitionItem, nameToPartition, retainedPayloadBytes(nameToPartition)); + } + + public PaimonPartitionInfo(Map nameToPartitionItem, + Map nameToPartition, long retainedPayloadBytes) { this.pruningStatus = PruningStatus.PRUNABLE; this.nameToPartitionItem = nameToPartitionItem; this.nameToPartition = nameToPartition; + this.retainedPayloadBytes = retainedPayloadBytes; } public Map getNameToPartitionItem() { @@ -74,4 +83,47 @@ public Map getNameToPartition() { public PruningStatus getPruningStatus() { return pruningStatus; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + static long addRetainedStringPayload(long bytes, String value) { + return addString(bytes, value); + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (Map.Entry entry : partitions.entrySet()) { + bytes = addString(bytes, entry.getKey()); + Partition partition = entry.getValue(); + if (partition == null) { + continue; + } + bytes = addStrings(bytes, partition.spec()); + bytes = addString(bytes, partition.createdBy()); + bytes = addString(bytes, partition.updatedBy()); + bytes = addStrings(bytes, partition.options()); + } + return bytes; + } + + private static long addStrings(long bytes, Map values) { + if (values == null) { + return bytes; + } + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java index 4eccb269c2fe56..49d5847e0e0469 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java @@ -23,13 +23,23 @@ import com.google.common.base.Objects; public class PaimonSchemaCacheKey extends SchemaCacheKey { + private final long tableGeneration; private final long schemaId; public PaimonSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, 0L, schemaId); + } + + public PaimonSchemaCacheKey(NameMapping nameMapping, long tableGeneration, long schemaId) { super(nameMapping); + this.tableGeneration = tableGeneration; this.schemaId = schemaId; } + public long getTableGeneration() { + return tableGeneration; + } + public long getSchemaId() { return schemaId; } @@ -46,11 +56,11 @@ public boolean equals(Object o) { return false; } PaimonSchemaCacheKey that = (PaimonSchemaCacheKey) o; - return schemaId == that.schemaId; + return tableGeneration == that.tableGeneration && schemaId == that.schemaId; } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableGeneration, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index 37be7c6a5f3585..e6b37d4c020b72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -17,21 +17,33 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; private final PaimonSnapshot snapshot; private final boolean schemaFromSnapshotTable; + private final long tableGeneration; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { - this(partitionInfo, snapshot, false); + this(partitionInfo, snapshot, false, 0L); } public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, boolean schemaFromSnapshotTable) { + this(partitionInfo, snapshot, schemaFromSnapshotTable, 0L); + } + + public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, + boolean schemaFromSnapshotTable, long tableGeneration) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; this.schemaFromSnapshotTable = schemaFromSnapshotTable; + this.tableGeneration = tableGeneration; } public PaimonPartitionInfo getPartitionInfo() { @@ -45,4 +57,29 @@ public PaimonSnapshot getSnapshot() { public boolean isSchemaFromSnapshotTable() { return schemaFromSnapshotTable; } + + public long getTableGeneration() { + return tableGeneration; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + MetaCacheSizeEstimate prepareForCachePublication(PaimonSnapshotEntryKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_snapshot_preparation_failed", + () -> { + retainedTablePayloadBytes = + PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); + return PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java new file mode 100644 index 00000000000000..d820d3c15e992b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.NameMapping; + +import java.util.Objects; + +/** Stable identity for a Paimon projection hydrated from one captured snapshot/schema fence. */ +public final class PaimonSnapshotEntryKey { + private final NameMapping nameMapping; + private final long snapshotId; + private final long schemaId; + private final long tableGeneration; + + public PaimonSnapshotEntryKey( + NameMapping nameMapping, long snapshotId, long schemaId, long tableGeneration) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.tableGeneration = tableGeneration; + } + + public static PaimonSnapshotEntryKey of( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return new PaimonSnapshotEntryKey( + nameMapping, fence.getSnapshotId(), fence.getSchemaId(), tableGeneration); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public long getSnapshotId() { + return snapshotId; + } + + public long getSchemaId() { + return schemaId; + } + + public long getTableGeneration() { + return tableGeneration; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PaimonSnapshotEntryKey)) { + return false; + } + PaimonSnapshotEntryKey that = (PaimonSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && tableGeneration == that.tableGeneration + && nameMapping.equals(that.nameMapping); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, snapshotId, schemaId, tableGeneration); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index 7539f28d770bf6..9e381602dcbc9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -17,28 +17,37 @@ package org.apache.doris.datasource.paimon; -import com.google.common.base.Suppliers; import org.apache.paimon.table.Table; -import java.util.function.Supplier; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; /** - * Cache value for Paimon table metadata and its latest runtime snapshot projection. + * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry so this + * value cannot grow after admission. */ public class PaimonTableCacheValue { + private static final AtomicLong NEXT_GENERATION = new AtomicLong(); + private final Table paimonTable; - private final Supplier latestSnapshotCacheValue; + private final long generation; - public PaimonTableCacheValue(Table paimonTable, Supplier latestSnapshotCacheValue) { + public PaimonTableCacheValue(Table paimonTable) { this.paimonTable = paimonTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + this.generation = NEXT_GENERATION.incrementAndGet(); + } + + public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue ignoredFence) { + this(paimonTable); + Objects.requireNonNull(ignoredFence, "latestSnapshotFence can not be null"); } public Table getPaimonTable() { return paimonTable; } - public PaimonSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + public long getGeneration() { + return generation; } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index eaa59b28c2f4c1..a1eaf3100cdc28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -176,6 +176,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List candidates = Lists.newArrayListWithExpectedSize(partitionEntries.size()); Map> displayNameToTypedSpec = Maps.newHashMap(); + long retainedPayloadBytes = 0L; for (PartitionEntry partitionEntry : partitionEntries) { Map typedSpec = getPartitionInfoMap( @@ -193,6 +194,10 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List previousTypedSpec = displayNameToTypedSpec.putIfAbsent( displayName, orderedTypedSpec); if (previousTypedSpec != null) { @@ -247,7 +254,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List 0L) { + return paimonExternalMetaCache(dorisTable).getPaimonSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), snapshotValue.getSnapshot().getSchemaId(), + snapshotValue.getTableGeneration(), snapshotValue.getSnapshot().getTable()); + } return getSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index b9e84c076905d5..607dde99476a5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -33,6 +33,7 @@ import mockit.MockUp; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.HashMap; @@ -40,6 +41,12 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; public class ExternalMetaCacheRouteResolverTest { @@ -51,6 +58,55 @@ public void testEngineAliasCompatibility() { Assert.assertEquals("maxcompute", metaCacheMgr.engine("max_compute").engine()); } + @Test + public void testCatalogCachePropertiesRejectUnknownEngineEntryAndAliasNamespace() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = new HashMap<>(); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 1L, "hms", null, Collections.emptyMap(), ""); + + properties.put("meta.cache.hvie.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + properties.clear(); + + properties.put("meta.cache.hms.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + properties.clear(); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + } + + @Test + public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = Collections.singletonMap( + "meta.cache.hive.partition_values.capacity", "10"); + PaimonExternalCatalog catalog = new PaimonExternalCatalog( + 1L, "paimon", null, Collections.emptyMap(), ""); + + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); + } + + @Test + public void testRuntimePreparationIgnoresInvalidPersistedCacheProperties() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = new HashMap<>(); + properties.put("meta.cache.max-weight", "1.5GB"); + properties.put("meta.cache.hive.partition_values.enable", "1"); + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + + metaCacheMgr.prepareCatalogByEngine(101L, "hive", properties); + + Assert.assertFalse(metaCacheMgr.getCatalogCacheStats(101L).isEmpty()); + metaCacheMgr.removeCatalog(101L); + } + @Test public void testRouteByCatalogType() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); @@ -110,6 +166,83 @@ public void testPrepareCatalogByEngineSkipsMissingCatalog() throws Exception { Assert.assertEquals(0, hive.initCatalogCalls); } + @Test + public void testPreparedEngineUsesLockFreeFastPath() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> true); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + long catalogId = 12L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + + Assert.assertEquals(1, hive.initCatalogCalls); + } + + @Test + public void testCatalogRemovalFencesInFlightFirstInitialization() throws Exception { + CountDownLatch initializationEntered = new CountDownLatch(1); + CountDownLatch releaseInitialization = new CountDownLatch(1); + BlockingRecordingExternalMetaCache hive = new BlockingRecordingExternalMetaCache( + "hive", initializationEntered, releaseInitialization); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + ExecutorService workers = Executors.newFixedThreadPool(2); + long catalogId = 13L; + Map oldProperties = Collections.singletonMap("generation", "old"); + Map newProperties = Collections.singletonMap("generation", "new"); + try { + Future initialization = workers.submit( + () -> metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", oldProperties)); + Assert.assertTrue(initializationEntered.await(3L, TimeUnit.SECONDS)); + + CountDownLatch removalStarted = new CountDownLatch(1); + Future removal = workers.submit(() -> { + removalStarted.countDown(); + metaCacheMgr.removeCatalogByEngine(catalogId, "hive"); + }); + Assert.assertTrue(removalStarted.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("removal must wait for the property snapshot publication", removal.isDone()); + + releaseInitialization.countDown(); + initialization.get(3L, TimeUnit.SECONDS); + removal.get(3L, TimeUnit.SECONDS); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", newProperties); + Assert.assertEquals("new", hive.lastCatalogProperties.get("generation")); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + } finally { + releaseInitialization.countDown(); + workers.shutdownNow(); + } + } + + @Test + public void testRollbackRetiresGroupInitializedFromRejectedCandidate() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + long catalogId = 14L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + Map oldProperties = Collections.singletonMap("generation", "old"); + + metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties); + + Mockito.verify(catalog).rollBackCatalogProps(oldProperties); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + Assert.assertEquals(1, hive.invalidateCatalogCalls); + } + @Test public void testGetSchemaCacheValueReturnsEmptyWhenCatalogMissing() throws Exception { MissingCatalogSchemaExternalMetaCache schemaCache = new MissingCatalogSchemaExternalMetaCache("default"); @@ -373,4 +506,35 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class throw new IllegalStateException("catalog " + catalogId + " is not initialized"); } } + + private static final class BlockingRecordingExternalMetaCache extends RecordingExternalMetaCache { + private final CountDownLatch initializationEntered; + private final CountDownLatch releaseInitialization; + private final AtomicBoolean blockNextInitialization = new AtomicBoolean(true); + private Map lastCatalogProperties = Collections.emptyMap(); + + private BlockingRecordingExternalMetaCache(String engine, + CountDownLatch initializationEntered, CountDownLatch releaseInitialization) { + super(engine, Collections.emptyList(), catalog -> true); + this.initializationEntered = initializationEntered; + this.releaseInitialization = releaseInitialization; + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (blockNextInitialization.compareAndSet(true, false)) { + initializationEntered.countDown(); + try { + if (!releaseInitialization.await(3L, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to publish catalog initialization"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + lastCatalogProperties = new HashMap<>(catalogProperties); + super.initCatalog(catalogId, catalogProperties); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index 151d46252d9084..df24c0c754d6a2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -17,23 +17,49 @@ package org.apache.doris.datasource.hive; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import com.google.common.collect.HashBiMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; public class HiveMetaStoreCacheTest { + @Test + public void testPartitionValueWeightScalesLinearlyToOneHundredThousandPartitions() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.singletonList(Type.STRING)); + long base = partitionValueWeight(key, 0); + long oneThousand = partitionValueWeight(key, 1_000); + long tenThousand = partitionValueWeight(key, 10_000); + long oneHundredThousand = partitionValueWeight(key, 100_000); + + long oneThousandPayload = oneThousand - base; + Assertions.assertTrue(oneThousandPayload > 0L); + Assertions.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assertions.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + @Test public void testInvalidateTableCache() { ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( @@ -144,6 +170,103 @@ public void testInvalidatePartitionCacheClearsStaleFileCacheOnPartitionMiss() { } } + @Test + public void testPartitionValuesEstimateIsPreparedAgainAfterCopy() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.emptyList()); + PartitionKey partitionKey = new PartitionKey(); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + partitionItem.setDefaultPartition(true); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("p", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, Collections.emptyList()); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + Assertions.assertTrue(values.getSizeEstimate().isComplete()); + Assertions.assertTrue(values.getSizeEstimate().getBytes() > 0L); + + HiveExternalMetaCache.HivePartitionValues copy = values.mutableCopy(); + Assertions.assertFalse(copy.getSizeEstimate().isComplete()); + copy.sealForPublication(); + copy.prepareSizeEstimate(new HiveExternalMetaCache.PartitionValueCacheKey( + key.getNameMapping(), null)); + Assertions.assertTrue(copy.getSizeEstimate().isComplete()); + Assertions.assertTrue(copy.getSizeEstimate().getBytes() > 0L); + ListPartitionItem publishedItem = (ListPartitionItem) values.getIdToPartitionItem().get(1L); + Assertions.assertSame(partitionItem, publishedItem, + "cache publication must not rewrite common catalog partition objects"); + Assertions.assertSame(partitionKey, publishedItem.getItems().get(0)); + } + + @Test + public void testPartitionValuesEstimateSupportsRealLiteralGraph() throws Exception { + List types = java.util.Arrays.asList(Type.STRING, Type.INT, Type.DATEV2, Type.DECIMALV2); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + java.util.Arrays.asList( + new PartitionValue("tail-value"), + new PartitionValue("42"), + new PartitionValue("2026-08-12"), + new PartitionValue("123456789.0123")), + types, true); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("s=tail-value/i=42/d=2026-08-12/n=123456789.0123", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, java.util.Arrays.asList( + "tail-value", "42", "2026-08-12", "123456789.0123")); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + + Assertions.assertTrue(values.getSizeEstimate().isComplete(), + values.getSizeEstimate().getIncompleteReason()); + long estimatedBytes = values.getSizeEstimate().getBytes(); + PartitionKey publishedKey = ((ListPartitionItem) values.getIdToPartitionItem().get(1L)).getItems().get(0); + StringLiteral publishedString = (StringLiteral) publishedKey.getKeys().get(0); + // Exercise normal read-only lazy paths after publication. Their bounded memoized state is + // covered by estimator headroom without changing or cloning common expression classes. + publishedString.getExprName(); + values.getSortedPartitionRanges().orElseThrow(AssertionError::new).sortedPartitions + .forEach(partition -> partition.range.toString()); + values.prepareSizeEstimate(key); + Assertions.assertEquals(estimatedBytes, values.getSizeEstimate().getBytes()); + Assertions.assertSame(partitionKey, publishedKey, + "cache publication must not rewrite common catalog partition objects"); + } + + @Test + public void testPartitionValuesFormulaAgainstJolOwnedGraph() throws Exception { + List types = Collections.singletonList(Type.STRING); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + HiveExternalMetaCache.HivePartitionValues empty = realPartitionValues(types, 0, 16); + HiveExternalMetaCache.HivePartitionValues populated = realPartitionValues(types, 32, 16); + HiveExternalMetaCache.HivePartitionValues shortTail = realPartitionValues(types, 1, 16); + HiveExternalMetaCache.HivePartitionValues longTail = realPartitionValues(types, 1, 4096); + + long emptyEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, empty).getBytes(); + long populatedEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, populated).getBytes(); + long shortTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, shortTail).getBytes(); + long longTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, longTail).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive partition values", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + private void putCache( MetaCacheEntry fileCache, MetaCacheEntry partitionCache, @@ -178,4 +301,42 @@ private long entrySize(MetaCacheEntry entry) { entry.forEach((k, v) -> count.incrementAndGet()); return count.get(); } + + private long partitionValueWeight( + HiveExternalMetaCache.PartitionValueCacheKey key, int partitionCount) { + Map items = sizeOnlyMap(partitionCount); + HiveExternalMetaCache.HivePartitionValues values = + new HiveExternalMetaCache.HivePartitionValues( + items, null, null, partitionCount * 16L, 1); + MetaCacheSizeEstimate estimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, values); + Assertions.assertTrue(estimate.isComplete(), estimate.getIncompleteReason()); + return estimate.getBytes(); + } + + private HiveExternalMetaCache.HivePartitionValues realPartitionValues( + List types, int partitionCount, int valueLength) throws Exception { + Map items = new HashMap<>(); + HashBiMap names = HashBiMap.create(); + Map> values = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = "p" + index + String.join("", Collections.nCopies(valueLength, "x")); + long id = index + 1L; + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + Collections.singletonList(new PartitionValue(value)), types, true); + items.put(id, new ListPartitionItem(Collections.singletonList(partitionKey))); + names.put("p=" + value, id); + values.put(id, Collections.singletonList(value)); + } + HiveExternalMetaCache.HivePartitionValues result = + new HiveExternalMetaCache.HivePartitionValues(items, names, values); + result.sealForPublication(); + return result; + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index af3be4475f71ac..9992238170c2c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -225,6 +225,8 @@ protected void runBeforeAll() throws Exception { } return invocation.callRealMethod(); }); + icebergUtilsMock.when(() -> IcebergUtils.getWritableIcebergTable( + ArgumentMatchers.any(ExternalTable.class))).thenReturn(mockedIcebergTable); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 00dde9f4cc0a74..5d98c753b82463 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -17,28 +17,953 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.GenericBlobMetadata; +import org.apache.iceberg.GenericStatisticsFile; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.types.Types; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class IcebergExternalMetaCacheTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testSnapshotAndManifestWeightsScaleLinearlyToOneHundredThousandItems() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, tableWithMetadataLocation("/metadata/linear-v1.json")).get(); + long snapshotBase = snapshotWeight(snapshotKey, 0); + long snapshotOneThousand = snapshotWeight(snapshotKey, 1_000); + assertLinearScale(snapshotBase, snapshotOneThousand, + snapshotWeight(snapshotKey, 10_000), snapshotWeight(snapshotKey, 100_000)); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey( + "/manifest/linear.avro", ManifestContent.DATA); + long manifestBase = manifestWeight(manifestKey, 0); + long manifestOneThousand = manifestWeight(manifestKey, 1_000); + assertLinearScale(manifestBase, manifestOneThousand, + manifestWeight(manifestKey, 10_000), manifestWeight(manifestKey, 100_000)); + } + + @Test + public void testWeightedEntriesAreRegistered() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.max-weight", "4MB"); + properties.put("meta.cache.iceberg.snapshot.max-weight", "8MB"); + properties.put("meta.cache.iceberg.manifest.enable", "true"); + properties.put("meta.cache.iceberg.manifest.max-weight", "16MB"); + cache.initCatalog(1L, properties); + + Map stats = cache.stats(1L); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_TABLE).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_MANIFEST).isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.enable", "false"); + properties.put("meta.cache.iceberg.table.ttl-second", "17"); + properties.put("meta.cache.iceberg.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(IcebergExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotKeyIncludesMetadataGeneration() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table first = tableWithMetadataLocation("/metadata/v1.json"); + Table second = tableWithMetadataLocation("/metadata/v2.json"); + Table recreated = tableWithMetadataLocation("/metadata/v1.json"); + + IcebergSnapshotEntryKey firstKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey sameKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey secondKey = IcebergSnapshotEntryKey.tryCreate(mapping, second).get(); + IcebergSnapshotEntryKey recreatedKey = IcebergSnapshotEntryKey.tryCreate(mapping, recreated).get(); + + Assert.assertEquals(firstKey, sameKey); + Assert.assertNotEquals(firstKey, secondKey); + Assert.assertNotEquals("drop/recreate may reuse HadoopCatalog's v1 path", firstKey, recreatedKey); + Assert.assertEquals("/metadata/v1.json", firstKey.getMetadataFileLocation()); + Assert.assertNotEquals(firstKey.getTableUuid(), recreatedKey.getTableUuid()); + Assert.assertFalse(IcebergSnapshotEntryKey.tryCreate(mapping, + newInterfaceProxy(Table.class)).isPresent()); + } + + @Test + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue first = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v1.json")); + IcebergTableCacheValue second = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v2.json")); + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + tables.put(mapping, first); + IcebergSnapshotEntryKey oldSnapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, first.getRetainedIcebergTable()).get(); + MetaCacheEntry snapshots = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); + IcebergSchemaCacheKey oldSchemaKey = new IcebergSchemaCacheKey( + mapping, first.getTableUuid().get(), 0L); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testOldGenerationSchemaLoadCannotRepopulateAfterTableReplacement() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue oldTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-old.json")); + IcebergTableCacheValue newTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-new.json")); + cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, newTable); + IcebergSchemaCacheKey staleKey = new IcebergSchemaCacheKey( + mapping, oldTable.getTableUuid().get(), 0L); + IcebergSchemaCacheValue staleValue = new IcebergSchemaCacheValue( + Collections.emptyList(), Collections.emptyList()); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(staleKey, staleValue); + + Assert.assertSame(staleValue, cache.getIcebergSchemaCacheValue( + mapping, 0L, oldTable.getRetainedIcebergTable())); + Assert.assertNull(schemas.peekIfPresent(staleKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testFrozenGenerationPreservesSparseEquivalentSchemaIds() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/sparse.json", "{" + + "\"format-version\":2,\"table-uuid\":\"sparse-schema-table\"," + + "\"location\":\"file:/warehouse/sparse\",\"last-sequence-number\":0," + + "\"last-updated-ms\":1,\"last-column-id\":2,\"current-schema-id\":2," + + "\"schemas\":[" + + "{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}," + + "{\"type\":\"struct\",\"schema-id\":1,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}," + + "{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"}]}," + + "{\"type\":\"struct\",\"schema-id\":2,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":-1,\"refs\":{},\"snapshots\":[]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[],\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + TableMetadata retainedMetadata = ((HasTableOperations) retained).operations().current(); + + Assert.assertEquals(2, retainedMetadata.currentSchemaId()); + Assert.assertEquals(java.util.Arrays.asList(0, 1, 2), retainedMetadata.schemas().stream() + .map(Schema::schemaId).collect(Collectors.toList())); + Assert.assertEquals(1, retainedMetadata.schemas().stream() + .filter(schema -> schema.schemaId() == 2).findFirst().get().columns().size()); + } + + @Test + public void testFrozenGenerationAcceptsSnapshotCreatedBeforeV3Upgrade() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/upgraded-v3.json", "{" + + "\"format-version\":3,\"table-uuid\":\"upgraded-v3-table\"," + + "\"location\":\"file:/warehouse/v3\",\"last-sequence-number\":1," + + "\"last-updated-ms\":2,\"last-column-id\":1,\"current-schema-id\":0," + + "\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":7,\"next-row-id\":0," + + "\"refs\":{\"main\":{\"snapshot-id\":7,\"type\":\"branch\"}}," + + "\"snapshots\":[{\"sequence-number\":0,\"snapshot-id\":7," + + "\"timestamp-ms\":1,\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[],\"schema-id\":0}]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[{\"timestamp-ms\":1,\"snapshot-id\":7}]," + + "\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + + Assert.assertEquals(7L, retained.currentSnapshot().snapshotId()); + Assert.assertEquals(3, ((HasTableOperations) retained).operations().current().formatVersion()); + } + + @Test + public void testTableSnapshotAndManifestEstimatesArePrecomputed() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table table = tableWithMetadataLocation("/metadata/v1.json"); + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(table); + tableValue.prepareForCachePublication(mapping); + Assert.assertTrue(tableValue.getSizeEstimate().isComplete()); + Assert.assertTrue(tableValue.getSizeEstimate().getBytes() > 0L); + + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), table); + snapshotValue.prepareForCachePublication(snapshotKey); + Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), + snapshotValue.getSizeEstimate().isComplete()); + Assert.assertTrue(snapshotValue.getSizeEstimate().getBytes() > 0L); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey("/manifest/a.avro", ManifestContent.DATA); + ManifestCacheValue manifestValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/a.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + MetaCacheSizeEstimate manifestEstimate = + IcebergCacheSizeEstimator.estimateManifestEntry(manifestKey, manifestValue); + Assert.assertTrue(manifestEstimate.getIncompleteReason(), manifestEstimate.isComplete()); + Assert.assertTrue(manifestEstimate.getBytes() > 0L); + + Table unsupportedTable = newInterfaceProxy(Table.class); + MetaCacheSizeEstimate unsupported = IcebergCacheSizeEstimator.estimateTableEntry( + mapping, new IcebergTableCacheValue(unsupportedTable)); + Assert.assertFalse(unsupported.isComplete()); + Assert.assertTrue(unsupported.getIncompleteReason().startsWith("unsupported_iceberg_table:")); + } + + @Test + public void testIcebergPreparationFailureIsFailClosed() { + TableMetadata brokenMetadata = Mockito.mock(TableMetadata.class); + Mockito.when(brokenMetadata.currentSnapshot()) + .thenThrow(new IllegalStateException("unsupported snapshot state")); + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(brokenMetadata); + Table brokenTable = new BaseTable(operations, "db.tbl"); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(brokenTable); + MetaCacheSizeEstimate tableEstimate = tableValue.prepareForCachePublication(mapping); + + Assert.assertFalse(tableEstimate.isComplete()); + Assert.assertTrue(tableEstimate.getIncompleteReason() + .startsWith("iceberg_table_preparation_failed:")); + Assert.assertSame(tableValue.getRetainedIcebergTable(), tableValue.newQueryScopedTable()); + + Table healthyTable = tableWithMetadataLocation("/metadata/fail-closed-key.json"); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, healthyTable).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), brokenTable); + MetaCacheSizeEstimate snapshotEstimate = snapshotValue.prepareForCachePublication(key); + + Assert.assertFalse(snapshotEstimate.isComplete()); + Assert.assertTrue(snapshotEstimate.getIncompleteReason() + .startsWith("iceberg_snapshot_preparation_failed:")); + } + + @Test + public void testManifestAccountingFailureKeepsFilesAndRejectsWeightedAdmission() { + DataFile file = Mockito.mock(DataFile.class); + Mockito.when(file.columnSizes()).thenThrow(new IllegalStateException("new metrics representation")); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles(Collections.singletonList(file)); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/fail-closed.avro", ManifestContent.DATA), value); + + Assert.assertEquals(Collections.singletonList(file), value.getDataFiles()); + Assert.assertFalse(value.isAccountingComplete()); + Assert.assertFalse(estimate.isComplete()); + Assert.assertEquals("iceberg_manifest_accounting_incomplete", estimate.getIncompleteReason()); + } + + @Test + public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { + String largePayload = repeatedCharacter('x', 64 * 1024); + Table smallTable = tableWithNestedSchemaAndProperty("x", "x"); + Table largeTable = tableWithNestedSchemaAndProperty(largePayload, largePayload); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(smallTable); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue(largeTable); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + long expectedPayloadDelta = (largePayload.length() - 1L) * 4L; + Assert.assertTrue(largeValue.getSizeEstimate().getBytes() + - smallValue.getSizeEstimate().getBytes() >= expectedPayloadDelta); + } + + @Test + public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { + List fields = IntStream.range(0, 100) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema largeSchema = new Schema(0, fields); + Schema smallSchema = new Schema(1, fields.get(0)); + TableMetadata schemaHistory = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + schemaHistory = TableMetadata.buildFrom(schemaHistory) + .addSchema(smallSchema) + .setCurrentSchema(smallSchema.schemaId()) + .discardChanges() + .build(); + TableMetadata smallSchemaOnly = TableMetadata.newTableMetadata( + smallSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(largeSchema).withSpecId(0); + SortOrder.Builder sortBuilder = SortOrder.builderFor(largeSchema).withOrderId(1); + for (Types.NestedField field : fields) { + specBuilder.identity(field.name()); + sortBuilder.asc(field.name()); + } + TableMetadata fieldHistory = TableMetadata.newTableMetadata( + largeSchema, specBuilder.build(), sortBuilder.build(), + "file:/warehouse/field-history", Collections.emptyMap()); + fieldHistory = TableMetadata.buildFrom(fieldHistory) + .setDefaultPartitionSpec( + PartitionSpec.builderFor(largeSchema).withSpecId(1).build()) + .setDefaultSortOrder(SortOrder.unsorted()) + .discardChanges() + .build(); + TableMetadata emptyFields = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), SortOrder.unsorted(), + "file:/warehouse/field-history", Collections.emptyMap()); + + long schemaDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(schemaHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(smallSchemaOnly)); + long specAndSortDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(fieldHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + + Assert.assertTrue(schemaDelta >= 99L * 512L); + Assert.assertTrue(specAndSortDelta >= 100L * (384L + 256L)); + } + + @Test + public void testTablePayloadAccountsForRetainedHistoricalMetadata() { + String largePayload = repeatedCharacter('x', 64 * 1024); + long smallBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload("x", 32))); + long largeBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload(largePayload, 64 * 1024))); + + Assert.assertTrue(largeBytes - smallBytes >= 64L * 1024L - 32L); + } + + @Test + public void testTableEstimateAccountsForRetainedBranchHistory() { + TableMetadata oneCommit = metadataWithSnapshotSequence(1L); + TableMetadata tenThousandCommits = metadataWithSnapshotSequence(10_000L); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(tableWithMetadata(oneCommit)); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue( + tableWithMetadata(tenThousandCommits)); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + Assert.assertTrue(smallValue.getSizeEstimate().getIncompleteReason(), + smallValue.getSizeEstimate().isComplete()); + Assert.assertTrue(largeValue.getSizeEstimate().getIncompleteReason(), + largeValue.getSizeEstimate().isComplete()); + Assert.assertEquals(smallValue.getSizeEstimate().getBytes(), + largeValue.getSizeEstimate().getBytes()); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshots(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshots(); + Mockito.verify(oneCommit, Mockito.never()).lastSequenceNumber(); + Mockito.verify(tenThousandCommits, Mockito.never()).lastSequenceNumber(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).refs(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).refs(); + } + + @Test + public void testWeightedTablePreparationRunsInsideCatalogAuthenticator() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + AtomicBoolean authenticated = new AtomicBoolean(); + AtomicBoolean firstPreparation = new AtomicBoolean(true); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Table table = tableWithMetadataLocation("/metadata/authenticated-v1.json"); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenReturn(table); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + Assert.assertTrue(authenticated.compareAndSet(false, true)); + try { + return task.call(); + } finally { + authenticated.set(false); + } + } + }); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + if (firstPreparation.compareAndSet(true, false)) { + Assert.assertTrue("publication preparation must retain Kerberos scope", + authenticated.get()); + } + return super.prepareTableForCachePublication(nameMapping, value); + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = new NameMapping( + 1L, "db", "tbl", "remote_db", "remote_tbl"); + + IcebergTableCacheValue value = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + IcebergTableCacheValue cached = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertSame(value, cached); + Mockito.verify(metadataOps, Mockito.times(1)).loadTable("remote_db", "remote_tbl"); + Assert.assertFalse(authenticated.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testQueryScopedMetadataReusesFrozenGenerationWithoutFileIo() throws Exception { + String tableLocation = temporaryFolder.newFolder("authenticated-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + AtomicInteger metadataReads = new AtomicInteger(); + FileIO trackingFileIO = Mockito.mock(FileIO.class); + Mockito.when(trackingFileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + metadataReads.incrementAndGet(); + return liveTable.io().newInputFile((String) invocation.getArgument(0)); + }); + TableMetadata metadata = ((HasTableOperations) liveTable).operations().current(); + Table trackedTable = new BaseTable( + new StaticTableOperations(metadata, trackingFileIO), liveTable.name()); + IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable); + countValue.getWritableIcebergTable(liveTable); + Assert.assertEquals("count-based writes must not add metadata FileIO", 0, metadataReads.get()); + IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable); + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Table statementTable = value.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(statementTable); + IcebergSnapshotCacheValue statementValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), statementTable); + Assert.assertEquals(statementTable.schema().asStruct(), + statementValue.getIcebergTable().get().schema().asStruct()); + com.google.common.collect.Lists.newArrayList( + statementValue.getIcebergTable().get().snapshots()); + Assert.assertEquals("statement handoff must reuse frozen metadata", 0, metadataReads.get()); + value.getWritableIcebergTable(liveTable); + + Assert.assertEquals(0, metadataReads.get()); + } + + @Test + public void testCountModeTimeTravelDoesNotEnableQueryIsolation() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/count-v1.json")); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, value); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertSame(value.getRetainedIcebergTable(), queryTable); + Assert.assertFalse(value.isQueryIsolationPrepared()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testTimeTravelGenerationBundleDoesNotMixReplacedTableValue() throws Exception { + String firstLocation = temporaryFolder.newFolder("bundle-first").toURI().toString(); + String secondLocation = temporaryFolder.newFolder("bundle-second").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table firstTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), firstLocation); + firstTable.newAppend().appendFile(DataFiles.builder(firstTable.spec()) + .withPath(firstLocation + "/data/a.parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()).commit(); + Table secondTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), secondLocation); + secondTable.newAppend().appendFile(DataFiles.builder(secondTable.spec()) + .withPath(secondLocation + "/data/b.parquet") + .withFileSizeInBytes(20L).withRecordCount(2L).build()).commit(); + long firstSnapshotId = firstTable.currentSnapshot().snapshotId(); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + MetaCacheEntry entry = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + entry.put(mapping, new IcebergTableCacheValue(firstTable)); + ExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(dorisTable); + entry.put(mapping, new IcebergTableCacheValue(secondTable)); + + Assert.assertEquals(firstSnapshotId, + queryTable.currentSnapshot().snapshotId()); + Assert.assertEquals(firstTable.location(), + queryTable.location()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testPinnedGenerationSurvivesMetadataFileRetirement() throws Exception { + String staleLocation = temporaryFolder.newFolder("stale-metadata").toURI().toString(); + String freshLocation = temporaryFolder.newFolder("fresh-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table staleTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), staleLocation); + Table freshTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), freshLocation); + String staleMetadataLocation = ((HasTableOperations) staleTable) + .operations().current().metadataFileLocation(); + IcebergTableCacheValue staleValue = new IcebergTableCacheValue(staleTable); + staleValue.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + staleTable.io().deleteFile(staleMetadataLocation); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + Mockito.when(metadataOps.loadTable("db", "tbl")).thenReturn(freshTable); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, staleValue); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertEquals(staleTable.schema().asStruct(), queryTable.schema().asStruct()); + Mockito.verify(metadataOps, Mockito.never()).loadTable("db", "tbl"); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testWeightedTablePublicationRetainsNonGrowingGeneration() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot currentSnapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/current-a.avro\"," + + "\"/manifest/current-b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .setBranchSnapshot(currentSnapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.when(fileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + InputFile inputFile = Mockito.mock(InputFile.class); + Mockito.when(inputFile.location()).thenReturn(invocation.getArgument(0)); + return inputFile; + }); + Table liveTable = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(mapping); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); + Table firstUse = value.getIcebergTable(); + Table secondUse = value.getIcebergTable(); + Assert.assertNotSame(retained, firstUse); + Assert.assertNotSame(firstUse, secondUse); + Assert.assertNotSame(retained.currentSnapshot(), firstUse.currentSnapshot()); + Assert.assertNotSame(firstUse.currentSnapshot(), secondUse.currentSnapshot()); + Assert.assertEquals(2, firstUse.snapshot(7L).dataManifests(firstUse.io()).size()); + Assert.assertEquals(2, secondUse.snapshot(7L).dataManifests(secondUse.io()).size()); + + IcebergSnapshotEntryKey snapshotKey = + IcebergSnapshotEntryKey.tryCreate(mapping, retained).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), + Optional.empty(), retained, value.getRetainedCurrentSnapshotJson()); + snapshotValue.prepareForCachePublication(snapshotKey); + Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), + snapshotValue.getSizeEstimate().isComplete()); + Table snapshotQuery = snapshotValue.getIcebergTable().get(); + Assert.assertEquals(2, + snapshotQuery.currentSnapshot().dataManifests(snapshotQuery.io()).size()); + } + + @Test + public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Exception { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + String tableLocation = temporaryFolder.newFolder("v2-table").toURI().toString(); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + liveTable.newAppend().appendFile( + DataFiles.builder(liveTable.spec()) + .withPath(tableLocation + "/data/a.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()).commit(); + Assert.assertNotNull(liveTable.currentSnapshot().manifestListLocation()); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Table firstQuery = value.getIcebergTable(); + Table secondQuery = value.getIcebergTable(); + List firstManifests = + firstQuery.currentSnapshot().dataManifests(firstQuery.io()); + List secondManifests = + secondQuery.currentSnapshot().dataManifests(secondQuery.io()); + Assert.assertEquals(1, firstManifests.size()); + Assert.assertEquals(1, secondManifests.size()); + Assert.assertNotSame(firstQuery.currentSnapshot(), secondQuery.currentSnapshot()); + Assert.assertNotSame(firstManifests, secondManifests); + Assert.assertNotSame(retained.currentSnapshot(), firstQuery.currentSnapshot()); + } + + @Test + public void testTablePublicationDoesNotReadHistoricalManifestLists() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot historical = SnapshotParser.fromJson("{\"snapshot-id\":6,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/manifest-list/history.avro\",\"schema-id\":0}"); + Snapshot current = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"},\"manifests\":[],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .addSnapshot(historical) + .setBranchSnapshot(current, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v2.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + IcebergTableCacheValue value = new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl")); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Mockito.verify(fileIO, Mockito.never()).newInputFile("/manifest-list/history.avro"); + } + + @Test + public void testManifestEstimateScalesWithFileCount() { + ManifestCacheValue oneFile = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/one.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + ManifestCacheValue twoFiles = ManifestCacheValue.forDataFiles(java.util.Arrays.asList( + oneFile.getDataFiles().get(0), oneFile.getDataFiles().get(0))); + IcebergManifestEntryKey key = new IcebergManifestEntryKey("/manifest/data.avro", ManifestContent.DATA); + + long oneFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, oneFile).getBytes(); + long twoFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, twoFiles).getBytes(); + + Assert.assertTrue(twoFileBytes > oneFileBytes); + } + + @Test + public void testManifestFormulaAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol.avro", ManifestContent.DATA); + ManifestCacheValue empty = ManifestCacheValue.forDataFiles(Collections.emptyList()); + ManifestCacheValue populated = ManifestCacheValue.forDataFiles( + IntStream.range(0, 32).mapToObj(this::dataFileWithMetrics) + .collect(Collectors.toList())); + ManifestCacheValue shortTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(16))); + ManifestCacheValue longTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(4096))); + + long emptyEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, empty).getBytes(); + long populatedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, populated).getBytes(); + long shortTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, shortTail).getBytes(); + long longTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, longTail).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest files", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg long-tail path", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + + @Test + public void testManifestEstimateAccountsForSkewedFilePaths() { + String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/x.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(largePath) + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/path-skew.avro", ManifestContent.DATA); + + long smallBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue).getBytes(); + long largeBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue).getBytes(); + + Assert.assertTrue(largeBytes - smallBytes >= (largePath.length() - "/data/x.parquet".length()) * 2L); + } + + @Test + public void testManifestEstimateAccountsForSkewedBufferPayload() { + Metrics smallMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(32)), Collections.emptyMap()); + Metrics largeMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(64 * 1024)), Collections.emptyMap()); + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(smallMetrics) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(largeMetrics) + .build())); + + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/encrypted.avro", ManifestContent.DATA); + MetaCacheSizeEstimate smallEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue); + MetaCacheSizeEstimate largeEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue); + + Assert.assertTrue(smallEstimate.getIncompleteReason(), smallEstimate.isComplete()); + Assert.assertTrue(largeEstimate.getIncompleteReason(), largeEstimate.isComplete()); + Assert.assertEquals(1L, smallValue.getDataFileMetricEntryCount()); + Assert.assertEquals(1L, largeValue.getDataFileMetricEntryCount()); + Assert.assertTrue(largeEstimate.getBytes() - smallEstimate.getBytes() >= 64 * 1024 - 32); + } + + @Test + public void testManifestEstimateAccountsForDeleteFileAuxiliaryPayload() { + String largeReference = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + List largeOffsets = IntStream.range(0, 4096) + .mapToObj(index -> (long) index).collect(Collectors.toList()); + DeleteFile smallPositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile("/data/x.parquet") + .withSplitOffsets(Collections.singletonList(0L)) + .build(); + DeleteFile largePositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile(largeReference) + .withSplitOffsets(largeOffsets) + .build(); + int[] largeEqualityIds = IntStream.range(0, 4096).toArray(); + DeleteFile smallEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(1) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + DeleteFile largeEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(largeEqualityIds) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/delete.avro", ManifestContent.DELETES); + + long smallPositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallPositionDelete))).getBytes(); + long largePositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largePositionDelete))).getBytes(); + long smallEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallEqualityDelete))).getBytes(); + long largeEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largeEqualityDelete))).getBytes(); + + Assert.assertTrue(largePositionBytes > smallPositionBytes); + Assert.assertTrue(largeEqualityBytes > smallEqualityBytes); + } + + @Test + public void testSnapshotPublicationDoesNotMaterializeManifestLists() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/a.avro\",\"/manifest/b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Table table = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), Optional.empty(), table); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate( + NameMapping.createForTest(1L, "db", "tbl"), table).get(); + + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Table queryTable = value.getIcebergTable().get(); + Assert.assertNotSame(table.currentSnapshot(), queryTable.currentSnapshot()); + Mockito.verifyNoInteractions(fileIO); + } @Test public void testInvalidateTableKeepsManifestCache() { @@ -52,10 +977,16 @@ public void testInvalidateTableKeepsManifestCache() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + + Table snapshotTable = tableWithMetadataLocation("/metadata/invalidate-v1.json"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(t1, snapshotTable).get(); + MetaCacheEntry snapshotEntry = cache.entry(catalogId, + IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, + new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); MetaCacheEntry viewEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_VIEW, NameMapping.class, org.apache.iceberg.view.View.class); @@ -77,6 +1008,7 @@ public void testInvalidateTableKeepsManifestCache() { Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); Assert.assertNull(viewEntry.getIfPresent(t1)); Assert.assertNotNull(viewEntry.getIfPresent(t2)); Assert.assertNotNull(manifestEntry.getIfPresent(m1)); @@ -98,10 +1030,8 @@ public void testInvalidateDbAndStats() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); MetaCacheEntry schemaEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class); @@ -228,6 +1158,167 @@ private Map manifestCacheEnabledProperties() { return properties; } + private long snapshotWeight(IcebergSnapshotEntryKey key, int partitionCount) { + IcebergPartitionInfo partitionInfo = Mockito.mock(IcebergPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Map> aliases = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToIcebergPartition()).thenReturn(partitions); + Mockito.when(partitionInfo.getNameToIcebergPartitionNames()).thenReturn(aliases); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(key.getSnapshotId(), key.getSchemaId())); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateSnapshotEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private long manifestWeight(IcebergManifestEntryKey key, int fileCount) { + ManifestCacheValue value = Mockito.mock(ManifestCacheValue.class); + List dataFiles = sizeOnlyList(fileCount); + Mockito.when(value.getDataFiles()).thenReturn(dataFiles); + Mockito.when(value.getDeleteFiles()).thenReturn(Collections.emptyList()); + Mockito.when(value.isAccountingComplete()).thenReturn(true); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private void assertLinearScale(long base, long oneThousand, long tenThousand, long oneHundredThousand) { + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + @SuppressWarnings("unchecked") + private List sizeOnlyList(int size) { + List list = Mockito.mock(List.class); + Mockito.when(list.size()).thenReturn(size); + return list; + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Table tableWithMetadataLocation(String metadataLocation) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation(metadataLocation).build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + + private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { + Map columnSizes = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> (long) index + column)); + Map valueCounts = new java.util.HashMap<>(columnSizes); + Map nullCounts = new java.util.HashMap<>(columnSizes); + Map nanCounts = new java.util.HashMap<>(columnSizes); + Map lowerBounds = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Map upperBounds = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Metrics metrics = new Metrics( + 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/jol-" + index + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build(); + } + + private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/" + repeatedCharacter('x', pathLength) + ".parquet") + .withFileSizeInBytes(1024L) + .withRecordCount(1L) + .build(); + } + + private Table tableWithMetadata(TableMetadata metadata) { + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(metadata); + return new BaseTable(operations, "db.tbl"); + } + + private TableMetadata metadataWithMaterializedPayload(String payload, int bufferBytes) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.uuid()).thenReturn("stable-uuid"); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + payload, Mockito.mock(SnapshotRef.class))); + TableMetadata.MetadataLogEntry metadataLogEntry = + Mockito.mock(TableMetadata.MetadataLogEntry.class); + Mockito.when(metadataLogEntry.file()).thenReturn(payload); + Mockito.when(metadata.previousFiles()).thenReturn( + Collections.singletonList(metadataLogEntry)); + GenericBlobMetadata blob = new GenericBlobMetadata( + payload, 1L, 1L, Collections.singletonList(1), + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.singletonList( + new GenericStatisticsFile(1L, payload, 1L, 1L, + Collections.singletonList(blob)))); + org.apache.iceberg.PartitionStatisticsFile partitionStatistics = + Mockito.mock(org.apache.iceberg.PartitionStatisticsFile.class); + Mockito.when(partitionStatistics.path()).thenReturn(payload); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn( + Collections.singletonList(partitionStatistics)); + EncryptedKey encryptedKey = Mockito.mock(EncryptedKey.class); + Mockito.when(encryptedKey.keyId()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedById()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedKeyMetadata()).thenReturn( + ByteBuffer.allocateDirect(bufferBytes)); + Mockito.when(encryptedKey.properties()).thenReturn( + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.singletonList(encryptedKey)); + return metadata; + } + + private TableMetadata metadataWithSnapshotSequence(long lastSequenceNumber) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.singletonList( + Mockito.mock(org.apache.iceberg.HistoryEntry.class))); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + "branch-tip", Mockito.mock(SnapshotRef.class))); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.lastSequenceNumber()).thenReturn(lastSequenceNumber); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/sequence.json"); + return metadata; + } + + private Table tableWithNestedSchemaAndProperty(String nestedFieldName, String propertyValue) { + Schema schema = new Schema(Types.NestedField.optional(1, "payload", + Types.StructType.of(Types.NestedField.optional( + 2, nestedFieldName, Types.StringType.get())))); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.singletonMap("payload", propertyValue)); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/nested.json").build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + private IcebergManifestEntryKey mockManifestKey(String path) { return IcebergManifestEntryKey.of(new TestingManifestFile(path, ManifestContent.DATA)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java index 720f66fd5f9e3f..de3ab8c9746397 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -92,9 +92,9 @@ public void setUp() throws IOException { Mockito.doReturn(db).when(catalog).getDbNullable(Mockito.any()); Mockito.doReturn(dorisTable).when(db).getTableNullable(Mockito.any()); - // mock IcebergUtils.getIcebergTable to return our test icebergTable + // Mock writable access used by branch and tag mutations. mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(Mockito.any())) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.any())) .thenReturn(icebergTable); // mock Env.getCurrentEnv().getEditLog().logBranchOrTag(info) to do nothing diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index a4b882c4497132..cc019c73ed12a7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -138,7 +138,8 @@ public void testTopLevelVariantModifyOnlyUpdatesMetadataOnOrcTable() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("payload"), column, ColumnPosition.FIRST, 1L); } @@ -165,7 +166,8 @@ public void testTopLevelVariantModifyRejectsTypeConversions() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("variant_col"), new Column("variant_col", Type.STRING, true), null, 1L), @@ -294,7 +296,7 @@ public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), new Column("new_field", Type.LARGEINT, true), null, 1L), @@ -328,7 +330,7 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); @@ -363,7 +365,7 @@ public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwab try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); } @@ -391,7 +393,7 @@ public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComm try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -433,7 +435,7 @@ public void testFullStructModifyPreservesOmittedChildComments() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), new Column("payload", payloadType, true), null, 1L); @@ -463,7 +465,7 @@ public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -489,7 +491,7 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("id"), new Column("id", Type.BIGINT, true), null, 1L); @@ -515,7 +517,7 @@ public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L), @@ -537,7 +539,7 @@ public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L); @@ -568,7 +570,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws T try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, @@ -603,7 +605,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); @@ -633,7 +635,7 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, @@ -661,7 +663,7 @@ public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdate try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, @@ -692,7 +694,7 @@ public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); // Iceberg schema columns are represented as keys in Doris, so the legacy API must not // interpret isKey as an explicit KEY clause. @@ -723,7 +725,7 @@ public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, column, null, 1L); } @@ -757,7 +759,7 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); @@ -810,7 +812,7 @@ public void execute(Runnable task) { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(staleTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(staleTable); try { conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), @@ -861,7 +863,7 @@ public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); icebergTable.refresh(); @@ -911,7 +913,7 @@ public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, "a", "renamed", 1L); icebergTable.refresh(); @@ -940,7 +942,7 @@ public void testNestedColumnOperationsRejectDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), nestedAddDefaultColumn, null, 1L), @@ -971,7 +973,7 @@ public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: id"); @@ -1002,7 +1004,7 @@ public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), @@ -1035,7 +1037,7 @@ public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); @@ -1082,7 +1084,7 @@ public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, new Column("info", infoType, true), null, 1L), @@ -1158,7 +1160,7 @@ public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn( dorisTable, new Column("id", Type.STRING, true), null, 1L), @@ -1194,7 +1196,7 @@ public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); } @@ -1216,7 +1218,7 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), null, 1L); @@ -1239,7 +1241,7 @@ public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), @@ -1270,7 +1272,7 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), "struct comment", 1L); @@ -1296,7 +1298,7 @@ public void testRejectsCommentsOnDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), @@ -1336,7 +1338,7 @@ public void testRejectsTopLevelRowLineageMutationsForV3Tables() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L), @@ -1392,8 +1394,10 @@ public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v3DorisTable)).thenReturn(v3IcebergTable); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v2DorisTable)).thenReturn(v2IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v3DorisTable)) + .thenReturn(v3IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v2DorisTable)) + .thenReturn(v2IcebergTable); ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), new Column("_last_updated_sequence_number", Type.BIGINT, true), null, 1L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java index 74c8c3f6954a97..84745f815f828c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java @@ -22,11 +22,28 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.Map; import java.util.Set; public class IcebergPartitionInfoTest { + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + String largeValue = repeatedCharacter('x', 64 * 1024); + IcebergPartition small = new IcebergPartition("p=x", 0, 0, 0, 0, 1, 101, + Collections.singletonList("x"), Collections.singletonList("identity")); + IcebergPartition large = new IcebergPartition("p=" + largeValue, 0, 0, 0, 0, 1, 101, + Collections.singletonList(largeValue), Collections.singletonList("identity")); + + Assertions.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= (largeValue.length() - 1L) * 4L); + IcebergPartitionInfo info = new IcebergPartitionInfo( + Collections.emptyMap(), Collections.singletonMap(large.getPartitionName(), large), + Collections.emptyMap()); + Assertions.assertEquals(large.getRetainedPayloadBytes(), info.getRetainedPayloadBytes()); + } + @Test public void testGetLatestSnapshotId() { IcebergPartition p1 = new IcebergPartition("p1", 0, 0, 0, 0, 1, 101, null, null); @@ -50,4 +67,10 @@ public void testGetLatestSnapshotId() { Assertions.assertEquals(102, snapshot2); Assertions.assertEquals(103, snapshot3); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java index 03d217c25d16e5..18969f0a535ebb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -18,6 +18,9 @@ package org.apache.doris.datasource.iceberg; import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -45,4 +48,29 @@ public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { sourceTable, MetadataTableType.DATA_FILES.name()); Assertions.assertTrue(dataFiles.supportsSnapshotSelection()); } + + @Test + public void testMetadataSchemaReloadsAfterSourceEvolution() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(catalog); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + Table firstGeneration = Mockito.mock(Table.class); + Table evolvedGeneration = Mockito.mock(Table.class); + Mockito.when(firstGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()))); + Mockito.when(evolvedGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.optional(2, "evolved_partition", Types.StringType.get()))); + IcebergSysExternalTable sysTable = Mockito.spy(new IcebergSysExternalTable( + sourceTable, MetadataTableType.PARTITIONS.name())); + Mockito.doReturn(firstGeneration, evolvedGeneration).when(sysTable).getSysIcebergTable(); + + Assertions.assertEquals(1, sysTable.getFullSchema().size()); + Assertions.assertEquals(2, sysTable.getFullSchema().size()); + Mockito.verify(sysTable, Mockito.times(2)).getSysIcebergTable(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index e2d923f3438863..2df4ad7229dbe7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -202,7 +202,7 @@ public void testPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); // Allow parsePartitionValueFromString to call the real implementation mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( @@ -318,7 +318,7 @@ public void testUnPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -429,7 +429,7 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -455,7 +455,7 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -500,7 +500,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -517,7 +517,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th checkPushDownByPartition(table, Expressions.equal("str1", "partition-b"), 1); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -590,7 +590,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(3); @@ -651,7 +651,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(formatVersion); @@ -724,7 +724,7 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -736,6 +736,26 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User TableIdentifier.of(dbName, tbWithoutPartition)).currentSnapshot()); } + @Test + public void testWeightedTableSupportsSchemaAndPartitionSpecCommits() { + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergTableCacheValue cacheValue = new IcebergTableCacheValue(liveTable); + cacheValue.prepareForCachePublication(NameMapping.createForTest(dbName, tbWithoutPartition)); + + Table writableTable = cacheValue.getWritableIcebergTable(liveTable); + writableTable.updateSchema() + .addColumn("new_col", Types.StringType.get()) + .commit(); + writableTable.updateSpec() + .addField("int1") + .commit(); + + Table refreshed = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + Assert.assertNotNull(refreshed.schema().findField("new_col")); + Assert.assertEquals(1, refreshed.spec().fields().size()); + Assert.assertEquals("int1", refreshed.spec().fields().get(0).name()); + } + @Test public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws UserException { Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); @@ -754,7 +774,7 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -783,7 +803,7 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index 6407d540d1ef03..3decfbef1b0483 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -28,8 +28,13 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; public class AbstractExternalMetaCacheTest { @@ -97,6 +102,34 @@ public void testEntryFailsFastAfterCatalogRemoved() { } } + @Test + public void testCapturedCatalogGroupReturnsClosedEntryDuringConcurrentRemoval() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newSingleThreadExecutor(); + CountDownLatch groupCaptured = new CountDownLatch(1); + CountDownLatch releaseEntryLookup = new CountDownLatch(1); + LookupRaceExternalMetaCache cache = + new LookupRaceExternalMetaCache(refreshExecutor, groupCaptured, releaseEntryLookup); + try { + cache.initCatalog(1L, Maps.newHashMap()); + Future> lookup = workers.submit( + () -> cache.entry(1L, "value", String.class, Integer.class)); + Assert.assertTrue(groupCaptured.await(3L, TimeUnit.SECONDS)); + + cache.invalidateCatalog(1L); + releaseEntryLookup.countDown(); + + MetaCacheEntry capturedClosedEntry = lookup.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(Integer.valueOf(1), capturedClosedEntry.get("k")); + Assert.assertNull(capturedClosedEntry.peekIfPresent("k")); + } finally { + releaseEntryLookup.countDown(); + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testEntryLevelInvalidationUsesRegisteredMatcher() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); @@ -121,6 +154,127 @@ public void testEntryLevelInvalidationUsesRegisteredMatcher() { } } + @Test + public void testGlobalWeightAutomaticallyActivatesEntriesWithEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(600L))); + try { + cache.initCatalog(1L, Maps.newHashMap()); + MetaCacheEntry entry = cache.entry(1L, "value", String.class, Integer.class); + + Assert.assertTrue(entry.isWeightBounded()); + Assert.assertEquals(600L, entry.stats().getMaxWeight()); + entry.put("first", 60); + entry.put("second", 60); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(60), entry.getIfPresent("second")); + Assert.assertEquals(60L + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES, + entry.stats().getGlobalEstimatedWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitIgnoresEntryWeightWithoutEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + TestExternalMetaCache cache = new TestExternalMetaCache(refreshExecutor); + Map properties = Maps.newHashMap(); + properties.put("meta.cache.test_engine.schema.max-weight", "1KB"); + + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertFalse(cache.entry(1L, "schema", SchemaCacheKey.class, SchemaCacheValue.class) + .isWeightBounded()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitIgnoresEntryWeightAboveCatalogWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(4L * 1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertEquals(1024L, cache.entry(1L, "value", String.class, Integer.class) + .stats().getMaxWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitClampsCatalogAcceptedOnLargerFe() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "4KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + Assert.assertThrows(IllegalArgumentException.class, + () -> cache.validateCatalogProperties(properties)); + + cache.initCatalog(1L, properties); + + MetaCacheEntryStats stats = cache.stats(1L).get("value"); + Assert.assertEquals(1024L, stats.getMaxWeight()); + Assert.assertEquals(1024L, stats.getCatalogMaxWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCatalogRemoveAndInitDoesNotDuplicateBudgetScope() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newFixedThreadPool(2); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(100L))); + CountDownLatch start = new CountDownLatch(1); + try { + Future first = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + Future second = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + start.countDown(); + first.get(10L, TimeUnit.SECONDS); + second.get(10L, TimeUnit.SECONDS); + cache.initCatalog(1L, Maps.newHashMap()); + Assert.assertTrue(cache.isCatalogInitialized(1L)); + } finally { + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + private static void repeatedlyRebuildCatalog(WeightedExternalMetaCache cache, CountDownLatch start) { + try { + Assert.assertTrue(start.await(3L, TimeUnit.SECONDS)); + for (int i = 0; i < 100; i++) { + cache.initCatalog(1L, Maps.newHashMap()); + cache.invalidateCatalog(1L); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + private static final class TestExternalMetaCache extends AbstractExternalMetaCache { private TestExternalMetaCache(ExecutorService refreshExecutor) { super("test_engine", refreshExecutor); @@ -135,4 +289,44 @@ private TestExternalMetaCache(ExecutorService refreshExecutor) { MetaCacheEntryInvalidation.forNameMapping(SchemaCacheKey::getNameMapping))); } } + + private static final class WeightedExternalMetaCache extends AbstractExternalMetaCache { + private WeightedExternalMetaCache( + ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super("weighted_test", refreshExecutor, budgetManager); + registerEntry(MetaCacheEntryDef.of( + "value", + String.class, + Integer.class, + key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(value.longValue()))); + } + } + + private static final class LookupRaceExternalMetaCache extends AbstractExternalMetaCache { + private final CountDownLatch groupCaptured; + private final CountDownLatch releaseEntryLookup; + + private LookupRaceExternalMetaCache(ExecutorService refreshExecutor, + CountDownLatch groupCaptured, CountDownLatch releaseEntryLookup) { + super("lookup_race", refreshExecutor); + this.groupCaptured = groupCaptured; + this.releaseEntryLookup = releaseEntryLookup; + registerEntry(MetaCacheEntryDef.of( + "value", String.class, Integer.class, key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L))); + } + + @Override + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + groupCaptured.countDown(); + try { + Assert.assertTrue(releaseEntryLookup.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java index 05acbb539a26d6..6ea171abd6d717 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.DdlException; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -68,6 +69,7 @@ public void testFromPropertiesWithPropertySpecBuilder() { public void testFromPropertiesWithEngineEntryKeys() { Map properties = Maps.newHashMap(); properties.put("meta.cache.hive.schema.ttl-second", "0"); + properties.put("meta.cache.hive.schema.max-weight", "2KB"); CacheSpec defaultSpec = CacheSpec.fromProperties( Maps.newHashMap(), @@ -79,6 +81,8 @@ public void testFromPropertiesWithEngineEntryKeys() { Assert.assertTrue(spec.isEnable()); Assert.assertEquals(0, spec.getTtlSecond()); Assert.assertEquals(100, spec.getCapacity()); + Assert.assertTrue(spec.isWeightBounded()); + Assert.assertEquals(2048L, spec.getMaxWeight().getAsLong()); } @Test @@ -108,6 +112,7 @@ public void testOfSemantics() { Assert.assertTrue(enabled.isEnable()); Assert.assertEquals(60, enabled.getTtlSecond()); Assert.assertEquals(100, enabled.getCapacity()); + Assert.assertFalse(enabled.isWeightBounded()); CacheSpec zeroTtl = CacheSpec.of(true, 0, 100); Assert.assertTrue(zeroTtl.isEnable()); @@ -147,6 +152,10 @@ public void testIsCacheEnabled() { Assert.assertFalse(CacheSpec.isCacheEnabled(false, CacheSpec.CACHE_NO_TTL, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, 0, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, CacheSpec.CACHE_NO_TTL, 0)); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 0L, 1L).isCacheEnabled()); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 1L, 0L).isCacheEnabled()); } @Test @@ -166,4 +175,51 @@ public void testToExpireAfterAccess() { Assert.assertTrue(negativeOther.isPresent()); Assert.assertEquals(0, negativeOther.getAsLong()); } + + @Test + public void testParseWeight() { + Assert.assertEquals(1L, CacheSpec.parseWeight("1", "weight", false, 0L)); + Assert.assertEquals(1024L, CacheSpec.parseWeight("1KB", "weight", false, 0L)); + Assert.assertEquals(2L * 1024L * 1024L, + CacheSpec.parseWeight("2 mb", "weight", false, 0L)); + Assert.assertEquals(250L, CacheSpec.parseWeight("25%", "weight", true, 1000L)); + Assert.assertEquals(0L, CacheSpec.parseWeight("0", "weight", false, 0L)); + + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1.5GB", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("101%", "weight", true, 1000L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1PB000", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("999999999999999999PB", "weight", false, 0L)); + } + + @Test + public void testStrictEnginePropertyAllowlist() { + Map properties = Maps.newHashMap(); + properties.put("meta.cache.hive.partition_values.enable", "true"); + properties.put("meta.cache.hive.partition_values.ttl-second", "-1"); + properties.put("meta.cache.hive.partition_values.capacity", "10"); + properties.put("meta.cache.hive.partition_values.max-weight", "2MB"); + CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values")); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partiton_values.capacity"); + + properties.put("meta.cache.hive.partition_values.enabel", "true"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partition_values.enabel"); + + properties.put("meta.cache.hive.schema.max-weight", "1MB"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java new file mode 100644 index 00000000000000..be4978d33c129d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.junit.Assert; +import org.openjdk.jol.info.GraphLayout; + +/** JOL oracle used only by estimator calibration tests. */ +public final class EstimatorCalibrationAssertions { + private static final long MAX_CONSERVATIVE_FACTOR = 8L; + private static final boolean PRINT_RESULT = Boolean.getBoolean( + "metacache.estimator.calibration.print"); + + static { + // Doris expression graphs contain JVM hidden lambda classes. JOL cannot obtain their + // offsets through the regular instrumentation path on JDK 17, so enable its Unsafe + // fallback for these test-only retained-graph measurements. Skip all attach attempts: + // Iceberg/Paimon calibration tests share their fork with Mockito's inline mock maker. + System.setProperty("jol.magicFieldOffset", "true"); + System.setProperty("jol.skipInstallAttach", "true"); + System.setProperty("jol.skipDynamicAttach", "true"); + System.setProperty("jol.skipHotspotSAAttach", "true"); + } + + private EstimatorCalibrationAssertions() { + } + + public static void assertConservativeDelta( + String fixture, long emptyEstimate, long populatedEstimate, + Object emptyGraph, Object populatedGraph) { + long actualDelta = GraphLayout.parseInstance(populatedGraph).totalSize() + - GraphLayout.parseInstance(emptyGraph).totalSize(); + long estimatedDelta = populatedEstimate - emptyEstimate; + if (PRINT_RESULT) { + System.out.printf("%s: estimated=%d, jol=%d, ratio=%.3f%n", + fixture, estimatedDelta, actualDelta, + actualDelta == 0L ? Double.NaN : (double) estimatedDelta / actualDelta); + } + Assert.assertTrue(fixture + " must add retained heap", actualDelta > 0L); + Assert.assertTrue(fixture + " underestimates retained heap: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta >= actualDelta); + Assert.assertTrue(fixture + " estimate is excessively conservative: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta <= MetaCacheWeightUtils.saturatedMultiply( + actualDelta, MAX_CONSERVATIVE_FACTOR)); + } + + public static long graphSize(Object graph) { + return GraphLayout.parseInstance(graph).totalSize(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java new file mode 100644 index 00000000000000..d4b76b97c0131e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java @@ -0,0 +1,291 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class ExternalMetaCacheBudgetManagerTest { + + @Test + public void testGlobalCatalogAndEntryLimits() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget first = manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(60L)); + EntryBudget second = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(80L), OptionalLong.of(50L)); + + AdmissionReservation firstReservation = first.tryReserve(60L).get(); + Assert.assertFalse(second.tryReserve(30L).isPresent()); + AdmissionReservation secondReservation = second.tryReserve(20L).get(); + Assert.assertEquals(80L, manager.getGlobalUsedWeight()); + Assert.assertFalse(secondReservation.tryResize(30L)); + + firstReservation.release(); + Assert.assertTrue(secondReservation.tryResize(30L)); + Assert.assertEquals(30L, manager.getGlobalUsedWeight()); + + secondReservation.release(); + first.close(); + second.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testConcurrentReservationNeverExceedsGlobalLimit() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "iceberg", "manifest", OptionalLong.empty(), OptionalLong.empty()); + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch start = new CountDownLatch(1); + List reservations = Collections.synchronizedList(new ArrayList<>()); + try { + for (int i = 0; i < 200; i++) { + executor.submit(() -> { + await(start); + Optional reservation = budget.tryReserve(1L); + reservation.ifPresent(reservations::add); + }); + } + start.countDown(); + executor.shutdown(); + Assert.assertTrue(executor.awaitTermination(10L, TimeUnit.SECONDS)); + Assert.assertEquals(100, reservations.size()); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + } finally { + executor.shutdownNow(); + reservations.forEach(AdmissionReservation::release); + budget.close(); + } + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testRejectChildLargerThanParent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(90L))); + + Map properties = new HashMap<>(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "120"); + Assert.assertEquals(120L, manager.parseCatalogMaxWeight(properties).getAsLong()); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.validateCatalogMaxWeight(properties)); + } + + @Test + public void testRuntimeBudgetClampsReplayedCatalogToLocalGlobalLimit() { + ExternalMetaCacheBudgetManager observerManager = manager(100L); + + EntryBudget budget = observerManager.createEntryBudget( + 1L, "iceberg", "table", OptionalLong.of(400L), OptionalLong.of(300L)); + + Assert.assertEquals(100L, budget.getEffectiveMaxWeight()); + Assert.assertEquals(100L, budget.getCatalogMaxWeight()); + AdmissionReservation reservation = budget.tryReserve(100L).get(); + Assert.assertFalse(budget.tryReserve(1L).isPresent()); + reservation.release(); + budget.close(); + } + + @Test + public void testGlobalConfigSupportsPercentageAndDisabledZero() { + String original = Config.external_meta_cache_max_weight; + try { + Config.external_meta_cache_max_weight = "25%"; + ExternalMetaCacheBudgetManager percentageManager = ExternalMetaCacheBudgetManager.fromConfig(); + Assert.assertEquals(Runtime.getRuntime().maxMemory() / 4L, + percentageManager.getGlobalMaxWeight().getAsLong()); + + Config.external_meta_cache_max_weight = "0"; + Assert.assertFalse(ExternalMetaCacheBudgetManager.fromConfig().getGlobalMaxWeight().isPresent()); + + Config.external_meta_cache_max_weight = "0%"; + Assert.assertThrows(IllegalArgumentException.class, ExternalMetaCacheBudgetManager::fromConfig); + } finally { + Config.external_meta_cache_max_weight = original; + } + } + + @Test + public void testReservationReleaseIsIdempotent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = budget.tryReserve(40L).get(); + + reservation.release(); + reservation.release(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + budget.close(); + } + + @Test + public void testClosedBudgetRejectsStaleHandleAndReservationResize() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation zeroByteReservation = staleBudget.tryReserve(0L).get(); + + staleBudget.close(); + staleBudget.close(); + + Assert.assertFalse(staleBudget.tryReserve(1L).isPresent()); + Assert.assertFalse(zeroByteReservation.tryResize(1L)); + Assert.assertEquals(0L, staleBudget.getRejectedCount()); + Assert.assertEquals(0L, manager.getGlobalRejectedCount()); + zeroByteReservation.release(); + + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation replacementReservation = replacement.tryReserve(100L).get(); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + replacementReservation.release(); + replacement.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testCloseForceReleasesOutstandingAccounting() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation staleReservation = staleBudget.tryReserve(40L).get(); + + staleBudget.close(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + staleReservation.release(); + Assert.assertFalse(staleReservation.isActive()); + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + replacement.close(); + } + + @Test + public void testPeerReclaimCoalescesConcurrentMissesToLargestAdmission() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget owner = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.empty(), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 2L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = owner.tryReserve(100L).get(); + CountDownLatch firstReclaimStarted = new CountDownLatch(1); + CountDownLatch releaseFirstReclaim = new CountDownLatch(1); + CountDownLatch secondReclaimFinished = new CountDownLatch(1); + AtomicInteger invocation = new AtomicInteger(); + List targets = Collections.synchronizedList(new ArrayList<>()); + owner.setReclaimer(target -> { + targets.add(target); + if (invocation.getAndIncrement() == 0) { + firstReclaimStarted.countDown(); + await(releaseFirstReclaim); + } else { + secondReclaimFinished.countDown(); + } + return 0L; + }); + try { + requester.requestPeerReclaim(10L); + Assert.assertTrue(firstReclaimStarted.await(3L, TimeUnit.SECONDS)); + + requester.requestPeerReclaim(10L); + requester.requestPeerReclaim(20L); + requester.requestPeerReclaim(15L); + releaseFirstReclaim.countDown(); + + Assert.assertTrue(secondReclaimFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(2, targets.size()); + Assert.assertEquals(Long.valueOf(10L), targets.get(0)); + Assert.assertEquals(Long.valueOf(20L), targets.get(1)); + } finally { + releaseFirstReclaim.countDown(); + reservation.release(); + owner.close(); + requester.close(); + } + } + + @Test + public void testCatalogOnlyDeficitReclaimsSiblingWithoutTouchingOtherCatalog() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(200L); + EntryBudget sibling = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget otherCatalog = manager.createEntryBudget( + 2L, "paimon", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + AdmissionReservation siblingReservation = sibling.tryReserve(100L).get(); + AdmissionReservation otherReservation = otherCatalog.tryReserve(50L).get(); + CountDownLatch siblingReclaimed = new CountDownLatch(1); + AtomicInteger otherCatalogReclaims = new AtomicInteger(); + sibling.setReclaimer(target -> { + siblingReservation.release(); + siblingReclaimed.countDown(); + return 100L; + }); + otherCatalog.setReclaimer(target -> { + otherCatalogReclaims.incrementAndGet(); + return 0L; + }); + try { + requester.requestPeerReclaim(20L); + + Assert.assertTrue(siblingReclaimed.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, otherCatalogReclaims.get()); + Assert.assertEquals(0L, sibling.getUsedWeight()); + Assert.assertEquals(50L, manager.getGlobalUsedWeight()); + } finally { + siblingReservation.release(); + otherReservation.release(); + sibling.close(); + requester.close(); + otherCatalog.close(); + } + } + + private static ExternalMetaCacheBudgetManager manager(long maxWeight) { + return new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + } + + private static void await(CountDownLatch latch) { + try { + Assert.assertTrue(latch.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index ef1090dd5300c6..41d575417654e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -20,18 +20,29 @@ import org.apache.doris.common.Config; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.lang.ref.WeakReference; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Map; +import java.util.OptionalLong; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class MetaCacheEntryTest { @@ -311,6 +322,943 @@ void beforeManualCachePutForTest(String key, Integer loaded) { } } + @Test + public void testExplicitPutWinsAgainstInFlightManualLoad() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePutStarted = new CountDownLatch(1); + CountDownLatch releaseBeforePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePutStarted.countDown(); + awaitLatch(releaseBeforePut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePutStarted.await(3L, TimeUnit.SECONDS)); + entry.put("k", 2); + releaseBeforePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + } finally { + releaseBeforePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedReplacementDoesNotQueueOldValuesOnRefreshExecutor() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "test", "value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "value", key -> new byte[1], CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), entryBudget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + entry.put("k", new byte[100]); + for (int i = 0; i < 100; i++) { + entry.put("k", new byte[100]); + } + + Assert.assertTrue("removal callbacks must not retain replaced values in the executor queue", + refreshExecutor.getQueue().isEmpty()); + Assert.assertEquals(accountedWeight(100L), entry.stats().getEstimatedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + } + } + + @Test + public void testWeightedFirstPublicationInvokesReplacementListener() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "publication", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger publications = new AtomicInteger(); + AtomicReference previous = new AtomicReference<>(); + byte[] value = new byte[10]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> value, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, loaded) -> MetaCacheSizeEstimate.complete(loaded.length), budget, + (key, oldValue, currentValue) -> { + publications.incrementAndGet(); + previous.set(oldValue); + Assert.assertSame(value, currentValue); + }); + try { + entry.put("k", value); + + Assert.assertEquals(1, publications.get()); + Assert.assertNull(previous.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCountEntryLoadAndRefreshInvokeReplacementListener() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loads = new AtomicInteger(); + AtomicInteger publications = new AtomicInteger(); + AtomicReference refreshPrevious = new AtomicReference<>(); + AtomicReference refreshCurrent = new AtomicReference<>(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> loads.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, null, null, + (key, previousValue, currentValue) -> { + publications.incrementAndGet(); + if (previousValue != null) { + refreshPrevious.set(previousValue); + refreshCurrent.set(currentValue); + } + }); + try { + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + Assert.assertEquals(1, publications.get()); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + Assert.assertEquals(2, loads.get()); + Assert.assertEquals(2, publications.get()); + Assert.assertEquals(Integer.valueOf(1), refreshPrevious.get()); + Assert.assertEquals(Integer.valueOf(2), refreshCurrent.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testQueuedWeightedRefreshDoesNotCaptureCurrentValue() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-capture", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-capture", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + + Runnable queuedRefresh = refreshExecutor.getQueue().peek(); + Assert.assertNotNull(queuedRefresh); + for (Field field : queuedRefresh.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("queued refresh must not directly retain the cached value", + currentValue, field.get(queuedRefresh)); + } + + entry.invalidateAll(); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, loaderCalls.get()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderWeightedRefresh() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-fence", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-fence", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertFalse(refreshExecutor.getQueue().isEmpty()); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + + Assert.assertEquals("the older refresh must be rejected before it calls the loader", + 0, loaderCalls.get()); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderCountRefresh() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch loaderEntered = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + CountDownLatch loaderFinished = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-refresh-fence", key -> { + loaderEntered.countDown(); + awaitLatch(releaseLoader); + loaderFinished.countDown(); + return "stale"; + }, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + String currentValue = new String("current"); + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertTrue(loaderEntered.await(3L, TimeUnit.SECONDS)); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseLoader.countDown(); + Assert.assertTrue(loaderFinished.await(3L, TimeUnit.SECONDS)); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertSame("the event fence must suppress refresh write-back", + currentValue, entry.peekIfPresent("k")); + } finally { + releaseLoader.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentWeightedRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-weighted-refresh", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-weighted-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte["a".equals(key) ? 2 : 3]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("a", new byte[1]); + entry.put("b", new byte[1]); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValueLength(entry, "a", 2); + awaitValueLength(entry, "b", 3); + Assert.assertEquals(accountedWeight(2L) + accountedWeight(3L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCountRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-count-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return key + "-refreshed"; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("a", "a-current"); + entry.put("b", "b-current"); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValue(entry, "a", "a-refreshed"); + awaitValue(entry, "b", "b-refreshed"); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testInvalidatingOneKeyDoesNotSuppressAnotherKeysConcurrentMissAdmission() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-miss", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-miss", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte[1]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + Future first = queryExecutor.submit(() -> entry.get("a")); + Future second = queryExecutor.submit(() -> entry.get("b")); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + entry.invalidateKey("a"); + releaseLoaders.countDown(); + first.get(3L, TimeUnit.SECONDS); + second.get(3L, TimeUnit.SECONDS); + + Assert.assertNull(entry.peekIfPresent("a")); + Assert.assertNotNull(entry.peekIfPresent("b")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRemovalCleanupDoesNotDeadlockWithInvalidateAll() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "deadlock", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch removalListenerEntered = new CountDownLatch(1); + CountDownLatch invalidateHasAdmissionLock = new CountDownLatch(1); + CountDownLatch releaseRemovalListener = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "deadlock", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + removalListenerEntered.countDown(); + awaitLatch(releaseRemovalListener); + } + } + + @Override + void beforeWeightedInvalidateAllForTest() { + invalidateHasAdmissionLock.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future eviction = queryExecutor.submit( + () -> loadingCache.policy().eviction().get().setMaximum(1L)); + Assert.assertTrue(removalListenerEntered.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertTrue(invalidateHasAdmissionLock.await(3L, TimeUnit.SECONDS)); + + releaseRemovalListener.countDown(); + eviction.get(3L, TimeUnit.SECONDS); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + releaseRemovalListener.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedRemovalDoesNotReleaseSameIdentityReinsert() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "same-identity-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "same-identity-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCountRemovalDoesNotDropSameIdentityRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry( + "count-same-identity-aba", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeNonWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("the replacement refresh owner must remain usable", 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentCountRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-expired-same-identity", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null); + try { + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new refresh owner", + 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentWeightedReservation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted-expired-same-identity", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted-expired-same-identity", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new reservation", + accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedCacheUsesSoftValuesAndReleasesCollectedReservation() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "soft-value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft-value", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] value = new byte[1]; + entry.put("k", value); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference valueReference = extractValueReference(loadingCache); + + Assert.assertTrue("weighted values must be held through Caffeine SoftReference", + valueReference instanceof SoftReference); + Map owners = (Map) readField(entry, "reservations"); + Object owner = owners.get("k"); + Assert.assertNotNull(owner); + for (Field field : owner.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("reservation ownership must not strongly retain V", value, + field.get(owner)); + } + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + + valueReference.clear(); + Assert.assertTrue(valueReference.enqueue()); + loadingCache.cleanUp(); + + awaitGlobalWeight(manager, 0L); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testStrongQueryReferenceSurvivesSoftValueCollectionChecks() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "query-reference", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "query-reference", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] queryReference = entry.get("k"); + WeakReference observed = new WeakReference<>(queryReference); + + for (int i = 0; i < 3; i++) { + System.gc(); + extractLoadingCache(entry).cleanUp(); + } + + Assert.assertSame(queryReference, observed.get()); + Assert.assertSame(queryReference, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCollectedCallbackCannotReleaseReplacementGeneration() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "collected-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch collectedBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseCollectedCallback = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch collectedCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "collected-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + collectedBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseCollectedCallback); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + collectedCleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference oldValueReference = extractValueReference(loadingCache); + armRemovalHook.set(true); + + Future collection = queryExecutor.submit(() -> { + oldValueReference.clear(); + oldValueReference.enqueue(); + loadingCache.cleanUp(); + }); + Assert.assertTrue(collectedBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + + byte[] replacement = new byte[1]; + armPutHook.set(true); + Future replacementPut = queryExecutor.submit(() -> entry.put("k", replacement)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseCollectedCallback.countDown(); + collection.get(3L, TimeUnit.SECONDS); + releaseReplacementPut.countDown(); + replacementPut.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(collectedCleanupFinished.await(3L, TimeUnit.SECONDS)); + + Assert.assertSame(replacement, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseCollectedCallback.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOwnershipRecordsHaveNoGenericValueReference() { + for (Class nested : MetaCacheEntry.class.getDeclaredClasses()) { + if (nested.getSimpleName().equals("ReservationRecord") + || nested.getSimpleName().equals("RefreshRecord")) { + Assert.assertFalse(nested.getSimpleName() + " must not retain V", + Arrays.stream(nested.getDeclaredFields()) + .anyMatch(field -> field.getType() == Object.class)); + } + } + } + + @Test + public void testRemovalCleanupRetriesAfterTransientFailure() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "removal-retry", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch firstAttempt = new CountDownLatch(1); + CountDownLatch cleanupFinished = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "removal-retry", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalCleanupLockForTest(String key) { + if (attempts.incrementAndGet() == 1) { + firstAttempt.countDown(); + throw new IllegalStateException("transient cleanup failure"); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + cleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + extractLoadingCache(entry).invalidate("k"); + + Assert.assertTrue(firstAttempt.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue(cleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue("cleanup should be retried", attempts.get() >= 2); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testBulkInvalidateDoesNotEnqueueOneCleanupPerEntry() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_000L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "bulk-invalidate", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger queuedCleanupCount = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "bulk-invalidate", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 2_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + queuedCleanupCount.incrementAndGet(); + } + }; + try { + for (int i = 0; i < 1_000; i++) { + entry.put("k-" + i, new byte[1]); + } + + entry.invalidateAll(); + + Assert.assertEquals(0, queuedCleanupCount.get()); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(() -> entry.invalidateKey("k")); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateAllLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testManualMissLoadAllowsNullWithoutCaching() { boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; @@ -373,6 +1321,449 @@ public void testManualMissLoadDoesNotCacheWhenEntryDisabled() { } } + @Test + public void testClosedEntryCanNotBeRepopulated() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + AtomicInteger loadCounter = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", + key -> loadCounter.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, + false); + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + + entry.close(); + + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(Integer.valueOf(2), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testManualMissLoadDoesNotWriteBackAcrossClose() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePut = new CountDownLatch(1); + CountDownLatch releasePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePut.countDown(); + awaitLatch(releasePut); + } + }; + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePut.await(3L, TimeUnit.SECONDS)); + entry.close(); + releasePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releasePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCompareAndReplaceUsesIdentityAndPeekDoesNotPolluteStats() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", key -> "loaded", CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false); + String current = new String("same"); + entry.put("k", current); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.NOT_CURRENT, + entry.tryReplace("k", new String("same"), "wrong")); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", current, "new")); + Assert.assertEquals("new", entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionAndReplacementAccounting() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + + entry.put("k", 40); + Assert.assertEquals(Integer.valueOf(40), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(40L), manager.getGlobalUsedWeight()); + + entry.put("k", 10); + Assert.assertEquals(Integer.valueOf(10), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(10L), manager.getGlobalUsedWeight()); + + entry.invalidateKey("k"); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCaffeineWeigherOnlyReadsPreparedReservationWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Integer first = Integer.valueOf(20); + entry.put("k", first); + Assert.assertEquals(1, estimateCalls.get()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", first, Integer.valueOf(30))); + Assert.assertEquals(2, estimateCalls.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIncompleteEstimateReturnsValueWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.incomplete("unclassified_field"), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNegativeEstimateFailsImmediately() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(-1L), + budget); + try { + Assert.assertThrows(IllegalArgumentException.class, () -> entry.put("k", 30)); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testZeroEstimateIsRejectedWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(0L), budget); + try { + entry.put("k", 1); + + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + Assert.assertEquals("invalid_estimate", entry.stats().getLastWeightRejectReason()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedEntryEvictsItsOwnColdestValueBeforeAdmission() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + entry.put("first", 30); + entry.put("second", 30); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(30), entry.getIfPresent("second")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getEvictionCount()); + Assert.assertEquals(accountedWeight(30L), entry.stats().getEvictionWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionCanReclaimMoreThanOneThousandSmallValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_500L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "many-small-values", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "many-small-values", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + for (int i = 0; i < 1_500; i++) { + entry.put("small-" + i, new byte[1]); + } + + byte[] large = new byte[600_000]; + entry.put("large", large); + + Assert.assertSame(large, entry.peekIfPresent("large")); + Assert.assertTrue(entry.stats().getEvictionCount() > 1_024L); + Assert.assertTrue(entry.stats().getEstimatedWeight() <= maxWeight); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOversizedValueIsRejectedWithoutEvictingUsefulValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_100L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_100L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + entry.put("first", 20); + entry.put("second", 20); + entry.put("oversized", 600); + + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("first")); + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("second")); + Assert.assertNull(entry.peekIfPresent("oversized")); + Assert.assertEquals(2L * accountedWeight(20L), manager.getGlobalUsedWeight()); + Assert.assertEquals(0L, entry.stats().getEvictionCount()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRejectedAtomicReplacementKeepsExpectedValueUntilConditionalInvalidation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(600L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + Integer current = Integer.valueOf(30); + entry.put("k", current); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REJECTED, + entry.tryReplace("k", current, Integer.valueOf(100))); + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(entry.invalidateKeyIfSame("k", current)); + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRejectedWeightedRefreshRetainsPreviousValue() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(600L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-reject", OptionalLong.empty(), OptionalLong.empty()); + byte[] current = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-reject", key -> new byte[100], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRefreshFailureRetainsPreviousValueAndExecutorThread() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + String current = new String("current"); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-failure", key -> { + throw new IllegalStateException("temporary metastore failure"); + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + AtomicBoolean executorStillAlive = new AtomicBoolean(); + refreshExecutor.submit(() -> executorStillAlive.set(true)).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(executorStillAlive.get()); + Assert.assertEquals(1L, entry.stats().getLoadFailureCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testPeerReclamationPreventsGlobalBudgetStarvation() throws Exception { + long valueWeight = accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(2L * valueWeight)); + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager.EntryBudget firstBudget = manager.createEntryBudget( + 1L, "test", "first", OptionalLong.empty(), OptionalLong.empty()); + ExternalMetaCacheBudgetManager.EntryBudget secondBudget = manager.createEntryBudget( + 2L, "test", "second", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry first = new MetaCacheEntry<>( + "first", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), firstBudget); + MetaCacheEntry second = new MetaCacheEntry<>( + "second", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), secondBudget); + try { + first.put("a", new byte[1]); + first.put("b", new byte[1]); + second.put("c", new byte[1]); + Assert.assertNull(second.peekIfPresent("c")); + + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() > valueWeight && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + second.put("c", new byte[1]); + + Assert.assertNotNull(second.peekIfPresent("c")); + Assert.assertTrue(manager.getGlobalUsedWeight() <= 2L * valueWeight); + } finally { + first.close(); + second.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDisabledWeightedEntrySkipsEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 30, CacheSpec.ofWeight(false, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(0, estimateCalls.get()); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + // Keep the loader blocking helper in one place so concurrent tests stay readable. private void awaitLatch(CountDownLatch latch) { try { @@ -383,12 +1774,87 @@ private void awaitLatch(CountDownLatch latch) { } } + private void awaitValueLength(MetaCacheEntry entry, String key, int expectedLength) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + byte[] value = entry.peekIfPresent(key); + if (value != null && value.length == expectedLength) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expectedLength, entry.peekIfPresent(key).length); + } + + private void awaitValue(MetaCacheEntry entry, String key, String expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + if (expected.equals(entry.peekIfPresent(key))) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expected, entry.peekIfPresent(key)); + } + + private void awaitGlobalWeight(ExternalMetaCacheBudgetManager manager, long expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() != expected && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + Assert.assertEquals(expected, manager.getGlobalUsedWeight()); + } + + private Reference extractValueReference(LoadingCache loadingCache) throws Exception { + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + Assert.assertEquals(1, nodes.size()); + Object node = nodes.values().iterator().next(); + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + Object valueReference = valueReferenceMethod.invoke(node); + Assert.assertTrue(valueReference instanceof Reference); + return (Reference) valueReference; + } + + private Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } + @SuppressWarnings("unchecked") - private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { + private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { Field dataField = MetaCacheEntry.class.getDeclaredField("loadingData"); dataField.setAccessible(true); Object raw = dataField.get(entry); Assert.assertTrue(raw instanceof LoadingCache); - return (LoadingCache) raw; + return (LoadingCache) raw; + } + + private static long accountedWeight(long estimatedPayloadBytes) { + return estimatedPayloadBytes + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 228bb112ff0016..fa016361c9ad3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -21,12 +21,16 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; @@ -38,6 +42,8 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.privilege.PrivilegeChecker; +import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -47,9 +53,12 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.StreamTableCommit; import org.apache.paimon.table.sink.StreamTableWrite; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.TableScan; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; import org.junit.Assert; import org.junit.Assume; import org.junit.Rule; @@ -58,22 +67,457 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; public class PaimonExternalMetaCacheTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testSnapshotWeightScalesLinearlyToOneHundredThousandPartitions() throws Exception { + FileStoreTable table = newPartitionedTable("linear_snapshot_estimate", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + long base = snapshotWeight(key, table, 0); + long oneThousand = snapshotWeight(key, table, 1_000); + long tenThousand = snapshotWeight(key, table, 10_000); + long oneHundredThousand = snapshotWeight(key, table, 100_000); + + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @Test + public void testSnapshotWeightAccountsForSkewedTableOptions() throws Exception { + FileStoreTable smallTable = newPartitionedTable( + "option_small", Collections.singletonMap("payload", "x")); + String largePayload = repeatedCharacter('x', 64 * 1024); + FileStoreTable largeTable = newPartitionedTable( + "option_large", Collections.singletonMap("payload", largePayload)); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largePayload.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightAccountsForNestedSchemaPayload() throws Exception { + String largeFieldName = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTableWithNestedField("nested_small", "x"); + FileStoreTable largeTable = newPartitionedTableWithNestedField( + "nested_large", largeFieldName); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largeFieldName.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightAccountsForTableComment() throws Exception { + String largeComment = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTable( + "comment_small", Collections.emptyMap(), "x"); + FileStoreTable largeTable = newPartitionedTable( + "comment_large", Collections.emptyMap(), largeComment); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + + long smallBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L), smallTable, 0); + long largeBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L), largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largeComment.length() - 1L) * 2L); + } + + @Test + public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable table = newPartitionedTable("jol_snapshot", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16); + PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions(table, 32, 16); + PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions(table, 1, 16); + PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions(table, 1, 4096); + + long emptyEstimate = empty.prepareForCachePublication(key).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(key).getBytes(); + long shortTailEstimate = shortTail.prepareForCachePublication(key).getBytes(); + long longTailEstimate = longTail.prepareForCachePublication(key).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon snapshot partitions", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + + @Test + public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.paimon.snapshot.max-weight", "8MB")); + Assert.assertTrue(cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + FileStoreTable table = newPartitionedTable("snapshot_estimate", Collections.emptyMap()); + Object lazyStoreBefore = readField(table, table.getClass(), "lazyStore"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertTrue(value.getSizeEstimate().getBytes() > 0L); + Assert.assertSame("cache admission must not materialize FileStoreTable.store()", + lazyStoreBefore, readField(table, table.getClass(), "lazyStore")); + + PaimonSnapshotCacheValue unsupportedValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, Mockito.mock(Table.class))); + unsupportedValue.prepareForCachePublication(new PaimonSnapshotEntryKey(mapping, 1L, 1L, 1L)); + Assert.assertFalse(unsupportedValue.getSizeEstimate().isComplete()); + Assert.assertTrue(unsupportedValue.getSizeEstimate().getIncompleteReason() + .startsWith("unsupported_paimon_table:")); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotKeySeparatesReloadedTableGenerations() { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, null); + PaimonSnapshotCacheValue fenceValue = new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue first = new PaimonTableCacheValue(null, fenceValue); + PaimonTableCacheValue reloaded = new PaimonTableCacheValue(null, fenceValue); + + PaimonSnapshotEntryKey firstKey = PaimonSnapshotEntryKey.of( + mapping, fence, first.getGeneration()); + PaimonSnapshotEntryKey reloadedKey = PaimonSnapshotEntryKey.of( + mapping, fence, reloaded.getGeneration()); + + Assert.assertNotEquals(firstKey, reloadedKey); + Assert.assertNotEquals(firstKey.getTableGeneration(), reloadedKey.getTableGeneration()); + } + + @Test + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "db", "tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(Mockito.mock(Table.class)); + PaimonTableCacheValue second = new PaimonTableCacheValue(Mockito.mock(Table.class)); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + PaimonSnapshotEntryKey oldSnapshotKey = new PaimonSnapshotEntryKey( + mapping, 1L, 2L, first.getGeneration()); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, first.getPaimonTable()))); + PaimonSchemaCacheKey oldSchemaKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 2L); + org.apache.doris.datasource.metacache.MetaCacheEntry schemas = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotHitRefreshesFenceWithoutReloadingProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "remote_db", "remote_tbl"); + FileStoreTable table = Mockito.mock(FileStoreTable.class); + FileStoreTable pinnedTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + Mockito.when(table.copyWithLatestSchema()).thenReturn(table); + Mockito.when(table.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(table.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(table.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(pinnedTable); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, pinnedTable); + PaimonSnapshotCacheValue snapshotValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + mapping, fence, tableValue.getGeneration()); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class).put(mapping, tableValue); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class).put(key, snapshotValue); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + + Mockito.verify(table, Mockito.times(4)).copyWithLatestSchema(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testContextualSnapshotAndSchemaMissesRunAuthenticated() { + AtomicInteger authenticationDepth = new AtomicInteger(); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + authenticationDepth.incrementAndGet(); + try { + return task.call(); + } finally { + authenticationDepth.decrementAndGet(); + } + } + }; + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Mockito.doAnswer(invocation -> { + Assert.assertTrue("schema history must be read under authentication", + authenticationDepth.get() > 0); + Column partitionColumn = new Column("part", Type.INT); + return new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null); + }).when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); + + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(baseTable.copyWithLatestSchema()).thenAnswer(invocation -> { + Assert.assertTrue("snapshot fence must be read under authentication", + authenticationDepth.get() > 0); + return latestSchemaTable; + }); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenAnswer(invocation -> { + Assert.assertTrue("snapshot pinning must run under authentication", + authenticationDepth.get() > 0); + return snapshotTable; + }); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenAnswer(invocation -> { + Assert.assertTrue("partition enumeration must run under authentication", + authenticationDepth.get() > 0); + return readBuilder; + }); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenAnswer(invocation -> { + Assert.assertTrue("partition manifest access must run under authentication", + authenticationDepth.get() > 0); + return Collections.emptyList(); + }); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(baseTable); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + PaimonSnapshotCacheValue snapshot = cache.getSnapshotCache(dorisTable); + + Assert.assertEquals(7L, snapshot.getSnapshot().getSnapshotId()); + Assert.assertEquals(0, authenticationDepth.get()); + + PaimonTableCacheValue second = new PaimonTableCacheValue(baseTable); + tables.put(mapping, second); + PaimonSchemaCacheKey staleKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 99L); + cache.getPaimonSchemaCacheValue(mapping, 99L, first.getGeneration(), baseTable); + Assert.assertNull("a concurrent old-generation schema load must not repopulate the cache", + cache.entry(1L, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class).peekIfPresent(staleKey)); + Assert.assertEquals(0, authenticationDepth.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotEstimateSupportsPrivilegedTableWrapper() throws Exception { + FileStoreTable table = newPartitionedTable("privileged_estimate", Collections.emptyMap()); + FileStoreTable privileged = PrivilegedFileStoreTable.wrap( + table, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "tbl")); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, table.schema().id(), privileged)); + + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + } + + @Test + public void testSnapshotEstimateDoesNotMaterializeNestedRowTypeIndexes() throws Exception { + RowType nested = DataTypes.ROW( + DataTypes.FIELD(10, "nested_id", DataTypes.INT()), + DataTypes.FIELD(11, "nested_name", DataTypes.STRING())); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "payload", nested)), + 11, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder("nested_row_estimate").toURI()), + schema, + CatalogEnvironment.empty()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, schema.id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, schema.id(), table)); + + Map stateBefore = new HashMap<>(); + for (String fieldName : java.util.Arrays.asList( + "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex")) { + stateBefore.put(fieldName, readField(nested, fieldName)); + } + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + for (Map.Entry entry : stateBefore.entrySet()) { + Assert.assertSame(entry.getKey() + " must not be changed by cache admission", + entry.getValue(), readField(nested, entry.getKey())); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + Map properties = new HashMap<>(); + properties.put("meta.cache.paimon.table.enable", "false"); + properties.put("meta.cache.paimon.table.ttl-second", "17"); + properties.put("meta.cache.paimon.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + @Test public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -113,7 +557,7 @@ public void testFullLatestProjectionCapsManifestParallelismBeforePartitionLoad() .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -153,7 +597,7 @@ public void testLatestFenceDoesNotLoadSchemaOrPartitions() { PaimonPartitionInfoLoader partitionLoader = Mockito.mock(PaimonPartitionInfoLoader.class); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> { + (nameMapping, schemaId, tableGeneration, retainedTable) -> { throw new AssertionError("a version-only fence must not load schema metadata"); }); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); @@ -187,7 +631,7 @@ public void testFenceHydrationKeepsCapturedTableGeneration() throws Exception { .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable captured = Mockito.mock(FileStoreTable.class); @@ -224,7 +668,7 @@ public void testTagProjectionKeepsOnlyRepinnedSnapshotSelector() throws Exceptio table, Collections.singletonMap(CoreOptions.SCAN_TAG_NAME.key(), "stable")); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); PaimonSnapshotCacheValue value = loader.load( @@ -320,6 +764,11 @@ public void testPartitionProjectionIgnoresReaderOnlyPhysicalOptions() throws Exc } private FileStoreTable newPartitionedTable(String name, Map options) throws Exception { + return newPartitionedTable(name, options, null); + } + + private FileStoreTable newPartitionedTable( + String name, Map options, String comment) throws Exception { TableSchema schema = new TableSchema( 0, java.util.Arrays.asList( @@ -329,6 +778,27 @@ private FileStoreTable newPartitionedTable(String name, Map opti Collections.singletonList("part"), Collections.emptyList(), options, + comment); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + + private FileStoreTable newPartitionedTableWithNestedField( + String name, String nestedFieldName) throws Exception { + RowType nestedType = new RowType(Collections.singletonList( + new DataField(2, nestedFieldName, DataTypes.STRING()))); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "payload", nestedType), + new DataField(1, "part", new IntType())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), null); return new AppendOnlyFileStoreTable( LocalFileIO.create(), @@ -337,6 +807,60 @@ private FileStoreTable newPartitionedTable(String name, Map opti CatalogEnvironment.empty()); } + private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, int partitionCount) { + PaimonPartitionInfo partitionInfo = Mockito.mock(PaimonPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToPartition()).thenReturn(partitions); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( + FileStoreTable table, int partitionCount, int valueLength) { + Map partitionItems = new HashMap<>(); + Map partitions = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = "p" + index + repeatedCharacter('x', valueLength); + String name = "part=" + value; + partitionItems.put(name, new org.apache.doris.catalog.ListPartitionItem( + new ArrayList<>())); + partitions.put(name, new org.apache.paimon.partition.Partition( + Collections.singletonMap("part", value), + 100L, 1024L, 1L, 1L, 1, true)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(partitionItems, partitions); + return new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + private Object readField(RowType rowType, String fieldName) throws Exception { + return readField(rowType, RowType.class, fieldName); + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Object readField(Object target, Class owner, String fieldName) throws Exception { + Field field = owner.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(target); + } + @Test public void testInvalidateTablePrecise() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -350,15 +874,24 @@ public void testInvalidateTablePrecise() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(t1, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(t2, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(t1, new PaimonTableCacheValue(null, fence)); + tableEntry.put(t2, new PaimonTableCacheValue(null, fence)); + + PaimonSnapshotEntryKey snapshotKey = new PaimonSnapshotEntryKey(t1, 1L, 2L, 1L); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshotEntry = cache.entry(catalogId, + PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, null))); cache.invalidateTable(catalogId, "db1", "tbl1"); Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); } finally { executor.shutdownNow(); } @@ -377,10 +910,10 @@ public void testInvalidateDbAndStats() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(db1Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(db2Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(db1Table, new PaimonTableCacheValue(null, fence)); + tableEntry.put(db2Table, new PaimonTableCacheValue(null, fence)); org.apache.doris.datasource.metacache.MetaCacheEntry schemaEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java index a0f01f25cdfaeb..7f5483928c8a97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java @@ -107,7 +107,7 @@ public void testStatementContextDefersPhysicalManifestValidationUntilRelationOpt PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( Mockito.mock(PaimonPartitionInfoLoader.class), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "db", "table"); Mockito.doAnswer(ignored -> new PaimonMvccSnapshot( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 09bcee53985f5d..17fa76bb5593bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -36,6 +36,7 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Timestamp; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; @@ -87,6 +88,19 @@ private static PartitionEntry partitionEntry(BinaryRow partition, long sequence) return new PartitionEntry(partition, sequence, sequence, sequence, sequence, 1); } + @Test + public void testCompatibilityConstructorDerivesRetainedPartitionPayload() { + String largeValue = repeatedCharacter('x', 64 * 1024); + Partition partition = new Partition( + Collections.singletonMap("part", largeValue), + 1L, 1L, 1L, 1L, 1, false); + + PaimonPartitionInfo info = new PaimonPartitionInfo( + Collections.emptyMap(), Collections.singletonMap("part=" + largeValue, partition)); + + Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 4L); + } + @Test public void testSchemaForVarcharAndChar() { DataField c1 = new DataField(1, "c1", new VarCharType(32)); @@ -247,6 +261,7 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + "/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1"; + Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length() * 2L); Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().values().iterator().next(); List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) @@ -257,6 +272,21 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { "s1"), actualValues); } + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + List partitionColumns = Collections.singletonList(new Column("part", Type.STRING)); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + PaimonPartitionInfo small = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow("x"), 1L))); + String largeValue = repeatedCharacter('x', 64 * 1024); + PaimonPartitionInfo large = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow(largeValue), 1L))); + + Assert.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= (largeValue.length() - 1L) * 4L); + } + @Test public void testGeneratePartitionInfoUsesPartitionColumnOrder() { List partitionColumns = Arrays.asList( @@ -532,4 +562,10 @@ public void testAuditLogHistorySchemaWithoutSequenceNumber() { Assert.assertEquals("id", fields.get(1).getFieldPtr().getName()); Assert.assertEquals("name", fields.get(2).getFieldPtr().getName()); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/pom.xml b/fe/pom.xml index 3783d2660e6b9b..f709024e526e32 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -272,6 +272,7 @@ under the License. 3.1.0 18.3.14-doris-SNAPSHOT 1.49 + 0.17 2.18.0 1.11.0 1.1.1 @@ -438,6 +439,12 @@ under the License. + + benchmark + + fe-benchmark + + @@ -1925,6 +1932,11 @@ under the License. mockito-inline ${mockito.version} + + org.openjdk.jol + jol-core + ${jol.version} + it.unimi.dsi fastutil-core diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 2e2a2ea8e9b5c9..6079a17ec402ca 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -28,8 +28,24 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}" String warehouse = "${default_fs}/warehouse" - // 1. test default catalog + // DDL validation must reject misspelled memory-governance options. sql """drop catalog if exists ${catalog_name};""" + test { + sql """ + create catalog ${catalog_name} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = '${default_fs}', + 'warehouse' = '${warehouse}', + 'meta.cache.iceberg.snapshot.max-weigth' = '16MB' + ); + """ + exception "Unknown external meta cache" + } + + // 1. test a catalog-level memory bound without a global bound. The existing + // create/insert/select/refresh flow below is the weighted-cache happy path. sql """ create catalog ${catalog_name} properties ( 'type'='iceberg', @@ -37,6 +53,7 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', 'fs.defaultFS' = '${default_fs}', 'warehouse' = '${warehouse}', + 'meta.cache.max-weight' = '128MB', 'meta.cache.iceberg.manifest.enable' = 'true' ); """