diff --git a/pom.xml b/pom.xml index eb49773..7c382c9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.tidesdb tidesdb-java - 0.7.0 + 0.8.0 jar TidesDB Java diff --git a/src/main/c/com_tidesdb_TidesDB.c b/src/main/c/com_tidesdb_TidesDB.c index 1aa8dcc..2c8f806 100644 --- a/src/main/c/com_tidesdb_TidesDB.c +++ b/src/main/c/com_tidesdb_TidesDB.c @@ -86,7 +86,8 @@ JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeOpen( jint oscMaxConcurrentUploads, jint oscMaxConcurrentDownloads, jlong oscMultipartThreshold, jlong oscMultipartPartSize, jboolean oscSyncManifestToObject, jboolean oscReplicateWal, jboolean oscWalUploadSync, jlong oscWalSyncThresholdBytes, jboolean oscWalSyncOnCommit, - jboolean oscReplicaMode, jlong oscReplicaSyncIntervalUs, jboolean oscReplicaReplayWal) + jboolean oscReplicaMode, jlong oscReplicaSyncIntervalUs, jboolean oscReplicaReplayWal, + jint maxConcurrentFlushes) { const char *path = (*env)->GetStringUTFChars(env, dbPath, NULL); if (path == NULL) @@ -149,7 +150,8 @@ JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeOpen( .unified_memtable_sync_mode = unifiedMemtableSyncMode, .unified_memtable_sync_interval_us = (uint64_t)unifiedMemtableSyncIntervalUs, .object_store = obj_store, - .object_store_config = obj_store != NULL ? &os_cfg : NULL}; + .object_store_config = obj_store != NULL ? &os_cfg : NULL, + .max_concurrent_flushes = maxConcurrentFlushes}; tidesdb_t *db = NULL; int result = tidesdb_open(&config, &db); @@ -189,7 +191,8 @@ JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCreateColumnFamily( jboolean enableBlockIndexes, jint indexSampleRatio, jint blockIndexPrefixLen, jint syncMode, jlong syncIntervalUs, jstring comparatorName, jint skipListMaxLevel, jfloat skipListProbability, jint defaultIsolationLevel, jlong minDiskSpace, jint l1FileCountTrigger, - jint l0QueueStallThreshold, jboolean useBtree, + jint l0QueueStallThreshold, jdouble tombstoneDensityTrigger, + jlong tombstoneDensityMinEntries, jboolean useBtree, jboolean objectLazyCompaction, jboolean objectPrefetchCompaction) { tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; @@ -226,6 +229,8 @@ JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCreateColumnFamily( .min_disk_space = (uint64_t)minDiskSpace, .l1_file_count_trigger = l1FileCountTrigger, .l0_queue_stall_threshold = l0QueueStallThreshold, + .tombstone_density_trigger = tombstoneDensityTrigger, + .tombstone_density_min_entries = (uint64_t)tombstoneDensityMinEntries, .use_btree = useBtree ? 1 : 0, .object_lazy_compaction = objectLazyCompaction ? 1 : 0, .object_prefetch_compaction = objectPrefetchCompaction ? 1 : 0}; @@ -570,17 +575,78 @@ JNIEXPORT jobject JNICALL Java_com_tidesdb_ColumnFamily_nativeGetStats(JNIEnv *e free(counts); } + jlongArray levelTombstoneCounts = (*env)->NewLongArray(env, stats->num_levels); + if (stats->level_tombstone_counts != NULL) + { + jlong *counts = malloc(stats->num_levels * sizeof(jlong)); + for (int i = 0; i < stats->num_levels; i++) + { + counts[i] = (jlong)stats->level_tombstone_counts[i]; + } + (*env)->SetLongArrayRegion(env, levelTombstoneCounts, 0, stats->num_levels, counts); + free(counts); + } + + /* Build ColumnFamilyConfig from stats->config so callers can round-trip CF settings */ + jobject cfConfigObj = NULL; + if (stats->config != NULL) + { + jclass cfConfigClass = (*env)->FindClass(env, "com/tidesdb/ColumnFamilyConfig"); + jmethodID fromNative = (*env)->GetStaticMethodID( + env, cfConfigClass, "fromNative", + "(JJIIJIZDZIIIJLjava/lang/String;IFIJIIDJZZZ)Lcom/tidesdb/ColumnFamilyConfig;"); + + jstring comparatorName = (*env)->NewStringUTF(env, stats->config->comparator_name); + + cfConfigObj = (*env)->CallStaticObjectMethod( + env, cfConfigClass, fromNative, + (jlong)stats->config->write_buffer_size, + (jlong)stats->config->level_size_ratio, + (jint)stats->config->min_levels, + (jint)stats->config->dividing_level_offset, + (jlong)stats->config->klog_value_threshold, + (jint)stats->config->compression_algorithm, + stats->config->enable_bloom_filter != 0 ? JNI_TRUE : JNI_FALSE, + (jdouble)stats->config->bloom_fpr, + stats->config->enable_block_indexes != 0 ? JNI_TRUE : JNI_FALSE, + (jint)stats->config->index_sample_ratio, + (jint)stats->config->block_index_prefix_len, + (jint)stats->config->sync_mode, + (jlong)stats->config->sync_interval_us, + comparatorName, + (jint)stats->config->skip_list_max_level, + (jfloat)stats->config->skip_list_probability, + (jint)stats->config->default_isolation_level, + (jlong)stats->config->min_disk_space, + (jint)stats->config->l1_file_count_trigger, + (jint)stats->config->l0_queue_stall_threshold, + (jdouble)stats->config->tombstone_density_trigger, + (jlong)stats->config->tombstone_density_min_entries, + stats->config->use_btree != 0 ? JNI_TRUE : JNI_FALSE, + stats->config->object_lazy_compaction != 0 ? JNI_TRUE : JNI_FALSE, + stats->config->object_prefetch_compaction != 0 ? JNI_TRUE : JNI_FALSE); + + (*env)->DeleteLocalRef(env, comparatorName); + (*env)->DeleteLocalRef(env, cfConfigClass); + } + jclass statsClass = (*env)->FindClass(env, "com/tidesdb/Stats"); jmethodID constructor = (*env)->GetMethodID(env, statsClass, "", - "(IJ[J[ILcom/tidesdb/ColumnFamilyConfig;JJDD[JDDZJID)V"); + "(IJ[J[ILcom/tidesdb/ColumnFamilyConfig;JJDD[JDDZJIDJD[JDI)V"); jobject statsObj = (*env)->NewObject(env, statsClass, constructor, stats->num_levels, (jlong)stats->memtable_size, levelSizes, levelNumSSTables, - NULL, (jlong)stats->total_keys, (jlong)stats->total_data_size, + cfConfigObj, (jlong)stats->total_keys, + (jlong)stats->total_data_size, stats->avg_key_size, stats->avg_value_size, levelKeyCounts, stats->read_amp, stats->hit_rate, stats->use_btree != 0, (jlong)stats->btree_total_nodes, - (jint)stats->btree_max_height, stats->btree_avg_height); + (jint)stats->btree_max_height, stats->btree_avg_height, + (jlong)stats->total_tombstones, + (jdouble)stats->tombstone_ratio, + levelTombstoneCounts, + (jdouble)stats->max_sst_density, + (jint)stats->max_sst_density_level); tidesdb_free_stats(stats); @@ -599,6 +665,55 @@ JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeCompact(JNIEnv *env, } } +JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeCompactRange(JNIEnv *env, jclass cls, + jlong handle, + jbyteArray startKey, + jbyteArray endKey) +{ + tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; + + /* Map null/empty byte arrays to NULL pointers for unbounded endpoints. The C API + rejects both-NULL with TDB_ERR_INVALID_ARGS, so we don't need to filter here. */ + jbyte *startBytes = NULL; + jsize startLen = 0; + if (startKey != NULL) + { + startLen = (*env)->GetArrayLength(env, startKey); + if (startLen > 0) + { + startBytes = (*env)->GetByteArrayElements(env, startKey, NULL); + } + } + + jbyte *endBytes = NULL; + jsize endLen = 0; + if (endKey != NULL) + { + endLen = (*env)->GetArrayLength(env, endKey); + if (endLen > 0) + { + endBytes = (*env)->GetByteArrayElements(env, endKey, NULL); + } + } + + int result = tidesdb_compact_range(cf, (const uint8_t *)startBytes, (size_t)startLen, + (const uint8_t *)endBytes, (size_t)endLen); + + if (startBytes != NULL) + { + (*env)->ReleaseByteArrayElements(env, startKey, startBytes, JNI_ABORT); + } + if (endBytes != NULL) + { + (*env)->ReleaseByteArrayElements(env, endKey, endBytes, JNI_ABORT); + } + + if (result != TDB_SUCCESS) + { + throwTidesDBException(env, result, getErrorMessage(result)); + } +} + JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeFlushMemtable(JNIEnv *env, jclass cls, jlong handle) { @@ -1307,6 +1422,27 @@ JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativePromoteToPrimary(JNIEnv *e } } +JNIEXPORT jint JNICALL Java_com_tidesdb_Config_nativeDefaultMaxConcurrentFlushes(JNIEnv *env, + jclass cls) +{ + tidesdb_config_t cfg = tidesdb_default_config(); + return (jint)cfg.max_concurrent_flushes; +} + +JNIEXPORT jdouble JNICALL Java_com_tidesdb_ColumnFamilyConfig_nativeDefaultTombstoneDensityTrigger( + JNIEnv *env, jclass cls) +{ + tidesdb_column_family_config_t cfg = tidesdb_default_column_family_config(); + return (jdouble)cfg.tombstone_density_trigger; +} + +JNIEXPORT jlong JNICALL Java_com_tidesdb_ColumnFamilyConfig_nativeDefaultTombstoneDensityMinEntries( + JNIEnv *env, jclass cls) +{ + tidesdb_column_family_config_t cfg = tidesdb_default_column_family_config(); + return (jlong)cfg.tombstone_density_min_entries; +} + JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDBIterator_nativeKeyValue(JNIEnv *env, jclass cls, jlong handle) { diff --git a/src/main/java/com/tidesdb/ColumnFamily.java b/src/main/java/com/tidesdb/ColumnFamily.java index 7a49143..e07a57f 100644 --- a/src/main/java/com/tidesdb/ColumnFamily.java +++ b/src/main/java/com/tidesdb/ColumnFamily.java @@ -64,6 +64,24 @@ public Stats getStats() throws TidesDBException { public void compact() throws TidesDBException { nativeCompact(nativeHandle); } + + /** + * Synchronously compacts every SSTable whose key range overlaps {@code [startKey, endKey)}. + * Blocks the calling thread until the merge commits or fails - does not enqueue work + * onto the compaction thread pool. + * + *

A {@code null} or empty endpoint means unbounded on that side. Both endpoints + * being {@code null} or empty is rejected with {@link TidesDBException}; callers + * wanting full-CF compaction must use {@link #compact()}.

+ * + * @param startKey lower bound of the range, or {@code null}/empty for unbounded + * @param endKey upper bound (exclusive), or {@code null}/empty for unbounded + * @throws TidesDBException if the range is invalid, another compaction is running, + * or the merge fails + */ + public void compactRange(byte[] startKey, byte[] endKey) throws TidesDBException { + nativeCompactRange(nativeHandle, startKey, endKey); + } /** * Manually triggers memtable flush for this column family. @@ -203,6 +221,7 @@ long getNativeHandle() { private static native Stats nativeGetStats(long handle) throws TidesDBException; private static native void nativeCompact(long handle) throws TidesDBException; + private static native void nativeCompactRange(long handle, byte[] startKey, byte[] endKey) throws TidesDBException; private static native void nativeFlushMemtable(long handle) throws TidesDBException; private static native boolean nativeIsFlushing(long handle); private static native boolean nativeIsCompacting(long handle); diff --git a/src/main/java/com/tidesdb/ColumnFamilyConfig.java b/src/main/java/com/tidesdb/ColumnFamilyConfig.java index fc383e6..39079b0 100644 --- a/src/main/java/com/tidesdb/ColumnFamilyConfig.java +++ b/src/main/java/com/tidesdb/ColumnFamilyConfig.java @@ -22,7 +22,11 @@ * Configuration for a column family. */ public class ColumnFamilyConfig { - + + static { + NativeLibrary.load(); + } + private long writeBufferSize; private long levelSizeRatio; private int minLevels; @@ -43,6 +47,8 @@ public class ColumnFamilyConfig { private long minDiskSpace; private int l1FileCountTrigger; private int l0QueueStallThreshold; + private double tombstoneDensityTrigger; + private long tombstoneDensityMinEntries; private boolean useBtree; private boolean objectLazyCompaction; private boolean objectPrefetchCompaction; @@ -68,13 +74,17 @@ private ColumnFamilyConfig(Builder builder) { this.minDiskSpace = builder.minDiskSpace; this.l1FileCountTrigger = builder.l1FileCountTrigger; this.l0QueueStallThreshold = builder.l0QueueStallThreshold; + this.tombstoneDensityTrigger = builder.tombstoneDensityTrigger; + this.tombstoneDensityMinEntries = builder.tombstoneDensityMinEntries; this.useBtree = builder.useBtree; this.objectLazyCompaction = builder.objectLazyCompaction; this.objectPrefetchCompaction = builder.objectPrefetchCompaction; } - + /** - * Creates a default column family configuration. + * Creates a default column family configuration. The tombstone density defaults + * are sourced from the underlying C library so that this binding tracks the + * engine's defaults automatically. * * @return a new ColumnFamilyConfig with default values */ @@ -100,12 +110,14 @@ public static ColumnFamilyConfig defaultConfig() { .minDiskSpace(100 * 1024 * 1024) .l1FileCountTrigger(4) .l0QueueStallThreshold(20) + .tombstoneDensityTrigger(nativeDefaultTombstoneDensityTrigger()) + .tombstoneDensityMinEntries(nativeDefaultTombstoneDensityMinEntries()) .useBtree(false) .objectLazyCompaction(false) .objectPrefetchCompaction(true) .build(); } - + /** * Creates a new builder for ColumnFamilyConfig. * @@ -114,7 +126,53 @@ public static ColumnFamilyConfig defaultConfig() { public static Builder builder() { return new Builder(); } - + + /** + * Constructs a ColumnFamilyConfig from raw native primitives. Used by the JNI + * layer when reading back the configuration embedded in tidesdb_stats_t. + */ + static ColumnFamilyConfig fromNative(long writeBufferSize, long levelSizeRatio, int minLevels, + int dividingLevelOffset, long klogValueThreshold, + int compressionAlgorithm, boolean enableBloomFilter, + double bloomFPR, boolean enableBlockIndexes, + int indexSampleRatio, int blockIndexPrefixLen, + int syncMode, long syncIntervalUs, String comparatorName, + int skipListMaxLevel, float skipListProbability, + int defaultIsolationLevel, long minDiskSpace, + int l1FileCountTrigger, int l0QueueStallThreshold, + double tombstoneDensityTrigger, + long tombstoneDensityMinEntries, boolean useBtree, + boolean objectLazyCompaction, + boolean objectPrefetchCompaction) { + return new Builder() + .writeBufferSize(writeBufferSize) + .levelSizeRatio(levelSizeRatio) + .minLevels(minLevels) + .dividingLevelOffset(dividingLevelOffset) + .klogValueThreshold(klogValueThreshold) + .compressionAlgorithm(CompressionAlgorithm.fromValue(compressionAlgorithm)) + .enableBloomFilter(enableBloomFilter) + .bloomFPR(bloomFPR) + .enableBlockIndexes(enableBlockIndexes) + .indexSampleRatio(indexSampleRatio) + .blockIndexPrefixLen(blockIndexPrefixLen) + .syncMode(SyncMode.fromValue(syncMode)) + .syncIntervalUs(syncIntervalUs) + .comparatorName(comparatorName == null ? "" : comparatorName) + .skipListMaxLevel(skipListMaxLevel) + .skipListProbability(skipListProbability) + .defaultIsolationLevel(IsolationLevel.fromValue(defaultIsolationLevel)) + .minDiskSpace(minDiskSpace) + .l1FileCountTrigger(l1FileCountTrigger) + .l0QueueStallThreshold(l0QueueStallThreshold) + .tombstoneDensityTrigger(tombstoneDensityTrigger) + .tombstoneDensityMinEntries(tombstoneDensityMinEntries) + .useBtree(useBtree) + .objectLazyCompaction(objectLazyCompaction) + .objectPrefetchCompaction(objectPrefetchCompaction) + .build(); + } + public long getWriteBufferSize() { return writeBufferSize; } public long getLevelSizeRatio() { return levelSizeRatio; } public int getMinLevels() { return minLevels; } @@ -135,10 +193,15 @@ public static Builder builder() { public long getMinDiskSpace() { return minDiskSpace; } public int getL1FileCountTrigger() { return l1FileCountTrigger; } public int getL0QueueStallThreshold() { return l0QueueStallThreshold; } + public double getTombstoneDensityTrigger() { return tombstoneDensityTrigger; } + public long getTombstoneDensityMinEntries() { return tombstoneDensityMinEntries; } public boolean isUseBtree() { return useBtree; } public boolean isObjectLazyCompaction() { return objectLazyCompaction; } public boolean isObjectPrefetchCompaction() { return objectPrefetchCompaction; } + private static native double nativeDefaultTombstoneDensityTrigger(); + private static native long nativeDefaultTombstoneDensityMinEntries(); + /** * Builder for ColumnFamilyConfig. */ @@ -163,6 +226,8 @@ public static class Builder { private long minDiskSpace = 100 * 1024 * 1024; private int l1FileCountTrigger = 4; private int l0QueueStallThreshold = 20; + private double tombstoneDensityTrigger = 0.0; + private long tombstoneDensityMinEntries = 1024; private boolean useBtree = false; private boolean objectLazyCompaction = false; private boolean objectPrefetchCompaction = true; @@ -171,102 +236,112 @@ public Builder writeBufferSize(long writeBufferSize) { this.writeBufferSize = writeBufferSize; return this; } - + public Builder levelSizeRatio(long levelSizeRatio) { this.levelSizeRatio = levelSizeRatio; return this; } - + public Builder minLevels(int minLevels) { this.minLevels = minLevels; return this; } - + public Builder dividingLevelOffset(int dividingLevelOffset) { this.dividingLevelOffset = dividingLevelOffset; return this; } - + public Builder klogValueThreshold(long klogValueThreshold) { this.klogValueThreshold = klogValueThreshold; return this; } - + public Builder compressionAlgorithm(CompressionAlgorithm compressionAlgorithm) { this.compressionAlgorithm = compressionAlgorithm; return this; } - + public Builder enableBloomFilter(boolean enableBloomFilter) { this.enableBloomFilter = enableBloomFilter; return this; } - + public Builder bloomFPR(double bloomFPR) { this.bloomFPR = bloomFPR; return this; } - + public Builder enableBlockIndexes(boolean enableBlockIndexes) { this.enableBlockIndexes = enableBlockIndexes; return this; } - + public Builder indexSampleRatio(int indexSampleRatio) { this.indexSampleRatio = indexSampleRatio; return this; } - + public Builder blockIndexPrefixLen(int blockIndexPrefixLen) { this.blockIndexPrefixLen = blockIndexPrefixLen; return this; } - + public Builder syncMode(SyncMode syncMode) { this.syncMode = syncMode; return this; } - + public Builder syncIntervalUs(long syncIntervalUs) { this.syncIntervalUs = syncIntervalUs; return this; } - + public Builder comparatorName(String comparatorName) { this.comparatorName = comparatorName; return this; } - + public Builder skipListMaxLevel(int skipListMaxLevel) { this.skipListMaxLevel = skipListMaxLevel; return this; } - + public Builder skipListProbability(float skipListProbability) { this.skipListProbability = skipListProbability; return this; } - + public Builder defaultIsolationLevel(IsolationLevel defaultIsolationLevel) { this.defaultIsolationLevel = defaultIsolationLevel; return this; } - + public Builder minDiskSpace(long minDiskSpace) { this.minDiskSpace = minDiskSpace; return this; } - + public Builder l1FileCountTrigger(int l1FileCountTrigger) { this.l1FileCountTrigger = l1FileCountTrigger; return this; } - + public Builder l0QueueStallThreshold(int l0QueueStallThreshold) { this.l0QueueStallThreshold = l0QueueStallThreshold; return this; } - + + public Builder tombstoneDensityTrigger(double tombstoneDensityTrigger) { + this.tombstoneDensityTrigger = tombstoneDensityTrigger; + return this; + } + + public Builder tombstoneDensityMinEntries(long tombstoneDensityMinEntries) { + this.tombstoneDensityMinEntries = tombstoneDensityMinEntries; + return this; + } + public Builder useBtree(boolean useBtree) { this.useBtree = useBtree; return this; diff --git a/src/main/java/com/tidesdb/Config.java b/src/main/java/com/tidesdb/Config.java index abf4ebe..a6169e0 100644 --- a/src/main/java/com/tidesdb/Config.java +++ b/src/main/java/com/tidesdb/Config.java @@ -22,7 +22,11 @@ * Configuration for opening a TidesDB instance. */ public class Config { - + + static { + NativeLibrary.load(); + } + private String dbPath; private int numFlushThreads; private int numCompactionThreads; @@ -40,6 +44,7 @@ public class Config { private long unifiedMemtableSyncIntervalUs; private String objectStoreFsPath; private ObjectStoreConfig objectStoreConfig; + private int maxConcurrentFlushes; private Config(Builder builder) { this.dbPath = builder.dbPath; @@ -59,10 +64,13 @@ private Config(Builder builder) { this.unifiedMemtableSyncIntervalUs = builder.unifiedMemtableSyncIntervalUs; this.objectStoreFsPath = builder.objectStoreFsPath; this.objectStoreConfig = builder.objectStoreConfig; + this.maxConcurrentFlushes = builder.maxConcurrentFlushes; } - + /** - * Creates a default configuration. + * Creates a default configuration. The {@code maxConcurrentFlushes} default is + * sourced from the underlying C library via {@code tidesdb_default_config()} so + * the binding tracks the engine's defaults automatically. * * @return a new Config with default values */ @@ -76,8 +84,11 @@ public static Config defaultConfig() { .logToFile(false) .logTruncationAt(24 * 1024 * 1024) .maxMemoryUsage(0) + .maxConcurrentFlushes(nativeDefaultMaxConcurrentFlushes()) .build(); } + + private static native int nativeDefaultMaxConcurrentFlushes(); /** * Creates a new builder for Config. @@ -157,6 +168,10 @@ public ObjectStoreConfig getObjectStoreConfig() { return objectStoreConfig; } + public int getMaxConcurrentFlushes() { + return maxConcurrentFlushes; + } + /** * Builder for Config. */ @@ -178,6 +193,7 @@ public static class Builder { private long unifiedMemtableSyncIntervalUs = 0; private String objectStoreFsPath = null; private ObjectStoreConfig objectStoreConfig = null; + private int maxConcurrentFlushes = 0; public Builder dbPath(String dbPath) { this.dbPath = dbPath; @@ -264,6 +280,11 @@ public Builder objectStoreConfig(ObjectStoreConfig objectStoreConfig) { return this; } + public Builder maxConcurrentFlushes(int maxConcurrentFlushes) { + this.maxConcurrentFlushes = maxConcurrentFlushes; + return this; + } + public Config build() { validate(); return new Config(this); diff --git a/src/main/java/com/tidesdb/Stats.java b/src/main/java/com/tidesdb/Stats.java index 8a01a2f..37b6f5b 100644 --- a/src/main/java/com/tidesdb/Stats.java +++ b/src/main/java/com/tidesdb/Stats.java @@ -22,7 +22,7 @@ * Statistics about a column family. */ public class Stats { - + private final int numLevels; private final long memtableSize; private final long[] levelSizes; @@ -39,12 +39,19 @@ public class Stats { private final long btreeTotalNodes; private final int btreeMaxHeight; private final double btreeAvgHeight; - - public Stats(int numLevels, long memtableSize, long[] levelSizes, int[] levelNumSSTables, - ColumnFamilyConfig config, long totalKeys, long totalDataSize, + private final long totalTombstones; + private final double tombstoneRatio; + private final long[] levelTombstoneCounts; + private final double maxSstDensity; + private final int maxSstDensityLevel; + + public Stats(int numLevels, long memtableSize, long[] levelSizes, int[] levelNumSSTables, + ColumnFamilyConfig config, long totalKeys, long totalDataSize, double avgKeySize, double avgValueSize, long[] levelKeyCounts, double readAmp, double hitRate, boolean useBtree, long btreeTotalNodes, - int btreeMaxHeight, double btreeAvgHeight) { + int btreeMaxHeight, double btreeAvgHeight, + long totalTombstones, double tombstoneRatio, long[] levelTombstoneCounts, + double maxSstDensity, int maxSstDensityLevel) { this.numLevels = numLevels; this.memtableSize = memtableSize; this.levelSizes = levelSizes; @@ -61,8 +68,13 @@ public Stats(int numLevels, long memtableSize, long[] levelSizes, int[] levelNum this.btreeTotalNodes = btreeTotalNodes; this.btreeMaxHeight = btreeMaxHeight; this.btreeAvgHeight = btreeAvgHeight; + this.totalTombstones = totalTombstones; + this.tombstoneRatio = tombstoneRatio; + this.levelTombstoneCounts = levelTombstoneCounts; + this.maxSstDensity = maxSstDensity; + this.maxSstDensityLevel = maxSstDensityLevel; } - + /** * Gets the number of levels. * @@ -71,7 +83,7 @@ public Stats(int numLevels, long memtableSize, long[] levelSizes, int[] levelNum public int getNumLevels() { return numLevels; } - + /** * Gets the memtable size in bytes. * @@ -80,7 +92,7 @@ public int getNumLevels() { public long getMemtableSize() { return memtableSize; } - + /** * Gets the sizes of each level in bytes. * @@ -89,7 +101,7 @@ public long getMemtableSize() { public long[] getLevelSizes() { return levelSizes; } - + /** * Gets the number of SSTables at each level. * @@ -98,7 +110,7 @@ public long[] getLevelSizes() { public int[] getLevelNumSSTables() { return levelNumSSTables; } - + /** * Gets the column family configuration. * @@ -107,7 +119,7 @@ public int[] getLevelNumSSTables() { public ColumnFamilyConfig getConfig() { return config; } - + /** * Gets the total number of keys across memtable and all SSTables. * @@ -116,7 +128,7 @@ public ColumnFamilyConfig getConfig() { public long getTotalKeys() { return totalKeys; } - + /** * Gets the total data size (klog + vlog) across all SSTables. * @@ -125,7 +137,7 @@ public long getTotalKeys() { public long getTotalDataSize() { return totalDataSize; } - + /** * Gets the average key size in bytes. * @@ -134,7 +146,7 @@ public long getTotalDataSize() { public double getAvgKeySize() { return avgKeySize; } - + /** * Gets the average value size in bytes. * @@ -143,7 +155,7 @@ public double getAvgKeySize() { public double getAvgValueSize() { return avgValueSize; } - + /** * Gets the number of keys per level. * @@ -152,7 +164,7 @@ public double getAvgValueSize() { public long[] getLevelKeyCounts() { return levelKeyCounts; } - + /** * Gets the read amplification (point lookup cost multiplier). * @@ -161,7 +173,7 @@ public long[] getLevelKeyCounts() { public double getReadAmp() { return readAmp; } - + /** * Gets the cache hit rate for this column family. * @@ -170,7 +182,7 @@ public double getReadAmp() { public double getHitRate() { return hitRate; } - + /** * Returns whether this column family uses B+tree format. * @@ -179,7 +191,7 @@ public double getHitRate() { public boolean isUseBtree() { return useBtree; } - + /** * Gets the total number of B+tree nodes across all SSTables. * Only populated when useBtree is true. @@ -189,7 +201,7 @@ public boolean isUseBtree() { public long getBtreeTotalNodes() { return btreeTotalNodes; } - + /** * Gets the maximum B+tree height across all SSTables. * Only populated when useBtree is true. @@ -199,7 +211,7 @@ public long getBtreeTotalNodes() { public int getBtreeMaxHeight() { return btreeMaxHeight; } - + /** * Gets the average B+tree height across all SSTables. * Only populated when useBtree is true. @@ -209,7 +221,56 @@ public int getBtreeMaxHeight() { public double getBtreeAvgHeight() { return btreeAvgHeight; } - + + /** + * Gets the total number of tombstones across every SSTable in the column family. + * + * @return total tombstone count + */ + public long getTotalTombstones() { + return totalTombstones; + } + + /** + * Gets the tombstone ratio (totalTombstones / totalKeys). + * Returns 0.0 when totalKeys is 0. Always within [0.0, 1.0]. + * + * @return tombstone ratio + */ + public double getTombstoneRatio() { + return tombstoneRatio; + } + + /** + * Gets the per-level tombstone counts. Length matches numLevels and parallels + * {@link #getLevelKeyCounts()}. + * + * @return per-level tombstone counts + */ + public long[] getLevelTombstoneCounts() { + return levelTombstoneCounts; + } + + /** + * Gets the worst per-SSTable tombstone density (tombstone_count / num_entries) + * observed in this column family. Always within [0.0, 1.0]. + * + * @return max per-SSTable tombstone density + */ + public double getMaxSstDensity() { + return maxSstDensity; + } + + /** + * Gets the 1-based level index where the worst per-SSTable tombstone density + * was observed. Returns 0 if no SSTable contributed to the measurement. + * + * @return 1-based level index of the worst SSTable, or 0 if none + */ + public int getMaxSstDensityLevel() { + return maxSstDensityLevel; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -227,6 +288,10 @@ public String toString() { sb.append(", btreeMaxHeight=").append(btreeMaxHeight); sb.append(", btreeAvgHeight=").append(btreeAvgHeight); } + sb.append(", totalTombstones=").append(totalTombstones); + sb.append(", tombstoneRatio=").append(tombstoneRatio); + sb.append(", maxSstDensity=").append(maxSstDensity); + sb.append(", maxSstDensityLevel=").append(maxSstDensityLevel); if (levelSizes != null) { sb.append(", levelSizes=["); for (int i = 0; i < levelSizes.length; i++) { @@ -251,6 +316,14 @@ public String toString() { } sb.append("]"); } + if (levelTombstoneCounts != null) { + sb.append(", levelTombstoneCounts=["); + for (int i = 0; i < levelTombstoneCounts.length; i++) { + if (i > 0) sb.append(", "); + sb.append(levelTombstoneCounts[i]); + } + sb.append("]"); + } sb.append("}"); return sb.toString(); } diff --git a/src/main/java/com/tidesdb/TidesDB.java b/src/main/java/com/tidesdb/TidesDB.java index 337b37d..635d0b1 100644 --- a/src/main/java/com/tidesdb/TidesDB.java +++ b/src/main/java/com/tidesdb/TidesDB.java @@ -86,9 +86,10 @@ public static TidesDB open(Config config) throws TidesDBException { osc != null ? osc.isWalSyncOnCommit() : false, osc != null ? osc.isReplicaMode() : false, osc != null ? osc.getReplicaSyncIntervalUs() : 5000000, - osc != null ? osc.isReplicaReplayWal() : true + osc != null ? osc.isReplicaReplayWal() : true, + config.getMaxConcurrentFlushes() ); - + return new TidesDB(handle); } @@ -141,6 +142,8 @@ public void createColumnFamily(String name, ColumnFamilyConfig config) throws Ti config.getMinDiskSpace(), config.getL1FileCountTrigger(), config.getL0QueueStallThreshold(), + config.getTombstoneDensityTrigger(), + config.getTombstoneDensityMinEntries(), config.isUseBtree(), config.isObjectLazyCompaction(), config.isObjectPrefetchCompaction() @@ -387,7 +390,8 @@ private static native long nativeOpen(String dbPath, int numFlushThreads, int nu boolean oscWalUploadSync, long oscWalSyncThresholdBytes, boolean oscWalSyncOnCommit, boolean oscReplicaMode, long oscReplicaSyncIntervalUs, - boolean oscReplicaReplayWal) throws TidesDBException; + boolean oscReplicaReplayWal, + int maxConcurrentFlushes) throws TidesDBException; private static native void nativeClose(long handle); @@ -397,7 +401,9 @@ private static native void nativeCreateColumnFamily(long handle, String name, double bloomFPR, boolean enableBlockIndexes, int indexSampleRatio, int blockIndexPrefixLen, int syncMode, long syncIntervalUs, String comparatorName, int skipListMaxLevel, float skipListProbability, int defaultIsolationLevel, long minDiskSpace, - int l1FileCountTrigger, int l0QueueStallThreshold, boolean useBtree, + int l1FileCountTrigger, int l0QueueStallThreshold, + double tombstoneDensityTrigger, long tombstoneDensityMinEntries, + boolean useBtree, boolean objectLazyCompaction, boolean objectPrefetchCompaction) throws TidesDBException; diff --git a/src/test/java/com/tidesdb/TidesDBTest.java b/src/test/java/com/tidesdb/TidesDBTest.java index 485128c..24378b8 100644 --- a/src/test/java/com/tidesdb/TidesDBTest.java +++ b/src/test/java/com/tidesdb/TidesDBTest.java @@ -1565,6 +1565,181 @@ void testTransactionSingleDelete() throws TidesDBException { } } + @Test + @Order(44) + void testTombstoneCfConfigRoundTrip() throws TidesDBException { + Config config = Config.builder(tempDir.resolve("testdb_tombstone_cfg").toString()) + .numFlushThreads(2) + .numCompactionThreads(2) + .logLevel(LogLevel.INFO) + .blockCacheSize(64 * 1024 * 1024) + .maxOpenSSTables(256) + .build(); + + try (TidesDB db = TidesDB.open(config)) { + ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() + .tombstoneDensityTrigger(0.5) + .tombstoneDensityMinEntries(256) + .build(); + + db.createColumnFamily("ts_cf", cfConfig); + ColumnFamily cf = db.getColumnFamily("ts_cf"); + + ColumnFamilyConfig readback = cf.getStats().getConfig(); + assertNotNull(readback); + assertEquals(0.5, readback.getTombstoneDensityTrigger(), 0.0); + assertEquals(256L, readback.getTombstoneDensityMinEntries()); + + // Defaults from the C library should be sensible (min entries ~= 1024) + ColumnFamilyConfig defaults = ColumnFamilyConfig.defaultConfig(); + assertTrue(defaults.getTombstoneDensityMinEntries() > 0, + "default tombstoneDensityMinEntries should be non-zero (sourced from C library)"); + assertTrue(defaults.getTombstoneDensityTrigger() >= 0.0 + && defaults.getTombstoneDensityTrigger() <= 1.0, + "default tombstoneDensityTrigger should be in [0.0, 1.0]"); + } + } + + @Test + @Order(45) + void testTombstoneStatsPopulated() throws TidesDBException, InterruptedException { + Config config = Config.builder(tempDir.resolve("testdb_tombstone_stats").toString()) + .numFlushThreads(2) + .numCompactionThreads(2) + .logLevel(LogLevel.INFO) + .blockCacheSize(64 * 1024 * 1024) + .maxOpenSSTables(256) + .build(); + + try (TidesDB db = TidesDB.open(config)) { + db.createColumnFamily("ts_stats_cf", ColumnFamilyConfig.defaultConfig()); + ColumnFamily cf = db.getColumnFamily("ts_stats_cf"); + + final int n = 200; + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < n; i++) { + txn.put(cf, ("key" + i).getBytes(StandardCharsets.UTF_8), + ("value" + i).getBytes(StandardCharsets.UTF_8)); + } + txn.commit(); + } + cf.flushMemtable(); + + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < n / 2; i++) { + txn.delete(cf, ("key" + i).getBytes(StandardCharsets.UTF_8)); + } + txn.commit(); + } + cf.flushMemtable(); + + // Wait for the flush to land so the stats include the tombstones + Thread.sleep(500); + + Stats stats = cf.getStats(); + assertNotNull(stats); + assertTrue(stats.getTotalTombstones() > 0, + "expected total_tombstones > 0 after deletes + flush"); + assertTrue(stats.getTombstoneRatio() >= 0.0 && stats.getTombstoneRatio() <= 1.0, + "tombstone_ratio must be within [0.0, 1.0]"); + assertTrue(stats.getMaxSstDensity() >= 0.0 && stats.getMaxSstDensity() <= 1.0, + "max_sst_density must be within [0.0, 1.0]"); + assertTrue(stats.getMaxSstDensityLevel() >= 0, + "max_sst_density_level must be non-negative"); + + long[] perLevel = stats.getLevelTombstoneCounts(); + assertNotNull(perLevel, "level_tombstone_counts must be populated"); + assertEquals(stats.getNumLevels(), perLevel.length, + "level_tombstone_counts length must match num_levels"); + } + } + + @Test + @Order(46) + void testCompactRange() throws TidesDBException { + Config config = Config.builder(tempDir.resolve("testdb_compact_range").toString()) + .numFlushThreads(2) + .numCompactionThreads(2) + .logLevel(LogLevel.INFO) + .blockCacheSize(64 * 1024 * 1024) + .maxOpenSSTables(256) + .build(); + + try (TidesDB db = TidesDB.open(config)) { + // Small write buffer keeps each batch falling into its own SSTable + ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() + .writeBufferSize(64 * 1024) + .build(); + db.createColumnFamily("range_cf", cfConfig); + ColumnFamily cf = db.getColumnFamily("range_cf"); + + // Multi-batch insert + flush to spread keys across several SSTables + for (int batch = 0; batch < 4; batch++) { + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 50; i++) { + int n = batch * 50 + i; + byte[] key = String.format("k%05d", n).getBytes(StandardCharsets.UTF_8); + byte[] value = ("value" + n).getBytes(StandardCharsets.UTF_8); + txn.put(cf, key, value); + } + txn.commit(); + } + cf.flushMemtable(); + } + + // Narrow range compaction over a slice of the keyspace + byte[] start = "k00050".getBytes(StandardCharsets.UTF_8); + byte[] end = "k00100".getBytes(StandardCharsets.UTF_8); + cf.compactRange(start, end); + + // Both endpoints null should be rejected with INVALID_ARGS + TidesDBException ex = assertThrows(TidesDBException.class, + () -> cf.compactRange(null, null)); + assertEquals(-2, ex.getErrorCode(), "expected TDB_ERR_INVALID_ARGS for both-null range"); + + // Both empty should also be rejected + assertThrows(TidesDBException.class, + () -> cf.compactRange(new byte[0], new byte[0])); + + // A key outside the compacted range must still read back unchanged + try (Transaction txn = db.beginTransaction()) { + byte[] outside = txn.get(cf, "k00150".getBytes(StandardCharsets.UTF_8)); + assertNotNull(outside); + assertArrayEquals("value150".getBytes(StandardCharsets.UTF_8), outside); + } + } + } + + @Test + @Order(47) + void testMaxConcurrentFlushes() throws TidesDBException { + Config config = Config.builder(tempDir.resolve("testdb_max_flushes").toString()) + .numFlushThreads(2) + .numCompactionThreads(2) + .logLevel(LogLevel.INFO) + .blockCacheSize(64 * 1024 * 1024) + .maxOpenSSTables(256) + .maxConcurrentFlushes(1) + .build(); + + try (TidesDB db = TidesDB.open(config)) { + db.createColumnFamily("flush_cf", ColumnFamilyConfig.defaultConfig()); + ColumnFamily cf = db.getColumnFamily("flush_cf"); + + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, "k".getBytes(StandardCharsets.UTF_8), + "v".getBytes(StandardCharsets.UTF_8)); + txn.commit(); + } + cf.flushMemtable(); + } + + // defaultConfig() should source maxConcurrentFlushes from the C library + Config defaults = Config.defaultConfig(); + assertTrue(defaults.getMaxConcurrentFlushes() > 0, + "default maxConcurrentFlushes should be non-zero (sourced from tidesdb_default_config())"); + } + @Test @Order(43) void testTransactionSingleDeleteNullArgs() throws TidesDBException {