Port the rest of asg017#268: delete a chunk once DELETE empties it

CHANGELOG.md

@@ -8,6 +8,7 @@
 - The linux-arm64 and linux-arm64-musl binaries use NEON, as the darwin-arm64 binary does: `make` enables it on Linux arm64, and `OMIT_SIMD=1` builds without it. An L2 KNN query on a 100,000-row `float[768]` table took 19 ms instead of 57 ms there. Hamming and `int8` L1 distances are unchanged; `float32` L1 distances can differ in the last digit when their double-precision sum rounds, and `float32` L2 distances can differ from 2.1.1's in the last digits, closer to the exact value (at 768 dimensions, a mean relative error of 3.0e-8 instead of 1.7e-7), so a KNN query can order rows with nearly equal distances differently; `int8` L2 distances whose squared differences sum past 2^24 are computed from the exact sum.
 - Cosine distance between `float32` vectors of more than 16 dimensions, or `int8` vectors of more than 15, uses NEON in NEON builds, which include the darwin-arm64 binary: a KNN query on a 100,000-row `float[768]` table took 26 ms instead of 94 ms on Linux arm64. `float32` distances can differ from the scalar code's in the last digits, as NEON adds the same double-precision products in a different order: differences are rare between random vectors and common between near-duplicate ones, and a vector's distance to itself is between -2.2e-16 and 2.2e-16 in either. `int8` distances match up to 1,024 dimensions; past that, vectors whose squared elements sum past 2^24 can get different distances, usually closer to the exact value, as NEON sums them exactly. Ported from [asg017@0de765f](https://github.com/asg017/sqlite-vec/commit/0de765f), except that `float32` sums stay in `double` ([asg017#324](https://github.com/asg017/sqlite-vec/issues/324)), where asg017's kernel sums them in `float32`, and `int8` sums are kept in 64 bits, where asg017's 32-bit sums overflow from 131,072 elements.
 - A KNN query picks each chunk's k nearest rows with the heap of upstream's `SQLITE_VEC_EXPERIMENTAL_MIN_IDX` build option ([asg017/sqlite-vec@0de765f](https://github.com/asg017/sqlite-vec/commit/0de765f4570c1d65a0699e8e38d6c5562da9e23e)) instead of one pass over the chunk for every row it keeps, ordered so that it returns the same rows in the same order as before: on a 10,000-row `float[768]` table, `k = 1000` and `k = 4096` take 9.3 ms instead of 17.8 and 18.0 ms, `k = 100` 8.8 ms instead of 9.2, and `k = 1` 8.7 ms either way. A NaN distance ranks after every number instead of where its position put it. A static blob can produce one, and so can a `vec0` table written before 2.1.0, which started rejecting NaN elements. On such a table a NaN row used to push nearer rows out of the result or behind farther ones; queries now return the nearest rows in order, which can leave out a NaN row that a query with `k` smaller than the row count used to return.
+- A `DELETE` that removes the last row of a chunk now deletes the chunk's rows from the `_chunks`, `_vector_chunksNN`, and `_metadatachunksNN` shadow tables, so their pages are free for reuse without `optimize`; the emptied chunk used to stay until `optimize` removed it. Ported from [asg017@cb147c8](https://github.com/asg017/sqlite-vec/commit/cb147c8) via [vlasky@3a64182](https://github.com/vlasky/sqlite-vec/commit/3a64182) ([asg017#268](https://github.com/asg017/sqlite-vec/pull/268)).
 
 ### Fixed
 

README.md

@@ -262,7 +262,9 @@ order by distance;
 
 ### Space Reclamation with Optimize
 
-`optimize` compacts vec shadow tables. To shrink the database file:
+A `DELETE` that removes the last row of a chunk deletes the chunk, and
+`optimize` compacts the chunks that deletes leave partly empty. To shrink the
+database file:
 
 ```sql
 -- Before creating vec tables: enable autovacuum and apply it (recommended)

sqlite-vec.c

@@ -11123,6 +11123,110 @@ int vec0Update_Delete_ClearMetadata(vec0_vtab *p, int metadata_idx, i64 rowid,
   return rc;
 }
 
+// Once a DELETE has cleared the last validity bit of its chunk, delete the
+// chunk's rows from _chunks, _vector_chunksNN and _metadatachunksNN, so their
+// pages are free without an optimize.
+// https://github.com/asg017/sqlite-vec/commit/cb147c8
+int vec0Update_Delete_DeleteChunkIfEmpty(vec0_vtab *p, i64 chunk_id) {
+  int rc;
+  sqlite3_blob *blobValidity = NULL;
+  sqlite3_stmt *stmt = NULL;
+  char *zSql;
+
+  rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, "validity",
+                         chunk_id, 0, &blobValidity);
+  if (rc != SQLITE_OK) {
+    vtab_set_error(&p->base, "could not open validity blob for %s.%s.%lld",
+                   p->schemaName, p->shadowChunksName, chunk_id);
+    return vec0_error_rc(rc);
+  }
+  int validitySize = sqlite3_blob_bytes(blobValidity);
+  u8 *validity = sqlite3_malloc(validitySize);
+  if (!validity) {
+    sqlite3_blob_close(blobValidity);
+    return SQLITE_NOMEM;
+  }
+  rc = sqlite3_blob_read(blobValidity, validity, validitySize, 0);
+  int brc = sqlite3_blob_close(blobValidity);
+  if (rc != SQLITE_OK) {
+    sqlite3_free(validity);
+    vtab_set_error(&p->base, "could not read validity blob for %s.%s.%lld",
+                   p->schemaName, p->shadowChunksName, chunk_id);
+    return rc;
+  }
+  if (brc != SQLITE_OK) {
+    sqlite3_free(validity);
+    vtab_set_error(&p->base, "could not close validity blob for %s.%s.%lld",
+                   p->schemaName, p->shadowChunksName, chunk_id);
+    return brc;
+  }
+  int empty = !bitmap_any(validity, validitySize * CHAR_BIT);
+  sqlite3_free(validity);
+  if (!empty) {
+    return SQLITE_OK;
+  }
+
+  zSql = sqlite3_mprintf("DELETE FROM " VEC0_SHADOW_CHUNKS_NAME
+                         " WHERE chunk_id = ?",
+                         p->schemaName, p->tableName);
+  if (!zSql) {
+    return SQLITE_NOMEM;
+  }
+  rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);
+  sqlite3_free(zSql);
+  if (rc != SQLITE_OK) {
+    return rc;
+  }
+  sqlite3_bind_int64(stmt, 1, chunk_id);
+  rc = sqlite3_step(stmt);
+  sqlite3_finalize(stmt);
+  if (rc != SQLITE_DONE) {
+    return vec0_error_rc(rc);
+  }
+
+  for (int i = 0; i < p->numVectorColumns; i++) {
+    zSql = sqlite3_mprintf("DELETE FROM " VEC0_SHADOW_VECTOR_N_NAME
+                           " WHERE rowid = ?",
+                           p->schemaName, p->tableName, i);
+    if (!zSql) {
+      return SQLITE_NOMEM;
+    }
+    rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);
+    sqlite3_free(zSql);
+    if (rc != SQLITE_OK) {
+      return rc;
+    }
+    sqlite3_bind_int64(stmt, 1, chunk_id);
+    rc = sqlite3_step(stmt);
+    sqlite3_finalize(stmt);
+    if (rc != SQLITE_DONE) {
+      return vec0_error_rc(rc);
+    }
+  }
+
+  for (int i = 0; i < p->numMetadataColumns; i++) {
+    zSql = sqlite3_mprintf("DELETE FROM " VEC0_SHADOW_METADATA_N_NAME
+                           " WHERE rowid = ?",
+                           p->schemaName, p->tableName, i);
+    if (!zSql) {
+      return SQLITE_NOMEM;
+    }
+    rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);
+    sqlite3_free(zSql);
+    if (rc != SQLITE_OK) {
+      return rc;
+    }
+    sqlite3_bind_int64(stmt, 1, chunk_id);
+    rc = sqlite3_step(stmt);
+    sqlite3_finalize(stmt);
+    if (rc != SQLITE_DONE) {
+      return vec0_error_rc(rc);
+    }
+  }
+
+  return SQLITE_OK;
+}
+
 int vec0Update_Delete(sqlite3_vtab *pVTab, sqlite3_value *idValue) {
   vec0_vtab *p = (vec0_vtab *)pVTab;
   int rc;
@@ -11184,15 +11288,16 @@ int vec0Update_Delete(sqlite3_vtab *pVTab, sqlite3_value *idValue) {
     }
   }
 
-  // 6. delete metadata
+  // 7. delete metadata
   for (int i = 0; i < p->numMetadataColumns; i++) {
     rc = vec0Update_Delete_ClearMetadata(p, i, rowid, chunk_id, chunk_offset);
     if (rc != SQLITE_OK) {
       return rc;
     }
   }
 
-  return SQLITE_OK;
+  // 8. delete the chunk if that was its last row
+  return vec0Update_Delete_DeleteChunkIfEmpty(p, chunk_id);
 }
 
 int vec0Update_UpdateAuxColumn(vec0_vtab *p, int auxiliary_column_idx,

tests/fuzz/vec0-delete-completeness.c

@@ -71,10 +71,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     }
   }
 
-  // Delete all remaining rows. This fork's DELETE leaves emptied chunks in
-  // place until optimize removes them.
+  // Delete all remaining rows
   sqlite3_exec(db, "DELETE FROM v", NULL, NULL, NULL);
-  sqlite3_exec(db, "INSERT INTO v(v) VALUES ('optimize')", NULL, NULL, NULL);
 
   // Assert all shadow tables are empty after full deletion
   sqlite3_prepare_v2(db,

tests/test-error-paths.py

@@ -386,7 +386,9 @@ def interrupt_template():
     db.load_extension(get_extension_path())
     # Text values are longer than the 12 bytes a metadata chunk holds, so
     # filters and reads go to the _metadatatext table. v's one chunk is full,
-    # so an INSERT adds a chunk; vp has a deleted row for optimize to skip.
+    # so an INSERT adds a chunk; vp has a deleted row for optimize to skip, and
+    # rowid 4 is the last row of its partition 0 chunk, so deleting it deletes
+    # the chunk.
     db.execute(
         "CREATE VIRTUAL TABLE v USING vec0(vector float[2], m integer, t text, "
         "+a text, chunk_size=8)"
@@ -500,6 +502,7 @@ def handler():
         ),
         pytest.param("DELETE FROM v WHERE rowid = 4", id="delete"),
         pytest.param("DELETE FROM vt WHERE id = 'id4'", id="delete-text-pk"),
+        pytest.param("DELETE FROM vp WHERE rowid = 4", id="delete-emptying-chunk"),
         pytest.param("INSERT INTO vp(vp) VALUES ('optimize')", id="optimize"),
     ],
 )