From c023d75158886117295481e222ce6713f46bb876 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 23 Oct 2026 02:14:33 -0500 Subject: [PATCH] Find no row, not an error, for a missing rowid on rescore and DiskANN columns A point query on a rowid the table does not hold returns no rows on a flat column: vec0_get_vector_data() looks the rowid up in _rowids first, and vec0Filter_point() reads its SQLITE_EMPTY as the end of the results. A rescore column read the rowid's row of _rescore_vectors, and a DiskANN column its row of _vectorsNN, before that lookup, so the same query failed: create virtual table t using vec0( embedding float[8] indexed by rescore(quantizer=bit)); insert into t(rowid, embedding) values (1, ?); select rowid from t where rowid = 999; -- Could not fetch vector data for 999 from rescore vectors A rowid deleted from the table fails the same way, and a DiskANN column reports "Could not fetch vector data for 999 from DiskANN vectors table". An IVF column already returns no rows. Every index type registers its rows in _rowids, an all-DiskANN table included, so vec0_get_vector_data() now looks the rowid up there before reading any index's vectors. tests/test-rescore.py, tests/test-diskann.py and tests/test-ivf.py check a never-inserted and a deleted rowid on each; the rescore and DiskANN tests fail on main. --- sqlite-vec.c | 29 +++++++++++++++-------------- tests/test-diskann.py | 20 ++++++++++++++++++++ tests/test-ivf.py | 15 +++++++++++++++ tests/test-rescore.py | 20 ++++++++++++++++++++ 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/sqlite-vec.c b/sqlite-vec.c index 7af3b6a7..39b97292 100644 --- a/sqlite-vec.c +++ b/sqlite-vec.c @@ -4042,6 +4042,21 @@ int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx, void **outVector, int *outVectorSize) { vec0_vtab *p = pVtab; int rc, brc; + i64 chunk_id; + i64 chunk_offset; + assert((vector_column_idx >= 0) && + (vector_column_idx < pVtab->numVectorColumns)); + + // Every index type registers its rows in _rowids, so a rowid with no + // position there is no row, whichever table holds the column's vectors. + rc = vec0_get_chunk_position(pVtab, rowid, NULL, &chunk_id, &chunk_offset); + if (rc == SQLITE_EMPTY) { + vtab_set_error(&pVtab->base, "Could not find a row with rowid %lld", rowid); + return rc; + } + if (rc != SQLITE_OK) { + return rc; + } #if SQLITE_VEC_ENABLE_DISKANN // DiskANN fast path: read from _vectors table @@ -4061,9 +4076,6 @@ int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx, } #endif - i64 chunk_id; - i64 chunk_offset; - #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // IVF-indexed columns store vectors in _ivf_cells, not _vector_chunks if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF) { @@ -4074,8 +4086,6 @@ int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx, void *buf = NULL; int blobOffset; sqlite3_blob *vectorBlob = NULL; - assert((vector_column_idx >= 0) && - (vector_column_idx < pVtab->numVectorColumns)); #if SQLITE_VEC_ENABLE_RESCORE // Rescore columns store float vectors in _rescore_vectors (rowid-keyed) @@ -4112,15 +4122,6 @@ int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx, } #endif /* SQLITE_VEC_ENABLE_RESCORE */ - rc = vec0_get_chunk_position(pVtab, rowid, NULL, &chunk_id, &chunk_offset); - if (rc == SQLITE_EMPTY) { - vtab_set_error(&pVtab->base, "Could not find a row with rowid %lld", rowid); - goto cleanup; - } - if (rc != SQLITE_OK) { - goto cleanup; - } - rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowVectorChunksNames[vector_column_idx], "vectors", chunk_id, 0, &vectorBlob); diff --git a/tests/test-diskann.py b/tests/test-diskann.py index 4e651604..987fc309 100644 --- a/tests/test-diskann.py +++ b/tests/test-diskann.py @@ -324,6 +324,26 @@ def test_diskann_delete_single(db): assert db.execute("SELECT count(*) FROM t_diskann_nodes00").fetchone()[0] == 2 +def test_diskann_point_query_missing_rowid(db): + """SELECT by a rowid the table does not hold returns no rows.""" + db.execute(""" + CREATE VIRTUAL TABLE t USING vec0( + emb float[8] INDEXED BY diskann(neighbor_quantizer=binary, n_neighbors=8) + ) + """) + for i in range(1, 3): + db.execute( + "INSERT INTO t(rowid, emb) VALUES (?, ?)", + [i, _f32([float(i)] * 8)], + ) + db.execute("DELETE FROM t WHERE rowid = 2") + + rows = db.execute("SELECT rowid FROM t WHERE rowid = 1").fetchall() + assert [r[0] for r in rows] == [1] + assert db.execute("SELECT rowid FROM t WHERE rowid = 2").fetchall() == [] + assert db.execute("SELECT rowid FROM t WHERE rowid = 999").fetchall() == [] + + def test_diskann_delete_no_stale_references(db): """After delete, no node should reference the deleted rowid.""" db.execute(""" diff --git a/tests/test-ivf.py b/tests/test-ivf.py index 8b7f566b..af28e266 100644 --- a/tests/test-ivf.py +++ b/tests/test-ivf.py @@ -167,6 +167,21 @@ def test_ivf_delete_flat_mode(db): assert db.execute("SELECT count(*) FROM t_ivf_rowid_map00").fetchone()[0] == 1 +def test_ivf_point_query_missing_rowid(db): + """SELECT by a rowid the table does not hold returns no rows.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))" + ) + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 2, 3, 4])]) + db.execute("INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([5, 6, 7, 8])]) + db.execute("DELETE FROM t WHERE rowid = 2") + + rows = db.execute("SELECT rowid FROM t WHERE rowid = 1").fetchall() + assert [r[0] for r in rows] == [1] + assert db.execute("SELECT rowid FROM t WHERE rowid = 2").fetchall() == [] + assert db.execute("SELECT rowid FROM t WHERE rowid = 999").fetchall() == [] + + # ============================================================================ # KNN flat mode tests # ============================================================================ diff --git a/tests/test-rescore.py b/tests/test-rescore.py index 9fda08a3..8d179201 100644 --- a/tests/test-rescore.py +++ b/tests/test-rescore.py @@ -261,6 +261,26 @@ def test_point_query_returns_float(db): assert abs(a - b) < 1e-6 +@pytest.mark.parametrize("quantizer", ["bit", "int8"]) +def test_point_query_missing_rowid(db, quantizer): + """SELECT by a rowid the table does not hold returns no rows.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + f" embedding float[8] indexed by rescore(quantizer={quantizer})" + ")" + ) + for rowid in (1, 2): + db.execute( + "INSERT INTO t(rowid, embedding) VALUES (?, ?)", + [rowid, float_vec([0.1 * rowid] * 8)], + ) + db.execute("DELETE FROM t WHERE rowid = 2") + rows = db.execute("SELECT rowid FROM t WHERE rowid = 1").fetchall() + assert [r["rowid"] for r in rows] == [1] + assert db.execute("SELECT rowid FROM t WHERE rowid = 2").fetchall() == [] + assert db.execute("SELECT rowid FROM t WHERE rowid = 999").fetchall() == [] + + # ============================================================================ # KNN tests # ============================================================================