Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
### Fixed

- A KNN query that takes its query vector, `k`, or `mmr_lambda` from another table of a join now uses it. `select v.rowid from p join v on v.embedding match ? and v.k = p.k` failed with `A LIMIT or 'k = ?' constraint is required`; the same join with `v.k = 3 and v.mmr_lambda = p.lambda` returned no rows; and `select v.rowid from v join p on p.id = v.tag where v.embedding match p.vector and v.k = 2`, with `p.id` an integer primary key, could return no rows. These joins now run one KNN per row of the other table, so a filter on the `vec0` table whose value comes from that table applies before the k nearest rows are chosen. Ported in part from the unmerged [vlasky#14](https://github.com/vlasky/sqlite-vec/pull/14) ([asg017#331](https://github.com/asg017/sqlite-vec/pull/331)), which also applies filters whose value comes from a joined table before the k nearest rows are chosen; that part is not ported, and ARCHITECTURE.md's idxStr section says why.
- Fixed a memory leak: closing a connection freed the `vec_static_blobs` registry but not the name of each blob registered in it.
- Fixed a write one byte past the end of a heap allocation in KNN queries on a `vec_static_blob_entries` table. SQLite's allocator rounds the allocation up, which hides the write unless the blob's vector count rounds up to a multiple of 64, as 64 vectors do.

## [2.1.1] - 2026-10-09

Expand Down
13 changes: 11 additions & 2 deletions sqlite-vec.c
Original file line number Diff line number Diff line change
Expand Up @@ -11756,6 +11756,15 @@ struct vec_static_blob_data {
static_blob static_blobs[MAX_STATIC_BLOBS];
};

/** Frees the registry and the names vec_static_blobsUpdate() copied in. */
static void vec_static_blob_data_free(void *p) {
vec_static_blob_data *data = p;
for (int i = 0; i < MAX_STATIC_BLOBS; i++) {
sqlite3_free(data->static_blobs[i].name);
}
sqlite3_free(data);
}

typedef struct vec_static_blobs_vtab vec_static_blobs_vtab;
struct vec_static_blobs_vtab {
sqlite3_vtab base;
Expand Down Expand Up @@ -12242,7 +12251,7 @@ static int vec_static_blob_entriesFilter(sqlite3_vtab_cursor *pVtabCursor,
}

bitmap_fill(candidates, bsize);
for (size_t i = bsize; i >= p->blob->nvectors; i--) {
for (size_t i = p->blob->nvectors; i < bsize; i++) {
bitmap_set(candidates, i, 0);
}
i32 k_used = 0;
Expand Down Expand Up @@ -12542,7 +12551,7 @@ sqlite3_vec_static_blobs_init(sqlite3 *db, char **pzErrMsg,
return rc;

rc = sqlite3_create_module_v2(db, "vec_static_blobs", &vec_static_blobsModule,
static_blob_data, sqlite3_free);
static_blob_data, vec_static_blob_data_free);
if (rc != SQLITE_OK)
return rc;
rc = sqlite3_create_module_v2(db, "vec_static_blob_entries",
Expand Down
84 changes: 84 additions & 0 deletions tests/memory-test.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ extern int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi);
extern int sqlite3_vec_numpy_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi);
extern int sqlite3_vec_static_blobs_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi);

#define CHECK_OK(rc, msg) \
do { \
Expand Down Expand Up @@ -986,6 +988,87 @@ static int test_unaligned_blob_vectors(void) {
return result;
}

static int test_static_blobs(void) {
printf("Testing static blobs...\n");
sqlite3 *db = NULL;
sqlite3_stmt *stmt = NULL;
int rc;
int result = 1;

// 64 vectors fill a KNN's candidate bitmap to its last byte, so a write past
// its end lands outside the allocation.
static float vectors[64 * 4];
for (int i = 0; i < 64; i++) {
for (int j = 0; j < 4; j++) {
vectors[i * 4 + j] = i / 64.0f;
}
}
float query[4] = {0.5f, 0.5f, 0.5f, 0.5f};

rc = sqlite3_open(":memory:", &db);
CHECK_OK(rc, "open database");
rc = sqlite3_vec_static_blobs_init(db, NULL, NULL);
CHECK_OK(rc, "register static blobs");

rc = sqlite3_prepare_v2(db,
"INSERT INTO temp.vec_static_blobs(name, data) "
"SELECT 'w', vec_static_blob_from_raw(?1, '<f4', 4, "
"64)",
-1, &stmt, NULL);
CHECK_OK(rc, "prepare static blob registration");
sqlite3_bind_int64(stmt, 1, (sqlite3_int64)(intptr_t)vectors);
rc = sqlite3_step(stmt);
CHECK_DONE(rc, "register static blob");
sqlite3_finalize(stmt);
stmt = NULL;

rc = sqlite3_exec(db,
"CREATE VIRTUAL TABLE w USING vec_static_blob_entries(w)",
NULL, NULL, NULL);
CHECK_OK(rc, "create static blob entries table");

rc = sqlite3_prepare_v2(db, "SELECT count(*) FROM w", -1, &stmt, NULL);
CHECK_OK(rc, "prepare static blob count");
rc = sqlite3_step(stmt);
CHECK_ROW(rc, "static blob count");
if (sqlite3_column_int64(stmt, 0) != 64) {
fprintf(stderr, "FAILED: static blob has %lld entries, not 64\n",
sqlite3_column_int64(stmt, 0));
goto cleanup;
}
sqlite3_finalize(stmt);
stmt = NULL;

rc = sqlite3_prepare_v2(db,
"SELECT rowid FROM w WHERE vector MATCH ?1 AND k = 3 "
"ORDER BY distance",
-1, &stmt, NULL);
CHECK_OK(rc, "prepare static blob KNN");
sqlite3_bind_blob(stmt, 1, query, sizeof(query), SQLITE_STATIC);
sqlite3_int64 expected[3] = {32, 33, 31};
for (int i = 0; i < 3; i++) {
rc = sqlite3_step(stmt);
CHECK_ROW(rc, "static blob KNN row");
if (sqlite3_column_int64(stmt, 0) != expected[i]) {
fprintf(stderr, "FAILED: static blob KNN row %d is %lld, not %lld\n", i,
sqlite3_column_int64(stmt, 0), expected[i]);
goto cleanup;
}
}
rc = sqlite3_step(stmt);
CHECK_DONE(rc, "static blob KNN end");

printf(" PASS: static blobs\n");
result = 0;

cleanup:
if (stmt)
sqlite3_finalize(stmt);
if (db)
sqlite3_close(db);
return result;
}

static int test_npy_truncated_headers(void) {
printf("Testing truncated numpy headers...\n");
sqlite3 *db = NULL;
Expand Down Expand Up @@ -1111,6 +1194,7 @@ int main(void) {
failures += test_insert_with_multiple_vectors();
failures += test_error_path_leaks();
failures += test_unaligned_blob_vectors();
failures += test_static_blobs();
failures += test_npy_truncated_headers();
failures += test_parser_edge_inputs();

Expand Down
13 changes: 13 additions & 0 deletions tests/test-loadable.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,19 @@ def test_vec_static_blob_entries():
]


def test_vec_static_blob_entries_knn_with_whole_candidate_bytes():
# 64 vectors fill the KNN's candidate bitmap to its last byte. The blob
# reads the array's memory, so the array stays referenced while it is queried.
db = connect(EXT_PATH, extra_entrypoint="sqlite3_vec_static_blobs_init")
vectors = np.array([[i / 64] * 4 for i in range(64)], np.float32)
register_numpy(db, "w", vectors)
rows = db.execute(
"select rowid from w where vector match ? and k = 3 order by distance",
[np.array([0.5] * 4, dtype=np.float32)],
).fetchall()
assert [r[0] for r in rows] == [32, 33, 31]


def test_limits():
db = connect(EXT_PATH)
with _raises(
Expand Down