镜像站点 · 本页由第三方 GitHub 只读镜像提供,非 GitHub 官方站点,不接受任何登录或凭据输入。前往 github.com
Skip to content

Commit 03480ab

Browse files
committed
fix(vec0): preserve full-range rowids in KNN rowid IN filters
The qsort/bsearch comparator for the KNN rowid IN (...) filter returned the i64 rowid difference narrowed to int: the subtraction can overflow and the narrowing keeps only the low 32 bits, so unequal rowids could compare as equal or with the wrong sign, silently dropping matching rowids from KNN results. Compare three-way instead. Also report explicit-rowid shadow-table failures from the vtab's database handle (the statement handle used before is NULL on that path, so every error read "out of memory"), and cover negative and large-magnitude rowids across vec0 operations.
1 parent 5db3df4 commit 03480ab

4 files changed

Lines changed: 199 additions & 3 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
# Changelog
22

3+
## [Unreleased]
4+
5+
### Fixed
6+
7+
- Fixed the rowid comparator used by the KNN `rowid IN (...)` filter: it returned the 64-bit rowid difference narrowed to `int`. The subtraction can overflow and the narrowing keeps only the low 32 bits, so unequal rowids could compare as equal or with the wrong sign (e.g. a difference of 2^31 wraps negative, and exactly 2^32 wraps to zero) — an inconsistent ordering for `qsort` (undefined behavior) and silent `bsearch` misses that dropped matching rowids from KNN results. Now uses a proper three-way comparison, verified with rowids spanning more than 2^31 including negatives.
8+
- Fixed error reporting for non-primary-key failures during explicit-rowid inserts: the message was built from a NULL statement handle, so it always read "out of memory" instead of the underlying SQLite error.
9+
- Added test coverage for negative and large-magnitude rowids (insert, point lookup, update, delete, KNN, metadata/auxiliary/partition columns, `rowid IN (...)`).
10+
311
## [1.2.0] - 2026-07-06
412

513
### Removed

‎sqlite-vec.c‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1183,7 +1183,11 @@ int ensure_vector_match(sqlite3_value *aValue, sqlite3_value *bValue, void **a,
11831183
return SQLITE_OK;
11841184
}
11851185

1186-
int _cmp(const void *a, const void *b) { return (*(i64 *)a - *(i64 *)b); }
1186+
static int _cmp(const void *a, const void *b) {
1187+
i64 x = *(i64 *)a;
1188+
i64 y = *(i64 *)b;
1189+
return (x > y) - (x < y);
1190+
}
11871191

11881192
struct VecNpyFile {
11891193
char *path;
@@ -4470,7 +4474,7 @@ int vec0_rowids_insert_rowid(vec0_vtab *p, i64 rowid) {
44704474
// IMP: V04679_21517
44714475
vtab_set_error(&p->base,
44724476
"Error inserting rowid into rowids shadow table: %s",
4473-
sqlite3_errmsg(sqlite3_db_handle(p->stmtRowidsInsertId)));
4477+
sqlite3_errmsg(p->db));
44744478
}
44754479
rc = SQLITE_ERROR;
44764480
goto cleanup;
@@ -4536,7 +4540,7 @@ int vec0_rowids_insert_id(vec0_vtab *p, sqlite3_value *idValue, i64 *rowid) {
45364540
// IMP: V15177_32015
45374541
vtab_set_error(&p->base,
45384542
"Error inserting id into rowids shadow table: %s",
4539-
sqlite3_errmsg(sqlite3_db_handle(p->stmtRowidsInsertId)));
4543+
sqlite3_errmsg(p->db));
45404544
}
45414545
rc = SQLITE_ERROR;
45424546
goto complete;

‎tests/test-error-paths.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,5 +261,23 @@ def test_repeated_error_operations(db):
261261
db.execute("DROP TABLE test")
262262

263263

264+
def test_rowids_shadow_insert_reports_real_error(db):
265+
"""
266+
Regression test: a non-primary-key failure during an explicit-rowid
267+
insert used sqlite3_errmsg(sqlite3_db_handle(stmtRowidsInsertId)), but
268+
that statement is never prepared on the explicit-rowid path, so the
269+
error surfaced as "out of memory" instead of the real message.
270+
"""
271+
db.execute("CREATE VIRTUAL TABLE test USING vec0(v float[1])")
272+
db.execute("INSERT INTO test(rowid, v) VALUES (1, '[1]')")
273+
# Break the rowids shadow table so the next explicit-rowid insert fails
274+
# with something other than a primary-key conflict.
275+
db.execute("DROP TABLE test_rowids")
276+
with pytest.raises(sqlite3.OperationalError) as excinfo:
277+
db.execute("INSERT INTO test(rowid, v) VALUES (2, '[2]')")
278+
assert "out of memory" not in str(excinfo.value)
279+
assert "no such table" in str(excinfo.value)
280+
281+
264282
if __name__ == "__main__":
265283
pytest.main([__file__, "-v"])

‎tests/test-negative-rowids.py‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
"""
2+
Coverage for negative and large-magnitude rowids in vec0 tables.
3+
4+
vec0 accepts any i64 rowid, and downstream applications assign
5+
deterministic negative rowids (e.g. rowid = -source_id). These tests
6+
exercise insert, point lookup, update, delete, KNN result reporting,
7+
metadata/auxiliary/partition columns, and the KNN `rowid IN (...)`
8+
filter with rowids that are negative and/or span more than 2^31.
9+
"""
10+
11+
import sqlite3
12+
13+
import pytest
14+
15+
INT64_MIN = -(2**63)
16+
INT64_MAX = 2**63 - 1
17+
18+
19+
def test_insert_and_point_lookup(db):
20+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[2])")
21+
rowids = [INT64_MIN, -4_000_000_000, -42, -1, 0, 1, 4_000_000_000, INT64_MAX]
22+
for i, rowid in enumerate(rowids):
23+
db.execute(
24+
"INSERT INTO v(rowid, embedding) VALUES (?, ?)", [rowid, f"[{i}, {i}]"]
25+
)
26+
27+
for i, rowid in enumerate(rowids):
28+
row = db.execute(
29+
"SELECT rowid, vec_to_json(embedding) AS e FROM v WHERE rowid = ?",
30+
[rowid],
31+
).fetchone()
32+
assert row["rowid"] == rowid
33+
assert row["e"] == f"[{i}.000000,{i}.000000]"
34+
35+
assert db.execute("SELECT count(*) FROM v").fetchone()[0] == len(rowids)
36+
37+
38+
def test_update(db):
39+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[2])")
40+
db.execute("INSERT INTO v(rowid, embedding) VALUES (-7, '[1, 1]')")
41+
db.execute("UPDATE v SET embedding = '[9, 9]' WHERE rowid = -7")
42+
row = db.execute(
43+
"SELECT vec_to_json(embedding) AS e FROM v WHERE rowid = -7"
44+
).fetchone()
45+
assert row["e"] == "[9.000000,9.000000]"
46+
47+
48+
def test_delete(db):
49+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[2])")
50+
for rowid in [-3, -2, -1, 1, 2]:
51+
db.execute(
52+
"INSERT INTO v(rowid, embedding) VALUES (?, ?)",
53+
[rowid, f"[{rowid}, {rowid}]"],
54+
)
55+
db.execute("DELETE FROM v WHERE rowid = -2")
56+
remaining = [r[0] for r in db.execute("SELECT rowid FROM v ORDER BY rowid")]
57+
assert remaining == [-3, -1, 1, 2]
58+
assert db.execute("SELECT count(*) FROM v WHERE rowid = -2").fetchone()[0] == 0
59+
# a deleted negative rowid can be re-inserted
60+
db.execute("INSERT INTO v(rowid, embedding) VALUES (-2, '[5, 5]')")
61+
assert db.execute("SELECT count(*) FROM v WHERE rowid = -2").fetchone()[0] == 1
62+
63+
64+
def test_duplicate_negative_rowid_rejected(db):
65+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[2])")
66+
db.execute("INSERT INTO v(rowid, embedding) VALUES (-7, '[1, 1]')")
67+
with pytest.raises(sqlite3.OperationalError, match="UNIQUE constraint failed"):
68+
db.execute("INSERT INTO v(rowid, embedding) VALUES (-7, '[2, 2]')")
69+
70+
71+
def test_knn_reports_negative_rowids(db):
72+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[2])")
73+
# rowid -i holds [i, i], so KNN order from origin is -1, -2, -3, ...
74+
for i in range(1, 11):
75+
db.execute("INSERT INTO v(rowid, embedding) VALUES (?, ?)", [-i, f"[{i}, {i}]"])
76+
rows = db.execute(
77+
"SELECT rowid, distance FROM v WHERE embedding MATCH '[0, 0]' AND k = 3 "
78+
"ORDER BY distance"
79+
).fetchall()
80+
assert [r["rowid"] for r in rows] == [-1, -2, -3]
81+
82+
83+
def test_knn_negative_rowids_across_chunks(db):
84+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[1], chunk_size=8)")
85+
for i in range(-25, 25):
86+
db.execute("INSERT INTO v(rowid, embedding) VALUES (?, ?)", [i, f"[{i}]"])
87+
db.execute("DELETE FROM v WHERE rowid IN (-25, -1, 0, 24)")
88+
rows = db.execute(
89+
"SELECT rowid FROM v WHERE embedding MATCH '[0]' AND k = 4 ORDER BY distance"
90+
).fetchall()
91+
assert sorted(r["rowid"] for r in rows) == [-3, -2, 1, 2]
92+
93+
94+
def test_metadata_filter_with_negative_rowids(db):
95+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[2], category TEXT)")
96+
for i in range(1, 11):
97+
category = "even" if i % 2 == 0 else "odd"
98+
db.execute(
99+
"INSERT INTO v(rowid, embedding, category) VALUES (?, ?, ?)",
100+
[-i, f"[{i}, {i}]", category],
101+
)
102+
rows = db.execute(
103+
"SELECT rowid, category FROM v "
104+
"WHERE embedding MATCH '[0, 0]' AND k = 3 AND category = 'even' "
105+
"ORDER BY distance"
106+
).fetchall()
107+
assert [r["rowid"] for r in rows] == [-2, -4, -6]
108+
assert all(r["category"] == "even" for r in rows)
109+
110+
111+
def test_auxiliary_and_partition_with_negative_rowids(db):
112+
db.execute(
113+
"CREATE VIRTUAL TABLE v USING vec0("
114+
" user_id INTEGER partition key, embedding float[2], +contents TEXT"
115+
")"
116+
)
117+
for i in range(1, 7):
118+
db.execute(
119+
"INSERT INTO v(rowid, user_id, embedding, contents) VALUES (?, ?, ?, ?)",
120+
[-i, i % 2, f"[{i}, {i}]", f"item {i}"],
121+
)
122+
rows = db.execute(
123+
"SELECT rowid, contents FROM v "
124+
"WHERE embedding MATCH '[0, 0]' AND k = 2 AND user_id = 0 "
125+
"ORDER BY distance"
126+
).fetchall()
127+
assert [r["rowid"] for r in rows] == [-2, -4]
128+
assert [r["contents"] for r in rows] == ["item 2", "item 4"]
129+
130+
131+
def test_knn_rowid_in_negative(db):
132+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[2])")
133+
for i in range(1, 11):
134+
db.execute("INSERT INTO v(rowid, embedding) VALUES (?, ?)", [-i, f"[{i}, {i}]"])
135+
rows = db.execute(
136+
"SELECT rowid FROM v WHERE embedding MATCH '[0, 0]' AND k = 10 "
137+
"AND rowid IN (-2, -5, -9) ORDER BY distance"
138+
).fetchall()
139+
assert [r["rowid"] for r in rows] == [-2, -5, -9]
140+
141+
142+
def test_knn_rowid_in_span_exceeding_2_31(db):
143+
"""Regression test: the qsort/bsearch comparator for `rowid IN (...)`
144+
returned the i64 rowid difference narrowed to int. The subtraction can
145+
overflow and the narrowing keeps only the low 32 bits, so unequal
146+
rowids could compare as equal or with the wrong sign (a difference of
147+
2^31 wraps negative), giving qsort an inconsistent ordering and making
148+
bsearch silently drop matching rowids from KNN results."""
149+
db.execute("CREATE VIRTUAL TABLE v USING vec0(embedding float[1])")
150+
rowids = [
151+
-4_000_000_000,
152+
-(2**31) - 7,
153+
0,
154+
2**31,
155+
2**32 + 5,
156+
4_000_000_000,
157+
]
158+
for i, rowid in enumerate(rowids):
159+
db.execute("INSERT INTO v(rowid, embedding) VALUES (?, ?)", [rowid, f"[{i}]"])
160+
placeholders = ",".join("?" * len(rowids))
161+
rows = db.execute(
162+
f"SELECT rowid FROM v WHERE embedding MATCH '[0]' AND k = {len(rowids)} "
163+
f"AND rowid IN ({placeholders}) ORDER BY distance",
164+
rowids,
165+
).fetchall()
166+
assert sorted(r["rowid"] for r in rows) == sorted(rowids)

0 commit comments

Comments
 (0)