From 60cb6c21e5dfb8983314f81b819e6310d27009db Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 19 Nov 2026 12:23:58 -0700 Subject: [PATCH] Stop boxing the row and iteration state in load!'s insert loop `load!` ran its insert loop inside the `DBInterface.transaction` closure and reassigned `row` and `st` there. Julia stores captured variables that a closure reassigns in a `Core.Box`, so every row was boxed and every `iterate`, `eachcolumn`, and bind call in the loop dispatched at run time. Loading 1M rows of one integer column took about 370 ms and allocated 150 MB. The loop now lives in `insertrows!`, which the closure calls with the first row and state as arguments, so they are ordinary typed locals. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/tables.jl | 51 +++++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index f8f4fce..e71b94b 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -492,31 +492,10 @@ function load!( "$kind INTO $(esc_id(string(name))) ($columns) VALUES ($params)"; register = false, ) - handle = _get_stmt_handle(stmt) # start a transaction for inserting rows try DBInterface.transaction(db) do - if row === nothing - state = iterate(rows) - state === nothing && return - row, st = state - end - while true - Tables.eachcolumn(sch, row) do val, col, _ - bind!(stmt, col, val) - end - r = Base.@lock db.lock GC.@preserve row C.sqlite3_step(handle) - if r == C.SQLITE_DONE - Base.@lock db.lock C.sqlite3_reset(handle) - elseif r != C.SQLITE_ROW - e = sqliteexception(db, stmt) - Base.@lock db.lock C.sqlite3_reset(handle) - throw(e) - end - state = iterate(rows, st) - state === nothing && break - row, st = state - end + insertrows!(stmt, sch, rows, row, st) end finally _close_stmt!(stmt) @@ -525,6 +504,34 @@ function load!( return name end +# Insert each row of `rows` with `stmt`, starting at `row` and iteration state `st` if given. +# The loop sits outside the transaction closure, because a closure that reassigns `row` and `st` boxes them. +function insertrows!(stmt::Stmt, sch::Tables.Schema, rows, row, st) + if row === nothing + state = iterate(rows) + state === nothing && return + row, st = state + end + db = stmt.db + handle = _get_stmt_handle(stmt) + while true + Tables.eachcolumn(sch, row) do val, col, _ + bind!(stmt, col, val) + end + r = Base.@lock db.lock GC.@preserve row C.sqlite3_step(handle) + if r == C.SQLITE_DONE + Base.@lock db.lock C.sqlite3_reset(handle) + elseif r != C.SQLITE_ROW + e = sqliteexception(db, stmt) + Base.@lock db.lock C.sqlite3_reset(handle) + throw(e) + end + state = iterate(rows, st) + state === nothing && return + row, st = state + end +end + # unknown schema case function load!( ::Nothing,