From efa9b1c75d55a5b8e041ca6c38783f8165a4f8f3 Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Tue, 7 Apr 2026 14:08:53 +0900 Subject: [PATCH 1/7] add opt-in statement cache --- sqlite3.go | 135 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 124 insertions(+), 11 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index 90d91ec..ef06c85 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -445,12 +445,15 @@ type SQLiteDriver struct { // SQLiteConn implements driver.Conn. type SQLiteConn struct { - mu sync.Mutex - db *C.sqlite3 - loc *time.Location - txlock string - funcs []*functionInfo - aggregators []*aggInfo + mu sync.Mutex + db *C.sqlite3 + loc *time.Location + txlock string + funcs []*functionInfo + aggregators []*aggInfo + stmtCache map[string][]*SQLiteStmt + stmtCacheSize int + stmtCacheCount int } // SQLiteTx implements driver.Tx. @@ -467,6 +470,7 @@ type SQLiteStmt struct { closed bool cls bool // True if the statement was created by SQLiteConn.Query namedParams map[string][3]int + cacheKey string } // SQLiteResult implements sql.Result. @@ -944,7 +948,7 @@ func (c *SQLiteConn) exec(ctx context.Context, query string, args []driver.Named start := 0 for { - s, err := c.prepare(ctx, query) + s, err := c.prepareWithCache(ctx, query, true) if err != nil { return nil, err } @@ -1009,7 +1013,7 @@ func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, erro func (c *SQLiteConn) query(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { start := 0 for { - s, err := c.prepare(ctx, query) + s, err := c.prepareWithCache(ctx, query, true) if err != nil { return nil, err } @@ -1185,6 +1189,7 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) { writableSchema := -1 vfsName := "" var cacheSize *int64 + stmtCacheSize := 0 pos := strings.IndexRune(dsn, '?') if pos >= 1 { @@ -1520,6 +1525,17 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) { cacheSize = &iv } + if val := params.Get("_stmt_cache_size"); val != "" { + iv, err := strconv.Atoi(val) + if err != nil { + return nil, fmt.Errorf("Invalid _stmt_cache_size: %v: %v", val, err) + } + if iv < 0 { + return nil, fmt.Errorf("Invalid _stmt_cache_size: %v, expecting non-negative integer", val) + } + stmtCacheSize = iv + } + if val := params.Get("vfs"); val != "" { vfsName = val } @@ -1592,7 +1608,10 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) { // // Create connection to SQLite - conn := &SQLiteConn{db: db, loc: loc, txlock: txlock} + conn := &SQLiteConn{db: db, loc: loc, txlock: txlock, stmtCacheSize: stmtCacheSize} + if stmtCacheSize > 0 { + conn.stmtCache = make(map[string][]*SQLiteStmt) + } // Password Cipher has to be registered before authentication if len(authCrypt) > 0 { @@ -1865,6 +1884,9 @@ func (c *SQLiteConn) Close() error { return nil } runtime.SetFinalizer(c, nil) + if err := c.closeCachedStmtsLocked(); err != nil { + return err + } rv := C.sqlite3_close_v2(c.db) if rv != C.SQLITE_OK { return lastError(c.db) @@ -1883,12 +1905,85 @@ func (c *SQLiteConn) dbConnOpen() bool { return c.db != nil } +func (c *SQLiteConn) takeCachedStmt(query string) *SQLiteStmt { + if c == nil || query == "" { + return nil + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.db == nil || c.stmtCacheSize <= 0 { + return nil + } + stmts := c.stmtCache[query] + if len(stmts) == 0 { + return nil + } + s := stmts[len(stmts)-1] + if len(stmts) == 1 { + delete(c.stmtCache, query) + } else { + c.stmtCache[query] = stmts[:len(stmts)-1] + } + c.stmtCacheCount-- + s.closed = false + s.cls = false + s.t = "" + return s +} + +func (c *SQLiteConn) putCachedStmt(s *SQLiteStmt) bool { + if c == nil || s == nil || s.s == nil || s.cacheKey == "" { + return false + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.db == nil || c.stmtCacheSize <= 0 || c.stmtCacheCount >= c.stmtCacheSize { + return false + } + c.stmtCache[s.cacheKey] = append(c.stmtCache[s.cacheKey], s) + c.stmtCacheCount++ + return true +} + +func (c *SQLiteConn) closeCachedStmtsLocked() error { + for key, stmts := range c.stmtCache { + for _, s := range stmts { + if s == nil || s.s == nil { + continue + } + runtime.SetFinalizer(s, nil) + if rv := C.sqlite3_finalize(s.s); rv != C.SQLITE_OK { + return lastError(c.db) + } + s.s = nil + s.c = nil + } + delete(c.stmtCache, key) + } + c.stmtCacheCount = 0 + return nil +} + // Prepare the query string. Return a new statement. func (c *SQLiteConn) Prepare(query string) (driver.Stmt, error) { return c.prepare(context.Background(), query) } func (c *SQLiteConn) prepare(ctx context.Context, query string) (driver.Stmt, error) { + return c.prepareWithCache(ctx, query, false) +} + +func (c *SQLiteConn) prepareWithCache(ctx context.Context, query string, useCache bool) (driver.Stmt, error) { + if useCache { + if stmt := c.takeCachedStmt(query); stmt != nil { + return stmt, nil + } + } + pquery := C.CString(query) defer C.free(unsafe.Pointer(pquery)) var s *C.sqlite3_stmt @@ -1902,6 +1997,9 @@ func (c *SQLiteConn) prepare(ctx context.Context, query string) (driver.Stmt, er t = strings.TrimSpace(C.GoString(tail)) } ss := &SQLiteStmt{c: c, s: s, t: t} + if useCache && t == "" { + ss.cacheKey = query + } runtime.SetFinalizer(ss, (*SQLiteStmt).Close) return ss, nil } @@ -2014,11 +2112,26 @@ func (s *SQLiteStmt) Close() error { runtime.SetFinalizer(s, nil) conn := s.c stmt := s.s - s.s = nil - s.c = nil + if stmt == nil { + s.c = nil + return nil + } if !conn.dbConnOpen() { return errors.New("sqlite statement with already closed database connection") } + if s.cacheKey != "" { + rv := C._sqlite3_reset_clear(stmt) + if rv != C.SQLITE_ROW && rv != C.SQLITE_OK && rv != C.SQLITE_DONE { + s.s = nil + s.c = nil + return conn.lastError() + } + if conn.putCachedStmt(s) { + return nil + } + } + s.s = nil + s.c = nil rv := C.sqlite3_finalize(stmt) if rv != C.SQLITE_OK { return conn.lastError() From 061c2a5f43ccfcdbe0bf04a9a5203f4fc31fc63b Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Wed, 8 Apr 2026 13:38:45 +0900 Subject: [PATCH 2/7] check stmtCacheSize before acquiring mutex in takeCachedStmt stmtCacheSize is immutable after connection open, so checking it before the lock avoids mutex overhead when cache is not enabled. --- sqlite3.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index ef06c85..5861a2f 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -1906,14 +1906,14 @@ func (c *SQLiteConn) dbConnOpen() bool { } func (c *SQLiteConn) takeCachedStmt(query string) *SQLiteStmt { - if c == nil || query == "" { + if c == nil || query == "" || c.stmtCacheSize <= 0 { return nil } c.mu.Lock() defer c.mu.Unlock() - if c.db == nil || c.stmtCacheSize <= 0 { + if c.db == nil { return nil } stmts := c.stmtCache[query] From 325cb8d5d939baa141c92a6a3bf5289cfef3b8b2 Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Wed, 8 Apr 2026 13:39:22 +0900 Subject: [PATCH 3/7] remove redundant stmtCacheSize check in putCachedStmt When stmtCacheSize <= 0, stmtCacheCount >= stmtCacheSize is always true, so the explicit check is unnecessary. --- sqlite3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3.go b/sqlite3.go index 5861a2f..a05cfd4 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -1941,7 +1941,7 @@ func (c *SQLiteConn) putCachedStmt(s *SQLiteStmt) bool { c.mu.Lock() defer c.mu.Unlock() - if c.db == nil || c.stmtCacheSize <= 0 || c.stmtCacheCount >= c.stmtCacheSize { + if c.db == nil || c.stmtCacheCount >= c.stmtCacheSize { return false } c.stmtCache[s.cacheKey] = append(c.stmtCache[s.cacheKey], s) From e9f47da5c530abc942bd05893581187a08271f49 Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Wed, 8 Apr 2026 13:40:21 +0900 Subject: [PATCH 4/7] do not bail out on finalize error in closeCachedStmtsLocked Finalize all cached statements even if one fails. Leaving a finalized statement in the cache map would be a use-after-finalize bug per SQLite documentation. --- sqlite3.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index a05cfd4..d634d6f 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -1884,9 +1884,7 @@ func (c *SQLiteConn) Close() error { return nil } runtime.SetFinalizer(c, nil) - if err := c.closeCachedStmtsLocked(); err != nil { - return err - } + c.closeCachedStmtsLocked() rv := C.sqlite3_close_v2(c.db) if rv != C.SQLITE_OK { return lastError(c.db) @@ -1949,23 +1947,20 @@ func (c *SQLiteConn) putCachedStmt(s *SQLiteStmt) bool { return true } -func (c *SQLiteConn) closeCachedStmtsLocked() error { +func (c *SQLiteConn) closeCachedStmtsLocked() { for key, stmts := range c.stmtCache { for _, s := range stmts { if s == nil || s.s == nil { continue } runtime.SetFinalizer(s, nil) - if rv := C.sqlite3_finalize(s.s); rv != C.SQLITE_OK { - return lastError(c.db) - } + C.sqlite3_finalize(s.s) s.s = nil s.c = nil } delete(c.stmtCache, key) } c.stmtCacheCount = 0 - return nil } // Prepare the query string. Return a new statement. From 0e58fa4d72585edf68b93f749ae5ebfa01cc04ff Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Wed, 8 Apr 2026 13:41:29 +0900 Subject: [PATCH 5/7] simplify prepareWithCache to call prepare instead of duplicating logic prepareWithCache now delegates to prepare and sets cacheKey afterward, removing the useCache boolean parameter. --- sqlite3.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index d634d6f..41dc811 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -948,7 +948,7 @@ func (c *SQLiteConn) exec(ctx context.Context, query string, args []driver.Named start := 0 for { - s, err := c.prepareWithCache(ctx, query, true) + s, err := c.prepareWithCache(ctx, query) if err != nil { return nil, err } @@ -1013,7 +1013,7 @@ func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, erro func (c *SQLiteConn) query(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { start := 0 for { - s, err := c.prepareWithCache(ctx, query, true) + s, err := c.prepareWithCache(ctx, query) if err != nil { return nil, err } @@ -1969,16 +1969,6 @@ func (c *SQLiteConn) Prepare(query string) (driver.Stmt, error) { } func (c *SQLiteConn) prepare(ctx context.Context, query string) (driver.Stmt, error) { - return c.prepareWithCache(ctx, query, false) -} - -func (c *SQLiteConn) prepareWithCache(ctx context.Context, query string, useCache bool) (driver.Stmt, error) { - if useCache { - if stmt := c.takeCachedStmt(query); stmt != nil { - return stmt, nil - } - } - pquery := C.CString(query) defer C.free(unsafe.Pointer(pquery)) var s *C.sqlite3_stmt @@ -1992,10 +1982,22 @@ func (c *SQLiteConn) prepareWithCache(ctx context.Context, query string, useCach t = strings.TrimSpace(C.GoString(tail)) } ss := &SQLiteStmt{c: c, s: s, t: t} - if useCache && t == "" { + runtime.SetFinalizer(ss, (*SQLiteStmt).Close) + return ss, nil +} + +func (c *SQLiteConn) prepareWithCache(ctx context.Context, query string) (driver.Stmt, error) { + if stmt := c.takeCachedStmt(query); stmt != nil { + return stmt, nil + } + stmt, err := c.prepare(ctx, query) + if err != nil { + return nil, err + } + ss := stmt.(*SQLiteStmt) + if ss.t == "" { ss.cacheKey = query } - runtime.SetFinalizer(ss, (*SQLiteStmt).Close) return ss, nil } From 867dcbfbdcd171255ab3df6b44dc292765799e0a Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Wed, 8 Apr 2026 13:42:26 +0900 Subject: [PATCH 6/7] move reset/clear into putCachedStmt and always finalize on failure This avoids an unnecessary reset when the cache is full, guarantees a statement cannot enter the cache without being reset/cleared, and fixes a leak where sqlite3_finalize was not called when reset failed. --- sqlite3.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index 41dc811..db865a7 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -1942,6 +1942,10 @@ func (c *SQLiteConn) putCachedStmt(s *SQLiteStmt) bool { if c.db == nil || c.stmtCacheCount >= c.stmtCacheSize { return false } + rv := C._sqlite3_reset_clear(s.s) + if rv != C.SQLITE_ROW && rv != C.SQLITE_OK && rv != C.SQLITE_DONE { + return false + } c.stmtCache[s.cacheKey] = append(c.stmtCache[s.cacheKey], s) c.stmtCacheCount++ return true @@ -2116,16 +2120,8 @@ func (s *SQLiteStmt) Close() error { if !conn.dbConnOpen() { return errors.New("sqlite statement with already closed database connection") } - if s.cacheKey != "" { - rv := C._sqlite3_reset_clear(stmt) - if rv != C.SQLITE_ROW && rv != C.SQLITE_OK && rv != C.SQLITE_DONE { - s.s = nil - s.c = nil - return conn.lastError() - } - if conn.putCachedStmt(s) { - return nil - } + if s.cacheKey != "" && conn.putCachedStmt(s) { + return nil } s.s = nil s.c = nil From e302e5cb8c4c737561a2ab8a86136095c676c2af Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Wed, 8 Apr 2026 13:43:47 +0900 Subject: [PATCH 7/7] document that _stmt_cache_size is per connection Clarify that each connection in the sql.DB pool maintains its own independent statement cache. --- sqlite3.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index db865a7..1a5433c 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -1525,6 +1525,9 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) { cacheSize = &iv } + // _stmt_cache_size sets the maximum number of prepared statements + // cached per connection. Note that sql.DB is a connection pool, so + // each connection maintains its own independent cache. if val := params.Get("_stmt_cache_size"); val != "" { iv, err := strconv.Atoi(val) if err != nil {