Replace namedValue with driver.NamedValue to avoid copying exec/query args (#1128)

Add support for Serialize and Deserialize, which wrap sqlite3_serialize and sqlite3_deserialize.

.github/workflows/docker.yaml

@@ -0,0 +1,22 @@
+name: dockerfile
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'v*'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  dockerfile:
+    name: Run Dockerfiles in examples
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run example - simple
+        run: |
+          cd ./_example/simple
+          docker build -t simple .
+          docker run simple | grep 99\ こんにちは世界099

.github/workflows/go.yaml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        go: ['1.15', '1.16', '1.17']
+        go: ['1.17', '1.18', '1.19']
       fail-fast: false
     env:
       OS: ${{ matrix.os }}
@@ -29,7 +29,7 @@ jobs:
 
       - name: Get Build Tools
         run: |
-          GO111MODULE=on go get github.com/ory/go-acc
+          GO111MODULE=on go install github.com/ory/go-acc@latest
 
       - name: Add $GOPATH/bin to $PATH
         run: |
@@ -44,7 +44,7 @@ jobs:
         run: go-acc . -- -race -v -tags "libsqlite3"
 
       - name: 'Tags: full'
-        run: go-acc . -- -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify sqlite_column_metadata"
+        run: go-acc . -- -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_column_metadata sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_math_functions sqlite_os_trace sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_unlock_notify sqlite_userauth sqlite_vacuum_incr sqlite_vtable"
 
       - name: 'Tags: vacuum'
         run: go-acc . -- -race -v -tags "sqlite_vacuum_full"
@@ -64,7 +64,7 @@ jobs:
 
     strategy:
       matrix:
-        go: ['1.15', '1.16', '1.17']
+        go: ['1.17', '1.18', '1.19']
       fail-fast: false
     env:
       OS: windows-latest
@@ -99,7 +99,7 @@ jobs:
       - name: 'Tags: full'
         run: |
           echo 'skip this test'
-          echo go build -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify"
+          echo go build -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_column_metadata sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_math_functions sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_unlock_notify sqlite_userauth sqlite_vacuum_incr sqlite_vtable"
         shell: msys2 {0}
 
       - name: 'Tags: vacuum'

README.md

@@ -1,7 +1,7 @@
 go-sqlite3
 ==========
 
-[![GoDoc Reference](https://godoc.org/github.com/mattn/go-sqlite3?status.svg)](http://godoc.org/github.com/mattn/go-sqlite3)
+[![Go Reference](https://pkg.go.dev/badge/github.com/mattn/go-sqlite3.svg)](https://pkg.go.dev/github.com/mattn/go-sqlite3)
 [![GitHub Actions](https://github.com/mattn/go-sqlite3/workflows/Go/badge.svg)](https://github.com/mattn/go-sqlite3/actions?query=workflow%3AGo)
 [![Financial Contributors on Open Collective](https://opencollective.com/mattn-go-sqlite3/all/badge.svg?label=financial+contributors)](https://opencollective.com/mattn-go-sqlite3) 
 [![codecov](https://codecov.io/gh/mattn/go-sqlite3/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-sqlite3)
@@ -165,6 +165,7 @@ go build --tags "icu json1 fts5 secure_delete"
 | Allow URI Authority | sqlite_allow_uri_authority | URI filenames normally throws an error if the authority section is not either empty or "localhost".<br><br>However, if SQLite is compiled with the SQLITE_ALLOW_URI_AUTHORITY compile-time option, then the URI is converted into a Uniform Naming Convention (UNC) filename and passed down to the underlying operating system that way |
 | App Armor | sqlite_app_armor | When defined, this C-preprocessor macro activates extra code that attempts to detect misuse of the SQLite API, such as passing in NULL pointers to required parameters or using objects after they have been destroyed. <br><br>App Armor is not available under `Windows`. |
 | Disable Load Extensions | sqlite_omit_load_extension | Loading of external extensions is enabled by default.<br><br>To disable extension loading add the build tag `sqlite_omit_load_extension`. |
+| Enable Serialization with `libsqlite3` | sqlite_serialize | Serialization and deserialization of a SQLite database is available by default, unless the build tag `libsqlite3` is set.<br><br>To enable this functionality even if `libsqlite3` is set, add the build tag `sqlite_serialize`. |
 | Foreign Keys | sqlite_foreign_keys | This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.<br><br>Each database connection can always turn enforcement of foreign key constraints on and off and run-time using the foreign_keys pragma.<br><br>Enforcement of foreign key constraints is normally off by default, but if this compile-time parameter is set to 1, enforcement of foreign key constraints will be on by default | 
 | Full Auto Vacuum | sqlite_vacuum_full | Set the default auto vacuum to full |
 | Incremental Auto Vacuum | sqlite_vacuum_incr | Set the default auto vacuum to incremental |
@@ -172,15 +173,18 @@ go build --tags "icu json1 fts5 secure_delete"
 |  International Components for Unicode | sqlite_icu | This option causes the International Components for Unicode or "ICU" extension to SQLite to be added to the build |
 | Introspect PRAGMAS | sqlite_introspect | This option adds some extra PRAGMA statements. <ul><li>PRAGMA function_list</li><li>PRAGMA module_list</li><li>PRAGMA pragma_list</li></ul> |
 | JSON SQL Functions | sqlite_json | When this option is defined in the amalgamation, the JSON SQL functions are added to the build automatically |
+| Math Functions | sqlite_math_functions | This compile-time option enables built-in scalar math functions. For more information see [Built-In Mathematical SQL Functions](https://www.sqlite.org/lang_mathfunc.html) |
+| OS Trace | sqlite_os_trace | This option enables OSTRACE() debug logging. This can be verbose and should not be used in production. |
 | Pre Update Hook | sqlite_preupdate_hook | Registers a callback function that is invoked prior to each INSERT, UPDATE, and DELETE operation on a database table. |
 | Secure Delete | sqlite_secure_delete | This compile-time option changes the default setting of the secure_delete pragma.<br><br>When this option is not used, secure_delete defaults to off. When this option is present, secure_delete defaults to on.<br><br>The secure_delete setting causes deleted content to be overwritten with zeros. There is a small performance penalty since additional I/O must occur.<br><br>On the other hand, secure_delete can prevent fragments of sensitive information from lingering in unused parts of the database file after it has been deleted. See the documentation on the secure_delete pragma for additional information |
 | Secure Delete (FAST) | sqlite_secure_delete_fast | For more information see [PRAGMA secure_delete](https://www.sqlite.org/pragma.html#pragma_secure_delete) |
 | Tracing / Debug | sqlite_trace | Activate trace functions |
 | User Authentication | sqlite_userauth | SQLite User Authentication see [User Authentication](#user-authentication) for more information. |
+| Virtual Tables | sqlite_vtable | SQLite Virtual Tables see [SQLite Official VTABLE Documentation](https://www.sqlite.org/vtab.html) for more information, and a [full example here](https://github.com/mattn/go-sqlite3/tree/master/_example/vtable) |
 
 # Compilation
 
-This package requires the `CGO_ENABLED=1` ennvironment variable if not set by default, and the presence of the `gcc` compiler.
+This package requires the `CGO_ENABLED=1` environment variable if not set by default, and the presence of the `gcc` compiler.
 
 If you need to add additional CFLAGS or LDFLAGS to the build command, and do not want to modify this package, then this can be achieved by using the `CGO_CFLAGS` and `CGO_LDFLAGS` environment variables.
 
@@ -216,14 +220,13 @@ This library can be cross-compiled.
 In some cases you are required to the `CC` environment variable with the cross compiler.
 
 ## Cross Compiling from MAC OSX
-The simplest way to cross compile from OSX is to use [xgo](https://github.com/karalabe/xgo).
+The simplest way to cross compile from OSX is to use [musl-cross](https://github.com/FiloSottile/homebrew-musl-cross).
 
 Steps:
-- Install [xgo](https://github.com/karalabe/xgo) (`go get github.com/karalabe/xgo`).
-- Ensure that your project is within your `GOPATH`.
-- Run `xgo local/path/to/project`.
+- Install [musl-cross](https://github.com/FiloSottile/homebrew-musl-cross) (`brew install FiloSottile/musl-cross/musl-cross`).
+- Run `CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ GOARCH=amd64 GOOS=linux CGO_ENABLED=1 go build -ldflags "-linkmode external -extldflags -static"`.
 
-Please refer to the project's [README](https://github.com/karalabe/xgo/blob/master/README.md) for further information.
+Please refer to the project's [README](https://github.com/FiloSottile/homebrew-musl-cross#readme) for further information.
 
 # Google Cloud Platform
 

_example/limit/limit.go

@@ -17,7 +17,7 @@ func createBulkInsertQuery(n int, start int) (query string, args []interface{})
 	for i := 0; i < n; i++ {
 		values[i] = "(?, ?)"
 		args[pos] = start + i
-		args[pos+1] = fmt.Sprintf("こんにちわ世界%03d", i)
+		args[pos+1] = fmt.Sprintf("こんにちは世界%03d", i)
 		pos += 2
 	}
 	query = fmt.Sprintf(

_example/simple/Dockerfile

@@ -0,0 +1,45 @@
+# =============================================================================
+#  Multi-stage Dockerfile Example
+# =============================================================================
+#  This is a simple Dockerfile that will build an image of scratch-base image.
+#  Usage:
+#    docker build -t simple:local . && docker run --rm simple:local
+# =============================================================================
+
+# -----------------------------------------------------------------------------
+#  Build Stage
+# -----------------------------------------------------------------------------
+FROM golang:alpine AS build
+
+# Important:
+#   Because this is a CGO enabled package, you are required to set it as 1.
+ENV CGO_ENABLED=1
+
+RUN apk add --no-cache \
+    # Important: required for go-sqlite3
+    gcc \
+    # Required for Alpine
+    musl-dev
+
+WORKDIR /workspace
+
+COPY . /workspace/
+
+RUN \
+    go mod init github.com/mattn/sample && \
+    go mod tidy && \
+    go install -ldflags='-s -w -extldflags "-static"' ./simple.go
+
+RUN \
+    # Smoke test
+    set -o pipefail; \
+    /go/bin/simple | grep 99\ こんにちは世界099
+
+# -----------------------------------------------------------------------------
+#  Main Stage
+# -----------------------------------------------------------------------------
+FROM scratch
+
+COPY --from=build /go/bin/simple /usr/local/bin/simple
+
+ENTRYPOINT [ "/usr/local/bin/simple" ]

_example/simple/simple.go

@@ -37,12 +37,15 @@ func main() {
 	}
 	defer stmt.Close()
 	for i := 0; i < 100; i++ {
-		_, err = stmt.Exec(i, fmt.Sprintf("こんにちわ世界%03d", i))
+		_, err = stmt.Exec(i, fmt.Sprintf("こんにちは世界%03d", i))
 		if err != nil {
 			log.Fatal(err)
 		}
 	}
-	tx.Commit()
+	err = tx.Commit()
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	rows, err := db.Query("select id, name from foo")
 	if err != nil {

_example/vtable/vtable.go

@@ -63,7 +63,12 @@ func (v *ghRepoTable) Open() (sqlite3.VTabCursor, error) {
 }
 
 func (v *ghRepoTable) BestIndex(cst []sqlite3.InfoConstraint, ob []sqlite3.InfoOrderBy) (*sqlite3.IndexResult, error) {
-	return &sqlite3.IndexResult{}, nil
+	used := make([]bool, len(csts))
+	return &sqlite3.IndexResult{
+		IdxNum: 0,
+		IdxStr: "default",
+		Used:   used,
+	}, nil
 }
 
 func (v *ghRepoTable) Disconnect() error { return nil }

callback.go

@@ -353,6 +353,20 @@ func callbackRetNil(ctx *C.sqlite3_context, v reflect.Value) error {
 	return nil
 }
 
+func callbackRetGeneric(ctx *C.sqlite3_context, v reflect.Value) error {
+	if v.IsNil() {
+		C.sqlite3_result_null(ctx)
+		return nil
+	}
+
+	cb, err := callbackRet(v.Elem().Type())
+        if err != nil {
+                return err
+        }
+
+        return cb(ctx, v.Elem())
+}
+
 func callbackRet(typ reflect.Type) (callbackRetConverter, error) {
 	switch typ.Kind() {
 	case reflect.Interface:
@@ -360,6 +374,11 @@ func callbackRet(typ reflect.Type) (callbackRetConverter, error) {
 		if typ.Implements(errorInterface) {
 			return callbackRetNil, nil
 		}
+
+		if typ.NumMethod() == 0 {
+			return callbackRetGeneric, nil
+		}
+
 		fallthrough
 	case reflect.Slice:
 		if typ.Elem().Kind() != reflect.Uint8 {

callback_test.go

@@ -102,3 +102,15 @@ func TestCallbackConverters(t *testing.T) {
 		}
 	}
 }
+
+func TestCallbackReturnAny(t *testing.T) {
+	udf := func() interface{} {
+		return 1
+	}
+
+	typ := reflect.TypeOf(udf)
+	_, err := callbackRet(typ.Out(0))
+	if err != nil {
+		t.Errorf("Expected valid callback for any return type, got: %s", err)
+	}
+}

go.mod

@@ -1,3 +1,7 @@
 module github.com/mattn/go-sqlite3
 
-go 1.12
+go 1.16
+
+retract (
+ [v2.0.0+incompatible, v2.0.6+incompatible] // Accidental; no major changes or features.
+)

sqlite3-binding.h

@@ -147,9 +147,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.37.0"
-#define SQLITE_VERSION_NUMBER 3037000
-#define SQLITE_SOURCE_ID      "2021-11-27 14:13:22 bd41822c7424d393a30e92ff6cb254d25c26769889c1499a18a0b9339f5d6c8a"
+#define SQLITE_VERSION        "3.39.4"
+#define SQLITE_VERSION_NUMBER 3039004
+#define SQLITE_SOURCE_ID      "2022-09-29 15:55:41 a29f9949895322123f7c38fbe94c649a9d6e6c9cd0c3b41c96d694552f26b309"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -567,7 +567,7 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_WARNING_AUTOINDEX       (SQLITE_WARNING | (1<<8))
 #define SQLITE_AUTH_USER               (SQLITE_AUTH | (1<<8))
 #define SQLITE_OK_LOAD_PERMANENTLY     (SQLITE_OK | (1<<8))
-#define SQLITE_OK_SYMLINK              (SQLITE_OK | (2<<8))
+#define SQLITE_OK_SYMLINK              (SQLITE_OK | (2<<8)) /* internal use only */
 
 /*
 ** CAPI3REF: Flags For File Open Operations
@@ -3825,13 +3825,14 @@ SQLITE_API void sqlite3_free_filename(char*);
 ** sqlite3_extended_errcode() might change with each API call.
 ** Except, there are some interfaces that are guaranteed to never
 ** change the value of the error code.  The error-code preserving
-** interfaces are:
+** interfaces include the following:
 **
 ** <ul>
 ** <li> sqlite3_errcode()
 ** <li> sqlite3_extended_errcode()
 ** <li> sqlite3_errmsg()
 ** <li> sqlite3_errmsg16()
+** <li> sqlite3_error_offset()
 ** </ul>
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
@@ -3846,6 +3847,13 @@ SQLITE_API void sqlite3_free_filename(char*);
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
+** ^If the most recent error references a specific token in the input
+** SQL, the sqlite3_error_offset() interface returns the byte offset
+** of the start of that token.  ^The byte offset returned by
+** sqlite3_error_offset() assumes that the input SQL is UTF8.
+** ^If the most recent error does not reference a specific token in the input
+** SQL, then the sqlite3_error_offset() function returns -1.
+**
 ** When the serialized [threading mode] is in use, it might be the
 ** case that a second error occurs on a separate thread in between
 ** the time of the first error and the call to these interfaces.
@@ -3865,6 +3873,7 @@ SQLITE_API int sqlite3_extended_errcode(sqlite3 *db);
 SQLITE_API const char *sqlite3_errmsg(sqlite3*);
 SQLITE_API const void *sqlite3_errmsg16(sqlite3*);
 SQLITE_API const char *sqlite3_errstr(int);
+SQLITE_API int sqlite3_error_offset(sqlite3 *db);
 
 /*
 ** CAPI3REF: Prepared Statement Object
@@ -4276,6 +4285,10 @@ SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
 ** be false.  ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a
 ** read-only no-op if the table already exists, but
 ** sqlite3_stmt_readonly() still returns false for such a statement.
+**
+** ^If prepared statement X is an [EXPLAIN] or [EXPLAIN QUERY PLAN]
+** statement, then sqlite3_stmt_readonly(X) returns the same value as
+** if the EXPLAIN or EXPLAIN QUERY PLAN prefix were omitted.
 */
 SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 
@@ -4344,6 +4357,8 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*);
 **
 ** ^The sqlite3_value objects that are passed as parameters into the
 ** implementation of [application-defined SQL functions] are protected.
+** ^The sqlite3_value objects returned by [sqlite3_vtab_rhs_value()]
+** are protected.
 ** ^The sqlite3_value object returned by
 ** [sqlite3_column_value()] is unprotected.
 ** Unprotected sqlite3_value objects may only be used as arguments
@@ -4965,6 +4980,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** even empty strings, are always zero-terminated.  ^The return
 ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer.
 **
+** ^Strings returned by sqlite3_column_text16() always have the endianness
+** which is native to the platform, regardless of the text encoding set
+** for the database.
+**
 ** <b>Warning:</b> ^The object returned by [sqlite3_column_value()] is an
 ** [unprotected sqlite3_value] object.  In a multithreaded environment,
 ** an unprotected sqlite3_value object may only be used safely with
@@ -4978,7 +4997,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** [application-defined SQL functions] or [virtual tables], not within
 ** top-level application code.
 **
-** The these routines may attempt to convert the datatype of the result.
+** These routines may attempt to convert the datatype of the result.
 ** ^For example, if the internal representation is FLOAT and a text result
 ** is requested, [sqlite3_snprintf()] is used internally to perform the
 ** conversion automatically.  ^(The following table details the conversions
@@ -5003,7 +5022,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** <tr><td>  TEXT    <td>   BLOB    <td> No change
 ** <tr><td>  BLOB    <td> INTEGER   <td> [CAST] to INTEGER
 ** <tr><td>  BLOB    <td>  FLOAT    <td> [CAST] to REAL
-** <tr><td>  BLOB    <td>   TEXT    <td> Add a zero terminator if needed
+** <tr><td>  BLOB    <td>   TEXT    <td> [CAST] to TEXT, ensure zero terminator
 ** </table>
 ** </blockquote>)^
 **
@@ -5575,7 +5594,8 @@ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*);
 ** object D and returns a pointer to that copy.  ^The [sqlite3_value] returned
 ** is a [protected sqlite3_value] object even if the input is not.
 ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a
-** memory allocation fails.
+** memory allocation fails. ^If V is a [pointer value], then the result
+** of sqlite3_value_dup(V) is a NULL value.
 **
 ** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object
 ** previously obtained from [sqlite3_value_dup()].  ^If V is a NULL pointer
@@ -6257,6 +6277,28 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*);
 */
 SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*);
 
+/*
+** CAPI3REF: Return The Schema Name For A Database Connection
+** METHOD: sqlite3
+**
+** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name
+** for the N-th database on database connection D, or a NULL pointer of N is
+** out of range.  An N value of 0 means the main database file.  An N of 1 is
+** the "temp" schema.  Larger values of N correspond to various ATTACH-ed
+** databases.
+**
+** Space to hold the string that is returned by sqlite3_db_name() is managed
+** by SQLite itself.  The string might be deallocated by any operation that
+** changes the schema, including [ATTACH] or [DETACH] or calls to
+** [sqlite3_serialize()] or [sqlite3_deserialize()], even operations that
+** occur on a different thread.  Applications that need to
+** remember the string long-term should make their own copy.  Applications that
+** are accessing the same database connection simultaneously on multiple
+** threads should mutex-protect calls to this API and should make their own
+** private copy of the result prior to releasing the mutex.
+*/
+SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N);
+
 /*
 ** CAPI3REF: Return The Filename For A Database Connection
 ** METHOD: sqlite3
@@ -7123,24 +7165,56 @@ struct sqlite3_index_info {
 **
 ** These macros define the allowed values for the
 ** [sqlite3_index_info].aConstraint[].op field.  Each value represents
-** an operator that is part of a constraint term in the wHERE clause of
+** an operator that is part of a constraint term in the WHERE clause of
 ** a query that uses a [virtual table].
+**
+** ^The left-hand operand of the operator is given by the corresponding
+** aConstraint[].iColumn field.  ^An iColumn of -1 indicates the left-hand
+** operand is the rowid.
+** The SQLITE_INDEX_CONSTRAINT_LIMIT and SQLITE_INDEX_CONSTRAINT_OFFSET
+** operators have no left-hand operand, and so for those operators the
+** corresponding aConstraint[].iColumn is meaningless and should not be
+** used.
+**
+** All operator values from SQLITE_INDEX_CONSTRAINT_FUNCTION through
+** value 255 are reserved to represent functions that are overloaded
+** by the [xFindFunction|xFindFunction method] of the virtual table
+** implementation.
+**
+** The right-hand operands for each constraint might be accessible using
+** the [sqlite3_vtab_rhs_value()] interface.  Usually the right-hand
+** operand is only available if it appears as a single constant literal
+** in the input SQL.  If the right-hand operand is another column or an
+** expression (even a constant expression) or a parameter, then the
+** sqlite3_vtab_rhs_value() probably will not be able to extract it.
+** ^The SQLITE_INDEX_CONSTRAINT_ISNULL and
+** SQLITE_INDEX_CONSTRAINT_ISNOTNULL operators have no right-hand operand
+** and hence calls to sqlite3_vtab_rhs_value() for those operators will
+** always return SQLITE_NOTFOUND.
+**
+** The collating sequence to be used for comparison can be found using
+** the [sqlite3_vtab_collation()] interface.  For most real-world virtual
+** tables, the collating sequence of constraints does not matter (for example
+** because the constraints are numeric) and so the sqlite3_vtab_collation()
+** interface is no commonly needed.
 */
-#define SQLITE_INDEX_CONSTRAINT_EQ         2
-#define SQLITE_INDEX_CONSTRAINT_GT         4
-#define SQLITE_INDEX_CONSTRAINT_LE         8
-#define SQLITE_INDEX_CONSTRAINT_LT        16
-#define SQLITE_INDEX_CONSTRAINT_GE        32
-#define SQLITE_INDEX_CONSTRAINT_MATCH     64
-#define SQLITE_INDEX_CONSTRAINT_LIKE      65
-#define SQLITE_INDEX_CONSTRAINT_GLOB      66
-#define SQLITE_INDEX_CONSTRAINT_REGEXP    67
-#define SQLITE_INDEX_CONSTRAINT_NE        68
-#define SQLITE_INDEX_CONSTRAINT_ISNOT     69
-#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70
-#define SQLITE_INDEX_CONSTRAINT_ISNULL    71
-#define SQLITE_INDEX_CONSTRAINT_IS        72
-#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150
+#define SQLITE_INDEX_CONSTRAINT_EQ          2
+#define SQLITE_INDEX_CONSTRAINT_GT          4
+#define SQLITE_INDEX_CONSTRAINT_LE          8
+#define SQLITE_INDEX_CONSTRAINT_LT         16
+#define SQLITE_INDEX_CONSTRAINT_GE         32
+#define SQLITE_INDEX_CONSTRAINT_MATCH      64
+#define SQLITE_INDEX_CONSTRAINT_LIKE       65
+#define SQLITE_INDEX_CONSTRAINT_GLOB       66
+#define SQLITE_INDEX_CONSTRAINT_REGEXP     67
+#define SQLITE_INDEX_CONSTRAINT_NE         68
+#define SQLITE_INDEX_CONSTRAINT_ISNOT      69
+#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL  70
+#define SQLITE_INDEX_CONSTRAINT_ISNULL     71
+#define SQLITE_INDEX_CONSTRAINT_IS         72
+#define SQLITE_INDEX_CONSTRAINT_LIMIT      73
+#define SQLITE_INDEX_CONSTRAINT_OFFSET     74
+#define SQLITE_INDEX_CONSTRAINT_FUNCTION  150
 
 /*
 ** CAPI3REF: Register A Virtual Table Implementation
@@ -7169,7 +7243,7 @@ struct sqlite3_index_info {
 ** destructor.
 **
 ** ^If the third parameter (the pointer to the sqlite3_module object) is
-** NULL then no new module is create and any existing modules with the
+** NULL then no new module is created and any existing modules with the
 ** same name are dropped.
 **
 ** See also: [sqlite3_drop_modules()]
@@ -7945,7 +8019,8 @@ SQLITE_API int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_SEEK_COUNT              30
 #define SQLITE_TESTCTRL_TRACEFLAGS              31
 #define SQLITE_TESTCTRL_TUNE                    32
-#define SQLITE_TESTCTRL_LAST                    32  /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_LOGEST                  33
+#define SQLITE_TESTCTRL_LAST                    33  /* Largest TESTCTRL */
 
 /*
 ** CAPI3REF: SQL Keyword Checking
@@ -8468,6 +8543,16 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
 ** The counter is incremented on the first [sqlite3_step()] call of each
 ** cycle.
 **
+** [[SQLITE_STMTSTATUS_FILTER_MISS]]
+** [[SQLITE_STMTSTATUS_FILTER HIT]]
+** <dt>SQLITE_STMTSTATUS_FILTER_HIT<br>
+** SQLITE_STMTSTATUS_FILTER_MISS</dt>
+** <dd>^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join
+** step was bypassed because a Bloom filter returned not-found.  The
+** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of
+** times that the Bloom filter returned a find, and thus the join step
+** had to be processed as normal.
+**
 ** [[SQLITE_STMTSTATUS_MEMUSED]] <dt>SQLITE_STMTSTATUS_MEMUSED</dt>
 ** <dd>^This is the approximate number of bytes of heap memory
 ** used to store the prepared statement.  ^This value is not actually
@@ -8482,6 +8567,8 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
 #define SQLITE_STMTSTATUS_VM_STEP           4
 #define SQLITE_STMTSTATUS_REPREPARE         5
 #define SQLITE_STMTSTATUS_RUN               6
+#define SQLITE_STMTSTATUS_FILTER_MISS       7
+#define SQLITE_STMTSTATUS_FILTER_HIT        8
 #define SQLITE_STMTSTATUS_MEMUSED           99
 
 /*
@@ -9450,19 +9537,276 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*);
 
 /*
 ** CAPI3REF: Determine The Collation For a Virtual Table Constraint
+** METHOD: sqlite3_index_info
 **
 ** This function may only be called from within a call to the [xBestIndex]
-** method of a [virtual table].
+** method of a [virtual table].  This function returns a pointer to a string
+** that is the name of the appropriate collation sequence to use for text
+** comparisons on the constraint identified by its arguments.
 **
-** The first argument must be the sqlite3_index_info object that is the
-** first parameter to the xBestIndex() method. The second argument must be
-** an index into the aConstraint[] array belonging to the sqlite3_index_info
-** structure passed to xBestIndex. This function returns a pointer to a buffer
-** containing the name of the collation sequence for the corresponding
-** constraint.
+** The first argument must be the pointer to the [sqlite3_index_info] object
+** that is the first parameter to the xBestIndex() method. The second argument
+** must be an index into the aConstraint[] array belonging to the
+** sqlite3_index_info structure passed to xBestIndex.
+**
+** Important:
+** The first parameter must be the same pointer that is passed into the
+** xBestMethod() method.  The first parameter may not be a pointer to a
+** different [sqlite3_index_info] object, even an exact copy.
+**
+** The return value is computed as follows:
+**
+** <ol>
+** <li><p> If the constraint comes from a WHERE clause expression that contains
+**         a [COLLATE operator], then the name of the collation specified by
+**         that COLLATE operator is returned.
+** <li><p> If there is no COLLATE operator, but the column that is the subject
+**         of the constraint specifies an alternative collating sequence via
+**         a [COLLATE clause] on the column definition within the CREATE TABLE
+**         statement that was passed into [sqlite3_declare_vtab()], then the
+**         name of that alternative collating sequence is returned.
+** <li><p> Otherwise, "BINARY" is returned.
+** </ol>
 */
 SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
 
+/*
+** CAPI3REF: Determine if a virtual table query is DISTINCT
+** METHOD: sqlite3_index_info
+**
+** This API may only be used from within an [xBestIndex|xBestIndex method]
+** of a [virtual table] implementation. The result of calling this
+** interface from outside of xBestIndex() is undefined and probably harmful.
+**
+** ^The sqlite3_vtab_distinct() interface returns an integer between 0 and
+** 3.  The integer returned by sqlite3_vtab_distinct()
+** gives the virtual table additional information about how the query
+** planner wants the output to be ordered. As long as the virtual table
+** can meet the ordering requirements of the query planner, it may set
+** the "orderByConsumed" flag.
+**
+** <ol><li value="0"><p>
+** ^If the sqlite3_vtab_distinct() interface returns 0, that means
+** that the query planner needs the virtual table to return all rows in the
+** sort order defined by the "nOrderBy" and "aOrderBy" fields of the
+** [sqlite3_index_info] object.  This is the default expectation.  If the
+** virtual table outputs all rows in sorted order, then it is always safe for
+** the xBestIndex method to set the "orderByConsumed" flag, regardless of
+** the return value from sqlite3_vtab_distinct().
+** <li value="1"><p>
+** ^(If the sqlite3_vtab_distinct() interface returns 1, that means
+** that the query planner does not need the rows to be returned in sorted order
+** as long as all rows with the same values in all columns identified by the
+** "aOrderBy" field are adjacent.)^  This mode is used when the query planner
+** is doing a GROUP BY.
+** <li value="2"><p>
+** ^(If the sqlite3_vtab_distinct() interface returns 2, that means
+** that the query planner does not need the rows returned in any particular
+** order, as long as rows with the same values in all "aOrderBy" columns
+** are adjacent.)^  ^(Furthermore, only a single row for each particular
+** combination of values in the columns identified by the "aOrderBy" field
+** needs to be returned.)^  ^It is always ok for two or more rows with the same
+** values in all "aOrderBy" columns to be returned, as long as all such rows
+** are adjacent.  ^The virtual table may, if it chooses, omit extra rows
+** that have the same value for all columns identified by "aOrderBy".
+** ^However omitting the extra rows is optional.
+** This mode is used for a DISTINCT query.
+** <li value="3"><p>
+** ^(If the sqlite3_vtab_distinct() interface returns 3, that means
+** that the query planner needs only distinct rows but it does need the
+** rows to be sorted.)^ ^The virtual table implementation is free to omit
+** rows that are identical in all aOrderBy columns, if it wants to, but
+** it is not required to omit any rows.  This mode is used for queries
+** that have both DISTINCT and ORDER BY clauses.
+** </ol>
+**
+** ^For the purposes of comparing virtual table output values to see if the
+** values are same value for sorting purposes, two NULL values are considered
+** to be the same.  In other words, the comparison operator is "IS"
+** (or "IS NOT DISTINCT FROM") and not "==".
+**
+** If a virtual table implementation is unable to meet the requirements
+** specified above, then it must not set the "orderByConsumed" flag in the
+** [sqlite3_index_info] object or an incorrect answer may result.
+**
+** ^A virtual table implementation is always free to return rows in any order
+** it wants, as long as the "orderByConsumed" flag is not set.  ^When the
+** the "orderByConsumed" flag is unset, the query planner will add extra
+** [bytecode] to ensure that the final results returned by the SQL query are
+** ordered correctly.  The use of the "orderByConsumed" flag and the
+** sqlite3_vtab_distinct() interface is merely an optimization.  ^Careful
+** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed"
+** flag might help queries against a virtual table to run faster.  Being
+** overly aggressive and setting the "orderByConsumed" flag when it is not
+** valid to do so, on the other hand, might cause SQLite to return incorrect
+** results.
+*/
+SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*);
+
+/*
+** CAPI3REF: Identify and handle IN constraints in xBestIndex
+**
+** This interface may only be used from within an
+** [xBestIndex|xBestIndex() method] of a [virtual table] implementation.
+** The result of invoking this interface from any other context is
+** undefined and probably harmful.
+**
+** ^(A constraint on a virtual table of the form
+** "[IN operator|column IN (...)]" is
+** communicated to the xBestIndex method as a
+** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^  If xBestIndex wants to use
+** this constraint, it must set the corresponding
+** aConstraintUsage[].argvIndex to a postive integer.  ^(Then, under
+** the usual mode of handling IN operators, SQLite generates [bytecode]
+** that invokes the [xFilter|xFilter() method] once for each value
+** on the right-hand side of the IN operator.)^  Thus the virtual table
+** only sees a single value from the right-hand side of the IN operator
+** at a time.
+**
+** In some cases, however, it would be advantageous for the virtual
+** table to see all values on the right-hand of the IN operator all at
+** once.  The sqlite3_vtab_in() interfaces facilitates this in two ways:
+**
+** <ol>
+** <li><p>
+**   ^A call to sqlite3_vtab_in(P,N,-1) will return true (non-zero)
+**   if and only if the [sqlite3_index_info|P->aConstraint][N] constraint
+**   is an [IN operator] that can be processed all at once.  ^In other words,
+**   sqlite3_vtab_in() with -1 in the third argument is a mechanism
+**   by which the virtual table can ask SQLite if all-at-once processing
+**   of the IN operator is even possible.
+**
+** <li><p>
+**   ^A call to sqlite3_vtab_in(P,N,F) with F==1 or F==0 indicates
+**   to SQLite that the virtual table does or does not want to process
+**   the IN operator all-at-once, respectively.  ^Thus when the third
+**   parameter (F) is non-negative, this interface is the mechanism by
+**   which the virtual table tells SQLite how it wants to process the
+**   IN operator.
+** </ol>
+**
+** ^The sqlite3_vtab_in(P,N,F) interface can be invoked multiple times
+** within the same xBestIndex method call.  ^For any given P,N pair,
+** the return value from sqlite3_vtab_in(P,N,F) will always be the same
+** within the same xBestIndex call.  ^If the interface returns true
+** (non-zero), that means that the constraint is an IN operator
+** that can be processed all-at-once.  ^If the constraint is not an IN
+** operator or cannot be processed all-at-once, then the interface returns
+** false.
+**
+** ^(All-at-once processing of the IN operator is selected if both of the
+** following conditions are met:
+**
+** <ol>
+** <li><p> The P->aConstraintUsage[N].argvIndex value is set to a positive
+** integer.  This is how the virtual table tells SQLite that it wants to
+** use the N-th constraint.
+**
+** <li><p> The last call to sqlite3_vtab_in(P,N,F) for which F was
+** non-negative had F>=1.
+** </ol>)^
+**
+** ^If either or both of the conditions above are false, then SQLite uses
+** the traditional one-at-a-time processing strategy for the IN constraint.
+** ^If both conditions are true, then the argvIndex-th parameter to the
+** xFilter method will be an [sqlite3_value] that appears to be NULL,
+** but which can be passed to [sqlite3_vtab_in_first()] and
+** [sqlite3_vtab_in_next()] to find all values on the right-hand side
+** of the IN constraint.
+*/
+SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle);
+
+/*
+** CAPI3REF: Find all elements on the right-hand side of an IN constraint.
+**
+** These interfaces are only useful from within the
+** [xFilter|xFilter() method] of a [virtual table] implementation.
+** The result of invoking these interfaces from any other context
+** is undefined and probably harmful.
+**
+** The X parameter in a call to sqlite3_vtab_in_first(X,P) or
+** sqlite3_vtab_in_next(X,P) must be one of the parameters to the
+** xFilter method which invokes these routines, and specifically
+** a parameter that was previously selected for all-at-once IN constraint
+** processing use the [sqlite3_vtab_in()] interface in the
+** [xBestIndex|xBestIndex method].  ^(If the X parameter is not
+** an xFilter argument that was selected for all-at-once IN constraint
+** processing, then these routines return [SQLITE_MISUSE])^ or perhaps
+** exhibit some other undefined or harmful behavior.
+**
+** ^(Use these routines to access all values on the right-hand side
+** of the IN constraint using code like the following:
+**
+** <blockquote><pre>
+** &nbsp;  for(rc=sqlite3_vtab_in_first(pList, &pVal);
+** &nbsp;      rc==SQLITE_OK && pVal
+** &nbsp;      rc=sqlite3_vtab_in_next(pList, &pVal)
+** &nbsp;  ){
+** &nbsp;    // do something with pVal
+** &nbsp;  }
+** &nbsp;  if( rc!=SQLITE_OK ){
+** &nbsp;    // an error has occurred
+** &nbsp;  }
+** </pre></blockquote>)^
+**
+** ^On success, the sqlite3_vtab_in_first(X,P) and sqlite3_vtab_in_next(X,P)
+** routines return SQLITE_OK and set *P to point to the first or next value
+** on the RHS of the IN constraint.  ^If there are no more values on the
+** right hand side of the IN constraint, then *P is set to NULL and these
+** routines return [SQLITE_DONE].  ^The return value might be
+** some other value, such as SQLITE_NOMEM, in the event of a malfunction.
+**
+** The *ppOut values returned by these routines are only valid until the
+** next call to either of these routines or until the end of the xFilter
+** method from which these routines were called.  If the virtual table
+** implementation needs to retain the *ppOut values for longer, it must make
+** copies.  The *ppOut values are [protected sqlite3_value|protected].
+*/
+SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut);
+SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut);
+
+/*
+** CAPI3REF: Constraint values in xBestIndex()
+** METHOD: sqlite3_index_info
+**
+** This API may only be used from within the [xBestIndex|xBestIndex method]
+** of a [virtual table] implementation. The result of calling this interface
+** from outside of an xBestIndex method are undefined and probably harmful.
+**
+** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within
+** the [xBestIndex] method of a [virtual table] implementation, with P being
+** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and
+** J being a 0-based index into P->aConstraint[], then this routine
+** attempts to set *V to the value of the right-hand operand of
+** that constraint if the right-hand operand is known.  ^If the
+** right-hand operand is not known, then *V is set to a NULL pointer.
+** ^The sqlite3_vtab_rhs_value(P,J,V) interface returns SQLITE_OK if
+** and only if *V is set to a value.  ^The sqlite3_vtab_rhs_value(P,J,V)
+** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th
+** constraint is not available.  ^The sqlite3_vtab_rhs_value() interface
+** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if
+** something goes wrong.
+**
+** The sqlite3_vtab_rhs_value() interface is usually only successful if
+** the right-hand operand of a constraint is a literal value in the original
+** SQL statement.  If the right-hand operand is an expression or a reference
+** to some other column or a [host parameter], then sqlite3_vtab_rhs_value()
+** will probably return [SQLITE_NOTFOUND].
+**
+** ^(Some constraints, such as [SQLITE_INDEX_CONSTRAINT_ISNULL] and
+** [SQLITE_INDEX_CONSTRAINT_ISNOTNULL], have no right-hand operand.  For such
+** constraints, sqlite3_vtab_rhs_value() always returns SQLITE_NOTFOUND.)^
+**
+** ^The [sqlite3_value] object returned in *V is a protected sqlite3_value
+** and remains valid for the duration of the xBestIndex method call.
+** ^When xBestIndex returns, the sqlite3_value object returned by
+** sqlite3_vtab_rhs_value() is automatically deallocated.
+**
+** The "_rhs_" in the name of this routine is an abbreviation for
+** "Right-Hand Side".
+*/
+SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal);
+
 /*
 ** CAPI3REF: Conflict resolution modes
 ** KEYWORDS: {conflict resolution mode}

sqlite3.go

@@ -494,10 +494,12 @@ func (ai *aggInfo) Done(ctx *C.sqlite3_context) {
 // Commit transaction.
 func (tx *SQLiteTx) Commit() error {
 	_, err := tx.c.exec(context.Background(), "COMMIT", nil)
-	if err != nil && err.(Error).Code == C.SQLITE_BUSY {
-		// sqlite3 will leave the transaction open in this scenario.
+	if err != nil {
+		// sqlite3 may leave the transaction open in this scenario.
 		// However, database/sql considers the transaction complete once we
 		// return from Commit() - we must clean up to honour its semantics.
+		// We don't know if the ROLLBACK is strictly necessary, but according
+		// to sqlite's docs, there is no harm in calling ROLLBACK unnecessarily.
 		tx.c.exec(context.Background(), "ROLLBACK", nil)
 	}
 	return err
@@ -835,9 +837,9 @@ func lastError(db *C.sqlite3) error {
 
 // Exec implements Execer.
 func (c *SQLiteConn) Exec(query string, args []driver.Value) (driver.Result, error) {
-	list := make([]namedValue, len(args))
+	list := make([]driver.NamedValue, len(args))
 	for i, v := range args {
-		list[i] = namedValue{
+		list[i] = driver.NamedValue{
 			Ordinal: i + 1,
 			Value:   v,
 		}
@@ -845,7 +847,7 @@ func (c *SQLiteConn) Exec(query string, args []driver.Value) (driver.Result, err
 	return c.exec(context.Background(), query, list)
 }
 
-func (c *SQLiteConn) exec(ctx context.Context, query string, args []namedValue) (driver.Result, error) {
+func (c *SQLiteConn) exec(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
 	start := 0
 	for {
 		s, err := c.prepare(ctx, query)
@@ -854,7 +856,7 @@ func (c *SQLiteConn) exec(ctx context.Context, query string, args []namedValue)
 		}
 		var res driver.Result
 		if s.(*SQLiteStmt).s != nil {
-			stmtArgs := make([]namedValue, 0, len(args))
+			stmtArgs := make([]driver.NamedValue, 0, len(args))
 			na := s.NumInput()
 			if len(args)-start < na {
 				s.Close()
@@ -892,17 +894,11 @@ func (c *SQLiteConn) exec(ctx context.Context, query string, args []namedValue)
 	}
 }
 
-type namedValue struct {
-	Name    string
-	Ordinal int
-	Value   driver.Value
-}
-
 // Query implements Queryer.
 func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
-	list := make([]namedValue, len(args))
+	list := make([]driver.NamedValue, len(args))
 	for i, v := range args {
-		list[i] = namedValue{
+		list[i] = driver.NamedValue{
 			Ordinal: i + 1,
 			Value:   v,
 		}
@@ -910,10 +906,10 @@ func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, erro
 	return c.query(context.Background(), query, list)
 }
 
-func (c *SQLiteConn) query(ctx context.Context, query string, args []namedValue) (driver.Rows, error) {
+func (c *SQLiteConn) query(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
 	start := 0
 	for {
-		stmtArgs := make([]namedValue, 0, len(args))
+		stmtArgs := make([]driver.NamedValue, 0, len(args))
 		s, err := c.prepare(ctx, query)
 		if err != nil {
 			return nil, err
@@ -1910,7 +1906,7 @@ func (s *SQLiteStmt) NumInput() int {
 
 var placeHolder = []byte{0}
 
-func (s *SQLiteStmt) bind(args []namedValue) error {
+func (s *SQLiteStmt) bind(args []driver.NamedValue) error {
 	rv := C.sqlite3_reset(s.s)
 	if rv != C.SQLITE_ROW && rv != C.SQLITE_OK && rv != C.SQLITE_DONE {
 		return s.c.lastError()
@@ -1980,9 +1976,9 @@ func (s *SQLiteStmt) bind(args []namedValue) error {
 
 // Query the statement with arguments. Return records.
 func (s *SQLiteStmt) Query(args []driver.Value) (driver.Rows, error) {
-	list := make([]namedValue, len(args))
+	list := make([]driver.NamedValue, len(args))
 	for i, v := range args {
-		list[i] = namedValue{
+		list[i] = driver.NamedValue{
 			Ordinal: i + 1,
 			Value:   v,
 		}
@@ -1990,7 +1986,7 @@ func (s *SQLiteStmt) Query(args []driver.Value) (driver.Rows, error) {
 	return s.query(context.Background(), list)
 }
 
-func (s *SQLiteStmt) query(ctx context.Context, args []namedValue) (driver.Rows, error) {
+func (s *SQLiteStmt) query(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
 	if err := s.bind(args); err != nil {
 		return nil, err
 	}
@@ -2020,9 +2016,9 @@ func (r *SQLiteResult) RowsAffected() (int64, error) {
 
 // Exec execute the statement with arguments. Return result object.
 func (s *SQLiteStmt) Exec(args []driver.Value) (driver.Result, error) {
-	list := make([]namedValue, len(args))
+	list := make([]driver.NamedValue, len(args))
 	for i, v := range args {
-		list[i] = namedValue{
+		list[i] = driver.NamedValue{
 			Ordinal: i + 1,
 			Value:   v,
 		}
@@ -2039,7 +2035,7 @@ func isInterruptErr(err error) bool {
 }
 
 // exec executes a query that doesn't return rows. Attempts to honor context timeout.
-func (s *SQLiteStmt) exec(ctx context.Context, args []namedValue) (driver.Result, error) {
+func (s *SQLiteStmt) exec(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
 	if ctx.Done() == nil {
 		return s.execSync(args)
 	}
@@ -2071,7 +2067,7 @@ func (s *SQLiteStmt) exec(ctx context.Context, args []namedValue) (driver.Result
 	return rv.r, rv.err
 }
 
-func (s *SQLiteStmt) execSync(args []namedValue) (driver.Result, error) {
+func (s *SQLiteStmt) execSync(args []driver.NamedValue) (driver.Result, error) {
 	if err := s.bind(args); err != nil {
 		C.sqlite3_reset(s.s)
 		C.sqlite3_clear_bindings(s.s)

sqlite3_go18.go

@@ -25,20 +25,12 @@ func (c *SQLiteConn) Ping(ctx context.Context) error {
 
 // QueryContext implement QueryerContext.
 func (c *SQLiteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
-	list := make([]namedValue, len(args))
-	for i, nv := range args {
-		list[i] = namedValue(nv)
-	}
-	return c.query(ctx, query, list)
+	return c.query(ctx, query, args)
 }
 
 // ExecContext implement ExecerContext.
 func (c *SQLiteConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
-	list := make([]namedValue, len(args))
-	for i, nv := range args {
-		list[i] = namedValue(nv)
-	}
-	return c.exec(ctx, query, list)
+	return c.exec(ctx, query, args)
 }
 
 // PrepareContext implement ConnPrepareContext.
@@ -53,18 +45,10 @@ func (c *SQLiteConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver
 
 // QueryContext implement QueryerContext.
 func (s *SQLiteStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
-	list := make([]namedValue, len(args))
-	for i, nv := range args {
-		list[i] = namedValue(nv)
-	}
-	return s.query(ctx, list)
+	return s.query(ctx, args)
 }
 
 // ExecContext implement ExecerContext.
 func (s *SQLiteStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
-	list := make([]namedValue, len(args))
-	for i, nv := range args {
-		list[i] = namedValue(nv)
-	}
-	return s.exec(ctx, list)
+	return s.exec(ctx, args)
 }

sqlite3_opt_json1.go

@@ -1,13 +0,0 @@
-// Copyright (C) 2019 Yasuhiro Matsumoto <mattn.jp@gmail.com>.
-//
-// Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// +build sqlite_json sqlite_json1 json1
-
-package sqlite3
-
-/*
-#cgo CFLAGS: -DSQLITE_ENABLE_JSON1
-*/
-import "C"

sqlite3_opt_math_functions.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2022 Yasuhiro Matsumoto <mattn.jp@gmail.com>.
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build sqlite_math_functions
+
+package sqlite3
+
+/*
+#cgo CFLAGS: -DSQLITE_ENABLE_MATH_FUNCTIONS
+#cgo LDFLAGS: -lm
+*/
+import "C"

sqlite3_opt_math_functions_test.go

@@ -0,0 +1,29 @@
+// +build sqlite_math_functions
+
+package sqlite3
+
+import (
+	"database/sql"
+	"testing"
+)
+
+func TestMathFunctions(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatal("Failed to open database:", err)
+	}
+	defer db.Close()
+
+	queries := []string{
+		`SELECT acos(1)`,
+		`SELECT log(10, 100)`,
+		`SELECT power(2, 2)`,
+	}
+
+	for _, query := range queries {
+		var result float64
+		if err := db.QueryRow(query).Scan(&result); err != nil {
+			t.Errorf("invoking math function query %q: %v", query, err)
+		}
+	}
+}

sqlite3_opt_os_trace.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2022 Yasuhiro Matsumoto <mattn.jp@gmail.com>.
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+//go:build sqlite_os_trace
+// +build sqlite_os_trace
+
+package sqlite3
+
+/*
+#cgo CFLAGS: -DSQLITE_FORCE_OS_TRACE=1
+#cgo CFLAGS: -DSQLITE_DEBUG_OS_TRACE=1
+*/
+import "C"

sqlite3_opt_preupdate_hook.go

@@ -33,7 +33,7 @@ import (
 // The callback is passed a SQLitePreUpdateData struct with the data for
 // the update, as well as methods for fetching copies of impacted data.
 //
-// If there is an existing update hook for this connection, it will be
+// If there is an existing preupdate hook for this connection, it will be
 // removed. If callback is nil the existing hook (if any) will be removed
 // without creating a new one.
 func (c *SQLiteConn) RegisterPreUpdateHook(callback func(SQLitePreUpdateData)) {

sqlite3_opt_preupdate_omit.go

@@ -13,7 +13,7 @@ package sqlite3
 // The callback is passed a SQLitePreUpdateData struct with the data for
 // the update, as well as methods for fetching copies of impacted data.
 //
-// If there is an existing update hook for this connection, it will be
+// If there is an existing preupdate hook for this connection, it will be
 // removed. If callback is nil the existing hook (if any) will be removed
 // without creating a new one.
 func (c *SQLiteConn) RegisterPreUpdateHook(callback func(SQLitePreUpdateData)) {

sqlite3_opt_serialize.go

@@ -0,0 +1,82 @@
+// +build !libsqlite3 sqlite_serialize
+
+package sqlite3
+
+/*
+#ifndef USE_LIBSQLITE3
+#include <sqlite3-binding.h>
+#else
+#include <sqlite3.h>
+#endif
+#include <stdlib.h>
+#include <stdint.h>
+*/
+import "C"
+
+import (
+	"fmt"
+	"math"
+	"reflect"
+	"unsafe"
+)
+
+// Serialize returns a byte slice that is a serialization of the database.
+//
+// See https://www.sqlite.org/c3ref/serialize.html
+func (c *SQLiteConn) Serialize(schema string) ([]byte, error) {
+	if schema == "" {
+		schema = "main"
+	}
+	var zSchema *C.char
+	zSchema = C.CString(schema)
+	defer C.free(unsafe.Pointer(zSchema))
+
+	var sz C.sqlite3_int64
+	ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0)
+	if ptr == nil {
+		return nil, fmt.Errorf("serialize failed")
+	}
+	defer C.sqlite3_free(unsafe.Pointer(ptr))
+
+	if sz > C.sqlite3_int64(math.MaxInt) {
+		return nil, fmt.Errorf("serialized database is too large (%d bytes)", sz)
+	}
+
+	cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
+		Data: uintptr(unsafe.Pointer(ptr)),
+		Len:  int(sz),
+		Cap:  int(sz),
+	}))
+
+	res := make([]byte, int(sz))
+	copy(res, cBuf)
+	return res, nil
+}
+
+// Deserialize causes the connection to disconnect from the current database and
+// then re-open as an in-memory database based on the contents of the byte slice.
+//
+// See https://www.sqlite.org/c3ref/deserialize.html
+func (c *SQLiteConn) Deserialize(b []byte, schema string) error {
+	if schema == "" {
+		schema = "main"
+	}
+	var zSchema *C.char
+	zSchema = C.CString(schema)
+	defer C.free(unsafe.Pointer(zSchema))
+
+	tmpBuf := (*C.uchar)(C.sqlite3_malloc64(C.sqlite3_uint64(len(b))))
+	cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
+		Data: uintptr(unsafe.Pointer(tmpBuf)),
+		Len:  len(b),
+		Cap:  len(b),
+	}))
+	copy(cBuf, b)
+
+	rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)),
+		C.sqlite3_int64(len(b)), C.SQLITE_DESERIALIZE_FREEONCLOSE)
+	if rc != C.SQLITE_OK {
+		return fmt.Errorf("deserialize failed with return %v", rc)
+	}
+	return nil
+}

sqlite3_opt_serialize_omit.go

@@ -0,0 +1,20 @@
+// +build libsqlite3,!sqlite_serialize
+
+package sqlite3
+
+import (
+	"errors"
+)
+
+/*
+#cgo CFLAGS: -DSQLITE_OMIT_DESERIALIZE
+*/
+import "C"
+
+func (c *SQLiteConn) Serialize(schema string) ([]byte, error) {
+	return nil, errors.New("sqlite3: Serialize requires the sqlite_serialize build tag when using the libsqlite3 build tag")
+}
+
+func (c *SQLiteConn) Deserialize(b []byte, schema string) error {
+	return errors.New("sqlite3: Deserialize requires the sqlite_serialize build tag when using the libsqlite3 build tag")
+}

sqlite3_opt_serialize_test.go

@@ -0,0 +1,99 @@
+// +build !libsqlite3 sqlite_serialize
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"os"
+	"testing"
+)
+
+func TestSerializeDeserialize(t *testing.T) {
+	// Connect to the source database.
+	srcTempFilename := TempFilename(t)
+	defer os.Remove(srcTempFilename)
+	srcDb, err := sql.Open(driverName, srcTempFilename)
+	if err != nil {
+		t.Fatal("Failed to open the source database:", err)
+	}
+	defer srcDb.Close()
+	err = srcDb.Ping()
+	if err != nil {
+		t.Fatal("Failed to connect to the source database:", err)
+	}
+
+	// Connect to the destination database.
+	destTempFilename := TempFilename(t)
+	defer os.Remove(destTempFilename)
+	destDb, err := sql.Open(driverName, destTempFilename)
+	if err != nil {
+		t.Fatal("Failed to open the destination database:", err)
+	}
+	defer destDb.Close()
+	err = destDb.Ping()
+	if err != nil {
+		t.Fatal("Failed to connect to the destination database:", err)
+	}
+
+	// Write data to source database.
+	_, err = srcDb.Exec(`CREATE TABLE foo (name string)`)
+	if err != nil {
+		t.Fatal("Failed to create table in source database:", err)
+	}
+	_, err = srcDb.Exec(`INSERT INTO foo(name) VALUES("alice")`)
+	if err != nil {
+		t.Fatal("Failed to insert data into source database", err)
+	}
+
+	// Serialize the source database
+	srcConn, err := srcDb.Conn(context.Background())
+	if err != nil {
+		t.Fatal("Failed to get connection to source database:", err)
+	}
+	defer srcConn.Close()
+
+	var serialized []byte
+	if err := srcConn.Raw(func(raw interface{}) error {
+		var err error
+		serialized, err = raw.(*SQLiteConn).Serialize("")
+		return err
+	}); err != nil {
+		t.Fatal("Failed to serialize source database:", err)
+	}
+	srcConn.Close()
+
+	// Confirm that the destination database is initially empty.
+	var destTableCount int
+	err = destDb.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'").Scan(&destTableCount)
+	if err != nil {
+		t.Fatal("Failed to check the destination table count:", err)
+	}
+	if destTableCount != 0 {
+		t.Fatalf("The destination database is not empty; %v table(s) found.", destTableCount)
+	}
+
+	// Deserialize to destination database
+	destConn, err := destDb.Conn(context.Background())
+	if err != nil {
+		t.Fatal("Failed to get connection to destination database:", err)
+	}
+	defer destConn.Close()
+
+	if err := destConn.Raw(func(raw interface{}) error {
+		return raw.(*SQLiteConn).Deserialize(serialized, "")
+	}); err != nil {
+		t.Fatal("Failed to deserialize source database:", err)
+	}
+	destConn.Close()
+
+	// Confirm that destination database has been loaded correctly.
+	var destRowCount int
+	err = destDb.QueryRow(`SELECT COUNT(*) FROM foo`).Scan(&destRowCount)
+	if err != nil {
+		t.Fatal("Failed to count rows in destination database table", err)
+	}
+	if destRowCount != 1 {
+		t.Fatalf("Destination table does not have the expected records")
+	}
+}

sqlite3_test.go

@@ -248,6 +248,43 @@ func TestForeignKeys(t *testing.T) {
 	}
 }
 
+func TestDeferredForeignKey(t *testing.T) {
+	fname := TempFilename(t)
+	uri := "file:" + fname + "?_foreign_keys=1"
+	db, err := sql.Open("sqlite3", uri)
+	if err != nil {
+		os.Remove(fname)
+		t.Errorf("sql.Open(\"sqlite3\", %q): %v", uri, err)
+	}
+	_, err = db.Exec("CREATE TABLE bar (id INTEGER PRIMARY KEY)")
+	if err != nil {
+		t.Errorf("failed creating tables: %v", err)
+	}
+	_, err = db.Exec("CREATE TABLE foo (bar_id INTEGER, FOREIGN KEY(bar_id) REFERENCES bar(id) DEFERRABLE INITIALLY DEFERRED)")
+	if err != nil {
+		t.Errorf("failed creating tables: %v", err)
+	}
+	tx, err := db.Begin()
+	if err != nil {
+		t.Errorf("Failed to begin transaction: %v", err)
+	}
+	_, err = tx.Exec("INSERT INTO foo (bar_id) VALUES (123)")
+	if err != nil {
+		t.Errorf("Failed to insert row: %v", err)
+	}
+	err = tx.Commit()
+	if err == nil {
+		t.Errorf("Expected an error: %v", err)
+	}
+	_, err = db.Begin()
+	if err != nil {
+		t.Errorf("Failed to begin transaction: %v", err)
+	}
+
+	db.Close()
+	os.Remove(fname)
+}
+
 func TestRecursiveTriggers(t *testing.T) {
 	cases := map[string]bool{
 		"?_recursive_triggers=1": true,
@@ -1063,34 +1100,44 @@ func TestQueryer(t *testing.T) {
 	defer db.Close()
 
 	_, err = db.Exec(`
-	create table foo (id integer);
+		create table foo (id integer);
 	`)
 	if err != nil {
 		t.Error("Failed to call db.Query:", err)
 	}
 
-	rows, err := db.Query(`
-	insert into foo(id) values(?);
-	insert into foo(id) values(?);
-	insert into foo(id) values(?);
-	select id from foo order by id;
+	_, err = db.Exec(`
+		insert into foo(id) values(?);
+		insert into foo(id) values(?);
+		insert into foo(id) values(?);
 	`, 3, 2, 1)
+	if err != nil {
+		t.Error("Failed to call db.Exec:", err)
+	}
+	rows, err := db.Query(`
+		select id from foo order by id;
+	`)
 	if err != nil {
 		t.Error("Failed to call db.Query:", err)
 	}
 	defer rows.Close()
-	n := 1
-	if rows != nil {
-		for rows.Next() {
-			var id int
-			err = rows.Scan(&id)
-			if err != nil {
-				t.Error("Failed to db.Query:", err)
-			}
-			if id != n {
-				t.Error("Failed to db.Query: not matched results")
-			}
+	n := 0
+	for rows.Next() {
+		var id int
+		err = rows.Scan(&id)
+		if err != nil {
+			t.Error("Failed to db.Query:", err)
 		}
+		if id != n + 1 {
+			t.Error("Failed to db.Query: not matched results")
+		}
+		n = n + 1
+	}
+	if err := rows.Err(); err != nil {
+		t.Errorf("Post-scan failed: %v\n", err)
+	}
+	if n != 3 {
+		t.Errorf("Expected 3 rows but retrieved %v", n)
 	}
 }
 
@@ -1449,6 +1496,63 @@ func TestAggregatorRegistration(t *testing.T) {
 	}
 }
 
+type mode struct {
+        counts   map[interface{}]int
+        top      interface{}
+        topCount int
+}
+
+func newMode() *mode {
+        return &mode{
+                counts: map[interface{}]int{},
+        }
+}
+
+func (m *mode) Step(x interface{}) {
+        m.counts[x]++
+        c := m.counts[x]
+        if c > m.topCount {
+                m.top = x
+                m.topCount = c
+        }
+}
+
+func (m *mode) Done() interface{} {
+        return m.top
+}
+
+func TestAggregatorRegistration_GenericReturn(t *testing.T) {
+	sql.Register("sqlite3_AggregatorRegistration_GenericReturn", &SQLiteDriver{
+		ConnectHook: func(conn *SQLiteConn) error {
+			return conn.RegisterAggregator("mode", newMode, true)
+		},
+	})
+	db, err := sql.Open("sqlite3_AggregatorRegistration_GenericReturn", ":memory:")
+	if err != nil {
+		t.Fatal("Failed to open database:", err)
+	}
+	defer db.Close()
+
+	_, err = db.Exec("create table foo (department integer, profits integer)")
+        if err != nil {
+                t.Fatal("Failed to create table:", err)
+        }
+        _, err = db.Exec("insert into foo values (1, 10), (1, 20), (1, 45), (2, 42), (2, 115), (2, 20)")
+        if err != nil {
+                t.Fatal("Failed to insert records:", err)
+        }
+
+	var mode int
+        err = db.QueryRow("select mode(profits) from foo").Scan(&mode)
+        if err != nil {
+                t.Fatal("MODE query error:", err)
+        }
+
+	if mode != 20 {
+		t.Fatal("Got incorrect mode. Wanted 20, got: ", mode)
+	}
+}
+
 func rot13(r rune) rune {
 	switch {
 	case r >= 'A' && r <= 'Z':

sqlite3_windows.go

@@ -12,7 +12,6 @@ package sqlite3
 #cgo CFLAGS: -fno-stack-check
 #cgo CFLAGS: -fno-stack-protector
 #cgo CFLAGS: -mno-stack-arg-probe
-#cgo LDFLAGS: -lmingwex -lmingw32
 #cgo windows,386 CFLAGS: -D_USE_32BIT_TIME_T
 */
 import "C"

sqlite3ext.h

@@ -349,6 +349,19 @@ struct sqlite3_api_routines {
   int (*autovacuum_pages)(sqlite3*,
      unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int),
      void*, void(*)(void*));
+  /* Version 3.38.0 and later */
+  int (*error_offset)(sqlite3*);
+  int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**);
+  int (*vtab_distinct)(sqlite3_index_info*);
+  int (*vtab_in)(sqlite3_index_info*,int,int);
+  int (*vtab_in_first)(sqlite3_value*,sqlite3_value**);
+  int (*vtab_in_next)(sqlite3_value*,sqlite3_value**);
+  /* Version 3.39.0 and later */
+  int (*deserialize)(sqlite3*,const char*,unsigned char*,
+                     sqlite3_int64,sqlite3_int64,unsigned);
+  unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*,
+                              unsigned int);
+  const char *(*db_name)(sqlite3*,int);
 };
 
 /*
@@ -660,6 +673,19 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_total_changes64        sqlite3_api->total_changes64
 /* Version 3.37.0 and later */
 #define sqlite3_autovacuum_pages       sqlite3_api->autovacuum_pages
+/* Version 3.38.0 and later */
+#define sqlite3_error_offset           sqlite3_api->error_offset
+#define sqlite3_vtab_rhs_value         sqlite3_api->vtab_rhs_value
+#define sqlite3_vtab_distinct          sqlite3_api->vtab_distinct
+#define sqlite3_vtab_in                sqlite3_api->vtab_in
+#define sqlite3_vtab_in_first          sqlite3_api->vtab_in_first
+#define sqlite3_vtab_in_next           sqlite3_api->vtab_in_next
+/* Version 3.39.0 and later */
+#ifndef SQLITE_OMIT_DESERIALIZE
+#define sqlite3_deserialize            sqlite3_api->deserialize
+#define sqlite3_serialize              sqlite3_api->serialize
+#endif
+#define sqlite3_db_name                sqlite3_api->db_name
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)