go-sqlite3/sqlite3.go

1685 lines
45 KiB
Go
Raw Normal View History

2014-08-18 11:56:31 +04:00
// Copyright (C) 2014 Yasuhiro Matsumoto <mattn.jp@gmail.com>.
2018-05-29 13:20:11 +03:00
// Copyright (C) 2018 G.J.R. Timmer <gjr.timmer@gmail.com>.
2014-08-18 11:56:31 +04:00
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
2014-08-18 12:00:59 +04:00
// +build cgo
2013-08-13 17:10:30 +04:00
package sqlite3
2011-11-11 16:36:22 +04:00
/*
#cgo CFLAGS: -std=gnu99
#cgo CFLAGS: -DSQLITE_ENABLE_RTREE
#cgo CFLAGS: -DSQLITE_THREADSAFE=1
#cgo CFLAGS: -DHAVE_USLEEP=1
#cgo CFLAGS: -DSQLITE_ENABLE_FTS3
#cgo CFLAGS: -DSQLITE_ENABLE_FTS3_PARENTHESIS
#cgo CFLAGS: -DSQLITE_ENABLE_FTS4_UNICODE61
2016-09-05 21:58:10 +03:00
#cgo CFLAGS: -DSQLITE_TRACE_SIZE_LIMIT=15
2018-05-24 00:10:05 +03:00
#cgo CFLAGS: -DSQLITE_OMIT_DEPRECATED
#cgo CFLAGS: -DSQLITE_DISABLE_INTRINSIC
2018-05-24 00:05:26 +03:00
#cgo CFLAGS: -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1
#cgo CFLAGS: -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT
2016-10-28 17:22:18 +03:00
#cgo CFLAGS: -Wno-deprecated-declarations
#cgo linux,!android CFLAGS: -DHAVE_PREAD64=1 -DHAVE_PWRITE64=1
#ifndef USE_LIBSQLITE3
#include <sqlite3-binding.h>
#else
#include <sqlite3.h>
#endif
2011-11-11 16:36:22 +04:00
#include <stdlib.h>
#include <string.h>
#ifdef __CYGWIN__
# include <errno.h>
#endif
2013-04-06 18:06:48 +04:00
#ifndef SQLITE_OPEN_READWRITE
# define SQLITE_OPEN_READWRITE 0
#endif
#ifndef SQLITE_OPEN_FULLMUTEX
# define SQLITE_OPEN_FULLMUTEX 0
#endif
2016-04-22 11:20:04 +03:00
#ifndef SQLITE_DETERMINISTIC
# define SQLITE_DETERMINISTIC 0
#endif
static int
_sqlite3_open_v2(const char *filename, sqlite3 **ppDb, int flags, const char *zVfs) {
#ifdef SQLITE_OPEN_URI
return sqlite3_open_v2(filename, ppDb, flags | SQLITE_OPEN_URI, zVfs);
#else
return sqlite3_open_v2(filename, ppDb, flags, zVfs);
#endif
}
2011-11-11 16:36:22 +04:00
static int
_sqlite3_bind_text(sqlite3_stmt *stmt, int n, char *p, int np) {
return sqlite3_bind_text(stmt, n, p, np, SQLITE_TRANSIENT);
}
static int
_sqlite3_bind_blob(sqlite3_stmt *stmt, int n, void *p, int np) {
return sqlite3_bind_blob(stmt, n, p, np, SQLITE_TRANSIENT);
}
#include <stdio.h>
#include <stdint.h>
static int
_sqlite3_exec(sqlite3* db, const char* pcmd, long long* rowid, long long* changes)
{
int rv = sqlite3_exec(db, pcmd, 0, 0, 0);
*rowid = (long long) sqlite3_last_insert_rowid(db);
*changes = (long long) sqlite3_changes(db);
return rv;
}
static int
_sqlite3_step(sqlite3_stmt* stmt, long long* rowid, long long* changes)
{
int rv = sqlite3_step(stmt);
sqlite3* db = sqlite3_db_handle(stmt);
*rowid = (long long) sqlite3_last_insert_rowid(db);
*changes = (long long) sqlite3_changes(db);
return rv;
}
void _sqlite3_result_text(sqlite3_context* ctx, const char* s) {
sqlite3_result_text(ctx, s, -1, &free);
}
void _sqlite3_result_blob(sqlite3_context* ctx, const void* b, int l) {
sqlite3_result_blob(ctx, b, l, SQLITE_TRANSIENT);
}
int _sqlite3_create_function(
sqlite3 *db,
const char *zFunctionName,
int nArg,
int eTextRep,
uintptr_t pApp,
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*)
) {
return sqlite3_create_function(db, zFunctionName, nArg, eTextRep, (void*) pApp, xFunc, xStep, xFinal);
}
void callbackTrampoline(sqlite3_context*, int, sqlite3_value**);
void stepTrampoline(sqlite3_context*, int, sqlite3_value**);
void doneTrampoline(sqlite3_context*);
int compareTrampoline(void*, int, char*, int, char*);
int commitHookTrampoline(void*);
void rollbackHookTrampoline(void*);
void updateHookTrampoline(void*, int, char*, char*, sqlite3_int64);
#ifdef SQLITE_LIMIT_WORKER_THREADS
# define _SQLITE_HAS_LIMIT
# define SQLITE_LIMIT_LENGTH 0
# define SQLITE_LIMIT_SQL_LENGTH 1
# define SQLITE_LIMIT_COLUMN 2
# define SQLITE_LIMIT_EXPR_DEPTH 3
# define SQLITE_LIMIT_COMPOUND_SELECT 4
# define SQLITE_LIMIT_VDBE_OP 5
# define SQLITE_LIMIT_FUNCTION_ARG 6
# define SQLITE_LIMIT_ATTACHED 7
# define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8
# define SQLITE_LIMIT_VARIABLE_NUMBER 9
# define SQLITE_LIMIT_TRIGGER_DEPTH 10
# define SQLITE_LIMIT_WORKER_THREADS 11
# else
# define SQLITE_LIMIT_WORKER_THREADS 11
#endif
static int _sqlite3_limit(sqlite3* db, int limitId, int newLimit) {
#ifndef _SQLITE_HAS_LIMIT
return -1;
#else
return sqlite3_limit(db, limitId, newLimit);
#endif
}
2011-11-11 16:36:22 +04:00
*/
import "C"
import (
"context"
2012-01-20 20:44:24 +04:00
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"io"
"net/url"
"reflect"
"runtime"
2015-01-02 09:31:46 +03:00
"strconv"
"strings"
"sync"
"time"
2011-11-11 16:36:22 +04:00
"unsafe"
)
2016-11-04 18:40:06 +03:00
// SQLiteTimestampFormats is timestamp formats understood by both this module
// and SQLite. The first format in the slice will be used when saving time
// values into the database. When parsing a string from a timestamp or datetime
// column, the formats are tried in order.
var SQLiteTimestampFormats = []string{
// By default, store timestamps with whatever timezone they come with.
// When parsed, they will be returned with the same timezone.
"2006-01-02 15:04:05.999999999-07:00",
"2006-01-02T15:04:05.999999999-07:00",
"2006-01-02 15:04:05.999999999",
"2006-01-02T15:04:05.999999999",
"2006-01-02 15:04:05",
"2006-01-02T15:04:05",
"2006-01-02 15:04",
"2006-01-02T15:04",
"2006-01-02",
}
2018-04-17 12:13:35 +03:00
const (
columnDate string = "date"
columnDatetime string = "datetime"
columnTimestamp string = "timestamp"
)
2011-11-11 16:36:22 +04:00
func init() {
sql.Register("sqlite3", &SQLiteDriver{})
2011-11-11 16:36:22 +04:00
}
2016-02-23 09:20:57 +03:00
// Version returns SQLite library version information.
2016-11-04 18:40:06 +03:00
func Version() (libVersion string, libVersionNumber int, sourceID string) {
libVersion = C.GoString(C.sqlite3_libversion())
libVersionNumber = int(C.sqlite3_libversion_number())
2016-11-04 18:40:06 +03:00
sourceID = C.GoString(C.sqlite3_sourceid())
return libVersion, libVersionNumber, sourceID
}
const (
SQLITE_DELETE = C.SQLITE_DELETE
SQLITE_INSERT = C.SQLITE_INSERT
SQLITE_UPDATE = C.SQLITE_UPDATE
)
2016-11-04 18:40:06 +03:00
// SQLiteDriver implement sql.Driver.
2011-11-11 16:36:22 +04:00
type SQLiteDriver struct {
Extensions []string
ConnectHook func(*SQLiteConn) error
2011-11-11 16:36:22 +04:00
}
2016-11-04 18:40:06 +03:00
// SQLiteConn implement sql.Conn.
2011-11-11 16:36:22 +04:00
type SQLiteConn struct {
2017-08-01 18:06:18 +03:00
mu sync.Mutex
db *C.sqlite3
loc *time.Location
txlock string
funcs []*functionInfo
aggregators []*aggInfo
2011-11-11 16:36:22 +04:00
}
2016-11-04 18:40:06 +03:00
// SQLiteTx implemen sql.Tx.
2011-11-11 16:36:22 +04:00
type SQLiteTx struct {
c *SQLiteConn
}
2016-11-04 18:40:06 +03:00
// SQLiteStmt implement sql.Stmt.
2013-01-31 11:48:30 +04:00
type SQLiteStmt struct {
2017-08-30 07:29:47 +03:00
mu sync.Mutex
2013-01-31 11:48:30 +04:00
c *SQLiteConn
s *C.sqlite3_stmt
t string
closed bool
cls bool
2013-01-31 11:48:30 +04:00
}
2016-11-04 18:40:06 +03:00
// SQLiteResult implement sql.Result.
2013-01-31 11:48:30 +04:00
type SQLiteResult struct {
2013-05-11 16:45:48 +04:00
id int64
changes int64
2013-01-31 11:48:30 +04:00
}
2016-11-04 18:40:06 +03:00
// SQLiteRows implement sql.Rows.
2013-01-31 11:48:30 +04:00
type SQLiteRows struct {
s *SQLiteStmt
nc int
cols []string
decltype []string
cls bool
2017-08-01 19:43:14 +03:00
closed bool
2016-11-06 14:43:53 +03:00
done chan struct{}
2013-01-31 11:48:30 +04:00
}
type functionInfo struct {
f reflect.Value
argConverters []callbackArgConverter
variadicConverter callbackArgConverter
retConverter callbackRetConverter
}
func (fi *functionInfo) Call(ctx *C.sqlite3_context, argv []*C.sqlite3_value) {
args, err := callbackConvertArgs(argv, fi.argConverters, fi.variadicConverter)
if err != nil {
callbackError(ctx, err)
return
}
ret := fi.f.Call(args)
if len(ret) == 2 && ret[1].Interface() != nil {
callbackError(ctx, ret[1].Interface().(error))
return
}
err = fi.retConverter(ctx, ret[0])
if err != nil {
callbackError(ctx, err)
return
}
}
type aggInfo struct {
constructor reflect.Value
// Active aggregator objects for aggregations in flight. The
// aggregators are indexed by a counter stored in the aggregation
// user data space provided by sqlite.
active map[int64]reflect.Value
next int64
stepArgConverters []callbackArgConverter
stepVariadicConverter callbackArgConverter
doneRetConverter callbackRetConverter
}
func (ai *aggInfo) agg(ctx *C.sqlite3_context) (int64, reflect.Value, error) {
aggIdx := (*int64)(C.sqlite3_aggregate_context(ctx, C.int(8)))
if *aggIdx == 0 {
*aggIdx = ai.next
ret := ai.constructor.Call(nil)
if len(ret) == 2 && ret[1].Interface() != nil {
return 0, reflect.Value{}, ret[1].Interface().(error)
}
if ret[0].IsNil() {
return 0, reflect.Value{}, errors.New("aggregator constructor returned nil state")
}
ai.next++
ai.active[*aggIdx] = ret[0]
}
return *aggIdx, ai.active[*aggIdx], nil
}
func (ai *aggInfo) Step(ctx *C.sqlite3_context, argv []*C.sqlite3_value) {
_, agg, err := ai.agg(ctx)
if err != nil {
callbackError(ctx, err)
return
}
args, err := callbackConvertArgs(argv, ai.stepArgConverters, ai.stepVariadicConverter)
if err != nil {
callbackError(ctx, err)
return
}
ret := agg.MethodByName("Step").Call(args)
if len(ret) == 1 && ret[0].Interface() != nil {
callbackError(ctx, ret[0].Interface().(error))
return
}
}
func (ai *aggInfo) Done(ctx *C.sqlite3_context) {
idx, agg, err := ai.agg(ctx)
if err != nil {
callbackError(ctx, err)
return
}
defer func() { delete(ai.active, idx) }()
ret := agg.MethodByName("Done").Call(nil)
if len(ret) == 2 && ret[1].Interface() != nil {
callbackError(ctx, ret[1].Interface().(error))
return
}
err = ai.doneRetConverter(ctx, ret[0])
if err != nil {
callbackError(ctx, err)
return
}
}
2013-01-31 11:48:30 +04:00
// Commit transaction.
2011-11-11 16:36:22 +04:00
func (tx *SQLiteTx) Commit() error {
2016-11-08 06:19:13 +03:00
_, err := tx.c.exec(context.Background(), "COMMIT", nil)
if err != nil && err.(Error).Code == C.SQLITE_BUSY {
// sqlite3 will leave the transaction open in this scenario.
// However, database/sql considers the transaction complete once we
// return from Commit() - we must clean up to honour its semantics.
2016-11-08 06:19:13 +03:00
tx.c.exec(context.Background(), "ROLLBACK", nil)
}
return err
2011-11-11 16:36:22 +04:00
}
2013-01-31 11:48:30 +04:00
// Rollback transaction.
2011-11-11 16:36:22 +04:00
func (tx *SQLiteTx) Rollback() error {
2016-11-08 06:19:13 +03:00
_, err := tx.c.exec(context.Background(), "ROLLBACK", nil)
return err
2011-11-11 16:36:22 +04:00
}
// RegisterCollation makes a Go function available as a collation.
//
// cmp receives two UTF-8 strings, a and b. The result should be 0 if
// a==b, -1 if a < b, and +1 if a > b.
//
// cmp must always return the same result given the same
// inputs. Additionally, it must have the following properties for all
// strings A, B and C: if A==B then B==A; if A==B and B==C then A==C;
// if A<B then B>A; if A<B and B<C then A<C.
//
// If cmp does not obey these constraints, sqlite3's behavior is
// undefined when the collation is used.
func (c *SQLiteConn) RegisterCollation(name string, cmp func(string, string) int) error {
handle := newHandle(c, cmp)
cname := C.CString(name)
defer C.free(unsafe.Pointer(cname))
rv := C.sqlite3_create_collation(c.db, cname, C.SQLITE_UTF8, unsafe.Pointer(handle), (*[0]byte)(unsafe.Pointer(C.compareTrampoline)))
if rv != C.SQLITE_OK {
return c.lastError()
}
return nil
}
// RegisterCommitHook sets the commit hook for a connection.
//
// If the callback returns non-zero the transaction will become a rollback.
//
// If there is an existing commit hook for this connection, it will be
// removed. If callback is nil the existing hook (if any) will be removed
// without creating a new one.
func (c *SQLiteConn) RegisterCommitHook(callback func() int) {
if callback == nil {
C.sqlite3_commit_hook(c.db, nil, nil)
} else {
2018-04-17 11:55:42 +03:00
C.sqlite3_commit_hook(c.db, (*[0]byte)(C.commitHookTrampoline), unsafe.Pointer(newHandle(c, callback)))
}
}
// RegisterRollbackHook sets the rollback hook for a connection.
//
// If there is an existing rollback hook for this connection, it will be
// removed. If callback is nil the existing hook (if any) will be removed
// without creating a new one.
func (c *SQLiteConn) RegisterRollbackHook(callback func()) {
if callback == nil {
C.sqlite3_rollback_hook(c.db, nil, nil)
} else {
2018-04-17 11:55:42 +03:00
C.sqlite3_rollback_hook(c.db, (*[0]byte)(C.rollbackHookTrampoline), unsafe.Pointer(newHandle(c, callback)))
}
}
// RegisterUpdateHook sets the update hook for a connection.
//
// The parameters to the callback are the operation (one of the constants
// SQLITE_INSERT, SQLITE_DELETE, or SQLITE_UPDATE), the database name, the
// table name, and the rowid.
//
// If there is an existing update hook for this connection, it will be
// removed. If callback is nil the existing hook (if any) will be removed
// without creating a new one.
func (c *SQLiteConn) RegisterUpdateHook(callback func(int, string, string, int64)) {
if callback == nil {
C.sqlite3_update_hook(c.db, nil, nil)
} else {
2018-04-17 11:55:42 +03:00
C.sqlite3_update_hook(c.db, (*[0]byte)(C.updateHookTrampoline), unsafe.Pointer(newHandle(c, callback)))
}
}
// RegisterFunc makes a Go function available as a SQLite function.
//
// The Go function can have arguments of the following types: any
// numeric type except complex, bool, []byte, string and
// interface{}. interface{} arguments are given the direct translation
// of the SQLite data type: int64 for INTEGER, float64 for FLOAT,
// []byte for BLOB, string for TEXT.
//
// The function can additionally be variadic, as long as the type of
// the variadic argument is one of the above.
//
// If pure is true. SQLite will assume that the function's return
// value depends only on its inputs, and make more aggressive
// optimizations in its queries.
//
// See _example/go_custom_funcs for a detailed example.
func (c *SQLiteConn) RegisterFunc(name string, impl interface{}, pure bool) error {
var fi functionInfo
fi.f = reflect.ValueOf(impl)
t := fi.f.Type()
if t.Kind() != reflect.Func {
return errors.New("Non-function passed to RegisterFunc")
}
if t.NumOut() != 1 && t.NumOut() != 2 {
return errors.New("SQLite functions must return 1 or 2 values")
}
if t.NumOut() == 2 && !t.Out(1).Implements(reflect.TypeOf((*error)(nil)).Elem()) {
return errors.New("Second return value of SQLite function must be error")
}
numArgs := t.NumIn()
if t.IsVariadic() {
numArgs--
}
for i := 0; i < numArgs; i++ {
conv, err := callbackArg(t.In(i))
if err != nil {
return err
}
fi.argConverters = append(fi.argConverters, conv)
}
if t.IsVariadic() {
conv, err := callbackArg(t.In(numArgs).Elem())
if err != nil {
return err
}
fi.variadicConverter = conv
// Pass -1 to sqlite so that it allows any number of
// arguments. The call helper verifies that the minimum number
// of arguments is present for variadic functions.
numArgs = -1
}
conv, err := callbackRet(t.Out(0))
if err != nil {
return err
}
fi.retConverter = conv
// fi must outlast the database connection, or we'll have dangling pointers.
c.funcs = append(c.funcs, &fi)
cname := C.CString(name)
defer C.free(unsafe.Pointer(cname))
opts := C.SQLITE_UTF8
if pure {
opts |= C.SQLITE_DETERMINISTIC
}
2017-03-04 18:45:41 +03:00
rv := sqlite3CreateFunction(c.db, cname, C.int(numArgs), C.int(opts), newHandle(c, &fi), C.callbackTrampoline, nil, nil)
if rv != C.SQLITE_OK {
return c.lastError()
}
return nil
}
2017-03-04 18:45:41 +03:00
func sqlite3CreateFunction(db *C.sqlite3, zFunctionName *C.char, nArg C.int, eTextRep C.int, pApp uintptr, xFunc unsafe.Pointer, xStep unsafe.Pointer, xFinal unsafe.Pointer) C.int {
2018-04-17 11:55:42 +03:00
return C._sqlite3_create_function(db, zFunctionName, nArg, eTextRep, C.uintptr_t(pApp), (*[0]byte)(xFunc), (*[0]byte)(xStep), (*[0]byte)(xFinal))
2016-11-08 06:19:13 +03:00
}
// RegisterAggregator makes a Go type available as a SQLite aggregation function.
//
// Because aggregation is incremental, it's implemented in Go with a
// type that has 2 methods: func Step(values) accumulates one row of
// data into the accumulator, and func Done() ret finalizes and
// returns the aggregate value. "values" and "ret" may be any type
// supported by RegisterFunc.
//
// RegisterAggregator takes as implementation a constructor function
// that constructs an instance of the aggregator type each time an
// aggregation begins. The constructor must return a pointer to a
// type, or an interface that implements Step() and Done().
//
// The constructor function and the Step/Done methods may optionally
// return an error in addition to their other return values.
//
// See _example/go_custom_funcs for a detailed example.
func (c *SQLiteConn) RegisterAggregator(name string, impl interface{}, pure bool) error {
var ai aggInfo
ai.constructor = reflect.ValueOf(impl)
t := ai.constructor.Type()
if t.Kind() != reflect.Func {
return errors.New("non-function passed to RegisterAggregator")
}
if t.NumOut() != 1 && t.NumOut() != 2 {
return errors.New("SQLite aggregator constructors must return 1 or 2 values")
}
if t.NumOut() == 2 && !t.Out(1).Implements(reflect.TypeOf((*error)(nil)).Elem()) {
return errors.New("Second return value of SQLite function must be error")
}
if t.NumIn() != 0 {
return errors.New("SQLite aggregator constructors must not have arguments")
}
agg := t.Out(0)
switch agg.Kind() {
case reflect.Ptr, reflect.Interface:
default:
return errors.New("SQlite aggregator constructor must return a pointer object")
}
stepFn, found := agg.MethodByName("Step")
if !found {
return errors.New("SQlite aggregator doesn't have a Step() function")
}
step := stepFn.Type
if step.NumOut() != 0 && step.NumOut() != 1 {
return errors.New("SQlite aggregator Step() function must return 0 or 1 values")
}
if step.NumOut() == 1 && !step.Out(0).Implements(reflect.TypeOf((*error)(nil)).Elem()) {
return errors.New("type of SQlite aggregator Step() return value must be error")
}
stepNArgs := step.NumIn()
start := 0
if agg.Kind() == reflect.Ptr {
// Skip over the method receiver
stepNArgs--
start++
}
if step.IsVariadic() {
stepNArgs--
}
for i := start; i < start+stepNArgs; i++ {
conv, err := callbackArg(step.In(i))
if err != nil {
return err
}
ai.stepArgConverters = append(ai.stepArgConverters, conv)
}
if step.IsVariadic() {
conv, err := callbackArg(t.In(start + stepNArgs).Elem())
if err != nil {
return err
}
ai.stepVariadicConverter = conv
// Pass -1 to sqlite so that it allows any number of
// arguments. The call helper verifies that the minimum number
// of arguments is present for variadic functions.
stepNArgs = -1
}
doneFn, found := agg.MethodByName("Done")
if !found {
return errors.New("SQlite aggregator doesn't have a Done() function")
}
done := doneFn.Type
doneNArgs := done.NumIn()
if agg.Kind() == reflect.Ptr {
// Skip over the method receiver
doneNArgs--
}
if doneNArgs != 0 {
return errors.New("SQlite aggregator Done() function must have no arguments")
}
if done.NumOut() != 1 && done.NumOut() != 2 {
return errors.New("SQLite aggregator Done() function must return 1 or 2 values")
}
if done.NumOut() == 2 && !done.Out(1).Implements(reflect.TypeOf((*error)(nil)).Elem()) {
return errors.New("second return value of SQLite aggregator Done() function must be error")
}
conv, err := callbackRet(done.Out(0))
if err != nil {
return err
}
ai.doneRetConverter = conv
ai.active = make(map[int64]reflect.Value)
ai.next = 1
// ai must outlast the database connection, or we'll have dangling pointers.
c.aggregators = append(c.aggregators, &ai)
cname := C.CString(name)
defer C.free(unsafe.Pointer(cname))
opts := C.SQLITE_UTF8
if pure {
opts |= C.SQLITE_DETERMINISTIC
}
rv := sqlite3CreateFunction(c.db, cname, C.int(stepNArgs), C.int(opts), newHandle(c, &ai), nil, C.stepTrampoline, C.doneTrampoline)
if rv != C.SQLITE_OK {
return c.lastError()
}
return nil
}
2013-09-09 05:44:44 +04:00
// AutoCommit return which currently auto commit or not.
2013-08-23 09:11:15 +04:00
func (c *SQLiteConn) AutoCommit() bool {
2013-08-23 09:26:33 +04:00
return int(C.sqlite3_get_autocommit(c.db)) != 0
2013-08-23 09:11:15 +04:00
}
func (c *SQLiteConn) lastError() error {
return lastError(c.db)
}
func lastError(db *C.sqlite3) error {
rv := C.sqlite3_errcode(db)
if rv == C.SQLITE_OK {
return nil
}
return Error{
Code: ErrNo(rv),
ExtendedCode: ErrNoExtended(C.sqlite3_extended_errcode(db)),
err: C.GoString(C.sqlite3_errmsg(db)),
}
}
2016-11-04 18:40:06 +03:00
// Exec implements Execer.
func (c *SQLiteConn) Exec(query string, args []driver.Value) (driver.Result, error) {
2016-11-04 08:24:22 +03:00
list := make([]namedValue, len(args))
for i, v := range args {
list[i] = namedValue{
Ordinal: i + 1,
Value: v,
}
}
return c.exec(context.Background(), query, list)
}
func (c *SQLiteConn) exec(ctx context.Context, query string, args []namedValue) (driver.Result, error) {
2016-11-04 09:00:29 +03:00
start := 0
2014-06-25 22:54:09 +04:00
for {
2016-11-08 19:13:34 +03:00
s, err := c.prepare(ctx, query)
2014-06-25 22:54:09 +04:00
if err != nil {
return nil, err
}
var res driver.Result
if s.(*SQLiteStmt).s != nil {
na := s.NumInput()
2014-08-18 13:48:48 +04:00
if len(args) < na {
2017-01-07 16:22:02 +03:00
s.Close()
2017-03-05 16:16:51 +03:00
return nil, fmt.Errorf("not enough args to execute query: want %d got %d", na, len(args))
2014-08-18 13:23:58 +04:00
}
2016-11-04 09:00:29 +03:00
for i := 0; i < na; i++ {
args[i].Ordinal -= start
}
2016-11-04 08:24:22 +03:00
res, err = s.(*SQLiteStmt).exec(ctx, args[:na])
2014-06-25 22:54:09 +04:00
if err != nil && err != driver.ErrSkip {
s.Close()
return nil, err
}
args = args[na:]
2016-11-04 09:00:29 +03:00
start += na
2014-06-25 22:54:09 +04:00
}
tail := s.(*SQLiteStmt).t
s.Close()
2014-06-25 22:54:09 +04:00
if tail == "" {
return res, nil
}
query = tail
}
}
2016-11-04 08:24:22 +03:00
type namedValue struct {
Name string
Ordinal int
Value driver.Value
}
2016-11-04 18:40:06 +03:00
// Query implements Queryer.
func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
2016-11-04 08:24:22 +03:00
list := make([]namedValue, len(args))
for i, v := range args {
list[i] = namedValue{
Ordinal: i + 1,
Value: v,
}
}
return c.query(context.Background(), query, list)
}
func (c *SQLiteConn) query(ctx context.Context, query string, args []namedValue) (driver.Rows, error) {
2016-11-04 09:00:29 +03:00
start := 0
2014-06-25 22:54:09 +04:00
for {
2016-11-08 19:13:34 +03:00
s, err := c.prepare(ctx, query)
2014-06-25 22:54:09 +04:00
if err != nil {
return nil, err
}
s.(*SQLiteStmt).cls = true
2014-06-25 22:54:09 +04:00
na := s.NumInput()
if len(args) < na {
2017-03-05 16:16:51 +03:00
return nil, fmt.Errorf("not enough args to execute query: want %d got %d", na, len(args))
}
2016-11-04 09:00:29 +03:00
for i := 0; i < na; i++ {
args[i].Ordinal -= start
}
2016-11-04 08:24:22 +03:00
rows, err := s.(*SQLiteStmt).query(ctx, args[:na])
2014-06-25 22:54:09 +04:00
if err != nil && err != driver.ErrSkip {
s.Close()
2016-11-08 07:22:46 +03:00
return rows, err
2014-06-25 22:54:09 +04:00
}
args = args[na:]
2016-11-04 09:00:29 +03:00
start += na
2014-06-25 22:54:09 +04:00
tail := s.(*SQLiteStmt).t
if tail == "" {
return rows, nil
}
2014-11-13 20:21:49 +03:00
rows.Close()
2014-06-25 22:54:09 +04:00
s.Close()
query = tail
}
}
2013-01-31 11:48:30 +04:00
// Begin transaction.
2011-11-11 16:36:22 +04:00
func (c *SQLiteConn) Begin() (driver.Tx, error) {
2016-11-04 09:15:16 +03:00
return c.begin(context.Background())
}
func (c *SQLiteConn) begin(ctx context.Context) (driver.Tx, error) {
2016-11-08 06:19:13 +03:00
if _, err := c.exec(ctx, c.txlock, nil); err != nil {
2011-11-11 16:36:22 +04:00
return nil, err
}
return &SQLiteTx{c}, nil
}
func errorString(err Error) string {
return C.GoString(C.sqlite3_errstr(C.int(err.Code)))
}
2013-01-31 11:48:30 +04:00
// Open database and return a new connection.
//
// A pragma can take either zero or one argument.
// The argument is may be either in parentheses or it may be separated from
// the pragma name by an equal sign. The two syntaxes yield identical results.
// In many pragmas, the argument is a boolean. The boolean can be one of:
// 1 yes true on
// 0 no false off
//
2016-02-23 09:20:57 +03:00
// You can specify a DSN string using a URI as the filename.
2013-01-31 11:48:30 +04:00
// test.db
// file:test.db?cache=shared&mode=memory
// :memory:
// file::memory:
//
// mode
// Access mode of the database.
// https://www.sqlite.org/c3ref/open.html
// Values:
// - ro
// - rw
// - rwc
// - memory
//
// shared
// SQLite Shared-Cache Mode
// https://www.sqlite.org/sharedcache.html
// Values:
// - shared
// - private
//
// immutable=Boolean
// The immutable parameter is a boolean query parameter that indicates
// that the database file is stored on read-only media. When immutable is set,
// SQLite assumes that the database file cannot be changed,
// even by a process with higher privilege,
// and so the database is opened read-only and all locking and change detection is disabled.
// Caution: Setting the immutable property on a database file that
// does in fact change can result in incorrect query results and/or SQLITE_CORRUPT errors.
//
2015-06-05 17:03:38 +03:00
// go-sqlite3 adds the following query parameters to those used by SQLite:
// _loc=XXX
// Specify location of time format. It's possible to specify "auto".
//
// _mutex=XXX
// Specify mutex mode. XXX can be "no", "full".
//
// _txlock=XXX
// Specify locking behavior for transactions. XXX can be "immediate",
// "deferred", "exclusive".
//
// _busy_timeout=XXX"| _timeout=XXX
// Specify value for sqlite3_busy_timeout.
//
// _cslike=Boolean
// Default or disabled the LIKE operation is case-insensitive.
// When enabling this options behaviour of LIKE will become case-sensitive.
//
2018-05-29 13:09:56 +03:00
// _defer_foreign_keys=Boolean | _defer_fk=Boolean
// Defer Foreign Keys until outermost transaction is committed.
//
// _foreign_keys=Boolean | _fk=Boolean
// Enable or disable enforcement of foreign keys.
//
2018-05-29 13:19:46 +03:00
// _ignore_check_constraints=Boolean
// This pragma enables or disables the enforcement of CHECK constraints.
// The default setting is off, meaning that CHECK constraints are enforced by default.
//
2018-05-29 14:13:38 +03:00
// _journal=MODE
// Set journal mode for the databases associated with the current connection.
// https://www.sqlite.org/pragma.html#pragma_journal_mode
//
2018-05-29 14:19:40 +03:00
// _locking=X
// Sets the database connection locking-mode.
// The locking-mode is either NORMAL or EXCLUSIVE.
// https://www.sqlite.org/pragma.html#pragma_locking_mode
//
2018-05-29 14:29:06 +03:00
// _query_only=Boolean
// The query_only pragma prevents all changes to database files when enabled.
//
// _recursive_triggers=Boolean | _rt=Boolean
// Enable or disable recursive triggers.
//
// _secure_delete=Boolean|FAST
// When secure_delete is on, SQLite overwrites deleted content with zeros.
// https://www.sqlite.org/pragma.html#pragma_secure_delete
//
// _vacuum=X
// 0 | none - Auto Vacuum disabled
// 1 | full - Auto Vacuum FULL
// 2 | incremental - Auto Vacuum Incremental
2011-11-11 16:36:22 +04:00
func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) {
if C.sqlite3_threadsafe() == 0 {
return nil, errors.New("sqlite library was not compiled for thread-safe operation")
}
var pkey string
// Options
var loc *time.Location
mutex := C.int(C.SQLITE_OPEN_FULLMUTEX)
txlock := "BEGIN"
// PRAGMA's
autoVacuum := -1
2016-11-04 18:40:06 +03:00
busyTimeout := 5000
caseSensitiveLike := -1
2018-05-29 13:09:56 +03:00
deferForeignKeys := -1
2018-05-29 14:19:40 +03:00
foreignKeys := -1
2018-05-29 13:19:46 +03:00
ignoreCheckConstraints := -1
2018-05-29 14:13:38 +03:00
journalMode := "DELETE"
2018-05-29 14:19:40 +03:00
lockingMode := "NORMAL"
2018-05-29 14:29:06 +03:00
queryOnly := -1
recursiveTriggers := -1
secureDelete := "DEFAULT"
2015-03-05 05:05:58 +03:00
pos := strings.IndexRune(dsn, '?')
if pos >= 1 {
params, err := url.ParseQuery(dsn[pos+1:])
if err != nil {
return nil, err
}
// _loc
if val := params.Get("_loc"); val != "" {
2018-05-29 14:06:07 +03:00
switch strings.ToLower(val) {
case "auto":
2015-03-05 05:05:58 +03:00
loc = time.Local
2018-05-29 14:06:07 +03:00
default:
2015-03-05 05:05:58 +03:00
loc, err = time.LoadLocation(val)
if err != nil {
return nil, fmt.Errorf("Invalid _loc: %v: %v", val, err)
}
}
}
2015-03-05 05:05:58 +03:00
// _mutex
if val := params.Get("_mutex"); val != "" {
2018-05-29 14:06:07 +03:00
switch strings.ToLower(val) {
case "no":
mutex = C.SQLITE_OPEN_NOMUTEX
case "full":
mutex = C.SQLITE_OPEN_FULLMUTEX
default:
return nil, fmt.Errorf("Invalid _mutex: %v", val)
}
}
// _txlock
if val := params.Get("_txlock"); val != "" {
2018-05-29 14:06:07 +03:00
switch strings.ToLower(val) {
case "immediate":
txlock = "BEGIN IMMEDIATE"
case "exclusive":
txlock = "BEGIN EXCLUSIVE"
case "deferred":
txlock = "BEGIN"
default:
return nil, fmt.Errorf("Invalid _txlock: %v", val)
}
}
// Auto Vacuum (_vacuum)
//
// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
//
if val := params.Get("_vacuum"); val != "" {
switch strings.ToLower(val) {
case "0", "none":
autoVacuum = 0
case "1", "full":
autoVacuum = 1
case "2", "incremental":
autoVacuum = 2
default:
return nil, fmt.Errorf("Invalid _vacuum: %v", val)
}
}
// Busy Timeout (_busy_timeout)
//
// https://www.sqlite.org/pragma.html#pragma_busy_timeout
//
pkey = "" // Reset pkey
if _, ok := params["_busy_timeout"]; ok {
pkey = "_busy_timeout"
}
if _, ok := params["_timeout"]; ok {
pkey = "_timeout"
}
if val := params.Get(pkey); val != "" {
iv, err := strconv.ParseInt(val, 10, 64)
if err != nil {
return nil, fmt.Errorf("Invalid _busy_timeout: %v: %v", val, err)
}
busyTimeout = int(iv)
}
// Case Sensitive Like (_cslike)
//
// https://www.sqlite.org/pragma.html#pragma_case_sensitive_like
//
if val := params.Get("_cslike"); val != "" {
switch strings.ToLower(val) {
case "0", "no", "false", "off":
caseSensitiveLike = 0
case "1", "yes", "true", "on":
caseSensitiveLike = 1
default:
return nil, fmt.Errorf("Invalid _cslike: %v, expecting boolean value of '0 1 false true no yes off on'", val)
}
}
2018-05-29 13:19:57 +03:00
// Defer Foreign Keys (_defer_foreign_keys | _defer_fk)
2018-05-29 13:09:56 +03:00
//
// https://www.sqlite.org/pragma.html#pragma_defer_foreign_keys
//
pkey = "" // Reset pkey
if _, ok := params["_defer_foreign_keys"]; ok {
pkey = "_defer_foreign_keys"
}
if _, ok := params["_defer_fk"]; ok {
pkey = "_defer_fk"
}
if val := params.Get(pkey); val != "" {
switch strings.ToLower(val) {
case "0", "no", "false", "off":
deferForeignKeys = 0
case "1", "yes", "true", "on":
deferForeignKeys = 1
default:
2018-05-29 13:19:57 +03:00
return nil, fmt.Errorf("Invalid _defer_foreign_keys: %v, expecting boolean value of '0 1 false true no yes off on'", val)
2018-05-29 13:09:56 +03:00
}
}
2018-05-29 13:19:57 +03:00
// Foreign Keys (_foreign_keys | _fk)
//
// https://www.sqlite.org/pragma.html#pragma_foreign_keys
//
pkey = "" // Reset pkey
if _, ok := params["_foreign_keys"]; ok {
pkey = "_foreign_keys"
}
if _, ok := params["_fk"]; ok {
pkey = "_fk"
}
if val := params.Get(pkey); val != "" {
switch strings.ToLower(val) {
case "0", "no", "false", "off":
foreignKeys = 0
case "1", "yes", "true", "on":
foreignKeys = 1
default:
return nil, fmt.Errorf("Invalid _foreign_keys: %v, expecting boolean value of '0 1 false true no yes off on'", val)
}
}
2018-05-29 13:19:46 +03:00
// Ignore CHECK Constrains (_ignore_check_constraints)
//
// https://www.sqlite.org/pragma.html#pragma_ignore_check_constraints
//
if val := params.Get("_ignore_check_constraints"); val != "" {
switch strings.ToLower(val) {
case "0", "no", "false", "off":
ignoreCheckConstraints = 0
case "1", "yes", "true", "on":
ignoreCheckConstraints = 1
default:
return nil, fmt.Errorf("Invalid _ignore_check_constraints: %v, expecting boolean value of '0 1 false true no yes off on'", val)
}
}
2018-05-29 14:13:38 +03:00
// Journal Mode (_journal)
//
// https://www.sqlite.org/pragma.html#pragma_journal_mode
//
if val := params.Get("_journal"); val != "" {
switch strings.ToUpper(val) {
case "DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF":
journalMode = strings.ToUpper(val)
default:
return nil, fmt.Errorf("Invalid _journal: %v, expecting value of 'DELETE TRUNCATE PERSIST MEMORY WAL OFF'", val)
}
}
2018-05-29 14:19:40 +03:00
// Locking Mode (_locking)
//
// https://www.sqlite.org/pragma.html#pragma_locking_mode
//
if val := params.Get("_locking"); val != "" {
switch strings.ToUpper(val) {
case "NORMAL", "EXCLUSIVE":
lockingMode = strings.ToUpper(val)
default:
return nil, fmt.Errorf("Invalid _locking: %v, expecting value of 'NORMAL EXCLUSIVE", val)
}
}
2018-05-29 14:29:06 +03:00
// Query Only (_query_only)
//
// https://www.sqlite.org/pragma.html#pragma_query_only
//
if val := params.Get("_query_only"); val != "" {
switch strings.ToLower(val) {
case "0", "no", "false", "off":
queryOnly = 0
case "1", "yes", "true", "on":
queryOnly = 1
default:
return nil, fmt.Errorf("Invalid _query_only: %v, expecting boolean value of '0 1 false true no yes off on'", val)
}
}
// Recursive Triggers (_recursive_triggers)
//
// https://www.sqlite.org/pragma.html#pragma_recursive_triggers
//
pkey = "" // Reset pkey
if _, ok := params["_recursive_triggers"]; ok {
pkey = "_recursive_triggers"
}
if _, ok := params["_rt"]; ok {
pkey = "_rt"
}
if val := params.Get(pkey); val != "" {
switch strings.ToLower(val) {
case "0", "no", "false", "off":
recursiveTriggers = 0
case "1", "yes", "true", "on":
recursiveTriggers = 1
default:
return nil, fmt.Errorf("Invalid _recursive_triggers: %v, expecting boolean value of '0 1 false true no yes off on'", val)
}
}
// Secure Delete (_secure_delete)
//
// https://www.sqlite.org/pragma.html#pragma_secure_delete
//
if val := params.Get("_secure_delete"); val != "" {
switch strings.ToLower(val) {
case "0", "no", "false", "off":
secureDelete = "OFF"
case "1", "yes", "true", "on":
secureDelete = "ON"
case "fast":
secureDelete = "FAST"
default:
return nil, fmt.Errorf("Invalid _recursive_triggers: %v, expecting boolean value of '0 1 false true no yes off on'", val)
}
}
2015-03-05 20:23:57 +03:00
if !strings.HasPrefix(dsn, "file:") {
2015-03-05 20:00:09 +03:00
dsn = dsn[:pos]
}
}
2011-11-11 16:36:22 +04:00
var db *C.sqlite3
name := C.CString(dsn)
defer C.free(unsafe.Pointer(name))
rv := C._sqlite3_open_v2(name, &db,
mutex|C.SQLITE_OPEN_READWRITE|C.SQLITE_OPEN_CREATE,
2011-11-11 16:36:22 +04:00
nil)
if rv != 0 {
return nil, Error{Code: ErrNo(rv)}
2011-11-11 16:36:22 +04:00
}
if db == nil {
return nil, errors.New("sqlite succeeded without returning a database")
}
2012-03-12 09:20:55 +04:00
2016-11-04 18:40:06 +03:00
rv = C.sqlite3_busy_timeout(db, C.int(busyTimeout))
2012-03-12 09:20:55 +04:00
if rv != C.SQLITE_OK {
C.sqlite3_close_v2(db)
return nil, Error{Code: ErrNo(rv)}
2012-03-12 09:20:55 +04:00
}
exec := func(s string) error {
cs := C.CString(s)
rv := C.sqlite3_exec(db, cs, nil, nil, nil)
C.free(unsafe.Pointer(cs))
if rv != C.SQLITE_OK {
return lastError(db)
}
return nil
}
// Auto Vacuum
if autoVacuum > -1 {
if err := exec(fmt.Sprintf("PRAGMA auto_vacuum = %d;", autoVacuum)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
2018-05-29 13:10:05 +03:00
// Case Sensitive LIKE
if caseSensitiveLike > -1 {
if err := exec(fmt.Sprintf("PRAGMA case_sensitive_like = %d;", caseSensitiveLike)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
2018-05-29 13:09:56 +03:00
// Defer Foreign Keys
if deferForeignKeys > -1 {
if err := exec(fmt.Sprintf("PRAGMA defer_foreign_keys = %d;", deferForeignKeys)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
// Forgein Keys
if foreignKeys > -1 {
if err := exec(fmt.Sprintf("PRAGMA foreign_keys = %d;", foreignKeys)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
2018-05-29 13:19:46 +03:00
// Ignore CHECK Constraints
if ignoreCheckConstraints > -1 {
if err := exec(fmt.Sprintf("PRAGMA ignore_check_constraints = %d;", ignoreCheckConstraints)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
2018-05-29 14:13:38 +03:00
// Journal Mode
// Because default Journal Mode is DELETE this PRAGMA can always be executed.
if err := exec(fmt.Sprintf("PRAGMA journal_mode = %s;", journalMode)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
2018-05-29 14:19:40 +03:00
// Locking Mode
// Because the default is NORMAL and this is not changed in this package
// by using the compile time SQLITE_DEFAULT_LOCKING_MODE this PRAGMA can always be executed
if err := exec(fmt.Sprintf("PRAGMA locking_mode = %s;", lockingMode)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
2018-05-29 14:29:06 +03:00
// Query Only
if queryOnly > 0 {
if err := exec(fmt.Sprintf("PRAGMA query_only = %d;", queryOnly)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
// Recursive Triggers
if recursiveTriggers > -1 {
if err := exec(fmt.Sprintf("PRAGMA recursive_triggers = %d;", recursiveTriggers)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
// Secure Delete
//
// Because this package can set the compile time flag SQLITE_SECURE_DELETE with a build tag
// the default value for secureDelete var is 'DEFAULT' this way
// you can compile with secure_delete 'ON' and disable it for a specific database connection.
if secureDelete != "DEFAULT" {
if err := exec(fmt.Sprintf("PRAGMA secure_delete = %s;", secureDelete)); err != nil {
C.sqlite3_close_v2(db)
return nil, err
}
}
conn := &SQLiteConn{db: db, loc: loc, txlock: txlock}
2013-08-23 08:58:54 +04:00
if len(d.Extensions) > 0 {
if err := conn.loadExtensions(d.Extensions); err != nil {
conn.Close()
return nil, err
}
}
2013-08-23 08:58:54 +04:00
if d.ConnectHook != nil {
if err := d.ConnectHook(conn); err != nil {
conn.Close()
return nil, err
}
2013-08-23 08:58:54 +04:00
}
runtime.SetFinalizer(conn, (*SQLiteConn).Close)
2013-08-23 08:58:54 +04:00
return conn, nil
2011-11-11 16:36:22 +04:00
}
2013-01-31 11:48:30 +04:00
// Close the connection.
2011-11-11 16:36:22 +04:00
func (c *SQLiteConn) Close() error {
2013-10-24 17:21:37 +04:00
rv := C.sqlite3_close_v2(c.db)
2011-11-11 16:36:22 +04:00
if rv != C.SQLITE_OK {
return c.lastError()
2011-11-11 16:36:22 +04:00
}
deleteHandles(c)
2017-08-01 18:06:18 +03:00
c.mu.Lock()
2011-11-11 16:36:22 +04:00
c.db = nil
2017-08-01 18:06:18 +03:00
c.mu.Unlock()
runtime.SetFinalizer(c, nil)
2011-11-11 16:36:22 +04:00
return nil
}
func (c *SQLiteConn) dbConnOpen() bool {
if c == nil {
return false
}
2017-08-01 18:06:18 +03:00
c.mu.Lock()
defer c.mu.Unlock()
return c.db != nil
}
2016-02-23 09:20:57 +03:00
// Prepare the query string. Return a new statement.
2011-11-11 16:36:22 +04:00
func (c *SQLiteConn) Prepare(query string) (driver.Stmt, error) {
2016-11-04 09:11:24 +03:00
return c.prepare(context.Background(), query)
}
func (c *SQLiteConn) prepare(ctx context.Context, query string) (driver.Stmt, error) {
2011-11-11 16:36:22 +04:00
pquery := C.CString(query)
defer C.free(unsafe.Pointer(pquery))
var s *C.sqlite3_stmt
2013-09-09 05:44:44 +04:00
var tail *C.char
rv := C.sqlite3_prepare_v2(c.db, pquery, -1, &s, &tail)
2011-11-11 16:36:22 +04:00
if rv != C.SQLITE_OK {
return nil, c.lastError()
2011-11-11 16:36:22 +04:00
}
var t string
2015-03-24 00:18:23 +03:00
if tail != nil && *tail != '\000' {
2013-09-09 05:44:44 +04:00
t = strings.TrimSpace(C.GoString(tail))
2011-11-11 16:36:22 +04:00
}
2016-11-04 09:00:29 +03:00
ss := &SQLiteStmt{c: c, s: s, t: t}
runtime.SetFinalizer(ss, (*SQLiteStmt).Close)
return ss, nil
2011-11-11 16:36:22 +04:00
}
// Run-Time Limit Categories.
// See: http://www.sqlite.org/c3ref/c_limit_attached.html
const (
SQLITE_LIMIT_LENGTH = C.SQLITE_LIMIT_LENGTH
SQLITE_LIMIT_SQL_LENGTH = C.SQLITE_LIMIT_SQL_LENGTH
SQLITE_LIMIT_COLUMN = C.SQLITE_LIMIT_COLUMN
SQLITE_LIMIT_EXPR_DEPTH = C.SQLITE_LIMIT_EXPR_DEPTH
SQLITE_LIMIT_COMPOUND_SELECT = C.SQLITE_LIMIT_COMPOUND_SELECT
SQLITE_LIMIT_VDBE_OP = C.SQLITE_LIMIT_VDBE_OP
SQLITE_LIMIT_FUNCTION_ARG = C.SQLITE_LIMIT_FUNCTION_ARG
SQLITE_LIMIT_ATTACHED = C.SQLITE_LIMIT_ATTACHED
SQLITE_LIMIT_LIKE_PATTERN_LENGTH = C.SQLITE_LIMIT_LIKE_PATTERN_LENGTH
SQLITE_LIMIT_VARIABLE_NUMBER = C.SQLITE_LIMIT_VARIABLE_NUMBER
SQLITE_LIMIT_TRIGGER_DEPTH = C.SQLITE_LIMIT_TRIGGER_DEPTH
SQLITE_LIMIT_WORKER_THREADS = C.SQLITE_LIMIT_WORKER_THREADS
)
// GetLimit returns the current value of a run-time limit.
// See: sqlite3_limit, http://www.sqlite.org/c3ref/limit.html
func (c *SQLiteConn) GetLimit(id int) int {
return int(C._sqlite3_limit(c.db, C.int(id), -1))
}
// SetLimit changes the value of a run-time limits.
// Then this method returns the prior value of the limit.
// See: sqlite3_limit, http://www.sqlite.org/c3ref/limit.html
func (c *SQLiteConn) SetLimit(id int, newVal int) int {
return int(C._sqlite3_limit(c.db, C.int(id), C.int(newVal)))
}
2013-01-31 11:48:30 +04:00
// Close the statement.
2011-11-11 16:36:22 +04:00
func (s *SQLiteStmt) Close() error {
2017-08-30 07:29:47 +03:00
s.mu.Lock()
defer s.mu.Unlock()
2012-02-20 11:14:49 +04:00
if s.closed {
return nil
}
s.closed = true
if !s.c.dbConnOpen() {
2013-02-13 05:32:40 +04:00
return errors.New("sqlite statement with already closed database connection")
}
2011-11-11 16:38:53 +04:00
rv := C.sqlite3_finalize(s.s)
2017-08-28 12:58:02 +03:00
s.s = nil
2011-11-11 16:36:22 +04:00
if rv != C.SQLITE_OK {
return s.c.lastError()
2011-11-11 16:36:22 +04:00
}
runtime.SetFinalizer(s, nil)
2011-11-11 16:36:22 +04:00
return nil
}
2016-11-04 18:40:06 +03:00
// NumInput return a number of parameters.
2011-11-11 16:36:22 +04:00
func (s *SQLiteStmt) NumInput() int {
2016-11-04 09:00:29 +03:00
return int(C.sqlite3_bind_parameter_count(s.s))
}
type bindArg struct {
n int
v driver.Value
2011-11-11 16:36:22 +04:00
}
var placeHolder = []byte{0}
2016-11-04 08:24:22 +03:00
func (s *SQLiteStmt) bind(args []namedValue) error {
2011-11-11 16:36:22 +04:00
rv := C.sqlite3_reset(s.s)
2011-11-11 16:38:53 +04:00
if rv != C.SQLITE_ROW && rv != C.SQLITE_OK && rv != C.SQLITE_DONE {
return s.c.lastError()
2011-11-11 16:36:22 +04:00
}
2016-11-04 09:00:29 +03:00
for i, v := range args {
if v.Name != "" {
2016-12-09 07:12:14 +03:00
cname := C.CString(":" + v.Name)
2016-11-04 09:00:29 +03:00
args[i].Ordinal = int(C.sqlite3_bind_parameter_index(s.s, cname))
C.free(unsafe.Pointer(cname))
}
}
2016-11-04 09:00:29 +03:00
for _, arg := range args {
n := C.int(arg.Ordinal)
switch v := arg.Value.(type) {
2011-11-11 16:36:22 +04:00
case nil:
rv = C.sqlite3_bind_null(s.s, n)
case string:
if len(v) == 0 {
rv = C._sqlite3_bind_text(s.s, n, (*C.char)(unsafe.Pointer(&placeHolder[0])), C.int(0))
} else {
b := []byte(v)
rv = C._sqlite3_bind_text(s.s, n, (*C.char)(unsafe.Pointer(&b[0])), C.int(len(b)))
}
2011-11-11 16:36:22 +04:00
case int64:
rv = C.sqlite3_bind_int64(s.s, n, C.sqlite3_int64(v))
case bool:
2018-04-17 11:55:42 +03:00
if v {
2012-03-12 09:20:55 +04:00
rv = C.sqlite3_bind_int(s.s, n, 1)
2011-11-11 16:36:22 +04:00
} else {
rv = C.sqlite3_bind_int(s.s, n, 0)
}
case float64:
rv = C.sqlite3_bind_double(s.s, n, C.double(v))
case []byte:
2017-08-21 23:30:07 +03:00
ln := len(v)
if ln == 0 {
v = placeHolder
2011-11-11 16:36:22 +04:00
}
2017-08-21 23:30:07 +03:00
rv = C._sqlite3_bind_blob(s.s, n, unsafe.Pointer(&v[0]), C.int(ln))
case time.Time:
b := []byte(v.Format(SQLiteTimestampFormats[0]))
2015-03-04 19:17:38 +03:00
rv = C._sqlite3_bind_text(s.s, n, (*C.char)(unsafe.Pointer(&b[0])), C.int(len(b)))
2011-11-11 16:36:22 +04:00
}
if rv != C.SQLITE_OK {
return s.c.lastError()
2011-11-11 16:36:22 +04:00
}
}
return nil
}
2014-02-18 04:06:30 +04:00
// Query the statement with arguments. Return records.
2012-02-20 11:14:49 +04:00
func (s *SQLiteStmt) Query(args []driver.Value) (driver.Rows, error) {
2016-11-04 08:24:22 +03:00
list := make([]namedValue, len(args))
for i, v := range args {
list[i] = namedValue{
Ordinal: i + 1,
Value: v,
}
}
return s.query(context.Background(), list)
}
func (s *SQLiteStmt) query(ctx context.Context, args []namedValue) (driver.Rows, error) {
2011-11-11 16:36:22 +04:00
if err := s.bind(args); err != nil {
return nil, err
}
2016-11-06 14:43:53 +03:00
rows := &SQLiteRows{
s: s,
nc: int(C.sqlite3_column_count(s.s)),
cols: nil,
decltype: nil,
cls: s.cls,
2017-08-01 19:43:14 +03:00
closed: false,
2016-11-06 14:43:53 +03:00
done: make(chan struct{}),
}
Don't spawn interrupt goroutine if we know that context cannot be canceled For a Go-only project the following code pattern go func() { select { case <-ctx.Done(): // call some cancel case <-done: // work finished ok } }() // do some work close(done) works good and fast - without high scheduling overhead because scheduler usually puts spawned goroutine into run queue on the same OS thread and so after done is closed control is passed to spawned goroutine without OS context switch. However in the presence of Cgo calls in "do some work" the situation can become different - Cgo calls are treated by go runtime similarly to system calls with the effect that goroutines spawned on original OS thread tend to be migrated by scheduler to be executed on another OS thread. This in turn can bring high overhead for communicating on "done", which ultimately can result in full context switch: if the spawned goroutine had chance to run, already checked done and ctx to be not ready, and went into sleep via wait on futex - showing as something like below in strace for one read query (note futex calls): 27867 00:38:39.782146 stat(".../neo.sqlite-journal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782165 pread64(3, "\0\0\0\33\0\0\10\235\0\0\10]\0\0\0\27", 16, 24) = 16 27871 00:38:39.782179 <... pselect6 resumed> ) = 0 (Timeout) 27868 00:38:39.782187 <... pselect6 resumed> ) = 0 (Timeout) 27871 00:38:39.782193 futex(0xc4200f8538, FUTEX_WAIT, 0, NULL <unfinished ...> 27868 00:38:39.782199 futex(0xc420013138, FUTEX_WAIT, 0, NULL <unfinished ...> 27867 00:38:39.782205 stat(".../neo.sqlite-wal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782224 fstat(3, {st_mode=S_IFREG|0644, st_size=9031680, ...}) = 0 27867 00:38:39.782247 futex(0xc420013138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782259 <... futex resumed> ) = 0 27867 00:38:39.782265 <... futex resumed> ) = 1 27868 00:38:39.782270 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782279 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=0, l_len=0}) = 0 27867 00:38:39.782315 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1}) = 0 27868 00:38:39.782336 <... pselect6 resumed> ) = 0 (Timeout) 27867 00:38:39.782342 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741826, l_len=510} <unfinished ...> 27868 00:38:39.782348 futex(0xc4200f8538, FUTEX_WAKE, 1 <unfinished ...> 27867 00:38:39.782355 <... fcntl resumed> ) = 0 27871 00:38:39.782360 <... futex resumed> ) = 0 27868 00:38:39.782367 <... futex resumed> ) = 1 27871 00:38:39.782372 futex(0xc4200f8138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782377 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27871 00:38:39.782384 <... futex resumed> ) = 1 27870 00:38:39.782389 <... futex resumed> ) = 0 27867 00:38:39.782394 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1} <unfinished ...> 27870 00:38:39.782400 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782408 <... fcntl resumed> ) = 0 Below link shows that go scheduler itself might be significantly improved for cases when there are several Cgo calls made for a request in a server: https://github.com/golang/go/issues/21827#issuecomment-329092317 in particular CGo-4 case should be closely related to this sqlite3 go package, because for one query many CGo calls are made to SQLite. However until there are proper scheduler fixes, let's make what could be made to improve time to do queries: If we know that the context under which a query is executed will never be canceled - we know we can safely skip spawning the interrupt goroutine and this was avoid ping-pong on done in between different OS threads. This brings the following speedup on my notebook with go1.10: name old req/s new req/s delta Exec 254k ± 1% 379k ± 1% +48.89% (p=0.000 n=10+10) Query 90.6k ± 2% 96.4k ± 1% +6.37% (p=0.000 n=10+10) Params 81.5k ± 1% 87.0k ± 1% +6.83% (p=0.000 n=10+10) Stmt 122k ± 2% 129k ± 1% +6.07% (p=0.000 n=10+9) Rows 2.98k ± 1% 3.06k ± 1% +2.77% (p=0.000 n=9+10) StmtRows 3.10k ± 1% 3.13k ± 1% +1.12% (p=0.000 n=9+10) name old time/op new time/op delta CustomFunctions-4 10.6µs ± 1% 10.1µs ± 1% -5.01% (p=0.000 n=10+10)
2018-02-16 19:35:14 +03:00
if ctxdone := ctx.Done(); ctxdone != nil {
go func(db *C.sqlite3) {
2017-02-11 15:47:11 +03:00
select {
Don't spawn interrupt goroutine if we know that context cannot be canceled For a Go-only project the following code pattern go func() { select { case <-ctx.Done(): // call some cancel case <-done: // work finished ok } }() // do some work close(done) works good and fast - without high scheduling overhead because scheduler usually puts spawned goroutine into run queue on the same OS thread and so after done is closed control is passed to spawned goroutine without OS context switch. However in the presence of Cgo calls in "do some work" the situation can become different - Cgo calls are treated by go runtime similarly to system calls with the effect that goroutines spawned on original OS thread tend to be migrated by scheduler to be executed on another OS thread. This in turn can bring high overhead for communicating on "done", which ultimately can result in full context switch: if the spawned goroutine had chance to run, already checked done and ctx to be not ready, and went into sleep via wait on futex - showing as something like below in strace for one read query (note futex calls): 27867 00:38:39.782146 stat(".../neo.sqlite-journal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782165 pread64(3, "\0\0\0\33\0\0\10\235\0\0\10]\0\0\0\27", 16, 24) = 16 27871 00:38:39.782179 <... pselect6 resumed> ) = 0 (Timeout) 27868 00:38:39.782187 <... pselect6 resumed> ) = 0 (Timeout) 27871 00:38:39.782193 futex(0xc4200f8538, FUTEX_WAIT, 0, NULL <unfinished ...> 27868 00:38:39.782199 futex(0xc420013138, FUTEX_WAIT, 0, NULL <unfinished ...> 27867 00:38:39.782205 stat(".../neo.sqlite-wal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782224 fstat(3, {st_mode=S_IFREG|0644, st_size=9031680, ...}) = 0 27867 00:38:39.782247 futex(0xc420013138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782259 <... futex resumed> ) = 0 27867 00:38:39.782265 <... futex resumed> ) = 1 27868 00:38:39.782270 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782279 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=0, l_len=0}) = 0 27867 00:38:39.782315 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1}) = 0 27868 00:38:39.782336 <... pselect6 resumed> ) = 0 (Timeout) 27867 00:38:39.782342 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741826, l_len=510} <unfinished ...> 27868 00:38:39.782348 futex(0xc4200f8538, FUTEX_WAKE, 1 <unfinished ...> 27867 00:38:39.782355 <... fcntl resumed> ) = 0 27871 00:38:39.782360 <... futex resumed> ) = 0 27868 00:38:39.782367 <... futex resumed> ) = 1 27871 00:38:39.782372 futex(0xc4200f8138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782377 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27871 00:38:39.782384 <... futex resumed> ) = 1 27870 00:38:39.782389 <... futex resumed> ) = 0 27867 00:38:39.782394 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1} <unfinished ...> 27870 00:38:39.782400 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782408 <... fcntl resumed> ) = 0 Below link shows that go scheduler itself might be significantly improved for cases when there are several Cgo calls made for a request in a server: https://github.com/golang/go/issues/21827#issuecomment-329092317 in particular CGo-4 case should be closely related to this sqlite3 go package, because for one query many CGo calls are made to SQLite. However until there are proper scheduler fixes, let's make what could be made to improve time to do queries: If we know that the context under which a query is executed will never be canceled - we know we can safely skip spawning the interrupt goroutine and this was avoid ping-pong on done in between different OS threads. This brings the following speedup on my notebook with go1.10: name old req/s new req/s delta Exec 254k ± 1% 379k ± 1% +48.89% (p=0.000 n=10+10) Query 90.6k ± 2% 96.4k ± 1% +6.37% (p=0.000 n=10+10) Params 81.5k ± 1% 87.0k ± 1% +6.83% (p=0.000 n=10+10) Stmt 122k ± 2% 129k ± 1% +6.07% (p=0.000 n=10+9) Rows 2.98k ± 1% 3.06k ± 1% +2.77% (p=0.000 n=9+10) StmtRows 3.10k ± 1% 3.13k ± 1% +1.12% (p=0.000 n=9+10) name old time/op new time/op delta CustomFunctions-4 10.6µs ± 1% 10.1µs ± 1% -5.01% (p=0.000 n=10+10)
2018-02-16 19:35:14 +03:00
case <-ctxdone:
select {
case <-rows.done:
default:
C.sqlite3_interrupt(db)
rows.Close()
}
2017-02-11 15:47:11 +03:00
case <-rows.done:
}
Don't spawn interrupt goroutine if we know that context cannot be canceled For a Go-only project the following code pattern go func() { select { case <-ctx.Done(): // call some cancel case <-done: // work finished ok } }() // do some work close(done) works good and fast - without high scheduling overhead because scheduler usually puts spawned goroutine into run queue on the same OS thread and so after done is closed control is passed to spawned goroutine without OS context switch. However in the presence of Cgo calls in "do some work" the situation can become different - Cgo calls are treated by go runtime similarly to system calls with the effect that goroutines spawned on original OS thread tend to be migrated by scheduler to be executed on another OS thread. This in turn can bring high overhead for communicating on "done", which ultimately can result in full context switch: if the spawned goroutine had chance to run, already checked done and ctx to be not ready, and went into sleep via wait on futex - showing as something like below in strace for one read query (note futex calls): 27867 00:38:39.782146 stat(".../neo.sqlite-journal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782165 pread64(3, "\0\0\0\33\0\0\10\235\0\0\10]\0\0\0\27", 16, 24) = 16 27871 00:38:39.782179 <... pselect6 resumed> ) = 0 (Timeout) 27868 00:38:39.782187 <... pselect6 resumed> ) = 0 (Timeout) 27871 00:38:39.782193 futex(0xc4200f8538, FUTEX_WAIT, 0, NULL <unfinished ...> 27868 00:38:39.782199 futex(0xc420013138, FUTEX_WAIT, 0, NULL <unfinished ...> 27867 00:38:39.782205 stat(".../neo.sqlite-wal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782224 fstat(3, {st_mode=S_IFREG|0644, st_size=9031680, ...}) = 0 27867 00:38:39.782247 futex(0xc420013138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782259 <... futex resumed> ) = 0 27867 00:38:39.782265 <... futex resumed> ) = 1 27868 00:38:39.782270 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782279 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=0, l_len=0}) = 0 27867 00:38:39.782315 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1}) = 0 27868 00:38:39.782336 <... pselect6 resumed> ) = 0 (Timeout) 27867 00:38:39.782342 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741826, l_len=510} <unfinished ...> 27868 00:38:39.782348 futex(0xc4200f8538, FUTEX_WAKE, 1 <unfinished ...> 27867 00:38:39.782355 <... fcntl resumed> ) = 0 27871 00:38:39.782360 <... futex resumed> ) = 0 27868 00:38:39.782367 <... futex resumed> ) = 1 27871 00:38:39.782372 futex(0xc4200f8138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782377 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27871 00:38:39.782384 <... futex resumed> ) = 1 27870 00:38:39.782389 <... futex resumed> ) = 0 27867 00:38:39.782394 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1} <unfinished ...> 27870 00:38:39.782400 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782408 <... fcntl resumed> ) = 0 Below link shows that go scheduler itself might be significantly improved for cases when there are several Cgo calls made for a request in a server: https://github.com/golang/go/issues/21827#issuecomment-329092317 in particular CGo-4 case should be closely related to this sqlite3 go package, because for one query many CGo calls are made to SQLite. However until there are proper scheduler fixes, let's make what could be made to improve time to do queries: If we know that the context under which a query is executed will never be canceled - we know we can safely skip spawning the interrupt goroutine and this was avoid ping-pong on done in between different OS threads. This brings the following speedup on my notebook with go1.10: name old req/s new req/s delta Exec 254k ± 1% 379k ± 1% +48.89% (p=0.000 n=10+10) Query 90.6k ± 2% 96.4k ± 1% +6.37% (p=0.000 n=10+10) Params 81.5k ± 1% 87.0k ± 1% +6.83% (p=0.000 n=10+10) Stmt 122k ± 2% 129k ± 1% +6.07% (p=0.000 n=10+9) Rows 2.98k ± 1% 3.06k ± 1% +2.77% (p=0.000 n=9+10) StmtRows 3.10k ± 1% 3.13k ± 1% +1.12% (p=0.000 n=9+10) name old time/op new time/op delta CustomFunctions-4 10.6µs ± 1% 10.1µs ± 1% -5.01% (p=0.000 n=10+10)
2018-02-16 19:35:14 +03:00
}(s.c.db)
}
2016-11-06 14:43:53 +03:00
return rows, nil
2011-11-11 16:36:22 +04:00
}
2016-11-04 18:40:06 +03:00
// LastInsertId teturn last inserted ID.
2011-11-14 17:10:13 +04:00
func (r *SQLiteResult) LastInsertId() (int64, error) {
return r.id, nil
2011-11-14 17:10:13 +04:00
}
2016-11-04 18:40:06 +03:00
// RowsAffected return how many rows affected.
2011-11-14 17:10:13 +04:00
func (r *SQLiteResult) RowsAffected() (int64, error) {
return r.changes, nil
2011-11-14 17:10:13 +04:00
}
2016-11-04 18:40:06 +03:00
// Exec execute the statement with arguments. Return result object.
2012-02-20 11:14:49 +04:00
func (s *SQLiteStmt) Exec(args []driver.Value) (driver.Result, error) {
2016-11-04 08:24:22 +03:00
list := make([]namedValue, len(args))
for i, v := range args {
list[i] = namedValue{
Ordinal: i + 1,
Value: v,
}
}
return s.exec(context.Background(), list)
}
func (s *SQLiteStmt) exec(ctx context.Context, args []namedValue) (driver.Result, error) {
2011-11-11 16:36:22 +04:00
if err := s.bind(args); err != nil {
2014-11-16 17:51:46 +03:00
C.sqlite3_reset(s.s)
C.sqlite3_clear_bindings(s.s)
2011-11-11 16:36:22 +04:00
return nil, err
}
2016-11-06 14:46:27 +03:00
Don't spawn interrupt goroutine if we know that context cannot be canceled For a Go-only project the following code pattern go func() { select { case <-ctx.Done(): // call some cancel case <-done: // work finished ok } }() // do some work close(done) works good and fast - without high scheduling overhead because scheduler usually puts spawned goroutine into run queue on the same OS thread and so after done is closed control is passed to spawned goroutine without OS context switch. However in the presence of Cgo calls in "do some work" the situation can become different - Cgo calls are treated by go runtime similarly to system calls with the effect that goroutines spawned on original OS thread tend to be migrated by scheduler to be executed on another OS thread. This in turn can bring high overhead for communicating on "done", which ultimately can result in full context switch: if the spawned goroutine had chance to run, already checked done and ctx to be not ready, and went into sleep via wait on futex - showing as something like below in strace for one read query (note futex calls): 27867 00:38:39.782146 stat(".../neo.sqlite-journal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782165 pread64(3, "\0\0\0\33\0\0\10\235\0\0\10]\0\0\0\27", 16, 24) = 16 27871 00:38:39.782179 <... pselect6 resumed> ) = 0 (Timeout) 27868 00:38:39.782187 <... pselect6 resumed> ) = 0 (Timeout) 27871 00:38:39.782193 futex(0xc4200f8538, FUTEX_WAIT, 0, NULL <unfinished ...> 27868 00:38:39.782199 futex(0xc420013138, FUTEX_WAIT, 0, NULL <unfinished ...> 27867 00:38:39.782205 stat(".../neo.sqlite-wal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782224 fstat(3, {st_mode=S_IFREG|0644, st_size=9031680, ...}) = 0 27867 00:38:39.782247 futex(0xc420013138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782259 <... futex resumed> ) = 0 27867 00:38:39.782265 <... futex resumed> ) = 1 27868 00:38:39.782270 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782279 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=0, l_len=0}) = 0 27867 00:38:39.782315 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1}) = 0 27868 00:38:39.782336 <... pselect6 resumed> ) = 0 (Timeout) 27867 00:38:39.782342 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741826, l_len=510} <unfinished ...> 27868 00:38:39.782348 futex(0xc4200f8538, FUTEX_WAKE, 1 <unfinished ...> 27867 00:38:39.782355 <... fcntl resumed> ) = 0 27871 00:38:39.782360 <... futex resumed> ) = 0 27868 00:38:39.782367 <... futex resumed> ) = 1 27871 00:38:39.782372 futex(0xc4200f8138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782377 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27871 00:38:39.782384 <... futex resumed> ) = 1 27870 00:38:39.782389 <... futex resumed> ) = 0 27867 00:38:39.782394 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1} <unfinished ...> 27870 00:38:39.782400 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782408 <... fcntl resumed> ) = 0 Below link shows that go scheduler itself might be significantly improved for cases when there are several Cgo calls made for a request in a server: https://github.com/golang/go/issues/21827#issuecomment-329092317 in particular CGo-4 case should be closely related to this sqlite3 go package, because for one query many CGo calls are made to SQLite. However until there are proper scheduler fixes, let's make what could be made to improve time to do queries: If we know that the context under which a query is executed will never be canceled - we know we can safely skip spawning the interrupt goroutine and this was avoid ping-pong on done in between different OS threads. This brings the following speedup on my notebook with go1.10: name old req/s new req/s delta Exec 254k ± 1% 379k ± 1% +48.89% (p=0.000 n=10+10) Query 90.6k ± 2% 96.4k ± 1% +6.37% (p=0.000 n=10+10) Params 81.5k ± 1% 87.0k ± 1% +6.83% (p=0.000 n=10+10) Stmt 122k ± 2% 129k ± 1% +6.07% (p=0.000 n=10+9) Rows 2.98k ± 1% 3.06k ± 1% +2.77% (p=0.000 n=9+10) StmtRows 3.10k ± 1% 3.13k ± 1% +1.12% (p=0.000 n=9+10) name old time/op new time/op delta CustomFunctions-4 10.6µs ± 1% 10.1µs ± 1% -5.01% (p=0.000 n=10+10)
2018-02-16 19:35:14 +03:00
if ctxdone := ctx.Done(); ctxdone != nil {
done := make(chan struct{})
defer close(done)
go func(db *C.sqlite3) {
select {
case <-done:
Don't spawn interrupt goroutine if we know that context cannot be canceled For a Go-only project the following code pattern go func() { select { case <-ctx.Done(): // call some cancel case <-done: // work finished ok } }() // do some work close(done) works good and fast - without high scheduling overhead because scheduler usually puts spawned goroutine into run queue on the same OS thread and so after done is closed control is passed to spawned goroutine without OS context switch. However in the presence of Cgo calls in "do some work" the situation can become different - Cgo calls are treated by go runtime similarly to system calls with the effect that goroutines spawned on original OS thread tend to be migrated by scheduler to be executed on another OS thread. This in turn can bring high overhead for communicating on "done", which ultimately can result in full context switch: if the spawned goroutine had chance to run, already checked done and ctx to be not ready, and went into sleep via wait on futex - showing as something like below in strace for one read query (note futex calls): 27867 00:38:39.782146 stat(".../neo.sqlite-journal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782165 pread64(3, "\0\0\0\33\0\0\10\235\0\0\10]\0\0\0\27", 16, 24) = 16 27871 00:38:39.782179 <... pselect6 resumed> ) = 0 (Timeout) 27868 00:38:39.782187 <... pselect6 resumed> ) = 0 (Timeout) 27871 00:38:39.782193 futex(0xc4200f8538, FUTEX_WAIT, 0, NULL <unfinished ...> 27868 00:38:39.782199 futex(0xc420013138, FUTEX_WAIT, 0, NULL <unfinished ...> 27867 00:38:39.782205 stat(".../neo.sqlite-wal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782224 fstat(3, {st_mode=S_IFREG|0644, st_size=9031680, ...}) = 0 27867 00:38:39.782247 futex(0xc420013138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782259 <... futex resumed> ) = 0 27867 00:38:39.782265 <... futex resumed> ) = 1 27868 00:38:39.782270 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782279 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=0, l_len=0}) = 0 27867 00:38:39.782315 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1}) = 0 27868 00:38:39.782336 <... pselect6 resumed> ) = 0 (Timeout) 27867 00:38:39.782342 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741826, l_len=510} <unfinished ...> 27868 00:38:39.782348 futex(0xc4200f8538, FUTEX_WAKE, 1 <unfinished ...> 27867 00:38:39.782355 <... fcntl resumed> ) = 0 27871 00:38:39.782360 <... futex resumed> ) = 0 27868 00:38:39.782367 <... futex resumed> ) = 1 27871 00:38:39.782372 futex(0xc4200f8138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782377 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27871 00:38:39.782384 <... futex resumed> ) = 1 27870 00:38:39.782389 <... futex resumed> ) = 0 27867 00:38:39.782394 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1} <unfinished ...> 27870 00:38:39.782400 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782408 <... fcntl resumed> ) = 0 Below link shows that go scheduler itself might be significantly improved for cases when there are several Cgo calls made for a request in a server: https://github.com/golang/go/issues/21827#issuecomment-329092317 in particular CGo-4 case should be closely related to this sqlite3 go package, because for one query many CGo calls are made to SQLite. However until there are proper scheduler fixes, let's make what could be made to improve time to do queries: If we know that the context under which a query is executed will never be canceled - we know we can safely skip spawning the interrupt goroutine and this was avoid ping-pong on done in between different OS threads. This brings the following speedup on my notebook with go1.10: name old req/s new req/s delta Exec 254k ± 1% 379k ± 1% +48.89% (p=0.000 n=10+10) Query 90.6k ± 2% 96.4k ± 1% +6.37% (p=0.000 n=10+10) Params 81.5k ± 1% 87.0k ± 1% +6.83% (p=0.000 n=10+10) Stmt 122k ± 2% 129k ± 1% +6.07% (p=0.000 n=10+9) Rows 2.98k ± 1% 3.06k ± 1% +2.77% (p=0.000 n=9+10) StmtRows 3.10k ± 1% 3.13k ± 1% +1.12% (p=0.000 n=9+10) name old time/op new time/op delta CustomFunctions-4 10.6µs ± 1% 10.1µs ± 1% -5.01% (p=0.000 n=10+10)
2018-02-16 19:35:14 +03:00
case <-ctxdone:
select {
case <-done:
default:
C.sqlite3_interrupt(db)
}
}
Don't spawn interrupt goroutine if we know that context cannot be canceled For a Go-only project the following code pattern go func() { select { case <-ctx.Done(): // call some cancel case <-done: // work finished ok } }() // do some work close(done) works good and fast - without high scheduling overhead because scheduler usually puts spawned goroutine into run queue on the same OS thread and so after done is closed control is passed to spawned goroutine without OS context switch. However in the presence of Cgo calls in "do some work" the situation can become different - Cgo calls are treated by go runtime similarly to system calls with the effect that goroutines spawned on original OS thread tend to be migrated by scheduler to be executed on another OS thread. This in turn can bring high overhead for communicating on "done", which ultimately can result in full context switch: if the spawned goroutine had chance to run, already checked done and ctx to be not ready, and went into sleep via wait on futex - showing as something like below in strace for one read query (note futex calls): 27867 00:38:39.782146 stat(".../neo.sqlite-journal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782165 pread64(3, "\0\0\0\33\0\0\10\235\0\0\10]\0\0\0\27", 16, 24) = 16 27871 00:38:39.782179 <... pselect6 resumed> ) = 0 (Timeout) 27868 00:38:39.782187 <... pselect6 resumed> ) = 0 (Timeout) 27871 00:38:39.782193 futex(0xc4200f8538, FUTEX_WAIT, 0, NULL <unfinished ...> 27868 00:38:39.782199 futex(0xc420013138, FUTEX_WAIT, 0, NULL <unfinished ...> 27867 00:38:39.782205 stat(".../neo.sqlite-wal", 0x7f83809c4a20) = -1 ENOENT (No such file or directory) 27867 00:38:39.782224 fstat(3, {st_mode=S_IFREG|0644, st_size=9031680, ...}) = 0 27867 00:38:39.782247 futex(0xc420013138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782259 <... futex resumed> ) = 0 27867 00:38:39.782265 <... futex resumed> ) = 1 27868 00:38:39.782270 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782279 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=0, l_len=0}) = 0 27867 00:38:39.782315 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1}) = 0 27868 00:38:39.782336 <... pselect6 resumed> ) = 0 (Timeout) 27867 00:38:39.782342 fcntl(3, F_SETLK, {l_type=F_RDLCK, l_whence=SEEK_SET, l_start=1073741826, l_len=510} <unfinished ...> 27868 00:38:39.782348 futex(0xc4200f8538, FUTEX_WAKE, 1 <unfinished ...> 27867 00:38:39.782355 <... fcntl resumed> ) = 0 27871 00:38:39.782360 <... futex resumed> ) = 0 27868 00:38:39.782367 <... futex resumed> ) = 1 27871 00:38:39.782372 futex(0xc4200f8138, FUTEX_WAKE, 1 <unfinished ...> 27868 00:38:39.782377 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27871 00:38:39.782384 <... futex resumed> ) = 1 27870 00:38:39.782389 <... futex resumed> ) = 0 27867 00:38:39.782394 fcntl(3, F_SETLK, {l_type=F_UNLCK, l_whence=SEEK_SET, l_start=1073741824, l_len=1} <unfinished ...> 27870 00:38:39.782400 pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=3000}, NULL <unfinished ...> 27867 00:38:39.782408 <... fcntl resumed> ) = 0 Below link shows that go scheduler itself might be significantly improved for cases when there are several Cgo calls made for a request in a server: https://github.com/golang/go/issues/21827#issuecomment-329092317 in particular CGo-4 case should be closely related to this sqlite3 go package, because for one query many CGo calls are made to SQLite. However until there are proper scheduler fixes, let's make what could be made to improve time to do queries: If we know that the context under which a query is executed will never be canceled - we know we can safely skip spawning the interrupt goroutine and this was avoid ping-pong on done in between different OS threads. This brings the following speedup on my notebook with go1.10: name old req/s new req/s delta Exec 254k ± 1% 379k ± 1% +48.89% (p=0.000 n=10+10) Query 90.6k ± 2% 96.4k ± 1% +6.37% (p=0.000 n=10+10) Params 81.5k ± 1% 87.0k ± 1% +6.83% (p=0.000 n=10+10) Stmt 122k ± 2% 129k ± 1% +6.07% (p=0.000 n=10+9) Rows 2.98k ± 1% 3.06k ± 1% +2.77% (p=0.000 n=9+10) StmtRows 3.10k ± 1% 3.13k ± 1% +1.12% (p=0.000 n=9+10) name old time/op new time/op delta CustomFunctions-4 10.6µs ± 1% 10.1µs ± 1% -5.01% (p=0.000 n=10+10)
2018-02-16 19:35:14 +03:00
}(s.c.db)
}
2016-11-06 14:46:27 +03:00
var rowid, changes C.longlong
rv := C._sqlite3_step(s.s, &rowid, &changes)
2011-11-11 16:38:53 +04:00
if rv != C.SQLITE_ROW && rv != C.SQLITE_OK && rv != C.SQLITE_DONE {
err := s.c.lastError()
2014-11-16 17:51:46 +03:00
C.sqlite3_reset(s.s)
C.sqlite3_clear_bindings(s.s)
return nil, err
2011-11-11 16:36:22 +04:00
}
2016-11-06 14:46:27 +03:00
2016-11-08 06:19:13 +03:00
return &SQLiteResult{id: int64(rowid), changes: int64(changes)}, nil
2011-11-11 16:36:22 +04:00
}
2013-01-31 11:48:30 +04:00
// Close the rows.
2011-11-11 16:36:22 +04:00
func (rc *SQLiteRows) Close() error {
2017-08-30 07:29:47 +03:00
rc.s.mu.Lock()
2017-08-01 19:43:14 +03:00
if rc.s.closed || rc.closed {
2017-08-30 07:29:47 +03:00
rc.s.mu.Unlock()
2013-09-09 07:28:34 +04:00
return nil
}
2017-08-01 19:43:14 +03:00
rc.closed = true
2016-11-06 14:43:53 +03:00
if rc.done != nil {
close(rc.done)
}
if rc.cls {
2017-08-30 07:29:47 +03:00
rc.s.mu.Unlock()
return rc.s.Close()
}
2012-03-12 09:20:55 +04:00
rv := C.sqlite3_reset(rc.s.s)
if rv != C.SQLITE_OK {
2017-08-30 07:29:47 +03:00
rc.s.mu.Unlock()
return rc.s.c.lastError()
2012-03-12 09:20:55 +04:00
}
2017-08-30 07:29:47 +03:00
rc.s.mu.Unlock()
2012-03-12 09:20:55 +04:00
return nil
2011-11-11 16:36:22 +04:00
}
2016-11-04 18:40:06 +03:00
// Columns return column names.
2011-11-11 16:36:22 +04:00
func (rc *SQLiteRows) Columns() []string {
2017-08-30 07:29:47 +03:00
rc.s.mu.Lock()
defer rc.s.mu.Unlock()
if rc.s.s != nil && rc.nc != len(rc.cols) {
2011-11-11 16:36:22 +04:00
rc.cols = make([]string, rc.nc)
for i := 0; i < rc.nc; i++ {
rc.cols[i] = C.GoString(C.sqlite3_column_name(rc.s.s, C.int(i)))
}
}
return rc.cols
}
2017-08-30 13:37:57 +03:00
func (rc *SQLiteRows) declTypes() []string {
2017-08-30 07:29:47 +03:00
if rc.s.s != nil && rc.decltype == nil {
2016-03-06 23:27:17 +03:00
rc.decltype = make([]string, rc.nc)
for i := 0; i < rc.nc; i++ {
rc.decltype[i] = strings.ToLower(C.GoString(C.sqlite3_column_decltype(rc.s.s, C.int(i))))
}
}
return rc.decltype
}
2017-08-30 13:37:57 +03:00
// DeclTypes return column types.
func (rc *SQLiteRows) DeclTypes() []string {
rc.s.mu.Lock()
defer rc.s.mu.Unlock()
return rc.declTypes()
}
2016-11-04 18:40:06 +03:00
// Next move cursor to next.
2012-02-20 11:14:49 +04:00
func (rc *SQLiteRows) Next(dest []driver.Value) error {
2017-08-30 07:29:47 +03:00
if rc.s.closed {
return io.EOF
}
rc.s.mu.Lock()
2017-08-30 13:37:57 +03:00
defer rc.s.mu.Unlock()
rv := C.sqlite3_step(rc.s.s)
if rv == C.SQLITE_DONE {
return io.EOF
}
if rv != C.SQLITE_ROW {
rv = C.sqlite3_reset(rc.s.s)
if rv != C.SQLITE_OK {
return rc.s.c.lastError()
2013-09-09 08:44:24 +04:00
}
return nil
2011-11-11 16:36:22 +04:00
}
2017-08-30 13:37:57 +03:00
rc.declTypes()
2011-11-11 16:36:22 +04:00
for i := range dest {
2011-11-11 16:38:53 +04:00
switch C.sqlite3_column_type(rc.s.s, C.int(i)) {
case C.SQLITE_INTEGER:
val := int64(C.sqlite3_column_int64(rc.s.s, C.int(i)))
switch rc.decltype[i] {
2018-04-17 12:13:35 +03:00
case columnTimestamp, columnDatetime, columnDate:
2015-03-05 05:05:58 +03:00
var t time.Time
// Assume a millisecond unix timestamp if it's 13 digits -- too
// large to be a reasonable timestamp in seconds.
if val > 1e12 || val < -1e12 {
val *= int64(time.Millisecond) // convert ms to nsec
t = time.Unix(0, val)
2015-01-02 09:31:46 +03:00
} else {
t = time.Unix(val, 0)
2015-03-05 05:05:58 +03:00
}
t = t.UTC()
2015-03-05 05:05:58 +03:00
if rc.s.c.loc != nil {
t = t.In(rc.s.c.loc)
2015-01-02 09:31:46 +03:00
}
2015-03-05 05:05:58 +03:00
dest[i] = t
case "boolean":
dest[i] = val > 0
default:
dest[i] = val
}
2011-11-11 16:38:53 +04:00
case C.SQLITE_FLOAT:
dest[i] = float64(C.sqlite3_column_double(rc.s.s, C.int(i)))
case C.SQLITE_BLOB:
p := C.sqlite3_column_blob(rc.s.s, C.int(i))
if p == nil {
dest[i] = nil
continue
}
n := int(C.sqlite3_column_bytes(rc.s.s, C.int(i)))
switch dest[i].(type) {
case sql.RawBytes:
2018-04-17 11:55:42 +03:00
dest[i] = (*[1 << 30]byte)(p)[0:n]
default:
slice := make([]byte, n)
2018-04-17 11:55:42 +03:00
copy(slice[:], (*[1 << 30]byte)(p)[0:n])
dest[i] = slice
}
2011-11-11 16:38:53 +04:00
case C.SQLITE_NULL:
dest[i] = nil
case C.SQLITE_TEXT:
var err error
var timeVal time.Time
n := int(C.sqlite3_column_bytes(rc.s.s, C.int(i)))
s := C.GoStringN((*C.char)(unsafe.Pointer(C.sqlite3_column_text(rc.s.s, C.int(i)))), C.int(n))
2012-12-30 02:20:27 +04:00
switch rc.decltype[i] {
2018-04-17 12:13:35 +03:00
case columnTimestamp, columnDatetime, columnDate:
2015-03-05 05:05:58 +03:00
var t time.Time
2015-04-15 10:26:27 +03:00
s = strings.TrimSuffix(s, "Z")
2015-03-04 16:58:32 +03:00
for _, format := range SQLiteTimestampFormats {
2015-03-04 19:17:38 +03:00
if timeVal, err = time.ParseInLocation(format, s, time.UTC); err == nil {
2015-03-05 05:05:58 +03:00
t = timeVal
2015-03-04 16:58:32 +03:00
break
}
}
if err != nil {
2012-12-30 02:20:27 +04:00
// The column is a time value, so return the zero time on parse failure.
2015-03-05 05:05:58 +03:00
t = time.Time{}
}
if rc.s.c.loc != nil {
t = t.In(rc.s.c.loc)
}
2015-03-05 05:05:58 +03:00
dest[i] = t
default:
dest[i] = []byte(s)
}
2011-11-11 16:36:22 +04:00
}
}
return nil
}