//go:build !cgo && upgrade && ignore // +build !cgo,upgrade,ignore package main import ( "archive/zip" "bufio" "bytes" "encoding/hex" "errors" "flag" "fmt" "io" "io/ioutil" "log" "net/http" "os" "os/exec" "path/filepath" "strings" "time" "github.com/PuerkitoBio/goquery" "golang.org/x/crypto/sha3" ) var ( cFlags = "-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1" cleanup = true ) func main() { flag.StringVar(&shellPath, "shell", shellPath, "path to shell executable") flag.StringVar(&makePath, "make", makePath, "path to make executable") flag.StringVar(&cFlags, "cflags", cFlags, "sqlite CFLAGS") flag.BoolVar(&cleanup, "cleanup", cleanup, "cleanup source") flag.Parse() err := func() error { fmt.Println("Go-SQLite3 Upgrade Tool") wd, err := os.Getwd() if err != nil { return err } if filepath.Base(wd) != "upgrade" { return fmt.Errorf("Current directory is %q but should run in upgrade directory", wd) } // Download Source source, hash, err := download("sqlite-src-") if err != nil { return fmt.Errorf("failed to download: sqlite-src; %v", err) } fmt.Printf("Download successful and verified hash %x\n", hash) // Extract Source baseDir, err := extractZip(source) if cleanup && baseDir != "" && !filepath.IsAbs(baseDir) { defer func() { fmt.Println("Cleaning up source: deleting", baseDir) os.RemoveAll(baseDir) }() } if err != nil { return fmt.Errorf("failed to extract source: %v", err) } fmt.Println("Extracted sqlite source to", baseDir) // Build amalgamation files fmt.Printf("Starting to generate amalgamation with CFLAGS: %s\n", cFlags) if err := buildAmalgamation(baseDir, cFlags); err != nil { return fmt.Errorf("failed to build amalgamation: %v", err) } fmt.Println("SQLite3 amalgamation built") // Patch bindings patchSource(baseDir, "sqlite3.c", "../sqlite3-binding.c", "ext/userauth/userauth.c") patchSource(baseDir, "sqlite3.h", "../sqlite3-binding.h", "ext/userauth/sqlite3userauth.h") patchSource(baseDir, "sqlite3ext.h", "../sqlite3ext.h") fmt.Println("Done patching amalgamation") return nil }() if err != nil { log.Fatalln("Returned with error:", err) } } func download(prefix string) (content, hash []byte, err error) { year := time.Now().Year() site := "https://www.sqlite.org/download.html" //fmt.Printf("scraping %v\n", site) doc, err := goquery.NewDocument(site) if err != nil { return nil, nil, err } url, hashString := "", "" doc.Find("tr").EachWithBreak(func(_ int, s *goquery.Selection) bool { found := false s.Find("a").Each(func(_ int, s *goquery.Selection) { if strings.HasPrefix(s.Text(), prefix) { found = true url = fmt.Sprintf("https://www.sqlite.org/%d/", year) + s.Text() } }) if found { s.Find("td").Each(func(_ int, s *goquery.Selection) { text := s.Text() split := strings.Split(text, "(sha3: ") if len(split) < 2 { return } text = split[1] hashString = strings.Split(text, ")")[0] }) } return !found }) targetHash, err := hex.DecodeString(hashString) if err != nil || len(targetHash) != 32 { return nil, nil, fmt.Errorf("unable to find valid sha3-256 hash on sqlite.org: %q", hashString) } if url == "" { return nil, nil, fmt.Errorf("unable to find prefix '%s' on sqlite.org", prefix) } fmt.Printf("Downloading %v\n", url) resp, err := http.Get(url) if err != nil { return nil, nil, err } // Ready Body Content shasum := sha3.New256() content, err = ioutil.ReadAll(io.TeeReader(resp.Body, shasum)) defer resp.Body.Close() if err != nil { return nil, nil, err } computedHash := shasum.Sum(nil) if !bytes.Equal(targetHash, computedHash) { return nil, nil, fmt.Errorf("invalid hash of file downloaded from %q: got %x instead of %x", url, computedHash, targetHash) } return content, computedHash, nil } func extractZip(data []byte) (string, error) { zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) if err != nil { return "", err } if len(zr.File) == 0 { return "", errors.New("no files in zip archive") } if !zr.File[0].Mode().IsDir() { return "", errors.New("expecting base directory at the top of zip archive") } baseDir := zr.File[0].Name for _, zf := range zr.File { if !strings.HasPrefix(zf.Name, baseDir) { return baseDir, fmt.Errorf("file %q in zip archive not in base directory %q", zf.Name, baseDir) } if zf.Mode().IsDir() { if err := os.Mkdir(zf.Name, zf.Mode()); err != nil { return baseDir, err } continue } f, err := os.OpenFile(zf.Name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, zf.Mode()) if err != nil { return baseDir, err } if zf.UncompressedSize == 0 { continue } zr, err := zf.Open() if err != nil { return baseDir, err } _, err = io.Copy(f, zr) if err != nil { return baseDir, err } if err := zr.Close(); err != nil { return baseDir, err } if err := f.Close(); err != nil { return baseDir, err } } return baseDir, nil } func patchSource(baseDir, src, dst string, extensions ...string) error { srcFile, err := os.Open(filepath.Join(baseDir, src)) if err != nil { return err } defer srcFile.Close() dstFile, err := os.Create(dst) if err != nil { return err } defer dstFile.Close() _, err = io.WriteString(dstFile, "#ifndef USE_LIBSQLITE3\n") if err != nil { return err } scanner := bufio.NewScanner(srcFile) for scanner.Scan() { text := scanner.Text() if text == `#include "sqlite3.h"` { text = `#include "sqlite3-binding.h"` } _, err = fmt.Fprintln(dstFile, text) if err != nil { break } } err = scanner.Err() if err != nil { return err } _, err = io.WriteString(dstFile, "#else // USE_LIBSQLITE3\n// If users really want to link against the system sqlite3 we\n// need to make this file a noop.\n#endif\n") if err != nil { return err } for _, ext := range extensions { ext = filepath.FromSlash(ext) fmt.Printf("Merging: %s into %s\n", ext, dst) extFile, err := os.Open(filepath.Join(baseDir, ext)) if err != nil { return err } _, err = io.Copy(dstFile, extFile) extFile.Close() if err != nil { return err } } if err := dstFile.Close(); err != nil { return err } fmt.Printf("Patched: %s -> %s\n", src, dst) return nil } func buildAmalgamation(baseDir, buildFlags string) error { configureScript := "./configure" if cFlags != "" { configureScript += fmt.Sprintf(" CFLAGS=%q", cFlags) } cmd := exec.Command(shellPath, "-c", configureScript) cmd.Dir = baseDir out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("configure failed: %v\n\n%s", err, out) } fmt.Println("Ran configure successfully") cmd = exec.Command(makePath, "sqlite3.c") cmd.Dir = baseDir out, err = cmd.CombinedOutput() if err != nil { return fmt.Errorf("make failed: %v\n\n%s", err, out) } fmt.Println("Ran make successfully") return nil }