go-sqlite3/upgrade/upgrade.go

//go:build !cgo && upgrade && ignore
// +build !cgo,upgrade,ignore

package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/csv"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"golang.org/x/crypto/sha3"
	"golang.org/x/net/html"
)

var (
	cFlags  = "-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1"
	cleanup = true

	errNotFound = errors.New("records not found")
)

const (
	sqliteAddress = "https://www.sqlite.org"
	downloadPage  = "download.html"

	sqliteDownloadDataTag = "Download product data for scripts to read"
	urlColumnName         = "RELATIVE-URL"
	hashColumnName        = "SHA3-HASH"
)

func main() {
	flag.StringVar(&shellPath, "shell", shellPath, "path to shell executable")
	flag.StringVar(&makePath, "make", makePath, "path to make executable")
	flag.StringVar(&cFlags, "cflags", cFlags, "sqlite CFLAGS")
	flag.BoolVar(&cleanup, "cleanup", cleanup, "cleanup source")
	flag.Parse()

	err := func() error {
		fmt.Println("Go-SQLite3 Upgrade Tool")

		wd, err := os.Getwd()
		if err != nil {
			return err
		}
		if filepath.Base(wd) != "upgrade" {
			return fmt.Errorf("Current directory is %q but should run in upgrade directory", wd)
		}

		// Download Source
		source, hash, err := download("sqlite-src-")
		if err != nil {
			return fmt.Errorf("failed to download: sqlite-src; %v", err)
		}
		fmt.Printf("Download successful and verified hash %x\n", hash)

		// Extract Source
		baseDir, err := extractZip(source)
		if cleanup && baseDir != "" && !filepath.IsAbs(baseDir) {
			defer func() {
				fmt.Println("Cleaning up source: deleting", baseDir)
				os.RemoveAll(baseDir)
			}()
		}
		if err != nil {
			return fmt.Errorf("failed to extract source: %v", err)
		}
		fmt.Println("Extracted sqlite source to", baseDir)

		// Build amalgamation files
		fmt.Printf("Starting to generate amalgamation with CFLAGS: %s\n", cFlags)
		if err := buildAmalgamation(baseDir, cFlags); err != nil {
			return fmt.Errorf("failed to build amalgamation: %v", err)
		}
		fmt.Println("SQLite3 amalgamation built")

		// Patch bindings
		patchSource(baseDir, "sqlite3.c", "../sqlite3-binding.c", "ext/userauth/userauth.c")
		patchSource(baseDir, "sqlite3.h", "../sqlite3-binding.h", "ext/userauth/sqlite3userauth.h")
		patchSource(baseDir, "sqlite3ext.h", "../sqlite3ext.h")

		fmt.Println("Done patching amalgamation")
		return nil
	}()
	if err != nil {
		log.Fatalln("Returned with error:", err)
	}
}

func findAddress(n *html.Node, prefix string) (string, string, error) {
	if n.Type == html.CommentNode && strings.Contains(n.Data, sqliteDownloadDataTag) {
		data := strings.TrimSpace(n.Data[strings.Index(n.Data, "\n")+1:])
		r := csv.NewReader(strings.NewReader(data))
		records, err := r.ReadAll()
		if err != nil {
			return "", "", err
		}

		for i, r := range records {
			if len(r) != 5 {
				return "", "", fmt.Errorf("expected record of length 5, got: %v", r)
			}
			if i == 0 && (r[2] != urlColumnName || r[4] != hashColumnName) {
				return "", "", fmt.Errorf("unexpected columns: %v", r)
			}
			if strings.Contains(r[2], prefix) {
				return r[2], r[4], nil
			}
		}
	}

	mainErr := errNotFound
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		path, hash, err := findAddress(c, prefix)
		if err == nil {
			return path, hash, nil
		}
		if err != errNotFound {
			mainErr = err
		}
	}
	return "", "", mainErr
}

func download(prefix string) (content, hash []byte, err error) {
	u, err := url.Parse(sqliteAddress)
	if err != nil {
		return nil, nil, err
	}

	dwnldPage, err := http.Get(u.JoinPath(downloadPage).String())
	if err != nil {
		return nil, nil, err
	}
	defer dwnldPage.Body.Close()
	node, err := html.Parse(dwnldPage.Body)
	if err != nil {
		return nil, nil, err
	}

	relPath, hashString, err := findAddress(node, prefix)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to find download info in the document: %w", err)
	}

	targetHash, err := hex.DecodeString(hashString)
	if err != nil || len(targetHash) != 32 {
		return nil, nil, fmt.Errorf("unable to find valid sha3-256 hash on download page: %q", hashString)
	}

	if relPath == "" {
		return nil, nil, fmt.Errorf("unable to find prefix '%s' on download page", prefix)
	}

	src := u.JoinPath(relPath).String()
	fmt.Printf("Downloading %v\n", src)
	resp, err := http.Get(src)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()

	// Ready Body Content
	shasum := sha3.New256()
	content, err = ioutil.ReadAll(io.TeeReader(resp.Body, shasum))
	if err != nil {
		return nil, nil, err
	}

	computedHash := shasum.Sum(nil)
	if !bytes.Equal(targetHash, computedHash) {
		return nil, nil, fmt.Errorf("invalid hash of file downloaded from %q: got %x instead of %x", src, computedHash, targetHash)
	}

	return content, computedHash, nil
}

func extractZip(data []byte) (string, error) {
	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if err != nil {
		return "", err
	}

	if len(zr.File) == 0 {
		return "", errors.New("no files in zip archive")
	}
	if !zr.File[0].Mode().IsDir() {
		return "", errors.New("expecting base directory at the top of zip archive")
	}
	baseDir := zr.File[0].Name

	for _, zf := range zr.File {
		if !strings.HasPrefix(zf.Name, baseDir) {
			return baseDir, fmt.Errorf("file %q in zip archive not in base directory %q", zf.Name, baseDir)
		}

		if zf.Mode().IsDir() {
			if err := os.Mkdir(zf.Name, zf.Mode()); err != nil {
				return baseDir, err
			}
			continue
		}
		f, err := os.OpenFile(zf.Name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, zf.Mode())
		if err != nil {
			return baseDir, err
		}
		if zf.UncompressedSize == 0 {
			continue
		}

		zr, err := zf.Open()
		if err != nil {
			return baseDir, err
		}

		_, err = io.Copy(f, zr)
		if err != nil {
			return baseDir, err
		}

		if err := zr.Close(); err != nil {
			return baseDir, err
		}
		if err := f.Close(); err != nil {
			return baseDir, err
		}
	}

	return baseDir, nil
}

func patchSource(baseDir, src, dst string, extensions ...string) error {
	srcFile, err := os.Open(filepath.Join(baseDir, src))
	if err != nil {
		return err
	}
	defer srcFile.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer dstFile.Close()

	_, err = io.WriteString(dstFile, "#ifndef USE_LIBSQLITE3\n")
	if err != nil {
		return err
	}
	scanner := bufio.NewScanner(srcFile)
	for scanner.Scan() {
		text := scanner.Text()
		if text == `#include "sqlite3.h"` {
			text = `#include "sqlite3-binding.h"
#ifdef __clang__
#define assert(condition) ((void)0)
#endif
`
		}
		_, err = fmt.Fprintln(dstFile, text)
		if err != nil {
			break
		}
	}
	err = scanner.Err()
	if err != nil {
		return err
	}
	_, err = io.WriteString(dstFile, "#else // USE_LIBSQLITE3\n// If users really want to link against the system sqlite3 we\n// need to make this file a noop.\n#endif\n")
	if err != nil {
		return err
	}

	for _, ext := range extensions {
		ext = filepath.FromSlash(ext)
		fmt.Printf("Merging: %s into %s\n", ext, dst)

		extFile, err := os.Open(filepath.Join(baseDir, ext))
		if err != nil {
			return err
		}
		_, err = io.Copy(dstFile, extFile)
		extFile.Close()
		if err != nil {
			return err
		}
	}

	if err := dstFile.Close(); err != nil {
		return err
	}

	fmt.Printf("Patched: %s -> %s\n", src, dst)

	return nil
}

func buildAmalgamation(baseDir, buildFlags string) error {
	configureScript := "./configure"
	if cFlags != "" {
		configureScript += fmt.Sprintf(" CFLAGS=%q", cFlags)
	}
	cmd := exec.Command(shellPath, "-c", configureScript)
	cmd.Dir = baseDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("configure failed: %v\n\n%s", err, out)
	}
	fmt.Println("Ran configure successfully")

	cmd = exec.Command(makePath, "sqlite3.c")
	cmd.Dir = baseDir
	out, err = cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("make failed: %v\n\n%s", err, out)
	}
	fmt.Println("Ran make successfully")

	return nil
}