mirror of
https://github.com/jkaninda/mysql-bkup.git
synced 2026-11-13 16:47:14 +00:00
feat: mysql-bkup v2 on bkupkit
MysQL backup and restore built on the shared bkupkit core: streaming end to end, with no dump held in memory or written to local disk.
This commit is contained in:
1 parent
ccb3fcdf26
commit
28de0d7b52
65 files changed
+3453
-3871
No files matched your search
@@ -0,0 +1,279 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
"github.com/jkaninda/bkupkit/config"
|
||||
"github.com/jkaninda/bkupkit/execx"
|
||||
)
|
||||
|
||||
// Client binaries. Debian links mysql/mysqldump to the MariaDB tools, so the
|
||||
// Oracle ones are recognized by their --version output, not by name.
|
||||
const (
|
||||
mariadbCLI = "mariadb"
|
||||
mariadbDump = "mariadb-dump"
|
||||
mysqlCLI = "mysql"
|
||||
mysqlDump = "mysqldump"
|
||||
)
|
||||
|
||||
// Client preferences (MYSQL_CLIENT).
|
||||
const (
|
||||
ClientAuto = "auto"
|
||||
ClientMySQL = "mysql"
|
||||
ClientMariaDB = "mariadb"
|
||||
)
|
||||
|
||||
// tool is a resolved client binary.
|
||||
type tool struct {
|
||||
name string // logical name (mysqldump, mariadb-dump, ...)
|
||||
path string
|
||||
mariadb bool // MariaDB client (option names differ)
|
||||
}
|
||||
|
||||
// server is what Ping learned about a server.
|
||||
type server struct {
|
||||
version string
|
||||
mariadb bool
|
||||
}
|
||||
|
||||
// IsMariaDB reports whether a SELECT VERSION() result is a MariaDB server.
|
||||
func IsMariaDB(version string) bool {
|
||||
return strings.Contains(strings.ToLower(version), "mariadb")
|
||||
}
|
||||
|
||||
// lookTool resolves a binary and its client family (cached).
|
||||
func (e *Engine) lookTool(ctx context.Context, name string) (tool, bool) {
|
||||
e.mu.Lock()
|
||||
t, ok := e.tools[name]
|
||||
e.mu.Unlock()
|
||||
if ok {
|
||||
return t, t.path != ""
|
||||
}
|
||||
t = tool{name: name}
|
||||
if p, err := e.lookPath(name); err == nil {
|
||||
t.path = p
|
||||
switch name {
|
||||
case mariadbCLI, mariadbDump:
|
||||
t.mariadb = true
|
||||
default:
|
||||
out, err := execx.Output(ctx, p, []string{"--version"}, nil)
|
||||
if err != nil {
|
||||
t.path = ""
|
||||
} else {
|
||||
t.mariadb = IsMariaDB(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.tools[name] = t
|
||||
e.mu.Unlock()
|
||||
return t, t.path != ""
|
||||
}
|
||||
|
||||
func (e *Engine) lookPath(name string) (string, error) {
|
||||
if e.LookPath != nil {
|
||||
return e.LookPath(name)
|
||||
}
|
||||
return exec.LookPath(name)
|
||||
}
|
||||
|
||||
// oracle returns the Oracle MySQL binary called name, if installed.
|
||||
func (e *Engine) oracle(ctx context.Context, name string) (tool, bool) {
|
||||
t, ok := e.lookTool(ctx, name)
|
||||
return t, ok && !t.mariadb
|
||||
}
|
||||
|
||||
// maria returns the MariaDB binary (mariadb-dump, or a mysqldump that is MariaDB's).
|
||||
func (e *Engine) maria(ctx context.Context, name, legacy string) (tool, bool) {
|
||||
if t, ok := e.lookTool(ctx, name); ok {
|
||||
return t, true
|
||||
}
|
||||
t, ok := e.lookTool(ctx, legacy)
|
||||
return t, ok && t.mariadb
|
||||
}
|
||||
|
||||
// pick chooses the dump (dump=true) or command-line client for a server:
|
||||
// mariadb-dump/mariadb for MariaDB, Oracle mysqldump/mysql for MySQL.
|
||||
func (e *Engine) pick(ctx context.Context, pref string, srv server, dump bool) (tool, error) {
|
||||
oName, mName := mysqlCLI, mariadbCLI
|
||||
if dump {
|
||||
oName, mName = mysqlDump, mariadbDump
|
||||
}
|
||||
switch pref {
|
||||
case ClientMySQL:
|
||||
if t, ok := e.oracle(ctx, oName); ok {
|
||||
return t, nil
|
||||
}
|
||||
return tool{}, fmt.Errorf("MYSQL_CLIENT=mysql but the Oracle MySQL %s client is not installed", oName)
|
||||
case ClientMariaDB:
|
||||
if t, ok := e.maria(ctx, mName, oName); ok {
|
||||
return t, nil
|
||||
}
|
||||
return tool{}, fmt.Errorf("MYSQL_CLIENT=mariadb but %s is not installed", mName)
|
||||
}
|
||||
if srv.mariadb {
|
||||
if t, ok := e.maria(ctx, mName, oName); ok {
|
||||
return t, nil
|
||||
}
|
||||
if t, ok := e.oracle(ctx, oName); ok {
|
||||
e.warnOnce("oracle-for-mariadb", "MariaDB client not installed; using the Oracle MySQL client against a MariaDB server", "client", t.name)
|
||||
return t, nil
|
||||
}
|
||||
return tool{}, fmt.Errorf("no %s or %s client installed", mName, oName)
|
||||
}
|
||||
if t, ok := e.oracle(ctx, oName); ok {
|
||||
return t, nil
|
||||
}
|
||||
if t, ok := e.maria(ctx, mName, oName); ok {
|
||||
e.warnOnce("mariadb-for-mysql",
|
||||
"Oracle MySQL client not available on this platform; using the MariaDB client against a MySQL server "+
|
||||
"(dumps of MySQL-only features may be incomplete)",
|
||||
"client", t.name, "server_version", srv.version)
|
||||
return t, nil
|
||||
}
|
||||
return tool{}, fmt.Errorf("no %s or %s client installed", oName, mName)
|
||||
}
|
||||
|
||||
func (e *Engine) warnOnce(key, msg string, args ...any) {
|
||||
e.mu.Lock()
|
||||
seen := e.warned[key]
|
||||
e.warned[key] = true
|
||||
e.mu.Unlock()
|
||||
if !seen {
|
||||
slog.Default().Warn(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// withOptionFile writes c's option file (0600, private directory) and calls fn
|
||||
// with its path. The file is removed when fn returns.
|
||||
func withOptionFile(v *config.Values, c Conn, t tool, fn func(path string) error) error {
|
||||
dir, cleanup, err := execx.PrivateDir(v.String(config.TmpDir))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create private directory: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
p, err := execx.WriteSecret(dir, "my.cnf", optionFile(c, t.mariadb))
|
||||
if err != nil {
|
||||
return fmt.Errorf("write client option file: %w", err)
|
||||
}
|
||||
return fn(p)
|
||||
}
|
||||
|
||||
// query runs one SQL statement with the command-line client and returns the
|
||||
// raw tab-separated output without headers.
|
||||
func query(ctx context.Context, v *config.Values, c Conn, t tool, sql string) (string, error) {
|
||||
var out string
|
||||
err := withOptionFile(v, c, t, func(cnf string) error {
|
||||
var err error
|
||||
out, err = execx.Output(ctx, t.path, []string{"--defaults-extra-file=" + cnf, "-N", "-B", "-r", "-e", sql}, nil)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// server detects the server version and flavour (cached per address and user).
|
||||
// It tries the MariaDB client first (it speaks every MySQL authentication
|
||||
// method), then the Oracle client.
|
||||
func (e *Engine) server(ctx context.Context, v *config.Values, c Conn) (server, error) {
|
||||
key := c.Addr() + "\x00" + c.User
|
||||
e.mu.Lock()
|
||||
s, ok := e.servers[key]
|
||||
e.mu.Unlock()
|
||||
if ok {
|
||||
return s, nil
|
||||
}
|
||||
var cands []tool
|
||||
if t, ok := e.maria(ctx, mariadbCLI, mysqlCLI); ok {
|
||||
cands = append(cands, t)
|
||||
}
|
||||
if t, ok := e.oracle(ctx, mysqlCLI); ok {
|
||||
cands = append(cands, t)
|
||||
}
|
||||
if len(cands) == 0 {
|
||||
return s, errors.New("no mariadb or mysql client installed")
|
||||
}
|
||||
var firstErr error
|
||||
for _, t := range cands {
|
||||
out, err := query(ctx, v, c, t, "SELECT VERSION()")
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
s = server{version: strings.TrimSpace(out), mariadb: IsMariaDB(out)}
|
||||
e.mu.Lock()
|
||||
e.servers[key] = s
|
||||
e.mu.Unlock()
|
||||
return s, nil
|
||||
}
|
||||
return s, fmt.Errorf("failed to connect to database server %s: %w", c.Addr(), firstErr)
|
||||
}
|
||||
|
||||
// systemDatabases are never backed up one by one (mysql-bkup 1.x list).
|
||||
var systemDatabases = map[string]bool{
|
||||
"information_schema": true, "performance_schema": true, "mysql": true, "sys": true, "innodb": true,
|
||||
}
|
||||
|
||||
// listDatabases returns the user databases, minus exclude.
|
||||
func (e *Engine) listDatabases(ctx context.Context, v *config.Values, c Conn, exclude []string) ([]string, error) {
|
||||
srv, err := e.server(ctx, v, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t, err := e.pick(ctx, v.String(SettingClient), srv, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := query(ctx, v, c, t, "SHOW DATABASES")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list databases: %w", err)
|
||||
}
|
||||
return filterDatabases(strings.Split(out, "\n"), exclude), nil
|
||||
}
|
||||
|
||||
func filterDatabases(names, exclude []string) []string {
|
||||
skip := map[string]bool{}
|
||||
for _, x := range exclude {
|
||||
skip[x] = true
|
||||
}
|
||||
var dbs []string
|
||||
for _, n := range names {
|
||||
n = strings.TrimRight(n, "\r")
|
||||
if n == "" || systemDatabases[strings.ToLower(n)] || skip[n] {
|
||||
continue
|
||||
}
|
||||
dbs = append(dbs, n)
|
||||
}
|
||||
return dbs
|
||||
}
|
||||
|
||||
// quoteIdent quotes a MySQL identifier.
|
||||
func quoteIdent(s string) string {
|
||||
return "`" + strings.ReplaceAll(s, "`", "``") + "`"
|
||||
}
|
||||
|
||||
// ensureDatabase creates db when it does not exist yet. The existence check
|
||||
// runs first so that users without CREATE privilege can restore into an
|
||||
// existing database.
|
||||
func ensureDatabase(ctx context.Context, v *config.Values, c Conn, t tool, db string) error {
|
||||
lit := "CONVERT(X'" + hex.EncodeToString([]byte(db)) + "' USING utf8mb4)"
|
||||
out, err := query(ctx, v, c, t, "SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = "+lit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check database %s: %w", db, err)
|
||||
}
|
||||
if strings.TrimSpace(out) != "0" {
|
||||
return nil
|
||||
}
|
||||
slog.Default().Info("Creating database", "database", db)
|
||||
if _, err := query(ctx, v, c, t, "CREATE DATABASE IF NOT EXISTS "+quoteIdent(db)); err != nil {
|
||||
return fmt.Errorf("create database %s: %w", db, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jkaninda/bkupkit/config"
|
||||
)
|
||||
|
||||
// TLS modes accepted in DB_SSL_MODE (MySQL --ssl-mode names, lower-case).
|
||||
const (
|
||||
SSLDisabled = "disabled"
|
||||
SSLPreferred = "preferred"
|
||||
SSLRequired = "required"
|
||||
SSLVerifyCA = "verify_ca"
|
||||
SSLVerifyIdentity = "verify_identity"
|
||||
)
|
||||
|
||||
// ParseSSLMode normalizes DB_SSL_MODE. mysql-bkup 1.x wrote the value into
|
||||
// my.cnf as "ssl=<value>", so "0"/"1" (and booleans) are still accepted.
|
||||
// An empty value means disabled, the 1.x default.
|
||||
func ParseSSLMode(s string) (string, error) {
|
||||
m := strings.ToLower(strings.TrimSpace(s))
|
||||
m = strings.ReplaceAll(m, "-", "_")
|
||||
switch m {
|
||||
case "", "0", "false", "off", "no", SSLDisabled, "disable":
|
||||
return SSLDisabled, nil
|
||||
case "1", "true", "on", "yes", SSLRequired, "require":
|
||||
return SSLRequired, nil
|
||||
case SSLPreferred, "prefer":
|
||||
return SSLPreferred, nil
|
||||
case SSLVerifyCA:
|
||||
return SSLVerifyCA, nil
|
||||
case SSLVerifyIdentity, "verify_full":
|
||||
return SSLVerifyIdentity, nil
|
||||
}
|
||||
return "", fmt.Errorf("invalid TLS mode %q: use disabled, preferred, required, verify_ca or verify_identity", s)
|
||||
}
|
||||
|
||||
// Conn is a resolved connection.
|
||||
type Conn struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
DB string
|
||||
SSLMode string
|
||||
SSLCA string
|
||||
SSLCert string
|
||||
SSLKey string
|
||||
}
|
||||
|
||||
// Addr is host:port (or the socket path), for logs and cache keys.
|
||||
func (c Conn) Addr() string {
|
||||
if strings.HasPrefix(c.Host, "/") {
|
||||
return c.Host
|
||||
}
|
||||
return c.Host + ":" + strconv.Itoa(c.Port)
|
||||
}
|
||||
|
||||
// ParseURL parses mysql://user:pass@host:port/db?ssl-mode=required (also
|
||||
// mariadb://). Query parameters: ssl-mode (sslmode, tls), ssl-ca, ssl-cert, ssl-key.
|
||||
func ParseURL(raw string) (Conn, error) {
|
||||
var c Conn
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return c, errors.New("invalid database URL")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "mysql", "mariadb", "mysqlx":
|
||||
default:
|
||||
return c, fmt.Errorf("unsupported database URL scheme %q (use mysql://user:password@host:port/database)", u.Scheme)
|
||||
}
|
||||
c.Host = u.Hostname()
|
||||
if p := u.Port(); p != "" {
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil || n <= 0 || n > 65535 {
|
||||
return c, fmt.Errorf("invalid port %q in database URL", p)
|
||||
}
|
||||
c.Port = n
|
||||
}
|
||||
if u.User != nil {
|
||||
c.User = u.User.Username()
|
||||
c.Password, _ = u.User.Password()
|
||||
}
|
||||
c.DB = strings.TrimPrefix(u.Path, "/")
|
||||
q := u.Query()
|
||||
for _, k := range []string{"ssl-mode", "sslmode", "ssl_mode", "tls"} {
|
||||
if v := q.Get(k); v != "" {
|
||||
if c.SSLMode, err = ParseSSLMode(v); err != nil {
|
||||
return c, err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
c.SSLCA = first(q.Get("ssl-ca"), q.Get("sslrootcert"))
|
||||
c.SSLCert = first(q.Get("ssl-cert"), q.Get("sslcert"))
|
||||
c.SSLKey = first(q.Get("ssl-key"), q.Get("sslkey"))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func first(s ...string) string {
|
||||
for _, v := range s {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// connKeys names the settings of one side (source or migrate target).
|
||||
type connKeys struct {
|
||||
URL, Host, Port, Name, User, Pass, SSLMode, SSLCA, SSLCert, SSLKey string
|
||||
}
|
||||
|
||||
var (
|
||||
sourceKeys = connKeys{config.DBURL, config.DBHost, config.DBPort, config.DBName, config.DBUser, config.DBPassword,
|
||||
config.DBSSLMode, config.DBSSLCA, config.DBSSLCert, config.DBSSLKey}
|
||||
targetKeys = connKeys{config.TargetDBURL, config.TargetDBHost, config.TargetDBPort, config.TargetDBName, config.TargetDBUser,
|
||||
config.TargetDBPass, config.TargetDBSSL, config.TargetDBSSLCA, "", ""}
|
||||
)
|
||||
|
||||
// connFrom resolves a connection: DB_URL first, then the individual settings
|
||||
// (which also fill in what the URL leaves out).
|
||||
func connFrom(v *config.Values, k connKeys) (Conn, error) {
|
||||
var c Conn
|
||||
if raw := v.String(k.URL); raw != "" {
|
||||
var err error
|
||||
if c, err = ParseURL(raw); err != nil {
|
||||
return c, fmt.Errorf("%s: %w", k.URL, err)
|
||||
}
|
||||
}
|
||||
str := func(dst *string, key string) {
|
||||
if *dst == "" && key != "" {
|
||||
*dst = v.String(key)
|
||||
}
|
||||
}
|
||||
str(&c.Host, k.Host)
|
||||
str(&c.User, k.User)
|
||||
str(&c.Password, k.Pass)
|
||||
str(&c.DB, k.Name)
|
||||
str(&c.SSLCA, k.SSLCA)
|
||||
str(&c.SSLCert, k.SSLCert)
|
||||
str(&c.SSLKey, k.SSLKey)
|
||||
if c.Port == 0 {
|
||||
c.Port = v.Int(k.Port)
|
||||
}
|
||||
if c.Port == 0 {
|
||||
c.Port = 3306
|
||||
}
|
||||
if c.SSLMode == "" {
|
||||
m, err := ParseSSLMode(v.String(k.SSLMode))
|
||||
if err != nil {
|
||||
return c, fmt.Errorf("%s: %w", k.SSLMode, err)
|
||||
}
|
||||
c.SSLMode = m
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// SourceConn returns the DB_* connection of v.
|
||||
func SourceConn(v *config.Values) (Conn, error) { return connFrom(v, sourceKeys) }
|
||||
|
||||
// TargetConn returns the TARGET_DB_* connection of v (migrate).
|
||||
func TargetConn(v *config.Values) (Conn, error) {
|
||||
c, err := connFrom(v, targetKeys)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if v.String(config.TargetDBURL) == "" && v.String(config.TargetDBHost) == "" {
|
||||
return c, errors.New("missing required configuration: TARGET_DB_HOST (or TARGET_DB_URL)")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c Conn) validate(prefix string) error {
|
||||
var missing []string
|
||||
if c.Host == "" {
|
||||
missing = append(missing, prefix+"DB_HOST")
|
||||
}
|
||||
if c.User == "" {
|
||||
missing = append(missing, prefix+"DB_USER")
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return fmt.Errorf("missing required configuration: %s", strings.Join(missing, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDBName rejects names that a client would parse as an option.
|
||||
func checkDBName(name string) error {
|
||||
if strings.HasPrefix(name, "-") {
|
||||
return fmt.Errorf("invalid database name %q", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
// Package mysql is the mysql-bkup engine for bkupkit: MySQL and MariaDB
|
||||
// logical backups with mysqldump / mariadb-dump, restored with mysql / mariadb.
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/jkaninda/bkupkit/cli"
|
||||
"github.com/jkaninda/bkupkit/config"
|
||||
"github.com/jkaninda/bkupkit/engine"
|
||||
"github.com/jkaninda/bkupkit/execx"
|
||||
)
|
||||
|
||||
// Engine-specific settings.
|
||||
const (
|
||||
SettingClient = "MYSQL_CLIENT"
|
||||
)
|
||||
|
||||
// AllInOneName is the target (and default artifact) name of --all-in-one backups.
|
||||
const AllInOneName = "all_databases"
|
||||
|
||||
// DefaultCA is the 1.x default of DB_SSL_CA.
|
||||
const DefaultCA = "/etc/ssl/certs/ca-certificates.crt"
|
||||
|
||||
// Engine implements engine.Engine and engine.Migrator.
|
||||
type Engine struct {
|
||||
// LookPath finds client binaries (default exec.LookPath); tests replace it.
|
||||
LookPath func(string) (string, error)
|
||||
|
||||
mu sync.Mutex
|
||||
tools map[string]tool
|
||||
servers map[string]server
|
||||
warned map[string]bool
|
||||
}
|
||||
|
||||
var (
|
||||
_ engine.Engine = (*Engine)(nil)
|
||||
_ engine.Migrator = (*Engine)(nil)
|
||||
)
|
||||
|
||||
// New returns the engine.
|
||||
func New() *Engine {
|
||||
return &Engine{tools: map[string]tool{}, servers: map[string]server{}, warned: map[string]bool{}}
|
||||
}
|
||||
|
||||
// Tool implements engine.Engine.
|
||||
func (e *Engine) Tool() string { return "mysql-bkup" }
|
||||
|
||||
// Kind implements engine.Engine. It is "mysql" for MySQL and MariaDB alike so
|
||||
// that {engine} in the index name does not depend on the server.
|
||||
func (e *Engine) Kind() string { return "mysql" }
|
||||
|
||||
// Settings implements engine.Engine.
|
||||
func (e *Engine) Settings(r *config.Registry) {
|
||||
config.RegisterDatabase(r, "3306")
|
||||
r.Override(config.DBURL, func(s *config.Setting) {
|
||||
s.Usage = "Connection URL mysql://user:password@host:port/database?ssl-mode=required (overrides host, port, user, password)"
|
||||
})
|
||||
r.Override(config.DBHost, func(s *config.Setting) {
|
||||
s.Usage = "Database host (a path starting with / is a Unix socket)"
|
||||
})
|
||||
r.Override(config.DBSSLMode, func(s *config.Setting) {
|
||||
s.Default = SSLDisabled
|
||||
s.Usage = "TLS mode: disabled, preferred, required, verify_ca, verify_identity (1.x values 0 and 1 mean disabled and required)"
|
||||
})
|
||||
r.Override(config.DBSSLCA, func(s *config.Setting) {
|
||||
s.Default = DefaultCA
|
||||
s.Usage = "CA certificate file, used with DB_SSL_MODE=verify_ca or verify_identity"
|
||||
})
|
||||
r.Override(config.TargetDBSSL, func(s *config.Setting) {
|
||||
s.Default = SSLDisabled
|
||||
s.Usage = "migrate: target TLS mode (same values as DB_SSL_MODE)"
|
||||
})
|
||||
r.Override(config.TargetDBSSLCA, func(s *config.Setting) {
|
||||
s.Default = DefaultCA
|
||||
})
|
||||
r.Override(config.TargetDBName, func(s *config.Setting) {
|
||||
s.Usage = "migrate: target database (default: the source name)"
|
||||
})
|
||||
// 1.x accepted --storage remote for SFTP.
|
||||
r.Override(config.Storage, func(s *config.Setting) {
|
||||
s.Enum = append(s.Enum, "remote")
|
||||
s.Usage = "Storage backend: local, s3, gcs, azure, sftp (ssh, remote), ftp"
|
||||
})
|
||||
r.Add(config.Setting{
|
||||
Key: SettingClient, Default: ClientAuto, Enum: []string{ClientAuto, ClientMySQL, ClientMariaDB},
|
||||
Group: config.GroupDatabase,
|
||||
Usage: "Client tools: auto (mariadb-dump for MariaDB, Oracle mysqldump for MySQL), mysql or mariadb",
|
||||
})
|
||||
}
|
||||
|
||||
// Configure implements engine.Engine. It only validates values: commands that
|
||||
// do not connect (list, prune, verify) must work without credentials.
|
||||
func (e *Engine) Configure(_ context.Context, v *config.Values) error {
|
||||
normalizeStorage(v)
|
||||
var errs []error
|
||||
if _, err := SourceConn(v); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if v.Has(config.TargetDBURL) || v.Has(config.TargetDBHost) {
|
||||
if _, err := TargetConn(v); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// normalizeStorage maps the 1.x storage name "remote" to sftp, and an
|
||||
// absolute local FILE_NAME (1.x: restore -f /backup/x.sql.gz) to a file name
|
||||
// relative to STORAGE_PATH (or to its own directory, as 1.x did).
|
||||
func normalizeStorage(v *config.Values) {
|
||||
if v.String(config.Storage) == "remote" {
|
||||
v.Set(config.Storage, "sftp")
|
||||
}
|
||||
f := v.String(config.FileName)
|
||||
if v.String(config.Storage) != "local" || !filepath.IsAbs(f) || v.Has(config.RemotePath) {
|
||||
return
|
||||
}
|
||||
root := filepath.Clean(v.String(config.StoragePath))
|
||||
if rel, err := filepath.Rel(root, filepath.Clean(f)); err == nil && !strings.HasPrefix(rel, "..") {
|
||||
v.Set(config.FileName, filepath.ToSlash(rel))
|
||||
return
|
||||
}
|
||||
v.Set(config.StoragePath, filepath.Dir(f))
|
||||
v.Set(config.FileName, filepath.Base(f))
|
||||
}
|
||||
|
||||
// restoreContext reports whether v belongs to a restore/verify/delete run,
|
||||
// where one database at most is involved. Outside the CLI (no command), a
|
||||
// FILE_NAME means the same.
|
||||
func restoreContext(v *config.Values) bool {
|
||||
switch v.Command {
|
||||
case "restore", "verify", "delete":
|
||||
return true
|
||||
case "":
|
||||
return v.Has(config.FileName) && !v.Bool(config.AllDatabases) && !v.Bool(config.AllInOne)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Targets implements engine.Engine.
|
||||
func (e *Engine) Targets(ctx context.Context, v *config.Values) ([]engine.Target, error) {
|
||||
normalizeStorage(v) // per-job values of a config file are resolved again
|
||||
c, err := SourceConn(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if restoreContext(v) {
|
||||
// restore -f FILE [-d DB]: without a database, the dump's own USE
|
||||
// statements decide (--all-in-one dumps).
|
||||
if err := checkDBName(c.DB); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []engine.Target{{Name: c.DB, Values: v}}, nil
|
||||
}
|
||||
switch {
|
||||
case v.Bool(config.AllInOne):
|
||||
return []engine.Target{{Name: AllInOneName, Values: v}}, nil
|
||||
case v.Bool(config.AllDatabases):
|
||||
if err := c.validate(""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dbs, err := e.listDatabases(ctx, v, c, v.List(config.ExcludeDBs))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(dbs) == 0 {
|
||||
return nil, errors.New("no databases found")
|
||||
}
|
||||
ts := make([]engine.Target, 0, len(dbs))
|
||||
for _, db := range dbs {
|
||||
ts = append(ts, engine.Target{Name: db, Values: v})
|
||||
}
|
||||
return ts, nil
|
||||
}
|
||||
// No database: fine for list, prune, verify and delete; Ping rejects it
|
||||
// for a backup.
|
||||
if err := checkDBName(c.DB); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []engine.Target{{Name: c.DB, Values: v}}, nil
|
||||
}
|
||||
|
||||
// errNoDatabase is a configuration error (exit code 2).
|
||||
var errNoDatabase = &cli.ExitError{Code: cli.ExitConfig, Err: errors.New("database name is required, use DB_NAME environment variable or -d flag")}
|
||||
|
||||
// Ext implements engine.Engine.
|
||||
func (e *Engine) Ext(engine.Target) string { return ".sql" }
|
||||
|
||||
// NativeCompression implements engine.Engine.
|
||||
func (e *Engine) NativeCompression(engine.Target) bool { return false }
|
||||
|
||||
// VerifyBeforeRestore makes RESTORE_VERIFY_FIRST=auto read and check the whole
|
||||
// dump before the client runs: MySQL cannot roll back the dump's DROP/CREATE
|
||||
// TABLE statements, so a backup that turns out to be damaged halfway would
|
||||
// otherwise leave the database half-restored.
|
||||
func (e *Engine) VerifyBeforeRestore(engine.Target, engine.ArtifactInfo) bool { return true }
|
||||
|
||||
var _ engine.RestoreVerifier = (*Engine)(nil)
|
||||
|
||||
// Ping implements engine.Engine: it connects, and detects the server flavour.
|
||||
func (e *Engine) Ping(ctx context.Context, t engine.Target) error {
|
||||
c, err := SourceConn(t.Values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t.Name == "" && !restoreContext(t.Values) {
|
||||
return errNoDatabase
|
||||
}
|
||||
if err := c.validate(""); err != nil {
|
||||
return &cli.ExitError{Code: cli.ExitConfig, Err: err}
|
||||
}
|
||||
srv, err := e.server(ctx, t.Values, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flavour := "MySQL"
|
||||
if srv.mariadb {
|
||||
flavour = "MariaDB"
|
||||
}
|
||||
slog.Default().Info("Connected to database server", "server", c.Addr(), "flavour", flavour, "version", srv.version)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ServerVersion implements engine.Engine.
|
||||
func (e *Engine) ServerVersion(ctx context.Context, t engine.Target) string {
|
||||
c, err := SourceConn(t.Values)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
srv, err := e.server(ctx, t.Values, c)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return srv.version
|
||||
}
|
||||
|
||||
// dumpSpec describes what a dump covers.
|
||||
type dumpSpec struct {
|
||||
db string // one database (no CREATE DATABASE / USE: restorable under another name)
|
||||
all bool // --all-databases
|
||||
databases []string // --databases list (all-in-one with exclusions)
|
||||
}
|
||||
|
||||
// DumpArgs returns the dump arguments after --defaults-extra-file.
|
||||
// mariadbTool is the client family, mariadbServer the server flavour.
|
||||
func DumpArgs(mariadbTool, mariadbServer bool, s dumpSpec) []string {
|
||||
args := []string{"--single-transaction", "--routines", "--triggers", "--events", "--hex-blob", "--quick"}
|
||||
if !mariadbServer {
|
||||
// MySQL: no tablespace statements (they need the PROCESS privilege
|
||||
// and are useless in logical backups).
|
||||
args = append(args, "--no-tablespaces")
|
||||
if !mariadbTool {
|
||||
args = append(args, "--set-gtid-purged=OFF")
|
||||
}
|
||||
} else if !mariadbTool {
|
||||
// Oracle mysqldump against MariaDB: no column_statistics table there.
|
||||
args = append(args, "--column-statistics=0")
|
||||
}
|
||||
switch {
|
||||
case s.all:
|
||||
args = append(args, "--all-databases")
|
||||
case len(s.databases) > 0:
|
||||
args = append(args, "--databases")
|
||||
args = append(args, s.databases...)
|
||||
default:
|
||||
args = append(args, s.db)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func (e *Engine) dumpCmd(ctx context.Context, t engine.Target, c Conn) (tool, []string, error) {
|
||||
v := t.Values
|
||||
srv, err := e.server(ctx, v, c)
|
||||
if err != nil {
|
||||
return tool{}, nil, err
|
||||
}
|
||||
tl, err := e.pick(ctx, v.String(SettingClient), srv, true)
|
||||
if err != nil {
|
||||
return tool{}, nil, err
|
||||
}
|
||||
spec := dumpSpec{db: t.Name}
|
||||
if t.Name == AllInOneName && v.Bool(config.AllInOne) {
|
||||
spec = dumpSpec{all: true}
|
||||
if ex := v.List(config.ExcludeDBs); len(ex) > 0 {
|
||||
dbs, err := e.listDatabases(ctx, v, c, ex)
|
||||
if err != nil {
|
||||
return tool{}, nil, err
|
||||
}
|
||||
if len(dbs) == 0 {
|
||||
return tool{}, nil, errors.New("no databases left after BACKUP_EXCLUDE_DATABASES")
|
||||
}
|
||||
spec = dumpSpec{databases: dbs}
|
||||
}
|
||||
} else if err := checkDBName(t.Name); err != nil || t.Name == "" {
|
||||
return tool{}, nil, fmt.Errorf("invalid database name %q", t.Name)
|
||||
}
|
||||
return tl, DumpArgs(tl.mariadb, srv.mariadb, spec), nil
|
||||
}
|
||||
|
||||
// Dump implements engine.Engine: the dump goes straight to w.
|
||||
func (e *Engine) Dump(ctx context.Context, t engine.Target, w io.Writer) error {
|
||||
c, err := SourceConn(t.Values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tl, args, err := e.dumpCmd(ctx, t, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Default().Info("Dumping database", "database", t.Name, "client", tl.name)
|
||||
return withOptionFile(t.Values, c, tl, func(cnf string) error {
|
||||
return execx.Run(ctx, execx.Cmd{
|
||||
Name: tl.path, Args: append([]string{"--defaults-extra-file=" + cnf}, args...),
|
||||
Stdout: w, Log: slog.Default(), StderrLevel: slog.LevelWarn,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// restoreDB returns the database a dump is replayed into ("" = none: the
|
||||
// dump's USE statements decide).
|
||||
func restoreDB(t engine.Target) string {
|
||||
if t.Name == AllInOneName && t.Values.Bool(config.AllInOne) {
|
||||
return ""
|
||||
}
|
||||
return t.Name
|
||||
}
|
||||
|
||||
func (e *Engine) restoreCmd(ctx context.Context, v *config.Values, c Conn, db string) (tool, []string, error) {
|
||||
if err := checkDBName(db); err != nil {
|
||||
return tool{}, nil, err
|
||||
}
|
||||
srv, err := e.server(ctx, v, c)
|
||||
if err != nil {
|
||||
return tool{}, nil, err
|
||||
}
|
||||
tl, err := e.pick(ctx, v.String(SettingClient), srv, false)
|
||||
if err != nil {
|
||||
return tool{}, nil, err
|
||||
}
|
||||
if db != "" {
|
||||
if err := ensureDatabase(ctx, v, c, tl, db); err != nil {
|
||||
return tool{}, nil, err
|
||||
}
|
||||
}
|
||||
// --binary-mode: dumps may contain \0 and \r in string literals.
|
||||
args := []string{"--binary-mode"}
|
||||
if db != "" {
|
||||
args = append(args, db)
|
||||
}
|
||||
return tl, args, nil
|
||||
}
|
||||
|
||||
// Restore implements engine.Engine. The client reads the dump from stdin and
|
||||
// stops at the first failing statement (non-zero exit). If r fails, execx
|
||||
// kills the client before it sees EOF. MySQL cannot roll back DDL, so a
|
||||
// failed restore can leave the statements before the failure applied.
|
||||
func (e *Engine) Restore(ctx context.Context, t engine.Target, r io.Reader, info engine.ArtifactInfo) error {
|
||||
c, err := SourceConn(t.Values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db := restoreDB(t)
|
||||
tl, args, err := e.restoreCmd(ctx, t.Values, c, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Default().Info("Restoring database", "database", db, "file", info.File, "client", tl.name)
|
||||
return withOptionFile(t.Values, c, tl, func(cnf string) error {
|
||||
return execx.Run(ctx, execx.Cmd{
|
||||
Name: tl.path, Args: append([]string{"--defaults-extra-file=" + cnf}, args...),
|
||||
Stdin: r, Log: slog.Default(), StderrLevel: slog.LevelWarn,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// MigrateTargets implements engine.Migrator: DB_NAME → TARGET_DB_NAME (default
|
||||
// the same name), or every database with --all-databases.
|
||||
func (e *Engine) MigrateTargets(ctx context.Context, v *config.Values) ([][2]engine.Target, error) {
|
||||
tc, err := TargetConn(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tc.validate("TARGET_"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sc, err := SourceConn(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := sc.validate(""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.Bool(config.AllDatabases) || v.Bool(config.AllInOne) {
|
||||
dbs, err := e.listDatabases(ctx, v, sc, v.List(config.ExcludeDBs))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pairs := make([][2]engine.Target, 0, len(dbs))
|
||||
for _, db := range dbs {
|
||||
pairs = append(pairs, [2]engine.Target{{Name: db, Values: v}, {Name: db, Values: v}})
|
||||
}
|
||||
return pairs, nil
|
||||
}
|
||||
if sc.DB == "" {
|
||||
return nil, errNoDatabase
|
||||
}
|
||||
name := tc.DB
|
||||
if name == "" {
|
||||
name = sc.DB
|
||||
}
|
||||
for _, n := range []string{sc.DB, name} {
|
||||
if err := checkDBName(n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return [][2]engine.Target{{{Name: sc.DB, Values: v}, {Name: name, Values: v}}}, nil
|
||||
}
|
||||
|
||||
// Migrate implements engine.Migrator: dump | restore through a pipe, no file.
|
||||
func (e *Engine) Migrate(ctx context.Context, source, target engine.Target) error {
|
||||
sc, err := SourceConn(source.Values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tc, err := TargetConn(target.Values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dumpTool, dumpArgs, err := e.dumpCmd(ctx, source, sc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("source: %w", err)
|
||||
}
|
||||
restTool, restArgs, err := e.restoreCmd(ctx, target.Values, tc, target.Name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("target: %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
// The first failure is the cause; the other side then fails because of it.
|
||||
var (
|
||||
once sync.Once
|
||||
cause error
|
||||
)
|
||||
fail := func(err error) {
|
||||
once.Do(func() { cause = err })
|
||||
cancel()
|
||||
}
|
||||
pr, pw := io.Pipe()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
err := withOptionFile(source.Values, sc, dumpTool, func(cnf string) error {
|
||||
return execx.Run(ctx, execx.Cmd{
|
||||
Name: dumpTool.path, Args: append([]string{"--defaults-extra-file=" + cnf}, dumpArgs...),
|
||||
Stdout: pw, Log: slog.Default(), StderrLevel: slog.LevelWarn,
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
err = fmt.Errorf("dump %s: %w", source.Name, err)
|
||||
fail(err)
|
||||
// The restore side must never see a clean EOF.
|
||||
_ = pw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
_ = pw.Close()
|
||||
}()
|
||||
err = withOptionFile(target.Values, tc, restTool, func(cnf string) error {
|
||||
return execx.Run(ctx, execx.Cmd{
|
||||
Name: restTool.path, Args: append([]string{"--defaults-extra-file=" + cnf}, restArgs...),
|
||||
Stdin: pr, Log: slog.Default(), StderrLevel: slog.LevelWarn,
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
fail(fmt.Errorf("restore into %s: %w", target.Name, err))
|
||||
_ = pr.CloseWithError(err)
|
||||
}
|
||||
<-done
|
||||
return cause
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jkaninda/bkupkit/cli"
|
||||
"github.com/jkaninda/bkupkit/config"
|
||||
"github.com/jkaninda/bkupkit/engine"
|
||||
)
|
||||
|
||||
func resolve(t *testing.T, env config.MapEnv) *config.Values {
|
||||
t.Helper()
|
||||
r := config.NewRegistry()
|
||||
config.RegisterCommon(r)
|
||||
New().Settings(r)
|
||||
v, err := r.Resolve(config.ResolveOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func TestParseSSLMode(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": SSLDisabled, "0": SSLDisabled, "false": SSLDisabled, "DISABLED": SSLDisabled,
|
||||
"1": SSLRequired, "true": SSLRequired, "Required": SSLRequired,
|
||||
"preferred": SSLPreferred, "VERIFY_CA": SSLVerifyCA, "verify-identity": SSLVerifyIdentity,
|
||||
}
|
||||
for in, want := range cases {
|
||||
got, err := ParseSSLMode(in)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("ParseSSLMode(%q) = %q, %v; want %q", in, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseSSLMode("2"); err == nil {
|
||||
t.Error("expected an error for 2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseURL(t *testing.T) {
|
||||
c, err := ParseURL("mysql://bob:p%40ss%3Aw%2Frd@db.example:3307/shop?ssl-mode=verify_identity&ssl-ca=/ca.pem")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := Conn{Host: "db.example", Port: 3307, User: "bob", Password: "p@ss:w/rd", DB: "shop", SSLMode: SSLVerifyIdentity, SSLCA: "/ca.pem"}
|
||||
if c != want {
|
||||
t.Fatalf("got %+v\nwant %+v", c, want)
|
||||
}
|
||||
if _, err := ParseURL("postgres://a@b/c"); err == nil {
|
||||
t.Fatal("expected scheme error")
|
||||
}
|
||||
if _, err := ParseURL("mysql://a@b:99999/c"); err == nil {
|
||||
t.Fatal("expected port error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceConnPrecedence(t *testing.T) {
|
||||
v := resolve(t, config.MapEnv{
|
||||
"DB_URL": "mariadb://u:pw@h1/db1", "DB_HOST": "ignored", "DB_PORT": "3310", "DB_NAME": "ignored",
|
||||
"DB_SSL_MODE": "1",
|
||||
})
|
||||
c, err := SourceConn(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Host != "h1" || c.Port != 3310 || c.User != "u" || c.Password != "pw" || c.DB != "db1" || c.SSLMode != SSLRequired {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
if c.SSLCA != DefaultCA {
|
||||
t.Fatalf("default CA = %q", c.SSLCA)
|
||||
}
|
||||
|
||||
v = resolve(t, config.MapEnv{"DB_HOST": "h", "DB_USER": "legacy", "DB_PASSWORD": "x", "DB_NAME": "d"})
|
||||
c, _ = SourceConn(v)
|
||||
if c.User != "legacy" || c.Port != 3306 || c.SSLMode != SSLDisabled {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
|
||||
v = resolve(t, config.MapEnv{"DB_SSL_MODE": "sometimes"})
|
||||
if err := New().Configure(context.Background(), v); err == nil {
|
||||
t.Fatal("invalid DB_SSL_MODE accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetConn(t *testing.T) {
|
||||
v := resolve(t, config.MapEnv{"DB_NAME": "a"})
|
||||
if _, err := TargetConn(v); err == nil {
|
||||
t.Fatal("expected missing TARGET_DB_HOST")
|
||||
}
|
||||
v = resolve(t, config.MapEnv{"TARGET_DB_HOST": "t", "TARGET_DB_USERNAME": "tu", "TARGET_DB_PASSWORD": "tp", "TARGET_DB_SSL_MODE": "verify_ca"})
|
||||
c, err := TargetConn(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Host != "t" || c.User != "tu" || c.Password != "tp" || c.Port != 3306 || c.SSLMode != SSLVerifyCA {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuoteOption(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
`plain`: `"plain"`,
|
||||
`a"b`: `"a\"b"`,
|
||||
`back\slash`: `"back\\slash"`,
|
||||
" # spaced ": `" # spaced "`,
|
||||
"new\nline\t": `"new\nline\t"`,
|
||||
`it's $x`: `"it's $x"`,
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := quoteOption(in); got != want {
|
||||
t.Errorf("quoteOption(%q) = %s, want %s", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionFile(t *testing.T) {
|
||||
c := Conn{Host: "db", Port: 3306, User: "root", Password: `p"w#`, SSLMode: SSLDisabled, SSLCA: DefaultCA}
|
||||
got := optionFile(c, false)
|
||||
for _, want := range []string{"[client]\n", "protocol=TCP\n", `host="db"` + "\n", "port=3306\n", `user="root"` + "\n",
|
||||
`password="p\"w#"` + "\n", "ssl-mode=DISABLED\n", "get-server-public-key=1\n"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("mysql option file lacks %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, "ssl-ca") {
|
||||
t.Errorf("CA written without verification:\n%s", got)
|
||||
}
|
||||
got = optionFile(c, true)
|
||||
if !strings.Contains(got, "ssl=0\n") || strings.Contains(got, "ssl-mode") || strings.Contains(got, "get-server-public-key") {
|
||||
t.Errorf("mariadb option file:\n%s", got)
|
||||
}
|
||||
|
||||
c.SSLMode = SSLVerifyIdentity
|
||||
if got := optionFile(c, false); !strings.Contains(got, "ssl-mode=VERIFY_IDENTITY\n") || !strings.Contains(got, `ssl-ca="`+DefaultCA+`"`) {
|
||||
t.Errorf("verify_identity:\n%s", got)
|
||||
}
|
||||
if got := optionFile(c, true); !strings.Contains(got, "ssl=1\nssl-verify-server-cert=1\n") || !strings.Contains(got, "ssl-ca=") {
|
||||
t.Errorf("mariadb verify:\n%s", got)
|
||||
}
|
||||
c.SSLMode = SSLRequired
|
||||
if got := optionFile(c, true); !strings.Contains(got, "ssl=1\nssl-verify-server-cert=0\n") || strings.Contains(got, "ssl-ca=") {
|
||||
t.Errorf("mariadb required:\n%s", got)
|
||||
}
|
||||
|
||||
c.Host = "/run/mysqld/mysqld.sock"
|
||||
if got := optionFile(c, true); !strings.Contains(got, "protocol=SOCKET\n") || !strings.Contains(got, `socket="/run/mysqld/mysqld.sock"`) || strings.Contains(got, "host=") {
|
||||
t.Errorf("socket:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpArgs(t *testing.T) {
|
||||
base := []string{"--single-transaction", "--routines", "--triggers", "--events", "--hex-blob", "--quick"}
|
||||
cases := []struct {
|
||||
name string
|
||||
toolMaria, srvMaria bool
|
||||
spec dumpSpec
|
||||
want []string
|
||||
}{
|
||||
{"oracle-mysql", false, false, dumpSpec{db: "shop"}, append(append([]string{}, base...), "--no-tablespaces", "--set-gtid-purged=OFF", "shop")},
|
||||
{"maria-maria", true, true, dumpSpec{db: "shop"}, append(append([]string{}, base...), "shop")},
|
||||
{"maria-mysql", true, false, dumpSpec{all: true}, append(append([]string{}, base...), "--no-tablespaces", "--all-databases")},
|
||||
{"oracle-maria", false, true, dumpSpec{databases: []string{"a", "b"}}, append(append([]string{}, base...), "--column-statistics=0", "--databases", "a", "b")},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := DumpArgs(c.toolMaria, c.srvMaria, c.spec); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s:\n got %v\nwant %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterDatabases(t *testing.T) {
|
||||
got := filterDatabases([]string{"information_schema", "shop", "mysql", "Performance_Schema", "sys", "blog", "", "tmp\r"}, []string{"tmp"})
|
||||
if !reflect.DeepEqual(got, []string{"shop", "blog"}) {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMariaDB(t *testing.T) {
|
||||
if !IsMariaDB("11.8.3-MariaDB-ubu2404") || IsMariaDB("8.4.6") {
|
||||
t.Fatal("flavour detection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuoteIdent(t *testing.T) {
|
||||
if got := quoteIdent("we`ird"); got != "`we``ird`" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargets(t *testing.T) {
|
||||
e := New()
|
||||
ctx := context.Background()
|
||||
for _, c := range []struct{ storagePath, file, wantRoot, wantFile string }{
|
||||
{"/backup", "/backup/x.sql.gz", "/backup", "x.sql.gz"},
|
||||
{"/backup", "/backup/sub/x.sql.gz", "/backup", "sub/x.sql.gz"},
|
||||
{"/backup", "/data/x.sql", "/data", "x.sql"},
|
||||
{"/backup", "x.sql", "/backup", "x.sql"},
|
||||
} {
|
||||
v := resolve(t, config.MapEnv{"STORAGE_PATH": c.storagePath, "FILE_NAME": c.file, "DB_NAME": "x"})
|
||||
if _, err := e.Targets(ctx, v); err != nil || v.String(config.StoragePath) != c.wantRoot || v.String(config.FileName) != c.wantFile {
|
||||
t.Errorf("%s: %q %q %v", c.file, v.String(config.StoragePath), v.String(config.FileName), err)
|
||||
}
|
||||
}
|
||||
v0 := resolve(t, config.MapEnv{"STORAGE": "Remote", "DB_NAME": "x"})
|
||||
if _, err := e.Targets(ctx, v0); err != nil || v0.String(config.Storage) != "sftp" {
|
||||
t.Fatalf("storage remote: %q %v", v0.String(config.Storage), err)
|
||||
}
|
||||
v := resolve(t, config.MapEnv{"DB_NAME": "shop"})
|
||||
ts, err := e.Targets(ctx, v)
|
||||
if err != nil || len(ts) != 1 || ts[0].Name != "shop" {
|
||||
t.Fatalf("%v %v", ts, err)
|
||||
}
|
||||
v = resolve(t, config.MapEnv{})
|
||||
ts, err = e.Targets(ctx, v)
|
||||
if err != nil || len(ts) != 1 || ts[0].Name != "" {
|
||||
t.Fatalf("no name: %v %v", ts, err)
|
||||
}
|
||||
var ee *cli.ExitError
|
||||
if err := e.Ping(ctx, ts[0]); !errors.As(err, &ee) || ee.Code != cli.ExitConfig || !strings.Contains(err.Error(), "DB_NAME") {
|
||||
t.Fatalf("backup without a database: %v", err)
|
||||
}
|
||||
// restore without a database (all-in-one dumps carry their own USE).
|
||||
v = resolve(t, config.MapEnv{"FILE_NAME": "all_databases_20260101_000000.sql.gz"})
|
||||
if ts, err := e.Targets(ctx, v); err != nil || ts[0].Name != "" {
|
||||
t.Fatalf("%v %v", ts, err)
|
||||
}
|
||||
v = resolve(t, config.MapEnv{"BACKUP_ALL_IN_ONE": "true"})
|
||||
if ts, err := e.Targets(ctx, v); err != nil || ts[0].Name != AllInOneName || restoreDB(ts[0]) != "" {
|
||||
t.Fatalf("%v %v", ts, err)
|
||||
}
|
||||
v = resolve(t, config.MapEnv{"DB_NAME": "-x"})
|
||||
if _, err := e.Targets(ctx, v); err == nil {
|
||||
t.Fatal("option-like database name accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeClients installs shell scripts standing in for the client binaries.
|
||||
// They log their argv and the option file, and answer the queries the engine runs.
|
||||
func fakeClients(t *testing.T, serverVersion string, oracle bool) (dir, logFile string) {
|
||||
t.Helper()
|
||||
dir = t.TempDir()
|
||||
logFile = filepath.Join(dir, "calls.log")
|
||||
script := `#!/bin/sh
|
||||
log="` + logFile + `"
|
||||
echo "ARGV $(basename "$0") $*" >> "$log"
|
||||
case "$1" in --defaults-extra-file=*) f="${1#--defaults-extra-file=}"; echo "MODE $(ls -ln "$f" | cut -c1-10)" >> "$log"; sed 's/^/CNF /' "$f" >> "$log";; esac
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--version) echo "$(basename "$0") Ver 8.4.6 for Linux (MySQL Community Server - GPL)"; exit 0;;
|
||||
"SELECT VERSION()") echo "` + serverVersion + `"; exit 0;;
|
||||
"SHOW DATABASES") printf 'information_schema\nmysql\nshop\nblog\nsys\n'; exit 0;;
|
||||
SELECT\ COUNT*) echo 0; exit 0;;
|
||||
CREATE\ DATABASE*) exit 0;;
|
||||
esac
|
||||
done
|
||||
case "$(basename "$0")" in
|
||||
*dump) echo "-- dump of $*";;
|
||||
*) cat > "` + filepath.Join(dir, "restored.sql") + `";;
|
||||
esac
|
||||
`
|
||||
names := []string{"mariadb", "mariadb-dump"}
|
||||
if oracle {
|
||||
names = append(names, "mysql", "mysqldump")
|
||||
}
|
||||
for _, n := range names {
|
||||
if err := os.WriteFile(filepath.Join(dir, n), []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir, logFile
|
||||
}
|
||||
|
||||
func engineWith(dir string) *Engine {
|
||||
e := New()
|
||||
e.LookPath = func(name string) (string, error) {
|
||||
p := filepath.Join(dir, name)
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func TestDumpAndRestoreUseOptionFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name, version string
|
||||
oracle bool
|
||||
wantDump, wantCL string
|
||||
}{
|
||||
{"mysql", "8.4.6", true, "mysqldump", "mysql"},
|
||||
{"mariadb", "11.8.3-MariaDB", true, "mariadb-dump", "mariadb"},
|
||||
{"mysql-without-oracle-client", "8.4.6", false, "mariadb-dump", "mariadb"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir, logFile := fakeClients(t, tc.version, tc.oracle)
|
||||
tmp := t.TempDir()
|
||||
v := resolve(t, config.MapEnv{"DB_HOST": "db", "DB_USERNAME": "root", "DB_PASSWORD": "s3cr3t pw", "DB_NAME": "shop", "TMP_DIR": tmp})
|
||||
e := engineWith(dir)
|
||||
tg := engine.Target{Name: "shop", Values: v}
|
||||
if err := e.Ping(ctx, tg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := e.ServerVersion(ctx, tg); got != tc.version {
|
||||
t.Fatalf("version %q", got)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := e.Dump(ctx, tg, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(out.String(), "-- dump of --defaults-extra-file=") || !strings.HasSuffix(strings.TrimSpace(out.String()), " shop") {
|
||||
t.Fatalf("dump output %q", out.String())
|
||||
}
|
||||
if err := e.Restore(ctx, tg, strings.NewReader("CREATE TABLE t (id int);\n"), engine.ArtifactInfo{File: "x.sql"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := os.ReadFile(logFile)
|
||||
calls := string(b)
|
||||
if strings.Contains(strings.Join(argvLines(calls), "\n"), "s3cr3t") {
|
||||
t.Fatalf("password in argv:\n%s", calls)
|
||||
}
|
||||
if !strings.Contains(calls, `CNF password="s3cr3t pw"`) {
|
||||
t.Fatalf("password not in option file:\n%s", calls)
|
||||
}
|
||||
if !strings.Contains(calls, "MODE -rw-------") || strings.Contains(calls, "MODE -rw-r") {
|
||||
t.Fatalf("option file mode:\n%s", calls)
|
||||
}
|
||||
for _, l := range argvLines(calls) {
|
||||
f := strings.Fields(l)
|
||||
if len(f) < 3 || !strings.HasPrefix(f[2], "--defaults-extra-file=") {
|
||||
t.Fatalf("--defaults-extra-file is not the first argument: %q", l)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(calls, "ARGV "+tc.wantDump+" ") || !strings.Contains(calls, "ARGV "+tc.wantCL+" --defaults-extra-file=") {
|
||||
t.Fatalf("clients:\n%s", calls)
|
||||
}
|
||||
if !strings.Contains(calls, "--binary-mode shop") {
|
||||
t.Fatalf("restore args:\n%s", calls)
|
||||
}
|
||||
if r, _ := os.ReadFile(filepath.Join(dir, "restored.sql")); string(r) != "CREATE TABLE t (id int);\n" {
|
||||
t.Fatalf("restored %q", r)
|
||||
}
|
||||
if ents, _ := os.ReadDir(tmp); len(ents) != 0 {
|
||||
t.Fatalf("TMP_DIR not cleaned: %v", ents)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func argvLines(log string) []string {
|
||||
var out []string
|
||||
for _, l := range strings.Split(log, "\n") {
|
||||
if strings.HasPrefix(l, "ARGV ") && !strings.Contains(l, "--version") {
|
||||
out = append(out, l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAllDatabasesTargets(t *testing.T) {
|
||||
dir, _ := fakeClients(t, "8.4.6", true)
|
||||
v := resolve(t, config.MapEnv{"DB_HOST": "db", "DB_USERNAME": "root", "BACKUP_ALL_DATABASES": "1", "BACKUP_EXCLUDE_DATABASES": "blog", "TMP_DIR": t.TempDir()})
|
||||
ts, err := engineWith(dir).Targets(context.Background(), v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ts) != 1 || ts[0].Name != "shop" {
|
||||
t.Fatalf("%v", ts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigratePipes(t *testing.T) {
|
||||
dir, logFile := fakeClients(t, "11.8.3-MariaDB", true)
|
||||
v := resolve(t, config.MapEnv{"DB_HOST": "src", "DB_USERNAME": "root", "DB_PASSWORD": "a", "DB_NAME": "shop",
|
||||
"TARGET_DB_HOST": "dst", "TARGET_DB_USERNAME": "root", "TARGET_DB_PASSWORD": "b", "TARGET_DB_NAME": "shop2", "TMP_DIR": t.TempDir()})
|
||||
e := engineWith(dir)
|
||||
pairs, err := e.MigrateTargets(context.Background(), v)
|
||||
if err != nil || len(pairs) != 1 || pairs[0][0].Name != "shop" || pairs[0][1].Name != "shop2" {
|
||||
t.Fatalf("%v %v", pairs, err)
|
||||
}
|
||||
if err := e.Migrate(context.Background(), pairs[0][0], pairs[0][1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, _ := os.ReadFile(filepath.Join(dir, "restored.sql"))
|
||||
if !strings.HasPrefix(string(r), "-- dump of --defaults-extra-file=") {
|
||||
t.Fatalf("restored %q", r)
|
||||
}
|
||||
b, _ := os.ReadFile(logFile)
|
||||
if !strings.Contains(string(b), `CNF host="dst"`) || !strings.Contains(string(b), "--binary-mode shop2") {
|
||||
t.Fatalf("calls:\n%s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifiesBeforeRestore(t *testing.T) {
|
||||
// MySQL cannot roll back DDL: a damaged dump must be caught first.
|
||||
if !New().VerifyBeforeRestore(engine.Target{}, engine.ArtifactInfo{}) {
|
||||
t.Fatal("restores must verify the dump first")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// quoteOption quotes a value for a MySQL/MariaDB option file. Both client
|
||||
// families parse option files with the same mysys code: a value wrapped in
|
||||
// double quotes keeps leading/trailing spaces and '#', and the escapes
|
||||
// \\ \" \n \r \t are decoded inside it.
|
||||
func quoteOption(s string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(s) + 2)
|
||||
b.WriteByte('"')
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '\\':
|
||||
b.WriteString(`\\`)
|
||||
case '"':
|
||||
b.WriteString(`\"`)
|
||||
case '\n':
|
||||
b.WriteString(`\n`)
|
||||
case '\r':
|
||||
b.WriteString(`\r`)
|
||||
case '\t':
|
||||
b.WriteString(`\t`)
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
b.WriteByte('"')
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// optionFile returns the [client] option file for c. mariadbClient selects
|
||||
// the MariaDB client option names (ssl, ssl-verify-server-cert) instead of
|
||||
// the Oracle MySQL ones (ssl-mode, get-server-public-key).
|
||||
//
|
||||
// The file holds the password; it is written 0600 in a private directory and
|
||||
// passed as --defaults-extra-file, never on the command line.
|
||||
func optionFile(c Conn, mariadbClient bool) string {
|
||||
var b strings.Builder
|
||||
line := func(k, v string) {
|
||||
b.WriteString(k)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(v)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
b.WriteString("[client]\n")
|
||||
if strings.HasPrefix(c.Host, "/") {
|
||||
line("protocol", "SOCKET")
|
||||
line("socket", quoteOption(c.Host))
|
||||
} else {
|
||||
// TCP even for "localhost" (a client would otherwise look for a socket).
|
||||
line("protocol", "TCP")
|
||||
line("host", quoteOption(c.Host))
|
||||
line("port", strconv.Itoa(c.Port))
|
||||
}
|
||||
line("user", quoteOption(c.User))
|
||||
line("password", quoteOption(c.Password))
|
||||
line("max-allowed-packet", "1G")
|
||||
|
||||
verify := c.SSLMode == SSLVerifyCA || c.SSLMode == SSLVerifyIdentity
|
||||
if mariadbClient {
|
||||
switch c.SSLMode {
|
||||
case SSLDisabled:
|
||||
line("ssl", "0")
|
||||
case SSLPreferred:
|
||||
// Leave TLS to the client default, without certificate checks.
|
||||
line("ssl-verify-server-cert", "0")
|
||||
case SSLRequired:
|
||||
line("ssl", "1")
|
||||
line("ssl-verify-server-cert", "0")
|
||||
default: // verify_ca, verify_identity: MariaDB always checks the host name too
|
||||
line("ssl", "1")
|
||||
line("ssl-verify-server-cert", "1")
|
||||
}
|
||||
} else {
|
||||
line("ssl-mode", strings.ToUpper(c.SSLMode))
|
||||
if c.SSLMode != SSLVerifyIdentity && c.SSLMode != SSLVerifyCA {
|
||||
// caching_sha2_password over a non-TLS connection needs the
|
||||
// server's RSA key (what the MariaDB client does by itself).
|
||||
line("get-server-public-key", "1")
|
||||
}
|
||||
}
|
||||
if c.SSLMode != SSLDisabled {
|
||||
// The CA is only used when certificates are verified: with the MySQL
|
||||
// client, a CA would otherwise silently turn "required" into "verify_ca".
|
||||
if verify && c.SSLCA != "" {
|
||||
line("ssl-ca", quoteOption(c.SSLCA))
|
||||
}
|
||||
if c.SSLCert != "" {
|
||||
line("ssl-cert", quoteOption(c.SSLCert))
|
||||
}
|
||||
if c.SSLKey != "" {
|
||||
line("ssl-key", quoteOption(c.SSLKey))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in new issue
Block a user