From cbb73ae89b8395d6637c6b88d44b4503f2891bc0 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Sat, 28 Sep 2024 07:26:33 +0200 Subject: [PATCH] chore: migrate backup scheduled mode from linux cron to go cron --- cmd/backup.go | 5 +- pkg/backup.go | 130 ++++++++++++++----------------------------------- pkg/config.go | 85 +++++++++++++++++++++++++++++++- pkg/helper.go | 4 ++ pkg/migrate.go | 4 +- pkg/restore.go | 25 +++------- pkg/var.go | 1 - 7 files changed, 137 insertions(+), 117 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 0eeff19..e1647b7 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -29,8 +29,9 @@ func init() { //Backup BackupCmd.PersistentFlags().StringP("storage", "s", "local", "Storage. local or s3") BackupCmd.PersistentFlags().StringP("path", "P", "", "AWS S3 path without file name. eg: /custom_path or ssh remote path `/home/foo/backup`") - BackupCmd.PersistentFlags().StringP("mode", "m", "default", "Execution mode. default or scheduled") - BackupCmd.PersistentFlags().StringP("period", "", "", "Schedule period time") + BackupCmd.PersistentFlags().StringP("mode", "m", "default", "Execution mode. | Deprecated") + BackupCmd.PersistentFlags().StringP("period", "", "", "Schedule period time | Deprecated") + BackupCmd.PersistentFlags().StringP("cron-expression", "", "", "Backup cron expression") BackupCmd.PersistentFlags().BoolP("prune", "", false, "Delete old backup, default disabled") BackupCmd.PersistentFlags().IntP("keep-last", "", 7, "Delete files created more than specified days ago, default 7 days") BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") diff --git a/pkg/backup.go b/pkg/backup.go index 96fe23f..f7bb3c0 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -8,126 +8,70 @@ package pkg import ( "fmt" - "github.com/hpcloud/tail" "github.com/jkaninda/mysql-bkup/utils" + "github.com/robfig/cron/v3" "github.com/spf13/cobra" "log" "os" "os/exec" "path/filepath" - "time" ) func StartBackup(cmd *cobra.Command) { intro() - //Set env - utils.SetEnv("STORAGE_PATH", storagePath) - utils.GetEnv(cmd, "period", "BACKUP_CRON_EXPRESSION") - - //Get flag value and set env - remotePath := utils.GetEnv(cmd, "path", "SSH_REMOTE_PATH") - storage = utils.GetEnv(cmd, "storage", "STORAGE") - file = utils.GetEnv(cmd, "file", "FILE_NAME") - backupRetention, _ := cmd.Flags().GetInt("keep-last") - prune, _ := cmd.Flags().GetBool("prune") - disableCompression, _ = cmd.Flags().GetBool("disable-compression") - executionMode, _ = cmd.Flags().GetString("mode") - gpqPassphrase := os.Getenv("GPG_PASSPHRASE") - _ = utils.GetEnv(cmd, "path", "AWS_S3_PATH") - cronExpression := os.Getenv("BACKUP_CRON_EXPRESSION") - - dbConf = getDbConfig(cmd) - - // - if gpqPassphrase != "" { - encryption = true - } - - //Generate file name - backupFileName := fmt.Sprintf("%s_%s.sql.gz", dbConf.dbName, time.Now().Format("20060102_150405")) - if disableCompression { - backupFileName = fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20060102_150405")) - } - - if cronExpression == "" { - switch storage { - case "s3": - s3Backup(dbConf, backupFileName, disableCompression, prune, backupRetention, encryption) - case "local": - localBackup(dbConf, backupFileName, disableCompression, prune, backupRetention, encryption) - case "ssh", "remote": - sshBackup(dbConf, backupFileName, remotePath, disableCompression, prune, backupRetention, encryption) - case "ftp": - utils.Fatal("Not supported storage type: %s", storage) - default: - localBackup(dbConf, backupFileName, disableCompression, prune, backupRetention, encryption) - } + dbConf = initDbConfig(cmd) + //Initialize backup configs + config := initBackupConfig(cmd) + if config.cronExpression == "" { + BackupTask(dbConf, config) } else { - if utils.IsValidCronExpression(cronExpression) { - scheduledMode(dbConf, storage) + if utils.IsValidCronExpression(config.cronExpression) { + scheduledMode(dbConf, config) } else { - utils.Fatal("Cron expression is not valid: %s", cronExpression) + utils.Fatal("Cron expression is not valid: %s", config.cronExpression) } } } // Run in scheduled mode -func scheduledMode(db *dbConfig, storage string) { - - fmt.Println() - fmt.Println("**********************************") - fmt.Println(" Starting MySQL Bkup... ") - fmt.Println("***********************************") +func scheduledMode(db *dbConfig, config *BackupConfig) { utils.Info("Running in Scheduled mode") - utils.Info("Execution period %s", os.Getenv("BACKUP_CRON_EXPRESSION")) + utils.Info("Backup cron expression: %s", os.Getenv("BACKUP_CRON_EXPRESSION")) utils.Info("Storage type %s ", storage) //Test database connexion testDatabaseConnection(db) - utils.Info("Creating backup job...") - CreateCrontabScript(disableCompression, storage) + utils.Info("Creating a new cron instance...") + // Create a new cron instance + c := cron.New() - //Set BACKUP_CRON_EXPRESSION to nil - err := os.Setenv("BACKUP_CRON_EXPRESSION", "") - if err != nil { - return - } - - supervisorConfig := "/etc/supervisor/supervisord.conf" - - // Start Supervisor - cmd := exec.Command("supervisord", "-c", supervisorConfig) - err = cmd.Start() - if err != nil { - utils.Fatal(fmt.Sprintf("Failed to start supervisord: %v", err)) - } - utils.Info("Backup job started") - defer func() { - if err := cmd.Process.Kill(); err != nil { - utils.Info("Failed to kill supervisord process: %v", err) - } else { - utils.Info("Supervisor stopped.") - } - }() - if _, err := os.Stat(cronLogFile); os.IsNotExist(err) { - utils.Fatal(fmt.Sprintf("Log file %s does not exist.", cronLogFile)) - } - t, err := tail.TailFile(cronLogFile, tail.Config{Follow: true}) - if err != nil { - utils.Fatal("Failed to tail file: %v", err) - } - - // Read and print new lines from the log file - for line := range t.Lines { - fmt.Println(line.Text) - } + // Add a cron job that runs every 10 seconds + c.AddFunc(config.cronExpression, func() { + BackupTask(db, config) + }) + // Start the cron scheduler + c.Start() + utils.Info("Creating a new cron instance...done") + defer c.Stop() + select {} } -func intro() { - utils.Info("Starting MySQL Backup...") - utils.Info("Copyright © 2024 Jonas Kaninda ") +func BackupTask(db *dbConfig, config *BackupConfig) { + utils.Info("Starting backup task...") + switch config.storage { + case "s3": + s3Backup(db, config.backupFileName, config.disableCompression, config.prune, config.backupRetention, config.encryption) + case "local": + localBackup(db, config.backupFileName, config.disableCompression, config.prune, config.backupRetention, config.encryption) + case "ssh", "remote": + sshBackup(db, config.backupFileName, config.remotePath, config.disableCompression, config.prune, config.backupRetention, config.encryption) + case "ftp": + utils.Fatal("Not supported storage type: %s", config.storage) + default: + localBackup(db, config.backupFileName, config.disableCompression, config.prune, config.backupRetention, config.encryption) + } } // BackupDatabase backup database diff --git a/pkg/config.go b/pkg/config.go index af0f10f..5c109ab 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -7,9 +7,11 @@ package pkg import ( + "fmt" "github.com/jkaninda/mysql-bkup/utils" "github.com/spf13/cobra" "os" + "time" ) type Config struct { @@ -30,7 +32,27 @@ type targetDbConfig struct { targetDbName string } -func getDbConfig(cmd *cobra.Command) *dbConfig { +type BackupConfig struct { + backupFileName string + backupRetention int + disableCompression bool + prune bool + encryption bool + remotePath string + gpqPassphrase string + storage string + cronExpression string +} +type RestoreConfig struct { + s3Path string + remotePath string + storage string + file string + bucket string + gpqPassphrase string +} + +func initDbConfig(cmd *cobra.Command) *dbConfig { //Set env utils.GetEnv(cmd, "dbname", "DB_NAME") dConf := dbConfig{} @@ -47,7 +69,66 @@ func getDbConfig(cmd *cobra.Command) *dbConfig { } return &dConf } -func getTargetDbConfig() *targetDbConfig { +func initBackupConfig(cmd *cobra.Command) *BackupConfig { + utils.SetEnv("STORAGE_PATH", storagePath) + utils.GetEnv(cmd, "period", "BACKUP_CRON_EXPRESSION") + + //Get flag value and set env + remotePath := utils.GetEnv(cmd, "path", "SSH_REMOTE_PATH") + storage = utils.GetEnv(cmd, "storage", "STORAGE") + backupRetention, _ := cmd.Flags().GetInt("keep-last") + prune, _ := cmd.Flags().GetBool("prune") + disableCompression, _ = cmd.Flags().GetBool("disable-compression") + _, _ = cmd.Flags().GetString("mode") + gpqPassphrase := os.Getenv("GPG_PASSPHRASE") + _ = utils.GetEnv(cmd, "path", "AWS_S3_PATH") + cronExpression := os.Getenv("BACKUP_CRON_EXPRESSION") + + if gpqPassphrase != "" { + encryption = true + } + //Generate file name + backupFileName := fmt.Sprintf("%s_%s.sql.gz", dbConf.dbName, time.Now().Format("20240102_150405")) + if disableCompression { + backupFileName = fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20240102_150405")) + } + + //Initialize backup configs + config := BackupConfig{} + config.backupFileName = backupFileName + config.backupRetention = backupRetention + config.disableCompression = disableCompression + config.prune = prune + config.storage = storage + config.encryption = encryption + config.remotePath = remotePath + config.gpqPassphrase = gpqPassphrase + config.cronExpression = cronExpression + return &config +} +func initRestoreConfig(cmd *cobra.Command) *RestoreConfig { + utils.SetEnv("STORAGE_PATH", storagePath) + + //Get flag value and set env + s3Path := utils.GetEnv(cmd, "path", "AWS_S3_PATH") + remotePath := utils.GetEnv(cmd, "path", "SSH_REMOTE_PATH") + storage = utils.GetEnv(cmd, "storage", "STORAGE") + file = utils.GetEnv(cmd, "file", "FILE_NAME") + _, _ = cmd.Flags().GetString("mode") + bucket := utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME") + gpqPassphrase := os.Getenv("GPG_PASSPHRASE") + //Initialize restore configs + rConfig := RestoreConfig{} + rConfig.s3Path = s3Path + rConfig.remotePath = remotePath + rConfig.storage = storage + rConfig.bucket = bucket + rConfig.file = file + rConfig.storage = storage + rConfig.gpqPassphrase = gpqPassphrase + return &rConfig +} +func initTargetDbConfig() *targetDbConfig { tdbConfig := targetDbConfig{} tdbConfig.targetDbHost = os.Getenv("TARGET_DB_HOST") tdbConfig.targetDbPort = os.Getenv("TARGET_DB_PORT") diff --git a/pkg/helper.go b/pkg/helper.go index 425b439..dcf3ecc 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -125,3 +125,7 @@ func testDatabaseConnection(db *dbConfig) { utils.Info("Successfully connected to %s database", db.dbName) } +func intro() { + utils.Info("Starting MySQL Backup...") + utils.Info("Copyright © 2024 Jonas Kaninda ") +} diff --git a/pkg/migrate.go b/pkg/migrate.go index 27e705f..7f52148 100644 --- a/pkg/migrate.go +++ b/pkg/migrate.go @@ -17,8 +17,8 @@ func StartMigration(cmd *cobra.Command) { intro() utils.Info("Starting database migration...") //Get DB config - dbConf = getDbConfig(cmd) - targetDbConf = getTargetDbConfig() + dbConf = initDbConfig(cmd) + targetDbConf = initTargetDbConfig() //Defining the target database variables newDbConfig := dbConfig{} diff --git a/pkg/restore.go b/pkg/restore.go index 16bd321..2a07529 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -17,33 +17,24 @@ import ( func StartRestore(cmd *cobra.Command) { intro() - //Set env - utils.SetEnv("STORAGE_PATH", storagePath) - - //Get flag value and set env - s3Path := utils.GetEnv(cmd, "path", "AWS_S3_PATH") - remotePath := utils.GetEnv(cmd, "path", "SSH_REMOTE_PATH") - storage = utils.GetEnv(cmd, "storage", "STORAGE") - file = utils.GetEnv(cmd, "file", "FILE_NAME") - executionMode, _ = cmd.Flags().GetString("mode") - bucket := utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME") - dbConf = getDbConfig(cmd) + dbConf = initDbConfig(cmd) + restoreConf := initRestoreConfig(cmd) switch storage { case "s3": - restoreFromS3(dbConf, file, bucket, s3Path) + restoreFromS3(dbConf, restoreConf.file, restoreConf.bucket, restoreConf.s3Path) case "local": utils.Info("Restore database from local") - copyToTmp(storagePath, file) - RestoreDatabase(dbConf, file) + copyToTmp(storagePath, restoreConf.file) + RestoreDatabase(dbConf, restoreConf.file) case "ssh": - restoreFromRemote(dbConf, file, remotePath) + restoreFromRemote(dbConf, restoreConf.file, restoreConf.remotePath) case "ftp": utils.Fatal("Restore from FTP is not yet supported") default: utils.Info("Restore database from local") - copyToTmp(storagePath, file) - RestoreDatabase(dbConf, file) + copyToTmp(storagePath, restoreConf.file) + RestoreDatabase(dbConf, restoreConf.file) } } diff --git a/pkg/var.go b/pkg/var.go index 837c8ec..fb9df35 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -16,7 +16,6 @@ const gpgExtension = "gpg" var ( storage = "local" file = "" - executionMode = "default" storagePath = "/backup" disableCompression = false encryption = false