feat: add multiple backup rescued mode for scheduled mode

pkg/azure.go

@@ -39,7 +39,11 @@ func azureBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Azure Blob Storage")
 
 	// Backup database
-	BackupDatabase(db, config.backupFileName, disableCompression)
+	err := BackupDatabase(db, config.backupFileName, disableCompression)
+	if err != nil {
+		recoverMode(err, "Error backing up database")
+		return
+	}
 	finalFileName := config.backupFileName
 	if config.encryption {
 		encryptBackup(config)

pkg/backup.go

@@ -72,13 +72,17 @@ func scheduledMode(db *dbConfig, config *BackupConfig) {
 
 	// Test backup
 	utils.Info("Testing backup configurations...")
-	testDatabaseConnection(db)
+	err := testDatabaseConnection(db)
+	if err != nil {
+		utils.Error("Error connecting to database: %s", db.dbName)
+		utils.Fatal("Error: %s", err)
+	}
 	utils.Info("Testing backup configurations...done")
 	utils.Info("Creating backup job...")
 	// Create a new cron instance
 	c := cron.New()
 
-	_, err := c.AddFunc(config.cronExpression, func() {
+	_, err = c.AddFunc(config.cronExpression, func() {
 		BackupTask(db, config)
 		utils.Info("Next backup time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat))
 
@@ -147,6 +151,7 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) {
 	if bkConfig.cronExpression == "" {
 		multiBackupTask(conf.Databases, bkConfig)
 	} else {
+		backupRescueMode = conf.BackupRescueMode
 		// Check if cronExpression is valid
 		if utils.IsValidCronExpression(bkConfig.cronExpression) {
 			utils.Info("Running backup in Scheduled mode")
@@ -157,7 +162,11 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) {
 			// Test backup
 			utils.Info("Testing backup configurations...")
 			for _, db := range conf.Databases {
-				testDatabaseConnection(getDatabase(db))
+				err = testDatabaseConnection(getDatabase(db))
+				if err != nil {
+					recoverMode(err, fmt.Sprintf("Error connecting to database: %s", db.Name))
+					continue
+				}
 			}
 			utils.Info("Testing backup configurations...done")
 			utils.Info("Creating backup job...")
@@ -187,16 +196,19 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) {
 }
 
 // BackupDatabase backup database
-func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) {
+func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) error {
 	storagePath = os.Getenv("STORAGE_PATH")
 
 	utils.Info("Starting database backup...")
 
 	err := os.Setenv("MYSQL_PWD", db.dbPassword)
 	if err != nil {
-		return
+		return fmt.Errorf("failed to set MYSQL_PWD environment variable: %v", err)
+	}
+	err = testDatabaseConnection(db)
+	if err != nil {
+		return fmt.Errorf(err.Error())
 	}
-	testDatabaseConnection(db)
 	// Backup Database database
 	utils.Info("Backing up database...")
 
@@ -211,24 +223,24 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 		)
 		output, err := cmd.Output()
 		if err != nil {
-			utils.Fatal(err.Error())
+			return fmt.Errorf("failed to backup database: %v", err)
 		}
 
 		// save output
 		file, err := os.Create(filepath.Join(tmpPath, backupFileName))
 		if err != nil {
-			utils.Fatal(err.Error())
+			return fmt.Errorf("failed to create backup file: %v", err)
 		}
 		defer func(file *os.File) {
 			err := file.Close()
 			if err != nil {
-				utils.Fatal(err.Error())
+				return
 			}
 		}(file)
 
 		_, err = file.Write(output)
 		if err != nil {
-			utils.Fatal(err.Error())
+			return err
 		}
 		utils.Info("Database has been backed up")
 
@@ -237,14 +249,14 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 		cmd := exec.Command("mysqldump", "-h", db.dbHost, "-P", db.dbPort, "-u", db.dbUserName, db.dbName)
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
-			log.Fatal(err)
+			return fmt.Errorf("failed to backup database: %v", err)
 		}
 		gzipCmd := exec.Command("gzip")
 		gzipCmd.Stdin = stdout
 		gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName))
 		err = gzipCmd.Start()
 		if err != nil {
-			return
+			return fmt.Errorf("failed to backup database: %v", err)
 		}
 		if err := cmd.Run(); err != nil {
 			log.Fatal(err)
@@ -252,13 +264,18 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 		if err := gzipCmd.Wait(); err != nil {
 			log.Fatal(err)
 		}
-		utils.Info("Database has been backed up")
 
 	}
+	utils.Info("Database has been backed up")
+	return nil
 }
 func localBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to local storage")
-	BackupDatabase(db, config.backupFileName, disableCompression)
+	err := BackupDatabase(db, config.backupFileName, disableCompression)
+	if err != nil {
+		recoverMode(err, "Error backing up database")
+		return
+	}
 	finalFileName := config.backupFileName
 	if config.encryption {
 		encryptBackup(config)
@@ -333,3 +350,17 @@ func encryptBackup(config *BackupConfig) {
 	}
 
 }
+func recoverMode(err error, msg string) {
+	if err != nil {
+		if backupRescueMode {
+			utils.NotifyError(fmt.Sprintf("%s : %v", msg, err))
+			utils.Error(msg)
+			utils.Error("Backup rescue mode is enabled")
+			utils.Error("Backup will continue")
+		} else {
+			utils.Error(msg)
+			utils.Fatal("Error: %v", err)
+		}
+	}
+
+}

pkg/config.go

@@ -42,8 +42,9 @@ type Database struct {
 	Path     string `yaml:"path"`
 }
 type Config struct {
-	Databases      []Database `yaml:"databases"`
 	CronExpression   string     `yaml:"cronExpression"`
+	BackupRescueMode bool       `yaml:"backupRescueMode"`
+	Databases        []Database `yaml:"databases"`
 }
 
 type dbConfig struct {

pkg/helper.go

@@ -66,10 +66,10 @@ func deleteTemp() {
 }
 
 // TestDatabaseConnection  tests the database connection
-func testDatabaseConnection(db *dbConfig) {
+func testDatabaseConnection(db *dbConfig) error {
 	err := os.Setenv("MYSQL_PWD", db.dbPassword)
 	if err != nil {
-		return
+		return fmt.Errorf("failed to set MYSQL_PWD environment variable: %v", err)
 	}
 	utils.Info("Connecting to %s database ...", db.dbName)
 	// Set database name for notification error
@@ -81,11 +81,11 @@ func testDatabaseConnection(db *dbConfig) {
 	cmd.Stderr = &out
 	err = cmd.Run()
 	if err != nil {
-		utils.Fatal("Error testing database connection: %v\nOutput: %s", err, out.String())
+		return fmt.Errorf("failed to connect to %s database: %v", db.dbName, err)
 
 	}
 	utils.Info("Successfully connected to %s database", db.dbName)
-
+	return nil
 }
 
 // checkPubKeyFile checks gpg public key

pkg/remote.go

@@ -39,7 +39,11 @@ import (
 func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Remote server")
 	// Backup database
-	BackupDatabase(db, config.backupFileName, disableCompression)
+	err := BackupDatabase(db, config.backupFileName, disableCompression)
+	if err != nil {
+		recoverMode(err, "Error backing up database")
+		return
+	}
 	finalFileName := config.backupFileName
 	if config.encryption {
 		encryptBackup(config)
@@ -156,7 +160,11 @@ func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to the remote FTP server")
 
 	// Backup database
-	BackupDatabase(db, config.backupFileName, disableCompression)
+	err := BackupDatabase(db, config.backupFileName, disableCompression)
+	if err != nil {
+		recoverMode(err, "Error backing up database")
+		return
+	}
 	finalFileName := config.backupFileName
 	if config.encryption {
 		encryptBackup(config)

pkg/s3.go

@@ -39,7 +39,11 @@ func s3Backup(db *dbConfig, config *BackupConfig) {
 
 	utils.Info("Backup database to s3 storage")
 	// Backup database
-	BackupDatabase(db, config.backupFileName, disableCompression)
+	err := BackupDatabase(db, config.backupFileName, disableCompression)
+	if err != nil {
+		recoverMode(err, "Error backing up database")
+		return
+	}
 	finalFileName := config.backupFileName
 	if config.encryption {
 		encryptBackup(config)

pkg/var.go

@@ -42,6 +42,7 @@ var (
 	usingKey                 = false
 	backupSize         int64 = 0
 	startTime                = time.Now()
+	backupRescueMode         = false
 )
 
 // dbHVars Required environment variables for database

templates/email-error.tmpl

@@ -60,10 +60,10 @@
 
     <p>We recommend investigating the issue as soon as possible to prevent potential data loss or service disruptions.</p>
 
-    <p>For more information, visit the <a href="https://jkaninda.github.io/pg-bkup">pg-bkup documentation</a>.</p>
+    <p>For more information, visit the <a href="https://jkaninda.github.io/mysql-bkup">mysql-bkup documentation</a>.</p>
 
     <footer>
-        &copy; 2024 <a href="https://github.com/jkaninda/pg-bkup">pg-bkup</a> | Automated Backup System
+        &copy; 2024 <a href="https://github.com/jkaninda/mysql-bkup">mysql-bkup</a> | Automated Backup System
     </footer>
 </body>
 </html>