feat: add backup all databases separately

cmd/backup.go

@@ -51,5 +51,6 @@ func init() {
 	BackupCmd.PersistentFlags().StringP("config", "c", "", "Configuration file for multi database backup. (e.g: `/backup/config.yaml`)")
 	BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression")
 	BackupCmd.PersistentFlags().BoolP("all", "a", false, "Backup all databases")
+	BackupCmd.PersistentFlags().BoolP("single-file", "", false, "Backup all databases in a single file")
 
 }

pkg/azure.go

@@ -39,7 +39,7 @@ func azureBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Azure Blob Storage")
 
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return

pkg/backup.go

@@ -26,6 +26,7 @@ SOFTWARE.
 package pkg
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/jkaninda/encryptor"
 	"github.com/jkaninda/go-storage/pkg/local"
@@ -36,6 +37,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 )
 
@@ -48,7 +50,7 @@ func StartBackup(cmd *cobra.Command) {
 	if err != nil {
 		dbConf = initDbConfig(cmd)
 		if config.cronExpression == "" {
-			BackupTask(dbConf, config)
+			createBackupTask(dbConf, config)
 		} else {
 			if utils.IsValidCronExpression(config.cronExpression) {
 				scheduledMode(dbConf, config)
@@ -82,7 +84,7 @@ func scheduledMode(db *dbConfig, config *BackupConfig) {
 	c := cron.New()
 
 	_, err = c.AddFunc(config.cronExpression, func() {
-		BackupTask(db, config)
+		createBackupTask(db, config)
 		utils.Info("Next backup time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat))
 
 	})
@@ -104,16 +106,41 @@ func multiBackupTask(databases []Database, bkConfig *BackupConfig) {
 		if db.Path != "" {
 			bkConfig.remotePath = db.Path
 		}
-		BackupTask(getDatabase(db), bkConfig)
+		createBackupTask(getDatabase(db), bkConfig)
 	}
 }
 
-// BackupTask backups database
-func BackupTask(db *dbConfig, config *BackupConfig) {
+// createBackupTask backup task
+func createBackupTask(db *dbConfig, config *BackupConfig) {
+	if config.all && !config.singleFile {
+		backupAll(db, config)
+	} else {
+		backupTask(db, config)
+	}
+}
+
+// backupAll backup all databases
+func backupAll(db *dbConfig, config *BackupConfig) {
+	databases, err := listDatabases(*db)
+	if err != nil {
+		utils.Fatal("Error listing databases: %s", err)
+	}
+	for _, dbName := range databases {
+		if dbName == "information_schema" || dbName == "performance_schema" || dbName == "mysql" || dbName == "sys" || dbName == "innodb" || dbName == "Database" {
+			continue
+		}
+		db.dbName = dbName
+		config.backupFileName = fmt.Sprintf("%s_%s.sql.gz", dbName, time.Now().Format("20060102_150405"))
+		backupTask(db, config)
+	}
+
+}
+
+func backupTask(db *dbConfig, config *BackupConfig) {
 	utils.Info("Starting backup task...")
 	startTime = time.Now()
 	prefix := db.dbName
-	if config.all {
+	if config.all && config.singleFile {
 		prefix = "all_databases"
 	}
 	// Generate file name
@@ -199,7 +226,7 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) {
 }
 
 // BackupDatabase backup database
-func BackupDatabase(db *dbConfig, backupFileName string, disableCompression, all bool) error {
+func BackupDatabase(db *dbConfig, backupFileName string, disableCompression, all, singleFile bool) error {
 	storagePath = os.Getenv("STORAGE_PATH")
 	utils.Info("Starting database backup...")
 
@@ -208,7 +235,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression, all
 	}
 
 	dumpArgs := []string{fmt.Sprintf("--defaults-file=%s", mysqlClientConfig)}
-	if all {
+	if all && singleFile {
 		dumpArgs = append(dumpArgs, "--all-databases", "--single-transaction", "--routines", "--triggers")
 	} else {
 		dumpArgs = append(dumpArgs, db.dbName)
@@ -267,7 +294,7 @@ func runCommandWithCompression(command string, args []string, outputPath string)
 }
 func localBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to local storage")
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return
@@ -346,6 +373,31 @@ func encryptBackup(config *BackupConfig) {
 	}
 
 }
+
+// listDatabases list all databases
+func listDatabases(db dbConfig) ([]string, error) {
+	databases := []string{}
+	// Create the mysql client config file
+	if err := createMysqlClientConfigFile(db); err != nil {
+		return databases, fmt.Errorf(err.Error())
+	}
+	utils.Info("Listing databases...")
+	// Step 1: List all databases
+	cmd := exec.Command("mariadb", fmt.Sprintf("--defaults-file=%s", mysqlClientConfig), "-e", "SHOW DATABASES;")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return databases, fmt.Errorf("failed to list databases: %s", err)
+	}
+	// Step 2: Parse the output
+	for _, _db := range strings.Split(out.String(), "\n") {
+		if _db != "" {
+			databases = append(databases, _db)
+		}
+	}
+	return databases, nil
+}
 func recoverMode(err error, msg string) {
 	if err != nil {
 		if backupRescueMode {

pkg/config.go

@@ -78,6 +78,7 @@ type BackupConfig struct {
 	storage            string
 	cronExpression     string
 	all                bool
+	singleFile         bool
 }
 type FTPConfig struct {
 	host       string
@@ -258,6 +259,7 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig {
 	}
 	disableCompression, _ = cmd.Flags().GetBool("disable-compression")
 	all, _ := cmd.Flags().GetBool("all")
+	singleFile, _ := cmd.Flags().GetBool("single-file")
 	_, _ = cmd.Flags().GetString("mode")
 	passphrase := os.Getenv("GPG_PASSPHRASE")
 	_ = utils.GetEnv(cmd, "path", "AWS_S3_PATH")
@@ -284,6 +286,7 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig {
 	config.usingKey = usingKey
 	config.cronExpression = cronExpression
 	config.all = all
+	config.singleFile = singleFile
 	return &config
 }
 

pkg/helper.go

@@ -195,8 +195,8 @@ func createMysqlClientConfigFile(db dbConfig) error {
 	sslMode := goutils.GetStringEnvWithDefault("DB_SSL_MODE", "0")
 	// Create the mysql client config file
 	mysqlClientConfigFile := filepath.Join(tmpPath, "my.cnf")
-	mysqlClientConfig := fmt.Sprintf("[client]\nhost=%s\nport=%s\nuser=%s\npassword=%s\nssl-ca=%s\nssl=%s\n", db.dbHost, db.dbPort, db.dbUserName, db.dbPassword, caCertPath, sslMode)
-	if err := os.WriteFile(mysqlClientConfigFile, []byte(mysqlClientConfig), 0644); err != nil {
+	mysqlCl := fmt.Sprintf("[client]\nhost=%s\nport=%s\nuser=%s\npassword=%s\nssl-ca=%s\nssl=%s\n", db.dbHost, db.dbPort, db.dbUserName, db.dbPassword, caCertPath, sslMode)
+	if err := os.WriteFile(mysqlClientConfigFile, []byte(mysqlCl), 0644); err != nil {
 		return fmt.Errorf("failed to create mysql client config file: %v", err)
 	}
 	return nil

pkg/migrate.go

@@ -51,7 +51,7 @@ func StartMigration(cmd *cobra.Command) {
 	conf := &RestoreConfig{}
 	conf.file = backupFileName
 	// Backup source Database
-	err := BackupDatabase(dbConf, backupFileName, true, false)
+	err := BackupDatabase(dbConf, backupFileName, true, false, false)
 	if err != nil {
 		utils.Fatal("Error backing up database: %s", err)
 	}

pkg/remote.go

@@ -39,7 +39,7 @@ import (
 func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Remote server")
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return
@@ -160,7 +160,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to the remote FTP server")
 
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return

pkg/s3.go

@@ -39,7 +39,7 @@ func s3Backup(db *dbConfig, config *BackupConfig) {
 
 	utils.Info("Backup database to s3 storage")
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return