From 1df1c46a964aa15f70b0c5268d5a0c37d9e50424 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 9 Oct 2024 08:32:51 +0200 Subject: [PATCH 1/3] feat: add multi backup --- go.mod | 1 + pkg/backup.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++----- pkg/config.go | 28 +++++++++++++++++++ pkg/helper.go | 44 +++++++++++++++++++++++++++++ pkg/var.go | 1 + 5 files changed, 144 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 8ae07ba..876dd6d 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.8.0 golang.org/x/crypto v0.28.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/pkg/backup.go b/pkg/backup.go index abca1ac..bb417f5 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -20,17 +20,22 @@ import ( func StartBackup(cmd *cobra.Command) { intro() - dbConf = initDbConfig(cmd) //Initialize backup configs config := initBackupConfig(cmd) - - if config.cronExpression == "" { - BackupTask(dbConf, config) + //Load backup configuration file + configFile, err := loadConfigFile() + if err == nil { + startMultiBackup(config, configFile) } else { - if utils.IsValidCronExpression(config.cronExpression) { - scheduledMode(dbConf, config) + dbConf = initDbConfig(cmd) + if config.cronExpression == "" { + BackupTask(dbConf, config) } else { - utils.Fatal("Cron expression is not valid: %s", config.cronExpression) + if utils.IsValidCronExpression(config.cronExpression) { + scheduledMode(dbConf, config) + } else { + utils.Fatal("Cron expression is not valid: %s", config.cronExpression) + } } } @@ -63,6 +68,15 @@ func scheduledMode(db *dbConfig, config *BackupConfig) { defer c.Stop() select {} } +func multiBackupTask(databases []Database, bkConfig *BackupConfig) { + for _, db := range databases { + //Check if path is defined in config file + if db.Path != "" { + bkConfig.remotePath = db.Path + } + BackupTask(getDatabase(db), bkConfig) + } +} func BackupTask(db *dbConfig, config *BackupConfig) { utils.Info("Starting backup task...") //Generate file name @@ -85,6 +99,55 @@ func BackupTask(db *dbConfig, config *BackupConfig) { localBackup(db, config) } } +func startMultiBackup(bkConfig *BackupConfig, configFile string) { + utils.Info("Starting multiple backup jobs...") + var conf = &Config{} + conf, err := readConf(configFile) + if err != nil { + utils.Fatal("Error reading config file: %s", err) + } + //Check if cronExpression is defined in config file + if conf.CronExpression != "" { + bkConfig.cronExpression = conf.CronExpression + } + // Check if cronExpression is defined + if bkConfig.cronExpression == "" { + multiBackupTask(conf.Databases, bkConfig) + } else { + // Check if cronExpression is valid + if utils.IsValidCronExpression(bkConfig.cronExpression) { + utils.Info("Running MultiBackup in Scheduled mode") + utils.Info("Backup cron expression: %s", bkConfig.cronExpression) + utils.Info("Storage type %s ", bkConfig.storage) + + //Test backup + utils.Info("Testing backup configurations...") + multiBackupTask(conf.Databases, bkConfig) + utils.Info("Testing backup configurations...done") + utils.Info("Creating multi backup job...") + // Create a new cron instance + c := cron.New() + + _, err := c.AddFunc(bkConfig.cronExpression, func() { + // Create a channel + multiBackupTask(conf.Databases, bkConfig) + }) + if err != nil { + return + } + // Start the cron scheduler + c.Start() + utils.Info("Creating multi backup job...done") + utils.Info("Backup job started") + defer c.Stop() + select {} + + } else { + utils.Fatal("Cron expression is not valid: %s", bkConfig.cronExpression) + } + } + +} func intro() { utils.Info("Starting PostgreSQL Backup...") utils.Info("Copyright (c) 2024 Jonas Kaninda ") diff --git a/pkg/config.go b/pkg/config.go index d898523..8cd457c 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -7,6 +7,7 @@ package pkg import ( + "errors" "fmt" "github.com/jkaninda/pg-bkup/utils" "github.com/spf13/cobra" @@ -14,7 +15,17 @@ import ( "strconv" ) +type Database struct { + Host string `yaml:"host"` + Port string `yaml:"port"` + Name string `yaml:"name"` + User string `yaml:"user"` + Password string `yaml:"password"` + Path string `yaml:"path"` +} type Config struct { + Databases []Database `yaml:"databases"` + CronExpression string `yaml:"cronExpression"` } type dbConfig struct { @@ -92,6 +103,16 @@ func initDbConfig(cmd *cobra.Command) *dbConfig { return &dConf } +func getDatabase(database Database) *dbConfig { + return &dbConfig{ + dbHost: database.Host, + dbPort: database.Port, + dbName: database.Name, + dbUserName: database.User, + dbPassword: database.Password, + } +} + // loadSSHConfig loads the SSH configuration from environment variables func loadSSHConfig() (*SSHConfig, error) { utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME") @@ -245,3 +266,10 @@ func initTargetDbConfig() *targetDbConfig { } return &tdbConfig } +func loadConfigFile() (string, error) { + backupConfigFile, err := checkConfigFile(os.Getenv("BACKUP_CONFIG_FILE")) + if err == nil { + return backupConfigFile, nil + } + return "", errors.New("backup config file not found") +} diff --git a/pkg/helper.go b/pkg/helper.go index 4f236ee..deb98f7 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -10,6 +10,7 @@ import ( "bytes" "fmt" "github.com/jkaninda/pg-bkup/utils" + "gopkg.in/yaml.v3" "os" "os/exec" "path/filepath" @@ -139,6 +140,8 @@ func testDatabaseConnection(db *dbConfig) { utils.Info("Successfully connected to %s database", db.dbName) } + +// checkPubKeyFile checks gpg public key func checkPubKeyFile(pubKey string) (string, error) { // Define possible key file names keyFiles := []string{filepath.Join(gpgHome, "public_key.asc"), filepath.Join(gpgHome, "public_key.gpg"), pubKey} @@ -160,6 +163,8 @@ func checkPubKeyFile(pubKey string) (string, error) { // Return an error if neither file exists return "", fmt.Errorf("no public key file found") } + +// checkPrKeyFile checks private key func checkPrKeyFile(prKey string) (string, error) { // Define possible key file names keyFiles := []string{filepath.Join(gpgHome, "private_key.asc"), filepath.Join(gpgHome, "private_key.gpg"), prKey} @@ -181,3 +186,42 @@ func checkPrKeyFile(prKey string) (string, error) { // Return an error if neither file exists return "", fmt.Errorf("no public key file found") } +func readConf(configFile string) (*Config, error) { + //configFile := filepath.Join("./", filename) + if utils.FileExists(configFile) { + buf, err := os.ReadFile(configFile) + if err != nil { + return nil, err + } + + c := &Config{} + err = yaml.Unmarshal(buf, c) + if err != nil { + return nil, fmt.Errorf("in file %q: %w", configFile, err) + } + + return c, err + } + return nil, fmt.Errorf("config file %q not found", configFile) +} +func checkConfigFile(filePath string) (string, error) { + // Define possible config file names + configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath} + + // Loop through config file names and check if they exist + for _, configFile := range configFiles { + if _, err := os.Stat(configFile); err == nil { + // File exists + return configFile, nil + } else if os.IsNotExist(err) { + // File does not exist, continue to the next one + continue + } else { + // An unexpected error occurred + return "", err + } + } + + // Return an error if neither file exists + return "", fmt.Errorf("no config file found") +} diff --git a/pkg/var.go b/pkg/var.go index 2702bd5..1541f94 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -16,6 +16,7 @@ var ( file = "" storagePath = "/backup" + workingDir = "/config" disableCompression = false encryption = false usingKey = false From ad9d7a00cca882f13973b8574de044b94a492046 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 9 Oct 2024 12:05:37 +0200 Subject: [PATCH 2/3] feat: add multi database backup --- docker/Dockerfile => Dockerfile | 0 docs/how-tos/deprecated-configs.md | 6 +++++ docs/how-tos/mutli-backup.md | 40 ++++++++++++++++++++++++++++++ pkg/backup.go | 26 +++++++++++-------- pkg/config.go | 3 +-- pkg/helper.go | 10 +++++++- utils/utils.go | 1 + 7 files changed, 72 insertions(+), 14 deletions(-) rename docker/Dockerfile => Dockerfile (100%) create mode 100644 docs/how-tos/deprecated-configs.md create mode 100644 docs/how-tos/mutli-backup.md diff --git a/docker/Dockerfile b/Dockerfile similarity index 100% rename from docker/Dockerfile rename to Dockerfile diff --git a/docs/how-tos/deprecated-configs.md b/docs/how-tos/deprecated-configs.md new file mode 100644 index 0000000..1813ff2 --- /dev/null +++ b/docs/how-tos/deprecated-configs.md @@ -0,0 +1,6 @@ +--- +title: Update deprecated configurations +layout: default +parent: How Tos +nav_order: 11 +--- \ No newline at end of file diff --git a/docs/how-tos/mutli-backup.md b/docs/how-tos/mutli-backup.md new file mode 100644 index 0000000..1a033a8 --- /dev/null +++ b/docs/how-tos/mutli-backup.md @@ -0,0 +1,40 @@ +--- +title: Run multiple database backup schedules in the same container +layout: default +parent: How Tos +nav_order: 11 +--- + +Multiple backup schedules with different configuration can be configured by mounting a configuration file into `/config/config.yaml` `/config/config.yml` or by defining an environment variable `BACKUP_CONFIG_FILE=/backup/config.yaml`. + +## Configuration file + +```yaml +#cronExpression: "@every 20m" //Optional, for scheduled backups +cronExpression: "" +databases: + - host: postgres1 + port: 5432 + name: database1 + user: database1 + password: password + path: /s3-path/database1 #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres2 + port: 5432 + name: lldap + user: lldap + password: password + path: /s3-path/lldap #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres3 + port: 5432 + name: keycloak + user: keycloak + password: password + path: /s3-path/keycloak #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres4 + port: 5432 + name: joplin + user: joplin + password: password + path: /s3-path/joplin #For SSH or FTP you need to define the full path (/home/toto/backup/) +``` \ No newline at end of file diff --git a/pkg/backup.go b/pkg/backup.go index bb417f5..6851564 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -24,9 +24,7 @@ func StartBackup(cmd *cobra.Command) { config := initBackupConfig(cmd) //Load backup configuration file configFile, err := loadConfigFile() - if err == nil { - startMultiBackup(config, configFile) - } else { + if err != nil { dbConf = initDbConfig(cmd) if config.cronExpression == "" { BackupTask(dbConf, config) @@ -37,11 +35,13 @@ func StartBackup(cmd *cobra.Command) { utils.Fatal("Cron expression is not valid: %s", config.cronExpression) } } + } else { + startMultiBackup(config, configFile) } } -// Run in scheduled mode +// scheduledMode Runs backup in scheduled mode func scheduledMode(db *dbConfig, config *BackupConfig) { utils.Info("Running in Scheduled mode") utils.Info("Backup cron expression: %s", config.cronExpression) @@ -68,6 +68,8 @@ func scheduledMode(db *dbConfig, config *BackupConfig) { defer c.Stop() select {} } + +// multiBackupTask backup multi database func multiBackupTask(databases []Database, bkConfig *BackupConfig) { for _, db := range databases { //Check if path is defined in config file @@ -100,7 +102,7 @@ func BackupTask(db *dbConfig, config *BackupConfig) { } } func startMultiBackup(bkConfig *BackupConfig, configFile string) { - utils.Info("Starting multiple backup jobs...") + utils.Info("Starting multiple backup job...") var conf = &Config{} conf, err := readConf(configFile) if err != nil { @@ -148,10 +150,6 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) { } } -func intro() { - utils.Info("Starting PostgreSQL Backup...") - utils.Info("Copyright (c) 2024 Jonas Kaninda ") -} // BackupDatabase backup database func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) { @@ -182,7 +180,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool log.Fatal(err) } // save output - file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + file, err := os.Create(filepath.Join(tmpPath, backupFileName)) if err != nil { log.Fatal(err) } @@ -208,7 +206,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool gzipCmd := exec.Command("gzip") gzipCmd.Stdin = stdout // save output - gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName)) gzipCmd.Start() if err != nil { log.Fatal(err) @@ -243,6 +241,7 @@ func localBackup(db *dbConfig, config *BackupConfig) { } //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } func s3Backup(db *dbConfig, config *BackupConfig) { @@ -283,6 +282,8 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func sshBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Remote server") @@ -318,6 +319,8 @@ func sshBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func ftpBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to the remote FTP server") @@ -353,6 +356,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } func encryptBackup(config *BackupConfig) { diff --git a/pkg/config.go b/pkg/config.go index 8cd457c..5015fce 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -7,7 +7,6 @@ package pkg import ( - "errors" "fmt" "github.com/jkaninda/pg-bkup/utils" "github.com/spf13/cobra" @@ -271,5 +270,5 @@ func loadConfigFile() (string, error) { if err == nil { return backupConfigFile, nil } - return "", errors.New("backup config file not found") + return "", fmt.Errorf("backup config file not found") } diff --git a/pkg/helper.go b/pkg/helper.go index deb98f7..8586631 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -17,6 +17,11 @@ import ( "time" ) +func intro() { + utils.Info("Starting PostgreSQL Backup...") + utils.Info("Copyright (c) 2024 Jonas Kaninda ") +} + // copyToTmp copy file to temporary directory func copyToTmp(sourcePath string, backupFileName string) { //Copy backup from storage to /tmp @@ -186,8 +191,9 @@ func checkPrKeyFile(prKey string) (string, error) { // Return an error if neither file exists return "", fmt.Errorf("no public key file found") } + +// readConf reads config file and returns Config func readConf(configFile string) (*Config, error) { - //configFile := filepath.Join("./", filename) if utils.FileExists(configFile) { buf, err := os.ReadFile(configFile) if err != nil { @@ -204,6 +210,8 @@ func readConf(configFile string) (*Config, error) { } return nil, fmt.Errorf("config file %q not found", configFile) } + +// checkConfigFile checks config files and returns one config file func checkConfigFile(filePath string) (string, error) { // Define possible config file names configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath} diff --git a/utils/utils.go b/utils/utils.go index 8ecb140..15392b8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -20,6 +20,7 @@ import ( "strconv" ) +// FileExists checks if the file does exist func FileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { From 431d0cdde90db64e6d66f4315e4f73755b6c518e Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 9 Oct 2024 12:06:41 +0200 Subject: [PATCH 3/3] ci: change Dockerfile path --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9242f3d..3530887 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: uses: docker/build-push-action@v3 with: push: true - file: "./docker/Dockerfile" + file: "./Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | appVersion=develop-${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cb417ee..c6ea3c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: uses: docker/build-push-action@v3 with: push: true - file: "./docker/Dockerfile" + file: "./Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | appVersion=${{ env.TAG_NAME }}