jkaninda/pg-bkup
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Migrate from s3fs to go aws s3 client

Browse Source
This commit is contained in:
Jonas Kaninda
2024-07-29 07:33:26 +02:00
parent 0aa38eec20
commit a8fa58dacb
11 changed files with 359 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
pkg/backup.go
View File

@@ -33,16 +33,35 @@ func StartBackup(cmd *cobra.Command) {
prune, _ := cmd.Flags().GetBool("prune")
disableCompression, _ = cmd.Flags().GetBool("disable-compression")
executionMode, _ = cmd.Flags().GetString("mode")
dbName = os.Getenv("DB_NAME")
storagePath = os.Getenv("STORAGE_PATH")
//Generate file name
backupFileName := fmt.Sprintf("%s_%s.sql.gz", dbName, time.Now().Format("20060102_150405"))
if disableCompression {
backupFileName = fmt.Sprintf("%s_%s.sql", dbName, time.Now().Format("20060102_150405"))
}
if executionMode == "default" {
if storage == "s3" {
switch storage {
case "s3":
utils.Info("Backup database to s3 storage")
s3Backup(disableCompression, s3Path, prune, keepLast)
} else {
BackupDatabase(backupFileName, disableCompression, prune, keepLast)
s3Upload(backupFileName, s3Path)
case "local":
utils.Info("Backup database to local storage")
BackupDatabase(disableCompression, prune, keepLast)
BackupDatabase(backupFileName, disableCompression, prune, keepLast)
moveToBackup(backupFileName, storagePath)
case "ssh":
fmt.Println("x is 2")
case "ftp":
fmt.Println("x is 3")
default:
utils.Info("Backup database to local storage")
BackupDatabase(backupFileName, disableCompression, prune, keepLast)
moveToBackup(backupFileName, storagePath)
}
} else if executionMode == "scheduled" {
scheduledMode()
} else {
@@ -98,7 +117,7 @@ func scheduledMode() {
}
// BackupDatabase backup database
func BackupDatabase(disableCompression bool, prune bool, keepLast int) {
func BackupDatabase(backupFileName string, disableCompression bool, prune bool, keepLast int) {
dbHost = os.Getenv("DB_HOST")
dbPassword = os.Getenv("DB_PASSWORD")
dbUserName = os.Getenv("DB_USERNAME")
@@ -117,12 +136,9 @@ func BackupDatabase(disableCompression bool, prune bool, keepLast int) {
utils.TestDatabaseConnection()
// Backup Database database
utils.Info("Backing up database...")
//Generate file name
bkFileName := fmt.Sprintf("%s_%s.sql.gz", dbName, time.Now().Format("20060102_150405"))
// Verify is compression is disabled
if disableCompression {
bkFileName = fmt.Sprintf("%s_%s.sql", dbName, time.Now().Format("20060102_150405"))
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
@@ -135,7 +151,7 @@ func BackupDatabase(disableCompression bool, prune bool, keepLast int) {
log.Fatal(err)
}
// save output
file, err := os.Create(fmt.Sprintf("%s/%s", storagePath, bkFileName))
file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
if err != nil {
log.Fatal(err)
}
@@ -145,7 +161,6 @@ func BackupDatabase(disableCompression bool, prune bool, keepLast int) {
if err != nil {
log.Fatal(err)
}
utils.Done("Database has been backed up")
} else {
// Execute pg_dump
@@ -162,7 +177,7 @@ func BackupDatabase(disableCompression bool, prune bool, keepLast int) {
gzipCmd := exec.Command("gzip")
gzipCmd.Stdin = stdout
// save output
gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", storagePath, bkFileName))
gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
gzipCmd.Start()
if err != nil {
log.Fatal(err)
@@ -173,30 +188,62 @@ func BackupDatabase(disableCompression bool, prune bool, keepLast int) {
if err := gzipCmd.Wait(); err != nil {
log.Fatal(err)
}
utils.Done("Database has been backed up")
}
utils.Done("Database has been backed up")
//Delete old backup
if prune {
deleteOldBackup(keepLast)
}
//if prune {
// deleteOldBackup(keepLast)
//}
historyFile, err := os.OpenFile(fmt.Sprintf("%s/history.txt", storagePath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
historyFile, err := os.OpenFile(fmt.Sprintf("%s/history.txt", tmpPath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Fatal(err)
}
defer historyFile.Close()
if _, err := historyFile.WriteString(bkFileName + "\n"); err != nil {
if _, err := historyFile.WriteString(backupFileName + "\n"); err != nil {
log.Fatal(err)
}
}
}
func moveToBackup(backupFileName string, destinationPath string) {
//Copy backup from tmp folder to storage destination
err := utils.CopyFile(filepath.Join(tmpPath, backupFileName), filepath.Join(destinationPath, backupFileName))
if err != nil {
utils.Fatal("Error copying file ", backupFileName, err)
func s3Backup(disableCompression bool, s3Path string, prune bool, keepLast int) {
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
}
utils.Done("Database has been backed up and copied to destination ")
}
func s3Upload(backupFileName string, s3Path string) {
bucket := os.Getenv("BUCKET_NAME")
utils.Info("Uploading file to S3 storage")
err := utils.UploadFileToS3(tmpPath, backupFileName, bucket, s3Path)
if err != nil {
utils.Fatalf("Error uploading file to S3: %s ", err)
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
}
utils.Done("Database has been backed up and uploaded to s3 ")
}
func s3Backup(backupFileName string, disableCompression bool, s3Path string, prune bool, keepLast int) {
// Backup Database to S3 storage
MountS3Storage(s3Path)
BackupDatabase(disableCompression, prune, keepLast)
//MountS3Storage(s3Path)
//BackupDatabase(backupFileName, disableCompression, prune, keepLast)
}
func deleteOldBackup(keepLast int) {
utils.Info("Deleting old backups...")

1
pkg/encrypt_archive.go Normal file
View File

@@ -0,0 +1 @@
package pkg

50
pkg/restore.go
View File

@@ -21,13 +21,34 @@ func StartRestore(cmd *cobra.Command) {
storage = utils.GetEnv(cmd, "storage", "STORAGE")
file = utils.GetEnv(cmd, "file", "FILE_NAME")
executionMode, _ = cmd.Flags().GetString("mode")
bucket := os.Getenv("BUCKET_NAME")
if storage == "s3" {
switch storage {
case "s3":
utils.Info("Restore database from s3")
s3Restore(file, s3Path)
} else {
err := utils.DownloadFile(tmpPath, file, bucket, s3Path)
if err != nil {
utils.Fatal("Error download file from s3 ", file, err)
}
RestoreDatabase(file)
case "local":
utils.Info("Restore database from local")
copyTmp(storagePath, file)
RestoreDatabase(file)
case "ssh":
fmt.Println("x is 2")
case "ftp":
fmt.Println("x is 3")
default:
utils.Info("Restore database from local")
RestoreDatabase(file)
}
}
func copyTmp(sourcePath string, backupFileName string) {
//Copy backup from tmp folder to storage destination
err := utils.CopyFile(filepath.Join(sourcePath, backupFileName), filepath.Join(tmpPath, backupFileName))
if err != nil {
utils.Fatal("Error copying file ", backupFileName, err)
}
}
@@ -39,7 +60,7 @@ func RestoreDatabase(file string) {
dbUserName = os.Getenv("DB_USERNAME")
dbName = os.Getenv("DB_NAME")
dbPort = os.Getenv("DB_PORT")
storagePath = os.Getenv("STORAGE_PATH")
//storagePath = os.Getenv("STORAGE_PATH")
if file == "" {
utils.Fatal("Error, file required")
}
@@ -48,7 +69,7 @@ func RestoreDatabase(file string) {
utils.Fatal("Please make sure all required environment variables are set")
} else {
if utils.FileExists(fmt.Sprintf("%s/%s", storagePath, file)) {
if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) {
err := os.Setenv("PGPASSWORD", dbPassword)
if err != nil {
@@ -56,10 +77,10 @@ func RestoreDatabase(file string) {
}
utils.TestDatabaseConnection()
extension := filepath.Ext(fmt.Sprintf("%s/%s", storagePath, file))
extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file))
// Restore from compressed file / .sql.gz
if extension == ".gz" {
str := "zcat " + fmt.Sprintf("%s/%s", storagePath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error, in restoring the database")
@@ -68,7 +89,7 @@ func RestoreDatabase(file string) {
} else if extension == ".sql" {
//Restore from sql file
str := "cat " + fmt.Sprintf("%s/%s", storagePath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error in restoring the database", err)
@@ -79,12 +100,13 @@ func RestoreDatabase(file string) {
}
} else {
utils.Fatal("File not found in ", fmt.Sprintf("%s/%s", storagePath, file))
utils.Fatal("File not found in ", fmt.Sprintf("%s/%s", tmpPath, file))
}
}
}
func s3Restore(file, s3Path string) {
// Restore database from S3
MountS3Storage(s3Path)
RestoreDatabase(file)
}
//func s3Restore(file, s3Path string) {
// // Restore database from S3
// MountS3Storage(s3Path)
// RestoreDatabase(file)
//}

1
pkg/var.go
View File

@@ -3,6 +3,7 @@ package pkg
const s3MountPath string = "/s3mnt"
const s3fsPasswdFile string = "/etc/passwd-s3fs"
const cronLogFile = "/var/log/pg-bkup.log"
const tmpPath = "/tmp/pg-bkup"
const backupCronFile = "/usr/local/bin/backup_cron.sh"
var (