jkaninda/pg-bkup
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Fix log , refactoring of code

Browse Source
This commit is contained in:
Jonas Kaninda
2024-08-04 01:20:30 +02:00
parent a81c7aa7b4
commit df2757fb1d
13 changed files with 293 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/release.yml vendored
View File

@@ -2,7 +2,7 @@ name: Release
on:
push:
tags:
- v**
- v1.**
env:
BUILDKIT_IMAGE: jkaninda/pg-bkup
jobs:
@@ -12,13 +12,13 @@ jobs:
packages: write
contents: read
steps:
-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v3
-
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
-
name: Login to DockerHub
uses: docker/login-action@v3
with:
@@ -30,6 +30,10 @@ jobs:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Get the tag name
id: get_tag_name
run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
-
name: Build and push
uses: docker/build-push-action@v3
@@ -38,8 +42,8 @@ jobs:
file: "./docker/Dockerfile"
platforms: linux/amd64,linux/arm64,linux/arm/v7
tags: |
"${{env.BUILDKIT_IMAGE}}:${{env.GITHUB_REF_NAME}}"
"${{env.BUILDKIT_IMAGE}}:latest"
"ghcr.io/${{env.BUILDKIT_IMAGE}}:${{env.GITHUB_REF_NAME}}"
"ghcr.io/${{env.BUILDKIT_IMAGE}}:latest"
"${{env.BUILDKIT_IMAGE}}:${{ env.TAG_NAME }}"
"${{env.BUILDKIT_IMAGE}}:latest"
"ghcr.io/${{env.BUILDKIT_IMAGE}}:${{ env.TAG_NAME }}"
"ghcr.io/${{env.BUILDKIT_IMAGE}}:latest"

20
Makefile
View File

@@ -1,4 +1,6 @@
BINARY_NAME=pg-bkup
IMAGE_NAME=jkaninda/pg-bkup
#IMAGE_NAME=ghcr.io/jkaninda/pg-bkup:v1.0
include .env
export
run:
@@ -14,33 +16,33 @@ compile:
GOOS=linux GOARCH=amd64 go build -o bin/${BINARY_NAME}-linux-amd64 .
docker-build:
docker build -f docker/Dockerfile -t jkaninda/pg-bkup:latest .
docker build -f docker/Dockerfile -t ${IMAGE_NAME}:latest .
docker-run: docker-build
docker run --rm --network internal --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup backup --prune --keep-last 2
docker run --rm --network web --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --prune --keep-last 2
docker-restore: docker-build
docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup restore -f ${FILE_NAME}
docker run --rm --network web --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup restore -f ${FILE_NAME}
docker-run-scheduled: docker-build
docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --mode scheduled --period "* * * * *"
docker run --rm --network web --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --mode scheduled --period "* * * * *"
docker-run-scheduled-s3: docker-build
docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage s3 --mode scheduled --path /custom-path --period "* * * * *"
docker run --rm --network web --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --storage s3 --mode scheduled --path /custom-path --period "* * * * *"
docker-run-s3: docker-build
docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage s3 #--path /custom-path
docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --storage s3 #--path /custom-path
docker-restore-s3: docker-build
docker run --rm --network internal --privileged --device /dev/fuse --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup restore --storage s3 -f ${FILE_NAME} #--path /custom-path
docker run --rm --network web --privileged --device /dev/fuse --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup restore --storage s3 -f ${FILE_NAME} #--path /custom-path
docker-run-ssh: docker-build
docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage ssh
docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --storage ssh
docker-restore-ssh: docker-build
docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" jkaninda/pg-bkup bkup restore --storage ssh -f data_20240731_200104.sql.gz.gpg
docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" ${IMAGE_NAME} bkup restore --storage ssh -f data_20240731_200104.sql.gz.gpg
run-docs:
cd docs && bundle exec jekyll serve -H 0.0.0.0 -t

2
go.mod
View File

@@ -13,8 +13,8 @@ require (
github.com/hpcloud/tail v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
golang.org/x/crypto v0.18.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/sys v0.22.0 // indirect
gopkg.in/fsnotify.v1 v1.4.7 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
)

150
pkg/backup.go
View File

@@ -55,7 +55,7 @@ func StartBackup(cmd *cobra.Command) {
case "ssh", "remote":
sshBackup(backupFileName, remotePath, disableCompression, prune, backupRetention, encryption)
case "ftp":
utils.Fatalf("Not supported storage type: %s", storage)
utils.Fatal("Not supported storage type: %s", storage)
default:
localBackup(backupFileName, disableCompression, prune, backupRetention, encryption)
}
@@ -76,7 +76,7 @@ func scheduledMode() {
fmt.Println(" Starting PostgreSQL Bkup... ")
fmt.Println("***********************************")
utils.Info("Running in Scheduled mode")
utils.Info("Execution period ", os.Getenv("SCHEDULE_PERIOD"))
utils.Info("Execution period %s ", os.Getenv("SCHEDULE_PERIOD"))
//Test database connexion
utils.TestDatabaseConnection()
@@ -93,6 +93,7 @@ func scheduledMode() {
utils.Fatal("Failed to start supervisord: %v", err)
}
utils.Info("Backup job started")
defer func() {
if err := cmd.Process.Kill(); err != nil {
utils.Info("Failed to kill supervisord process: %v", err)
@@ -105,13 +106,14 @@ func scheduledMode() {
}
t, err := tail.TailFile(cronLogFile, tail.Config{Follow: true})
if err != nil {
utils.Fatalf("Failed to tail file: %v", err)
utils.Fatal("Failed to tail file: %v", err)
}
// Read and print new lines from the log file
for line := range t.Lines {
fmt.Println(line.Text)
}
}
// BackupDatabase backup database
@@ -124,73 +126,75 @@ func BackupDatabase(backupFileName string, disableCompression bool) {
storagePath = os.Getenv("STORAGE_PATH")
utils.Info("Starting database backup...")
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" {
utils.Fatal("Please make sure all required environment variables for database are set")
} else {
err := os.Setenv("PGPASSWORD", dbPassword)
err := utils.CheckEnvVars(dbHVars)
if err != nil {
utils.Error("Please make sure all required environment variables for database are set")
utils.Fatal("Error checking environment variables: %s", err)
}
err = os.Setenv("PGPASSWORD", dbPassword)
if err != nil {
return
}
utils.TestDatabaseConnection()
// Backup Database database
utils.Info("Backing up database...")
// Verify is compression is disabled
if disableCompression {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
output, err := cmd.Output()
if err != nil {
return
log.Fatal(err)
}
utils.TestDatabaseConnection()
// Backup Database database
utils.Info("Backing up database...")
// Verify is compression is disabled
if disableCompression {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
output, err := cmd.Output()
if err != nil {
log.Fatal(err)
}
// save output
file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
if err != nil {
log.Fatal(err)
}
defer file.Close()
_, err = file.Write(output)
if err != nil {
log.Fatal(err)
}
} else {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
gzipCmd := exec.Command("gzip")
gzipCmd.Stdin = stdout
// save output
gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
gzipCmd.Start()
if err != nil {
log.Fatal(err)
}
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
if err := gzipCmd.Wait(); err != nil {
log.Fatal(err)
}
// save output
file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
if err != nil {
log.Fatal(err)
}
defer file.Close()
_, err = file.Write(output)
if err != nil {
log.Fatal(err)
}
} else {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
gzipCmd := exec.Command("gzip")
gzipCmd.Stdin = stdout
// save output
gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
gzipCmd.Start()
if err != nil {
log.Fatal(err)
}
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
if err := gzipCmd.Wait(); err != nil {
log.Fatal(err)
}
utils.Info("Database has been backed up")
}
utils.Info("Database has been backed up")
}
func localBackup(backupFileName string, disableCompression bool, prune bool, backupRetention int, encrypt bool) {
@@ -201,7 +205,7 @@ func localBackup(backupFileName string, disableCompression bool, prune bool, bac
encryptBackup(backupFileName)
finalFileName = fmt.Sprintf("%s.%s", backupFileName, gpgExtension)
}
utils.Info("Backup name is ", finalFileName)
utils.Info("Backup name is %s", finalFileName)
moveToBackup(finalFileName, storagePath)
//Delete old backup
if prune {
@@ -220,24 +224,24 @@ func s3Backup(backupFileName string, s3Path string, disableCompression bool, pru
finalFileName = fmt.Sprintf("%s.%s", backupFileName, "gpg")
}
utils.Info("Uploading backup file to S3 storage...")
utils.Info("Backup name is ", finalFileName)
utils.Info("Backup name is %s", finalFileName)
err := utils.UploadFileToS3(tmpPath, finalFileName, bucket, s3Path)
if err != nil {
utils.Fatalf("Error uploading file to S3: %s ", err)
utils.Fatal("Error uploading file to S3: %s ", err)
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
fmt.Println("Error deleting file: ", err)
}
// Delete old backup
if prune {
err := utils.DeleteOldBackup(bucket, s3Path, backupRetention)
if err != nil {
utils.Fatalf("Error deleting old backup from S3: %s ", err)
utils.Fatal("Error deleting old backup from S3: %s ", err)
}
}
utils.Done("Database has been backed up and uploaded to s3 ")
@@ -255,14 +259,14 @@ func sshBackup(backupFileName, remotePath string, disableCompression bool, prune
utils.Info("Backup name is ", backupFileName)
err := CopyToRemote(finalFileName, remotePath)
if err != nil {
utils.Fatalf("Error uploading file to the remote server: %s ", err)
utils.Fatal("Error uploading file to the remote server: %s ", err)
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
utils.Error("Error deleting file:", err)
}
if prune {
@@ -278,7 +282,7 @@ func encryptBackup(backupFileName string) {
gpgPassphrase := os.Getenv("GPG_PASSPHRASE")
err := Encrypt(filepath.Join(tmpPath, backupFileName), gpgPassphrase)
if err != nil {
utils.Fatalf("Error during encrypting backup %s", err)
utils.Fatal("Error during encrypting backup %s", err)
}
}

5
pkg/encrypt.go
View File

@@ -1,7 +1,6 @@
package pkg
import (
"fmt"
"github.com/jkaninda/pg-bkup/utils"
"os"
"os/exec"
@@ -9,14 +8,13 @@ import (
)
func Decrypt(inputFile string, passphrase string) error {
utils.Info("Decrypting backup file: " + inputFile + " ...")
utils.Info("Decrypting backup file: %s...", inputFile)
cmd := exec.Command("gpg", "--batch", "--passphrase", passphrase, "--output", RemoveLastExtension(inputFile), "--decrypt", inputFile)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
return err
}
@@ -32,7 +30,6 @@ func Encrypt(inputFile string, passphrase string) error {
err := cmd.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
return err
}

11
pkg/helper.go
View File

@@ -1,7 +1,6 @@
package pkg
import (
"fmt"
"github.com/jkaninda/pg-bkup/utils"
"os"
"path/filepath"
@@ -12,7 +11,7 @@ func copyToTmp(sourcePath string, backupFileName string) {
//Copy backup from storage to /tmp
err := utils.CopyFile(filepath.Join(sourcePath, backupFileName), filepath.Join(tmpPath, backupFileName))
if err != nil {
utils.Fatal("Error copying file ", backupFileName, err)
utils.Fatal("Error copying file %s %v", backupFileName, err)
}
}
@@ -20,16 +19,16 @@ func moveToBackup(backupFileName string, destinationPath string) {
//Copy backup from tmp folder to storage destination
err := utils.CopyFile(filepath.Join(tmpPath, backupFileName), filepath.Join(destinationPath, backupFileName))
if err != nil {
utils.Fatal("Error copying file ", backupFileName, err)
utils.Fatal("Error copying file %s %v", backupFileName, err)
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
utils.Error("Error deleting file: %s", err)
}
utils.Done("Database has been backed up and copied to ", filepath.Join(destinationPath, backupFileName))
utils.Done("Database has been backed up and copied to %s", filepath.Join(destinationPath, backupFileName))
}
func deleteOldBackup(retentionDays int) {
utils.Info("Deleting old backups...")
@@ -44,7 +43,7 @@ func deleteOldBackup(retentionDays int) {
if err != nil {
utils.Fatal("Error:", err)
} else {
utils.Done("File ", filePath, " deleted successfully")
utils.Done("File %s deleted successfully", filePath)
}
return err
}

77
pkg/restore.go
View File

@@ -33,7 +33,7 @@ func StartRestore(cmd *cobra.Command) {
case "ssh":
restoreFromRemote(file, remotePath)
case "ftp":
utils.Fatalf("Restore from FTP is not yet supported")
utils.Fatal("Restore from FTP is not yet supported")
default:
utils.Info("Restore database from local")
RestoreDatabase(file)
@@ -44,7 +44,7 @@ func restoreFromS3(file, bucket, s3Path string) {
utils.Info("Restore database from s3")
err := utils.DownloadFile(tmpPath, file, bucket, s3Path)
if err != nil {
utils.Fatal("Error download file from s3 ", file, err)
utils.Fatal("Error download file from s3 %s %v ", file, err)
}
RestoreDatabase(file)
}
@@ -52,7 +52,7 @@ func restoreFromRemote(file, remotePath string) {
utils.Info("Restore database from remote server")
err := CopyFromRemote(file, remotePath)
if err != nil {
utils.Fatal("Error download file from remote server: ", filepath.Join(remotePath, file), err)
utils.Fatal("Error download file from remote server: %s %v", filepath.Join(remotePath, file), err)
}
RestoreDatabase(file)
}
@@ -77,7 +77,7 @@ func RestoreDatabase(file string) {
//Decrypt file
err := Decrypt(filepath.Join(tmpPath, file), gpgPassphrase)
if err != nil {
utils.Fatal("Error decrypting file ", file, err)
utils.Fatal("Error decrypting file %s %v", file, err)
}
//Update file name
file = RemoveLastExtension(file)
@@ -85,42 +85,43 @@ func RestoreDatabase(file string) {
}
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" || file == "" {
utils.Fatal("Please make sure all required environment variables are set")
} else {
err := utils.CheckEnvVars(dbHVars)
if err != nil {
utils.Error("Please make sure all required environment variables for database are set")
utils.Fatal("Error checking environment variables: %s", err)
}
if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) {
if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) {
err := os.Setenv("PGPASSWORD", dbPassword)
if err != nil {
return
}
utils.TestDatabaseConnection()
extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file))
// Restore from compressed file / .sql.gz
if extension == ".gz" {
str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error, in restoring the database ", err)
}
utils.Done("Database has been restored")
} else if extension == ".sql" {
//Restore from sql file
str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error in restoring the database", err)
}
utils.Done("Database has been restored")
} else {
utils.Fatal("Unknown file extension ", extension)
}
} else {
utils.Fatal("File not found in ", fmt.Sprintf("%s/%s", tmpPath, file))
err := os.Setenv("PGPASSWORD", dbPassword)
if err != nil {
return
}
utils.TestDatabaseConnection()
extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file))
// Restore from compressed file / .sql.gz
if extension == ".gz" {
str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error, in restoring the database %v", err)
}
utils.Done("Database has been restored")
} else if extension == ".sql" {
//Restore from sql file
str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error in restoring the database %v", err)
}
utils.Done("Database has been restored")
} else {
utils.Fatal("Unknown file extension: %s", extension)
}
} else {
utils.Fatal("File not found in %s", fmt.Sprintf("%s/%s", tmpPath, file))
}
}

25
pkg/scp.go
View File

@@ -8,7 +8,6 @@ import (
"github.com/bramvdbogaerde/go-scp/auth"
"github.com/jkaninda/pg-bkup/utils"
"golang.org/x/crypto/ssh"
"golang.org/x/exp/slog"
"os"
"path/filepath"
)
@@ -20,15 +19,20 @@ func CopyToRemote(fileName, remotePath string) error {
sshPort := os.Getenv("SSH_PORT")
sshIdentifyFile := os.Getenv("SSH_IDENTIFY_FILE")
err := utils.CheckEnvVars(sshVars)
if err != nil {
utils.Fatal(fmt.Sprintf("Error checking environment variables\n: %s", err))
}
clientConfig, _ := auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
if sshIdentifyFile != "" && utils.FileExists(sshIdentifyFile) {
clientConfig, _ = auth.PrivateKey(sshUser, sshIdentifyFile, ssh.InsecureIgnoreHostKey())
} else {
if sshPassword == "" {
return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n")
return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty")
}
slog.Warn("Accessing the remote server using password, password is not recommended\n")
utils.Warn("Accessing the remote server using password, password is not recommended")
clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
}
@@ -36,9 +40,9 @@ func CopyToRemote(fileName, remotePath string) error {
client := scp.NewClient(fmt.Sprintf("%s:%s", sshHostName, sshPort), &clientConfig)
// Connect to the remote server
err := client.Connect()
err = client.Connect()
if err != nil {
return errors.New("Couldn't establish a connection to the remote server\n")
return errors.New("couldn't establish a connection to the remote server")
}
// Open a file
@@ -64,6 +68,11 @@ func CopyFromRemote(fileName, remotePath string) error {
sshPort := os.Getenv("SSH_PORT")
sshIdentifyFile := os.Getenv("SSH_IDENTIFY_FILE")
err := utils.CheckEnvVars(sshVars)
if err != nil {
utils.Fatal("Error checking environment variables\n: %s", err)
}
clientConfig, _ := auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
if sshIdentifyFile != "" && utils.FileExists(sshIdentifyFile) {
clientConfig, _ = auth.PrivateKey(sshUser, sshIdentifyFile, ssh.InsecureIgnoreHostKey())
@@ -72,7 +81,7 @@ func CopyFromRemote(fileName, remotePath string) error {
if sshPassword == "" {
return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n")
}
slog.Warn("Accessing the remote server using password, password is not recommended\n")
utils.Warn("Accessing the remote server using password, password is not recommended\n")
clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
}
@@ -80,7 +89,7 @@ func CopyFromRemote(fileName, remotePath string) error {
client := scp.NewClient(fmt.Sprintf("%s:%s", sshHostName, sshPort), &clientConfig)
// Connect to the remote server
err := client.Connect()
err = client.Connect()
if err != nil {
return errors.New("Couldn't establish a connection to the remote server\n")
}
@@ -96,7 +105,7 @@ func CopyFromRemote(fileName, remotePath string) error {
err = client.CopyFromRemote(context.Background(), file, filepath.Join(remotePath, fileName))
if err != nil {
fmt.Println("Error while copying file ", err)
utils.Error("Error while copying file %s ", err)
return err
}
return nil

14
pkg/scripts.go
View File

@@ -15,7 +15,7 @@ func CreateCrontabScript(disableCompression bool, storage string) {
//task := "/usr/local/bin/backup_cron.sh"
touchCmd := exec.Command("touch", backupCronFile)
if err := touchCmd.Run(); err != nil {
utils.Fatalf("Error creating file %s: %v\n", backupCronFile, err)
utils.Fatal("Error creating file %s: %v\n", backupCronFile, err)
}
var disableC = ""
if disableCompression {
@@ -37,36 +37,36 @@ bkup backup --dbname %s --port %s %v
}
if err := utils.WriteToFile(backupCronFile, scriptContent); err != nil {
utils.Fatalf("Error writing to %s: %v\n", backupCronFile, err)
utils.Fatal("Error writing to %s: %v\n", backupCronFile, err)
}
chmodCmd := exec.Command("chmod", "+x", "/usr/local/bin/backup_cron.sh")
if err := chmodCmd.Run(); err != nil {
utils.Fatalf("Error changing permissions of %s: %v\n", backupCronFile, err)
utils.Fatal("Error changing permissions of %s: %v\n", backupCronFile, err)
}
lnCmd := exec.Command("ln", "-s", "/usr/local/bin/backup_cron.sh", "/usr/local/bin/backup_cron")
if err := lnCmd.Run(); err != nil {
utils.Fatalf("Error creating symbolic link: %v\n", err)
utils.Fatal("Error creating symbolic link: %v\n", err)
}
touchLogCmd := exec.Command("touch", cronLogFile)
if err := touchLogCmd.Run(); err != nil {
utils.Fatalf("Error creating file %s: %v\n", cronLogFile, err)
utils.Fatal("Error creating file %s: %v\n", cronLogFile, err)
}
cronJob := "/etc/cron.d/backup_cron"
touchCronCmd := exec.Command("touch", cronJob)
if err := touchCronCmd.Run(); err != nil {
utils.Fatalf("Error creating file %s: %v\n", cronJob, err)
utils.Fatal("Error creating file %s: %v\n", cronJob, err)
}
cronContent := fmt.Sprintf(`%s root exec /bin/bash -c ". /run/supervisord.env; /usr/local/bin/backup_cron.sh >> %s"
`, os.Getenv("SCHEDULE_PERIOD"), cronLogFile)
if err := utils.WriteToFile(cronJob, cronContent); err != nil {
utils.Fatalf("Error writing to %s: %v\n", cronJob, err)
utils.Fatal("Error writing to %s: %v\n", cronJob, err)
}
utils.ChangePermission("/etc/cron.d/backup_cron", 0644)

16
pkg/var.go
View File

@@ -21,3 +21,19 @@ var (
disableCompression = false
encryption = false
)
// dbHVars Required environment variables for database
var dbHVars = []string{
"DB_HOST",
"DB_PASSWORD",
"DB_USERNAME",
"DB_NAME",
}
// sshVars Required environment variables for SSH remote server storage
var sshVars = []string{
"SSH_USER",
"SSH_REMOTE_PATH",
"SSH_HOST_NAME",
"SSH_PORT",
}

58
utils/logger.go Normal file
View File

@@ -0,0 +1,58 @@
package utils
import (
"fmt"
"os"
"time"
)
var currentTime = time.Now().Format("2006/01/02 15:04:05")
// Info message
func Info(msg string, args ...any) {
formattedMessage := fmt.Sprintf(msg, args...)
if len(args) == 0 {
fmt.Printf("%s INFO: %s\n", currentTime, msg)
} else {
fmt.Printf("%s INFO: %s\n", currentTime, formattedMessage)
}
}
// Warn Warning message
func Warn(msg string, args ...any) {
formattedMessage := fmt.Sprintf(msg, args...)
if len(args) == 0 {
fmt.Printf("%s WARN: %s\n", currentTime, msg)
} else {
fmt.Printf("%s WARN: %s\n", currentTime, formattedMessage)
}
}
// Error error message
func Error(msg string, args ...any) {
formattedMessage := fmt.Sprintf(msg, args...)
if len(args) == 0 {
fmt.Printf("%s ERROR: %s\n", currentTime, msg)
} else {
fmt.Printf("%s ERROR: %s\n", currentTime, formattedMessage)
}
}
func Done(msg string, args ...any) {
formattedMessage := fmt.Sprintf(msg, args...)
if len(args) == 0 {
fmt.Printf("%s INFO: %s\n", currentTime, msg)
} else {
fmt.Printf("%s INFO: %s\n", currentTime, formattedMessage)
}
}
func Fatal(msg string, args ...any) {
// Fatal logs an error message and exits the program.
formattedMessage := fmt.Sprintf(msg, args...)
if len(args) == 0 {
fmt.Printf("%s ERROR: %s\n", currentTime, msg)
} else {
fmt.Printf("%s ERROR: %s\n", currentTime, formattedMessage)
}
os.Exit(1)
}

35
utils/s3.go
View File

@@ -2,13 +2,11 @@ package utils
import (
"bytes"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"log"
"net/http"
"os"
"path/filepath"
@@ -19,13 +17,32 @@ import (
// CreateSession creates a new AWS session
func CreateSession() (*session.Session, error) {
// AwsVars Required environment variables for AWS S3 storage
var awsVars = []string{
"AWS_S3_ENDPOINT",
"AWS_S3_BUCKET_NAME",
"AWS_ACCESS_KEY",
"AWS_SECRET_KEY",
"AWS_REGION",
"AWS_REGION",
"AWS_REGION",
}
endPoint := GetEnvVariable("AWS_S3_ENDPOINT", "S3_ENDPOINT")
accessKey := GetEnvVariable("AWS_ACCESS_KEY", "ACCESS_KEY")
secretKey := GetEnvVariable("AWS_SECRET_KEY", "SECRET_KEY")
_ = GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME")
region := os.Getenv("AWS_REGION")
awsDisableSsl, err := strconv.ParseBool(os.Getenv("AWS_DISABLE_SSL"))
if err != nil {
Fatalf("Unable to parse AWS_DISABLE_SSL env var: %s", err)
Fatal("Unable to parse AWS_DISABLE_SSL env var: %s", err)
}
err = CheckEnvVars(awsVars)
if err != nil {
Error("Error checking environment variables\n: %s", err)
os.Exit(1)
}
// Configure to use MinIO Server
s3Config := &aws.Config{
@@ -88,7 +105,7 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error {
Info("Download backup from S3 storage...")
file, err := os.Create(filepath.Join(destinationPath, key))
if err != nil {
fmt.Println("Failed to create file", err)
Error("Failed to create file", err)
return err
}
defer file.Close()
@@ -102,7 +119,7 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error {
Key: aws.String(objectKey),
})
if err != nil {
fmt.Println("Failed to download file", err)
Error("Failed to download file", err)
return err
}
Info("Backup downloaded: ", file.Name(), " bytes size ", numBytes)
@@ -135,18 +152,18 @@ func DeleteOldBackup(bucket, prefix string, retention int) error {
Key: object.Key,
})
if err != nil {
log.Printf("Failed to delete object %s: %v", *object.Key, err)
Info("Failed to delete object %s: %v", *object.Key, err)
} else {
fmt.Printf("Deleted object %s\n", *object.Key)
Info("Deleted object %s\n", *object.Key)
}
}
}
return !lastPage
})
if err != nil {
log.Fatalf("Failed to list objects: %v", err)
Error("Failed to list objects: %v", err)
}
fmt.Println("Finished deleting old files.")
Info("Finished deleting old files.")
return nil
}

47
utils/utils.go
View File

@@ -10,31 +10,12 @@ import (
"bytes"
"fmt"
"github.com/spf13/cobra"
"golang.org/x/exp/slog"
"io"
"io/fs"
"os"
"os/exec"
)
func Info(v ...any) {
fmt.Println("⒤ ", fmt.Sprint(v...))
}
func Worn(msg string, v ...any) {
slog.Warn(fmt.Sprintf(msg, v))
}
func Done(v ...any) {
fmt.Println("✔ ", fmt.Sprint(v...))
}
func Fatal(v ...any) {
fmt.Println("✘ ", fmt.Sprint(v...))
os.Exit(1)
}
func Fatalf(msg string, v ...any) {
fmt.Printf("✘ "+msg, v...)
os.Exit(1)
}
func FileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
@@ -91,7 +72,7 @@ func CopyFile(src, dst string) error {
}
func ChangePermission(filePath string, mod int) {
if err := os.Chmod(filePath, fs.FileMode(mod)); err != nil {
Fatalf("Error changing permissions of %s: %v\n", filePath, err)
Fatal("Error changing permissions of %s: %v\n", filePath, err)
}
}
@@ -145,7 +126,7 @@ func TestDatabaseConnection() {
// Run the command and capture any errors
err = cmd.Run()
if err != nil {
fmt.Printf("Error running psql command: %v\nOutput: %s\n", err, out.String())
Error("Error running psql command: %v\nOutput: %s\n", err, out.String())
return
}
Info("Successfully connected to database")
@@ -187,11 +168,29 @@ func GetEnvVariable(envName, oldEnvName string) string {
if value == "" {
value = os.Getenv(oldEnvName)
if value != "" {
slog.Warn(fmt.Sprintf("%s is deprecated, please use %s instead!\n", oldEnvName, envName))
err := os.Setenv(envName, value)
if err != nil {
return value
}
Warn("%s is deprecated, please use %s instead! ", oldEnvName, envName)
}
}
return value
}
func ShowHistory() {
// CheckEnvVars checks if all the specified environment variables are set
func CheckEnvVars(vars []string) error {
missingVars := []string{}
for _, v := range vars {
if os.Getenv(v) == "" {
missingVars = append(missingVars, v)
}
}
if len(missingVars) > 0 {
return fmt.Errorf("missing environment variables: %v", missingVars)
}
return nil
}