jkaninda/pg-bkup
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

chore: integrate external storage module

Browse Source
This commit is contained in:
Jonas Kaninda
2024-10-23 04:08:39 +02:00
parent 49d3ffe6b2
commit 5242bf1b48
9 changed files with 11 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -22,7 +22,7 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jkaninda/go-storage v0.0.0-20241022140446-c79ba2b4300d // indirect
github.com/jkaninda/go-storage v0.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect

2
go.sum
View File

@@ -39,6 +39,8 @@ github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221 h1:AwkCf7el1kze
github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221/go.mod h1:9F8ZJ+ZXE8DZBo77+aneGj8LMjrYXX6eFUCC/uqZOUo=
github.com/jkaninda/go-storage v0.0.0-20241022140446-c79ba2b4300d h1:AFmLusMhR9TOpkZIFt7+dSSflenGWvTl26RttBo71ds=
github.com/jkaninda/go-storage v0.0.0-20241022140446-c79ba2b4300d/go.mod h1:7VK5gQISQaLxtLfBtc+een8spcgLVSBAKTRuyF1N81I=
github.com/jkaninda/go-storage v0.1.1 h1:vjpdD/fh39S5HGyfHvLE5HGYOEPIukINlOX3OnM3GW4=
github.com/jkaninda/go-storage v0.1.1/go.mod h1:7VK5gQISQaLxtLfBtc+een8spcgLVSBAKTRuyF1N81I=
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=

8
pkg/backup.go
View File

@@ -9,10 +9,10 @@ package pkg
import (
"fmt"
"github.com/jkaninda/encryptor"
"github.com/jkaninda/pg-bkup/pkg/storage/ftp"
"github.com/jkaninda/pg-bkup/pkg/storage/local"
"github.com/jkaninda/pg-bkup/pkg/storage/s3"
"github.com/jkaninda/pg-bkup/pkg/storage/ssh"
"github.com/jkaninda/go-storage/pkg/ftp"
"github.com/jkaninda/go-storage/pkg/local"
"github.com/jkaninda/go-storage/pkg/s3"
"github.com/jkaninda/go-storage/pkg/ssh"
"github.com/jkaninda/pg-bkup/utils"
"github.com/robfig/cron/v3"
"github.com/spf13/cobra"

8
pkg/restore.go
View File

@@ -8,10 +8,10 @@ package pkg
import (
"github.com/jkaninda/encryptor"
"github.com/jkaninda/pg-bkup/pkg/storage/ftp"
"github.com/jkaninda/pg-bkup/pkg/storage/local"
"github.com/jkaninda/pg-bkup/pkg/storage/s3"
"github.com/jkaninda/pg-bkup/pkg/storage/ssh"
"github.com/jkaninda/go-storage/pkg/ftp"
"github.com/jkaninda/go-storage/pkg/local"
"github.com/jkaninda/go-storage/pkg/s3"
"github.com/jkaninda/go-storage/pkg/ssh"
"github.com/jkaninda/pg-bkup/utils"
"github.com/spf13/cobra"
"os"

118
pkg/storage/ftp/ftp.go
View File

@@ -1,118 +0,0 @@
package ftp
import (
"fmt"
"github.com/jkaninda/pg-bkup/pkg/storage"
"github.com/jkaninda/pg-bkup/utils"
"github.com/jlaffaye/ftp"
"io"
"os"
"path/filepath"
"time"
)
type ftpStorage struct {
*storage.Backend
client *ftp.ServerConn
}
// Config holds the SSH connection details
type Config struct {
Host string
User string
Password string
Port string
LocalPath string
RemotePath string
}
// createClient creates FTP Client
func createClient(conf Config) (*ftp.ServerConn, error) {
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
if err != nil {
return nil, fmt.Errorf("failed to connect to FTP: %w", err)
}
err = ftpClient.Login(conf.User, conf.Password)
if err != nil {
return nil, fmt.Errorf("failed to log in to FTP: %w", err)
}
return ftpClient, nil
}
// NewStorage creates new Storage
func NewStorage(conf Config) (storage.Storage, error) {
client, err := createClient(conf)
if err != nil {
return nil, err
}
return &ftpStorage{
client: client,
Backend: &storage.Backend{
RemotePath: conf.RemotePath,
LocalPath: conf.LocalPath,
},
}, nil
}
// Copy copies file to the remote server
func (s ftpStorage) Copy(fileName string) error {
ftpClient := s.client
defer ftpClient.Quit()
filePath := filepath.Join(s.LocalPath, fileName)
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", fileName, err)
}
defer file.Close()
remoteFilePath := filepath.Join(s.RemotePath, fileName)
err = ftpClient.Stor(remoteFilePath, file)
if err != nil {
return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err)
}
return nil
}
// CopyFrom copies a file from the remote server to local storage
func (s ftpStorage) CopyFrom(fileName string) error {
ftpClient := s.client
defer ftpClient.Quit()
remoteFilePath := filepath.Join(s.RemotePath, fileName)
r, err := ftpClient.Retr(remoteFilePath)
if err != nil {
return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
}
defer r.Close()
localFilePath := filepath.Join(s.LocalPath, fileName)
outFile, err := os.Create(localFilePath)
if err != nil {
return fmt.Errorf("failed to create local file %s: %w", fileName, err)
}
defer outFile.Close()
_, err = io.Copy(outFile, r)
if err != nil {
return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err)
}
return nil
}
// Prune deletes old backup created more than specified days
func (s ftpStorage) Prune(retentionDays int) error {
utils.Info("Deleting old backup from a remote server is not implemented yet")
return nil
}
// Name returns the storage name
func (s ftpStorage) Name() string {
return "ftp"
}

108
pkg/storage/local/local.go
View File

@@ -1,108 +0,0 @@
package local
import (
"github.com/jkaninda/pg-bkup/pkg/storage"
"github.com/jkaninda/pg-bkup/utils"
"io"
"os"
"path/filepath"
"time"
)
type localStorage struct {
*storage.Backend
}
type Config struct {
LocalPath string
RemotePath string
}
func NewStorage(conf Config) storage.Storage {
return &localStorage{
Backend: &storage.Backend{
LocalPath: conf.LocalPath,
RemotePath: conf.RemotePath,
},
}
}
func (l localStorage) Copy(file string) error {
if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) {
return err
}
err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file))
if err != nil {
return err
}
return nil
}
func (l localStorage) CopyFrom(file string) error {
if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) {
return err
}
err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file))
if err != nil {
return err
}
return nil
}
// Prune deletes old backup created more than specified days
func (l localStorage) Prune(retentionDays int) error {
currentTime := time.Now()
// Delete file
deleteFile := func(filePath string) error {
err := os.Remove(filePath)
if err != nil {
utils.Fatal("Error:", err)
} else {
utils.Info("File %s deleted successfully", filePath)
}
return err
}
// Walk through the directory and delete files modified more than specified days ago
err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error {
if err != nil {
return err
}
// Check if it's a regular file and if it was modified more than specified days ago
if fileInfo.Mode().IsRegular() {
timeDiff := currentTime.Sub(fileInfo.ModTime())
if timeDiff.Hours() > 24*float64(retentionDays) {
err := deleteFile(filePath)
if err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
return nil
}
func (l localStorage) Name() string {
return "local"
}
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
_, err = io.Copy(out, in)
if err != nil {
out.Close()
return err
}
return out.Close()
}

162
pkg/storage/s3/s3.go
View File

@@ -1,162 +0,0 @@
package s3
import (
"bytes"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/jkaninda/pg-bkup/pkg/storage"
"github.com/jkaninda/pg-bkup/utils"
"net/http"
"os"
"path/filepath"
"time"
)
type s3Storage struct {
*storage.Backend
client *session.Session
bucket string
}
type Config struct {
Endpoint string
Bucket string
AccessKey string
SecretKey string
Region string
DisableSsl bool
ForcePathStyle bool
LocalPath string
RemotePath string
}
// CreateSession creates a new AWS session
func createSession(conf Config) (*session.Session, error) {
s3Config := &aws.Config{
Credentials: credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""),
Endpoint: aws.String(conf.Endpoint),
Region: aws.String(conf.Region),
DisableSSL: aws.Bool(conf.DisableSsl),
S3ForcePathStyle: aws.Bool(conf.ForcePathStyle),
}
return session.NewSession(s3Config)
}
func NewStorage(conf Config) (storage.Storage, error) {
sess, err := createSession(conf)
if err != nil {
return nil, err
}
return &s3Storage{
client: sess,
bucket: conf.Bucket,
Backend: &storage.Backend{
RemotePath: conf.RemotePath,
LocalPath: conf.LocalPath,
},
}, nil
}
func (s s3Storage) Copy(fileName string) error {
svc := s3.New(s.client)
file, err := os.Open(filepath.Join(s.LocalPath, fileName))
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
objectKey := filepath.Join(s.RemotePath, fileName)
buffer := make([]byte, fileInfo.Size())
file.Read(buffer)
fileBytes := bytes.NewReader(buffer)
fileType := http.DetectContentType(buffer)
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(objectKey),
Body: fileBytes,
ContentLength: aws.Int64(fileInfo.Size()),
ContentType: aws.String(fileType),
})
if err != nil {
return err
}
return nil
}
// CopyFrom copies a file from the remote server to local storage
func (s s3Storage) CopyFrom(fileName string) error {
file, err := os.Create(filepath.Join(s.LocalPath, fileName))
if err != nil {
return err
}
defer file.Close()
objectKey := filepath.Join(s.RemotePath, fileName)
downloader := s3manager.NewDownloader(s.client)
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(objectKey),
})
if err != nil {
utils.Error("Failed to download file %s", fileName)
return err
}
utils.Info("Backup downloaded: %s , bytes size: %d ", file.Name(), uint64(numBytes))
return nil
}
// Prune deletes old backup created more than specified days
func (s s3Storage) Prune(retentionDays int) error {
svc := s3.New(s.client)
// Get the current time
now := time.Now()
backupRetentionDays := now.AddDate(0, 0, -retentionDays)
// List objects in the bucket
listObjectsInput := &s3.ListObjectsV2Input{
Bucket: aws.String(s.bucket),
Prefix: aws.String(s.RemotePath),
}
err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, object := range page.Contents {
if object.LastModified.Before(backupRetentionDays) {
utils.Info("Deleting old backup: %s", *object.Key)
// Object is older than retention days, delete it
_, err := svc.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: object.Key,
})
if err != nil {
utils.Info("Failed to delete object %s: %v", *object.Key, err)
} else {
utils.Info("Deleted object %s", *object.Key)
}
}
}
return !lastPage
})
if err != nil {
utils.Error("Failed to list objects: %v", err)
}
utils.Info("Deleting old backups...done")
return nil
}
// Name returns the storage name
func (s s3Storage) Name() string {
return "s3"
}

116
pkg/storage/ssh/ssh.go
View File

@@ -1,116 +0,0 @@
package ssh
import (
"context"
"errors"
"fmt"
"github.com/bramvdbogaerde/go-scp"
"github.com/bramvdbogaerde/go-scp/auth"
"github.com/jkaninda/pg-bkup/pkg/storage"
"github.com/jkaninda/pg-bkup/utils"
"golang.org/x/crypto/ssh"
"os"
"path/filepath"
)
type sshStorage struct {
*storage.Backend
client scp.Client
}
// Config holds the SSH connection details
type Config struct {
Host string
User string
Password string
Port string
IdentifyFile string
LocalPath string
RemotePath string
}
func createClient(conf Config) (scp.Client, error) {
if conf.IdentifyFile != "" && utils.FileExists(conf.IdentifyFile) {
clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
} else {
if conf.Password == "" {
return scp.Client{}, errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty")
}
utils.Warn("Accessing the remote server using password, which is not recommended.")
clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
}
}
func NewStorage(conf Config) (storage.Storage, error) {
client, err := createClient(conf)
if err != nil {
return nil, err
}
return &sshStorage{
client: client,
Backend: &storage.Backend{
RemotePath: conf.RemotePath,
LocalPath: conf.LocalPath,
},
}, nil
}
func (s sshStorage) Copy(fileName string) error {
client := s.client
// Connect to the remote server
err := client.Connect()
if err != nil {
return errors.New("couldn't establish a connection to the remote server")
}
// Open the local file
filePath := filepath.Join(s.LocalPath, fileName)
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", filePath, err)
}
defer client.Close()
// Copy file to the remote server
err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655")
if err != nil {
return fmt.Errorf("failed to copy file to remote server: %w", err)
}
return nil
}
// CopyFrom copies a file from the remote server to local storage
func (s sshStorage) CopyFrom(fileName string) error {
// Create a new SCP client
client := s.client
// Connect to the remote server
err := client.Connect()
if err != nil {
return errors.New("couldn't establish a connection to the remote server")
}
// Close client connection after the file has been copied
defer client.Close()
file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777)
if err != nil {
return errors.New("couldn't open the output file")
}
defer file.Close()
err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
if err != nil {
return err
}
return nil
}
// Prune deletes old backup created more than specified days
func (s sshStorage) Prune(retentionDays int) error {
utils.Info("Deleting old backup from a remote server is not implemented yet")
return nil
}
func (s sshStorage) Name() string {
return "ssh"
}

14
pkg/storage/storage.go
View File

@@ -1,14 +0,0 @@
package storage
type Storage interface {
Copy(fileName string) error
CopyFrom(fileName string) error
Prune(retentionDays int) error
Name() string
}
type Backend struct {
//Local Path
LocalPath string
//Remote path or Destination path
RemotePath string
}