From 5242bf1b48cbda0be99c53ded4ec51fece317403 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 23 Oct 2024 04:08:39 +0200 Subject: [PATCH] chore: integrate external storage module --- go.mod | 2 +- go.sum | 2 + pkg/backup.go | 8 +- pkg/restore.go | 8 +- pkg/storage/ftp/ftp.go | 118 --------------------------- pkg/storage/local/local.go | 108 ------------------------- pkg/storage/s3/s3.go | 162 ------------------------------------- pkg/storage/ssh/ssh.go | 116 -------------------------- pkg/storage/storage.go | 14 ---- 9 files changed, 11 insertions(+), 527 deletions(-) delete mode 100644 pkg/storage/ftp/ftp.go delete mode 100644 pkg/storage/local/local.go delete mode 100644 pkg/storage/s3/s3.go delete mode 100644 pkg/storage/ssh/ssh.go delete mode 100644 pkg/storage/storage.go diff --git a/go.mod b/go.mod index bfa281a..03231ab 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jkaninda/go-storage v0.0.0-20241022140446-c79ba2b4300d // indirect + github.com/jkaninda/go-storage v0.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index dc1b5dd..8ebd348 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221 h1:AwkCf7el1kze github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221/go.mod h1:9F8ZJ+ZXE8DZBo77+aneGj8LMjrYXX6eFUCC/uqZOUo= github.com/jkaninda/go-storage v0.0.0-20241022140446-c79ba2b4300d h1:AFmLusMhR9TOpkZIFt7+dSSflenGWvTl26RttBo71ds= github.com/jkaninda/go-storage v0.0.0-20241022140446-c79ba2b4300d/go.mod h1:7VK5gQISQaLxtLfBtc+een8spcgLVSBAKTRuyF1N81I= +github.com/jkaninda/go-storage v0.1.1 h1:vjpdD/fh39S5HGyfHvLE5HGYOEPIukINlOX3OnM3GW4= +github.com/jkaninda/go-storage v0.1.1/go.mod h1:7VK5gQISQaLxtLfBtc+een8spcgLVSBAKTRuyF1N81I= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= diff --git a/pkg/backup.go b/pkg/backup.go index 2860674..ab54a63 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -9,10 +9,10 @@ package pkg import ( "fmt" "github.com/jkaninda/encryptor" - "github.com/jkaninda/pg-bkup/pkg/storage/ftp" - "github.com/jkaninda/pg-bkup/pkg/storage/local" - "github.com/jkaninda/pg-bkup/pkg/storage/s3" - "github.com/jkaninda/pg-bkup/pkg/storage/ssh" + "github.com/jkaninda/go-storage/pkg/ftp" + "github.com/jkaninda/go-storage/pkg/local" + "github.com/jkaninda/go-storage/pkg/s3" + "github.com/jkaninda/go-storage/pkg/ssh" "github.com/jkaninda/pg-bkup/utils" "github.com/robfig/cron/v3" "github.com/spf13/cobra" diff --git a/pkg/restore.go b/pkg/restore.go index 80e0f53..fb4378d 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -8,10 +8,10 @@ package pkg import ( "github.com/jkaninda/encryptor" - "github.com/jkaninda/pg-bkup/pkg/storage/ftp" - "github.com/jkaninda/pg-bkup/pkg/storage/local" - "github.com/jkaninda/pg-bkup/pkg/storage/s3" - "github.com/jkaninda/pg-bkup/pkg/storage/ssh" + "github.com/jkaninda/go-storage/pkg/ftp" + "github.com/jkaninda/go-storage/pkg/local" + "github.com/jkaninda/go-storage/pkg/s3" + "github.com/jkaninda/go-storage/pkg/ssh" "github.com/jkaninda/pg-bkup/utils" "github.com/spf13/cobra" "os" diff --git a/pkg/storage/ftp/ftp.go b/pkg/storage/ftp/ftp.go deleted file mode 100644 index 53274ac..0000000 --- a/pkg/storage/ftp/ftp.go +++ /dev/null @@ -1,118 +0,0 @@ -package ftp - -import ( - "fmt" - "github.com/jkaninda/pg-bkup/pkg/storage" - "github.com/jkaninda/pg-bkup/utils" - "github.com/jlaffaye/ftp" - "io" - "os" - "path/filepath" - "time" -) - -type ftpStorage struct { - *storage.Backend - client *ftp.ServerConn -} - -// Config holds the SSH connection details -type Config struct { - Host string - User string - Password string - Port string - LocalPath string - RemotePath string -} - -// createClient creates FTP Client -func createClient(conf Config) (*ftp.ServerConn, error) { - ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second)) - if err != nil { - return nil, fmt.Errorf("failed to connect to FTP: %w", err) - } - - err = ftpClient.Login(conf.User, conf.Password) - if err != nil { - return nil, fmt.Errorf("failed to log in to FTP: %w", err) - } - - return ftpClient, nil -} - -// NewStorage creates new Storage -func NewStorage(conf Config) (storage.Storage, error) { - client, err := createClient(conf) - if err != nil { - return nil, err - } - return &ftpStorage{ - client: client, - Backend: &storage.Backend{ - RemotePath: conf.RemotePath, - LocalPath: conf.LocalPath, - }, - }, nil -} - -// Copy copies file to the remote server -func (s ftpStorage) Copy(fileName string) error { - ftpClient := s.client - defer ftpClient.Quit() - - filePath := filepath.Join(s.LocalPath, fileName) - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", fileName, err) - } - defer file.Close() - - remoteFilePath := filepath.Join(s.RemotePath, fileName) - err = ftpClient.Stor(remoteFilePath, file) - if err != nil { - return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err) - } - - return nil -} - -// CopyFrom copies a file from the remote server to local storage -func (s ftpStorage) CopyFrom(fileName string) error { - ftpClient := s.client - - defer ftpClient.Quit() - - remoteFilePath := filepath.Join(s.RemotePath, fileName) - r, err := ftpClient.Retr(remoteFilePath) - if err != nil { - return fmt.Errorf("failed to retrieve file %s: %w", fileName, err) - } - defer r.Close() - - localFilePath := filepath.Join(s.LocalPath, fileName) - outFile, err := os.Create(localFilePath) - if err != nil { - return fmt.Errorf("failed to create local file %s: %w", fileName, err) - } - defer outFile.Close() - - _, err = io.Copy(outFile, r) - if err != nil { - return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err) - } - - return nil -} - -// Prune deletes old backup created more than specified days -func (s ftpStorage) Prune(retentionDays int) error { - utils.Info("Deleting old backup from a remote server is not implemented yet") - return nil - -} - -// Name returns the storage name -func (s ftpStorage) Name() string { - return "ftp" -} diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go deleted file mode 100644 index 4fcb70a..0000000 --- a/pkg/storage/local/local.go +++ /dev/null @@ -1,108 +0,0 @@ -package local - -import ( - "github.com/jkaninda/pg-bkup/pkg/storage" - "github.com/jkaninda/pg-bkup/utils" - "io" - "os" - "path/filepath" - "time" -) - -type localStorage struct { - *storage.Backend -} -type Config struct { - LocalPath string - RemotePath string -} - -func NewStorage(conf Config) storage.Storage { - return &localStorage{ - Backend: &storage.Backend{ - LocalPath: conf.LocalPath, - RemotePath: conf.RemotePath, - }, - } -} -func (l localStorage) Copy(file string) error { - if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) { - return err - } - err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file)) - if err != nil { - return err - } - return nil -} - -func (l localStorage) CopyFrom(file string) error { - if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) { - return err - } - err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file)) - if err != nil { - return err - } - return nil -} - -// Prune deletes old backup created more than specified days -func (l localStorage) Prune(retentionDays int) error { - currentTime := time.Now() - // Delete file - deleteFile := func(filePath string) error { - err := os.Remove(filePath) - if err != nil { - utils.Fatal("Error:", err) - } else { - utils.Info("File %s deleted successfully", filePath) - } - return err - } - // Walk through the directory and delete files modified more than specified days ago - err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - // Check if it's a regular file and if it was modified more than specified days ago - if fileInfo.Mode().IsRegular() { - timeDiff := currentTime.Sub(fileInfo.ModTime()) - if timeDiff.Hours() > 24*float64(retentionDays) { - err := deleteFile(filePath) - if err != nil { - return err - } - } - } - return nil - }) - if err != nil { - return err - } - return nil -} - -func (l localStorage) Name() string { - return "local" -} - -func copyFile(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return err - } - - _, err = io.Copy(out, in) - if err != nil { - out.Close() - return err - } - return out.Close() -} diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go deleted file mode 100644 index 454618e..0000000 --- a/pkg/storage/s3/s3.go +++ /dev/null @@ -1,162 +0,0 @@ -package s3 - -import ( - "bytes" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/jkaninda/pg-bkup/pkg/storage" - "github.com/jkaninda/pg-bkup/utils" - "net/http" - "os" - "path/filepath" - "time" -) - -type s3Storage struct { - *storage.Backend - client *session.Session - bucket string -} -type Config struct { - Endpoint string - Bucket string - AccessKey string - SecretKey string - Region string - DisableSsl bool - ForcePathStyle bool - LocalPath string - RemotePath string -} - -// CreateSession creates a new AWS session -func createSession(conf Config) (*session.Session, error) { - s3Config := &aws.Config{ - Credentials: credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""), - Endpoint: aws.String(conf.Endpoint), - Region: aws.String(conf.Region), - DisableSSL: aws.Bool(conf.DisableSsl), - S3ForcePathStyle: aws.Bool(conf.ForcePathStyle), - } - - return session.NewSession(s3Config) -} - -func NewStorage(conf Config) (storage.Storage, error) { - sess, err := createSession(conf) - if err != nil { - return nil, err - } - return &s3Storage{ - client: sess, - bucket: conf.Bucket, - Backend: &storage.Backend{ - RemotePath: conf.RemotePath, - LocalPath: conf.LocalPath, - }, - }, nil -} -func (s s3Storage) Copy(fileName string) error { - svc := s3.New(s.client) - file, err := os.Open(filepath.Join(s.LocalPath, fileName)) - if err != nil { - return err - } - defer file.Close() - - fileInfo, err := file.Stat() - if err != nil { - return err - } - objectKey := filepath.Join(s.RemotePath, fileName) - buffer := make([]byte, fileInfo.Size()) - file.Read(buffer) - fileBytes := bytes.NewReader(buffer) - fileType := http.DetectContentType(buffer) - - _, err = svc.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(objectKey), - Body: fileBytes, - ContentLength: aws.Int64(fileInfo.Size()), - ContentType: aws.String(fileType), - }) - if err != nil { - return err - } - - return nil -} - -// CopyFrom copies a file from the remote server to local storage -func (s s3Storage) CopyFrom(fileName string) error { - file, err := os.Create(filepath.Join(s.LocalPath, fileName)) - if err != nil { - return err - } - defer file.Close() - - objectKey := filepath.Join(s.RemotePath, fileName) - - downloader := s3manager.NewDownloader(s.client) - numBytes, err := downloader.Download(file, - &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(objectKey), - }) - if err != nil { - utils.Error("Failed to download file %s", fileName) - return err - } - utils.Info("Backup downloaded: %s , bytes size: %d ", file.Name(), uint64(numBytes)) - - return nil -} - -// Prune deletes old backup created more than specified days -func (s s3Storage) Prune(retentionDays int) error { - svc := s3.New(s.client) - - // Get the current time - now := time.Now() - backupRetentionDays := now.AddDate(0, 0, -retentionDays) - - // List objects in the bucket - listObjectsInput := &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(s.RemotePath), - } - err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool { - for _, object := range page.Contents { - if object.LastModified.Before(backupRetentionDays) { - utils.Info("Deleting old backup: %s", *object.Key) - // Object is older than retention days, delete it - _, err := svc.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: object.Key, - }) - if err != nil { - utils.Info("Failed to delete object %s: %v", *object.Key, err) - } else { - utils.Info("Deleted object %s", *object.Key) - } - } - } - return !lastPage - }) - if err != nil { - utils.Error("Failed to list objects: %v", err) - } - - utils.Info("Deleting old backups...done") - return nil - -} - -// Name returns the storage name -func (s s3Storage) Name() string { - return "s3" -} diff --git a/pkg/storage/ssh/ssh.go b/pkg/storage/ssh/ssh.go deleted file mode 100644 index 2aebb73..0000000 --- a/pkg/storage/ssh/ssh.go +++ /dev/null @@ -1,116 +0,0 @@ -package ssh - -import ( - "context" - "errors" - "fmt" - "github.com/bramvdbogaerde/go-scp" - "github.com/bramvdbogaerde/go-scp/auth" - "github.com/jkaninda/pg-bkup/pkg/storage" - "github.com/jkaninda/pg-bkup/utils" - "golang.org/x/crypto/ssh" - "os" - "path/filepath" -) - -type sshStorage struct { - *storage.Backend - client scp.Client -} - -// Config holds the SSH connection details -type Config struct { - Host string - User string - Password string - Port string - IdentifyFile string - LocalPath string - RemotePath string -} - -func createClient(conf Config) (scp.Client, error) { - if conf.IdentifyFile != "" && utils.FileExists(conf.IdentifyFile) { - clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey()) - return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err - } else { - if conf.Password == "" { - return scp.Client{}, errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty") - } - utils.Warn("Accessing the remote server using password, which is not recommended.") - clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey()) - return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err - - } -} - -func NewStorage(conf Config) (storage.Storage, error) { - client, err := createClient(conf) - if err != nil { - return nil, err - } - return &sshStorage{ - client: client, - Backend: &storage.Backend{ - RemotePath: conf.RemotePath, - LocalPath: conf.LocalPath, - }, - }, nil -} -func (s sshStorage) Copy(fileName string) error { - client := s.client - // Connect to the remote server - err := client.Connect() - if err != nil { - return errors.New("couldn't establish a connection to the remote server") - } - // Open the local file - filePath := filepath.Join(s.LocalPath, fileName) - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", filePath, err) - } - defer client.Close() - // Copy file to the remote server - err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655") - if err != nil { - return fmt.Errorf("failed to copy file to remote server: %w", err) - } - - return nil -} - -// CopyFrom copies a file from the remote server to local storage -func (s sshStorage) CopyFrom(fileName string) error { - // Create a new SCP client - client := s.client - // Connect to the remote server - err := client.Connect() - if err != nil { - return errors.New("couldn't establish a connection to the remote server") - } - // Close client connection after the file has been copied - defer client.Close() - file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777) - if err != nil { - return errors.New("couldn't open the output file") - } - defer file.Close() - - err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName)) - - if err != nil { - return err - } - return nil -} - -// Prune deletes old backup created more than specified days -func (s sshStorage) Prune(retentionDays int) error { - utils.Info("Deleting old backup from a remote server is not implemented yet") - return nil -} - -func (s sshStorage) Name() string { - return "ssh" -} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go deleted file mode 100644 index 6d6fd8d..0000000 --- a/pkg/storage/storage.go +++ /dev/null @@ -1,14 +0,0 @@ -package storage - -type Storage interface { - Copy(fileName string) error - CopyFrom(fileName string) error - Prune(retentionDays int) error - Name() string -} -type Backend struct { - //Local Path - LocalPath string - //Remote path or Destination path - RemotePath string -}