refactor: create storage interface, refactor local, s3, ftp and ssh storage

pkg/storage/ftp/ftp.go

@@ -0,0 +1,118 @@
+package ftp
+
+import (
+	"fmt"
+	"github.com/jkaninda/pg-bkup/pkg/storage"
+	"github.com/jkaninda/pg-bkup/utils"
+	"github.com/jlaffaye/ftp"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type ftpStorage struct {
+	*storage.Backend
+	client *ftp.ServerConn
+}
+
+// Config holds the SSH connection details
+type Config struct {
+	Host       string
+	User       string
+	Password   string
+	Port       string
+	LocalPath string
+	RemotePath string
+}
+
+// createClient creates FTP Client
+func createClient(conf Config) (*ftp.ServerConn, error) {
+	ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to FTP: %w", err)
+	}
+
+	err = ftpClient.Login(conf.User, conf.Password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to log in to FTP: %w", err)
+	}
+
+	return ftpClient, nil
+}
+
+// NewStorage creates new Storage
+func NewStorage(conf Config) (storage.Storage, error) {
+	client, err := createClient(conf)
+	if err != nil {
+		return nil, err
+	}
+	return &ftpStorage{
+		client: client,
+		Backend: &storage.Backend{
+			RemotePath: conf.RemotePath,
+			LocalPath: conf.LocalPath,
+		},
+	}, nil
+}
+
+// Copy copies file to the remote server
+func (s ftpStorage) Copy(fileName string) error {
+	ftpClient := s.client
+	defer ftpClient.Quit()
+
+	filePath := filepath.Join(s.LocalPath, fileName)
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to open file %s: %w", fileName, err)
+	}
+	defer file.Close()
+
+	remoteFilePath := filepath.Join(s.RemotePath, fileName)
+	err = ftpClient.Stor(remoteFilePath, file)
+	if err != nil {
+		return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err)
+	}
+
+	return nil
+}
+
+// CopyFrom copies a file from the remote server to local storage
+func (s ftpStorage) CopyFrom(fileName string) error {
+	ftpClient := s.client
+
+	defer ftpClient.Quit()
+
+	remoteFilePath := filepath.Join(s.RemotePath, fileName)
+	r, err := ftpClient.Retr(remoteFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
+	}
+	defer r.Close()
+
+	localFilePath := filepath.Join(s.LocalPath, fileName)
+	outFile, err := os.Create(localFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to create local file %s: %w", fileName, err)
+	}
+	defer outFile.Close()
+
+	_, err = io.Copy(outFile, r)
+	if err != nil {
+		return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err)
+	}
+
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (s ftpStorage) Prune(retentionDays int) error {
+	utils.Info("Deleting old backup from a remote server is not implemented yet")
+	return nil
+
+}
+
+// Name returns the storage name
+func (s ftpStorage) Name() string {
+	return "ftp"
+}

pkg/storage/local/local.go

@@ -0,0 +1,108 @@
+package local
+
+import (
+	"github.com/jkaninda/pg-bkup/pkg/storage"
+	"github.com/jkaninda/pg-bkup/utils"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type localStorage struct {
+	*storage.Backend
+}
+type Config struct {
+	LocalPath string
+	RemotePath string
+}
+
+func NewStorage(conf Config) storage.Storage {
+	return &localStorage{
+		Backend: &storage.Backend{
+			LocalPath: conf.LocalPath,
+			RemotePath: conf.RemotePath,
+		},
+	}
+}
+func (l localStorage) Copy(file string) error {
+	if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) {
+		return err
+	}
+	err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (l localStorage) CopyFrom(file string) error {
+	if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) {
+		return err
+	}
+	err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (l localStorage) Prune(retentionDays int) error {
+	currentTime := time.Now()
+	// Delete file
+	deleteFile := func(filePath string) error {
+		err := os.Remove(filePath)
+		if err != nil {
+			utils.Fatal("Error:", err)
+		} else {
+			utils.Info("File %s deleted successfully", filePath)
+		}
+		return err
+	}
+	// Walk through the directory and delete files modified more than specified days ago
+	err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// Check if it's a regular file and if it was modified more than specified days ago
+		if fileInfo.Mode().IsRegular() {
+			timeDiff := currentTime.Sub(fileInfo.ModTime())
+			if timeDiff.Hours() > 24*float64(retentionDays) {
+				err := deleteFile(filePath)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (l localStorage) Name() string {
+	return "local"
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}

pkg/storage/s3/s3.go

@@ -0,0 +1,163 @@
+package s3
+
+import (
+	"bytes"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/jkaninda/pg-bkup/pkg/storage"
+	"github.com/jkaninda/pg-bkup/utils"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type s3Storage struct {
+	*storage.Backend
+	client *session.Session
+	bucket string
+}
+type Config struct {
+	Endpoint       string
+	Bucket         string
+	AccessKey      string
+	SecretKey      string
+	Region         string
+	DisableSsl     bool
+	ForcePathStyle bool
+	LocalPath string
+	RemotePath string
+}
+
+// CreateSession creates a new AWS session
+func createSession(conf Config) (*session.Session, error) {
+	s3Config := &aws.Config{
+		Credentials:      credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""),
+		Endpoint:         aws.String(conf.Endpoint),
+		Region:           aws.String(conf.Region),
+		DisableSSL:       aws.Bool(conf.DisableSsl),
+		S3ForcePathStyle: aws.Bool(conf.ForcePathStyle),
+	}
+
+	return session.NewSession(s3Config)
+}
+
+func NewStorage(conf Config) (storage.Storage, error) {
+	sess, err := createSession(conf)
+	if err != nil {
+		return nil, err
+	}
+	return &s3Storage{
+		client: sess,
+		bucket: conf.Bucket,
+		Backend: &storage.Backend{
+			RemotePath: conf.RemotePath,
+			LocalPath: conf.LocalPath,
+		},
+	}, nil
+}
+func (s s3Storage) Copy(fileName string) error {
+	svc := s3.New(s.client)
+	file, err := os.Open(filepath.Join(s.LocalPath, fileName))
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return err
+	}
+	objectKey := filepath.Join(s.RemotePath, fileName)
+	buffer := make([]byte, fileInfo.Size())
+	file.Read(buffer)
+	fileBytes := bytes.NewReader(buffer)
+	fileType := http.DetectContentType(buffer)
+
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket:        aws.String(s.bucket),
+		Key:           aws.String(objectKey),
+		Body:          fileBytes,
+		ContentLength: aws.Int64(fileInfo.Size()),
+		ContentType:   aws.String(fileType),
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CopyFrom copies a file from the remote server to local storage
+func (s s3Storage) CopyFrom(fileName string) error {
+	utils.Info("Copy data from S3 storage...")
+	file, err := os.Create(filepath.Join(s.LocalPath, fileName))
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	objectKey := filepath.Join(s.RemotePath, fileName)
+
+	downloader := s3manager.NewDownloader(s.client)
+	numBytes, err := downloader.Download(file,
+		&s3.GetObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(objectKey),
+		})
+	if err != nil {
+		utils.Error("Failed to download file %s", fileName)
+		return err
+	}
+	utils.Info("Backup downloaded:  %s , bytes size: %d ", file.Name(), uint64(numBytes))
+
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (s s3Storage) Prune(retentionDays int) error {
+	svc := s3.New(s.client)
+
+	// Get the current time
+	now := time.Now()
+	backupRetentionDays := now.AddDate(0, 0, -retentionDays)
+
+	// List objects in the bucket
+	listObjectsInput := &s3.ListObjectsV2Input{
+		Bucket: aws.String(s.bucket),
+		Prefix: aws.String(s.RemotePath),
+	}
+	err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, object := range page.Contents {
+			if object.LastModified.Before(backupRetentionDays) {
+				utils.Info("Deleting old backup: %s", *object.Key)
+				// Object is older than retention days, delete it
+				_, err := svc.DeleteObject(&s3.DeleteObjectInput{
+					Bucket: aws.String(s.bucket),
+					Key:    object.Key,
+				})
+				if err != nil {
+					utils.Info("Failed to delete object %s: %v", *object.Key, err)
+				} else {
+					utils.Info("Deleted object %s", *object.Key)
+				}
+			}
+		}
+		return !lastPage
+	})
+	if err != nil {
+		utils.Error("Failed to list objects: %v", err)
+	}
+
+	utils.Info("Deleting old backups...done")
+	return nil
+
+}
+
+// Name returns the storage name
+func (s s3Storage) Name() string {
+	return "s3"
+}

pkg/storage/ssh/ssh.go

@@ -0,0 +1,116 @@
+package ssh
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/bramvdbogaerde/go-scp"
+	"github.com/bramvdbogaerde/go-scp/auth"
+	"github.com/jkaninda/pg-bkup/pkg/storage"
+	"github.com/jkaninda/pg-bkup/utils"
+	"golang.org/x/crypto/ssh"
+	"os"
+	"path/filepath"
+)
+
+type sshStorage struct {
+	*storage.Backend
+	client scp.Client
+}
+
+// Config holds the SSH connection details
+type Config struct {
+	Host         string
+	User         string
+	Password     string
+	Port         string
+	IdentifyFile string
+	LocalPath string
+	RemotePath string
+}
+
+func createClient(conf Config) (scp.Client, error) {
+	if conf.IdentifyFile != "" && utils.FileExists(conf.IdentifyFile) {
+		clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
+		return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
+	} else {
+		if conf.Password == "" {
+			return scp.Client{}, errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty")
+		}
+		utils.Warn("Accessing the remote server using password, which is not recommended.")
+		clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
+		return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
+
+	}
+}
+
+func NewStorage(conf Config) (storage.Storage, error) {
+	client, err := createClient(conf)
+	if err != nil {
+		return nil, err
+	}
+	return &sshStorage{
+		client: client,
+		Backend: &storage.Backend{
+			RemotePath: conf.RemotePath,
+			LocalPath: conf.LocalPath,
+		},
+	}, nil
+}
+func (s sshStorage) Copy(fileName string) error {
+	client := s.client
+	// Connect to the remote server
+	err := client.Connect()
+	if err != nil {
+		return errors.New("couldn't establish a connection to the remote server")
+	}
+	// Open the local file
+	filePath := filepath.Join(s.LocalPath, fileName)
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to open file %s: %w", filePath, err)
+	}
+	defer client.Close()
+	// Copy file to the remote server
+	err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655")
+	if err != nil {
+		return fmt.Errorf("failed to copy file to remote server: %w", err)
+	}
+
+	return nil
+}
+
+// CopyFrom copies a file from the remote server to local storage
+func (s sshStorage) CopyFrom(fileName string) error {
+	// Create a new SCP client
+	client := s.client
+	// Connect to the remote server
+	err := client.Connect()
+	if err != nil {
+		return errors.New("couldn't establish a connection to the remote server")
+	}
+	// Close client connection after the file has been copied
+	defer client.Close()
+	file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777)
+	if err != nil {
+		return errors.New("couldn't open the output file")
+	}
+	defer file.Close()
+
+	err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
+
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (s sshStorage) Prune(retentionDays int) error {
+	utils.Info("Deleting old backup from a remote server is not implemented yet")
+	return nil
+}
+
+func (s sshStorage) Name() string {
+	return "ssh"
+}

pkg/storage/storage.go

@@ -0,0 +1,14 @@
+package storage
+
+type Storage interface {
+	Copy(fileName string) error
+	CopyFrom(fileName string) error
+	Prune(retentionDays int) error
+	Name() string
+}
+type Backend struct {
+	//Local Path
+	LocalPath string
+	//Remote path or Destination path
+	RemotePath string
+}