Migrate storage

go.mod

@@ -22,7 +22,6 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/jkaninda/go-storage v0.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

internal/backup.go

@@ -3,15 +3,14 @@ package internal
 import (
 	"fmt"
 	"github.com/jkaninda/encryptor"
-	"github.com/jkaninda/go-storage/pkg/ftp"
-	"github.com/jkaninda/go-storage/pkg/local"
-	"github.com/jkaninda/go-storage/pkg/s3"
-	"github.com/jkaninda/go-storage/pkg/ssh"
 	"github.com/jkaninda/pg-bkup/pkg/logger"
+	"github.com/jkaninda/pg-bkup/pkg/storage/ftp"
+	"github.com/jkaninda/pg-bkup/pkg/storage/local"
+	"github.com/jkaninda/pg-bkup/pkg/storage/s3"
+	"github.com/jkaninda/pg-bkup/pkg/storage/ssh"
 	"github.com/jkaninda/pg-bkup/utils"
 	"github.com/robfig/cron/v3"
 	"github.com/spf13/cobra"
-
 	"log"
 	"os"
 	"os/exec"

internal/restore.go

@@ -9,15 +9,15 @@ package internal
 
 import (
 	"github.com/jkaninda/pg-bkup/pkg/logger"
+	"github.com/jkaninda/pg-bkup/pkg/storage/ftp"
+	"github.com/jkaninda/pg-bkup/pkg/storage/local"
+	"github.com/jkaninda/pg-bkup/pkg/storage/s3"
+	"github.com/jkaninda/pg-bkup/pkg/storage/ssh"
 	"os"
 	"os/exec"
 	"path/filepath"
 
 	"github.com/jkaninda/encryptor"
-	"github.com/jkaninda/go-storage/pkg/ftp"
-	"github.com/jkaninda/go-storage/pkg/local"
-	"github.com/jkaninda/go-storage/pkg/s3"
-	"github.com/jkaninda/go-storage/pkg/ssh"
 	"github.com/jkaninda/pg-bkup/utils"
 	"github.com/spf13/cobra"
 )

pkg/storage/ftp/ftp.go

@@ -0,0 +1,142 @@
+package ftp
+
+import (
+	"fmt"
+	pkg "github.com/jkaninda/pg-bkup/pkg/storage"
+	"github.com/jlaffaye/ftp"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type ftpStorage struct {
+	*pkg.Backend
+	client *ftp.ServerConn
+}
+
+// Config holds the SSH connection details
+type Config struct {
+	Host       string
+	User       string
+	Password   string
+	Port       string
+	LocalPath  string
+	RemotePath string
+}
+
+// createClient creates FTP Client
+func createClient(conf Config) (*ftp.ServerConn, error) {
+	ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to FTP: %w", err)
+	}
+
+	err = ftpClient.Login(conf.User, conf.Password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to log in to FTP: %w", err)
+	}
+
+	return ftpClient, nil
+}
+
+// NewStorage creates new Storage
+func NewStorage(conf Config) (pkg.Storage, error) {
+	client, err := createClient(conf)
+	if err != nil {
+		return nil, err
+	}
+	return &ftpStorage{
+		client: client,
+		Backend: &pkg.Backend{
+			RemotePath: conf.RemotePath,
+			LocalPath:  conf.LocalPath,
+		},
+	}, nil
+}
+
+// Copy copies file to the remote server
+func (s ftpStorage) Copy(fileName string) error {
+	ftpClient := s.client
+	defer func(ftpClient *ftp.ServerConn) {
+		err := ftpClient.Quit()
+		if err != nil {
+			return
+		}
+	}(ftpClient)
+
+	filePath := filepath.Join(s.LocalPath, fileName)
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to open file %s: %w", fileName, err)
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+			return
+		}
+	}(file)
+
+	remoteFilePath := filepath.Join(s.RemotePath, fileName)
+	err = ftpClient.Stor(remoteFilePath, file)
+	if err != nil {
+		return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err)
+	}
+
+	return nil
+}
+
+// CopyFrom copies a file from the remote server to local storage
+func (s ftpStorage) CopyFrom(fileName string) error {
+	ftpClient := s.client
+
+	defer func(ftpClient *ftp.ServerConn) {
+		err := ftpClient.Quit()
+		if err != nil {
+			return
+		}
+	}(ftpClient)
+
+	remoteFilePath := filepath.Join(s.RemotePath, fileName)
+	r, err := ftpClient.Retr(remoteFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
+	}
+	defer func(r *ftp.Response) {
+		err := r.Close()
+		if err != nil {
+			return
+		}
+	}(r)
+
+	localFilePath := filepath.Join(s.LocalPath, fileName)
+	outFile, err := os.Create(localFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to create local file %s: %w", fileName, err)
+	}
+	defer func(outFile *os.File) {
+		err := outFile.Close()
+		if err != nil {
+			return
+		}
+	}(outFile)
+
+	_, err = io.Copy(outFile, r)
+	if err != nil {
+		return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err)
+	}
+
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (s ftpStorage) Prune(retentionDays int) error {
+	fmt.Println("Deleting old backup from a remote server is not implemented yet")
+	return nil
+
+}
+
+// Name returns the storage name
+func (s ftpStorage) Name() string {
+	return "ftp"
+}

pkg/storage/local/local.go

@@ -0,0 +1,116 @@
+package local
+
+import (
+	pkg "github.com/jkaninda/pg-bkup/pkg/storage"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type localStorage struct {
+	*pkg.Backend
+}
+type Config struct {
+	LocalPath  string
+	RemotePath string
+}
+
+// NewStorage creates new Storage
+func NewStorage(conf Config) pkg.Storage {
+	return &localStorage{
+		Backend: &pkg.Backend{
+			LocalPath:  conf.LocalPath,
+			RemotePath: conf.RemotePath,
+		},
+	}
+}
+
+// Copy copies file to the local destination path
+func (l localStorage) Copy(file string) error {
+	if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) {
+		return err
+	}
+	err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// CopyFrom copies file from a Path to local path
+func (l localStorage) CopyFrom(file string) error {
+	if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) {
+		return err
+	}
+	err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (l localStorage) Prune(retentionDays int) error {
+	currentTime := time.Now()
+	// Delete file
+	deleteFile := func(filePath string) error {
+		err := os.Remove(filePath)
+		return err
+	}
+	// Walk through the directory and delete files modified more than specified days ago
+	err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// Check if it's a regular file and if it was modified more than specified days ago
+		if fileInfo.Mode().IsRegular() {
+			timeDiff := currentTime.Sub(fileInfo.ModTime())
+			if timeDiff.Hours() > 24*float64(retentionDays) {
+				err := deleteFile(filePath)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Name returns the storage name
+func (l localStorage) Name() string {
+	return "local"
+}
+
+// copyFile copies file
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer func(in *os.File) {
+		err := in.Close()
+		if err != nil {
+			return
+		}
+	}(in)
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		err := out.Close()
+		if err != nil {
+			return err
+		}
+		return err
+	}
+	return out.Close()
+}

pkg/storage/local/local_test.go

@@ -0,0 +1,66 @@
+package local
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const content = "Lorem ipsum dolor sit amet. Eum eius voluptas sit vitae vitae aut sequi molestias hic accusamus consequatur"
+const inputFile = "file.txt"
+const localPath = "./tests/local"
+const RemotePath = "./tests/remote"
+
+func TestCopy(t *testing.T) {
+
+	err := os.MkdirAll(localPath, 0777)
+	if err != nil {
+		t.Error(err)
+	}
+	err = os.MkdirAll(RemotePath, 0777)
+	if err != nil {
+		t.Error(err)
+	}
+
+	_, err = createFile(filepath.Join(localPath, inputFile), content)
+	if err != nil {
+		t.Error(err)
+	}
+
+	l := NewStorage(Config{
+		LocalPath:  "./tests/local",
+		RemotePath: "./tests/remote",
+	})
+	err = l.Copy(inputFile)
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Printf("File copied to %s\n", filepath.Join(RemotePath, inputFile))
+}
+func createFile(fileName, content string) ([]byte, error) {
+	// Create a file named hello.txt
+	file, err := os.Create(fileName)
+	if err != nil {
+		fmt.Println("Error creating file:", err)
+		return nil, err
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+			fmt.Println("Error closing file:", err)
+			return
+		}
+	}(file)
+
+	// Write the message to the file
+	_, err = file.WriteString(content)
+	if err != nil {
+		fmt.Println("Error writing to file:", err)
+		return nil, err
+	}
+
+	fmt.Printf("Successfully wrote to %s\n", fileName)
+	fileBytes, err := os.ReadFile(fileName)
+	return fileBytes, err
+}

pkg/storage/s3/s3.go

@@ -0,0 +1,176 @@
+package s3
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	pkg "github.com/jkaninda/pg-bkup/pkg/storage"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type s3Storage struct {
+	*pkg.Backend
+	client *session.Session
+	bucket string
+}
+
+// Config holds the AWS S3 config
+type Config struct {
+	Endpoint       string
+	Bucket         string
+	AccessKey      string
+	SecretKey      string
+	Region         string
+	DisableSsl     bool
+	ForcePathStyle bool
+	LocalPath      string
+	RemotePath     string
+}
+
+// CreateSession creates a new AWS session
+func createSession(conf Config) (*session.Session, error) {
+	s3Config := &aws.Config{
+		Credentials:      credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""),
+		Endpoint:         aws.String(conf.Endpoint),
+		Region:           aws.String(conf.Region),
+		DisableSSL:       aws.Bool(conf.DisableSsl),
+		S3ForcePathStyle: aws.Bool(conf.ForcePathStyle),
+	}
+
+	return session.NewSession(s3Config)
+}
+
+// NewStorage creates new Storage
+func NewStorage(conf Config) (pkg.Storage, error) {
+	sess, err := createSession(conf)
+	if err != nil {
+		return nil, err
+	}
+	return &s3Storage{
+		client: sess,
+		bucket: conf.Bucket,
+		Backend: &pkg.Backend{
+			RemotePath: conf.RemotePath,
+			LocalPath:  conf.LocalPath,
+		},
+	}, nil
+}
+
+// Copy copies file to S3 storage
+func (s s3Storage) Copy(fileName string) error {
+	svc := s3.New(s.client)
+	file, err := os.Open(filepath.Join(s.LocalPath, fileName))
+	if err != nil {
+		return err
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+			return
+		}
+	}(file)
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return err
+	}
+	objectKey := filepath.Join(s.RemotePath, fileName)
+	buffer := make([]byte, fileInfo.Size())
+	_, err = file.Read(buffer)
+	if err != nil {
+		return err
+	}
+	fileBytes := bytes.NewReader(buffer)
+	fileType := http.DetectContentType(buffer)
+
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket:        aws.String(s.bucket),
+		Key:           aws.String(objectKey),
+		Body:          fileBytes,
+		ContentLength: aws.Int64(fileInfo.Size()),
+		ContentType:   aws.String(fileType),
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CopyFrom copies a file from S3 to local storage
+func (s s3Storage) CopyFrom(fileName string) error {
+	file, err := os.Create(filepath.Join(s.LocalPath, fileName))
+	if err != nil {
+		return err
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+			fmt.Printf("Error closing file: %v\n", err)
+			return
+		}
+	}(file)
+
+	objectKey := filepath.Join(s.RemotePath, fileName)
+
+	downloader := s3manager.NewDownloader(s.client)
+	_, err = downloader.Download(file,
+		&s3.GetObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(objectKey),
+		})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (s s3Storage) Prune(retentionDays int) error {
+	svc := s3.New(s.client)
+
+	// Get the current time
+	now := time.Now()
+	backupRetentionDays := now.AddDate(0, 0, -retentionDays)
+
+	// List objects in the bucket
+	listObjectsInput := &s3.ListObjectsV2Input{
+		Bucket: aws.String(s.bucket),
+		Prefix: aws.String(s.RemotePath),
+	}
+	err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, object := range page.Contents {
+			if object.LastModified.Before(backupRetentionDays) {
+				// Object is older than retention days, delete it
+				_, err := svc.DeleteObject(&s3.DeleteObjectInput{
+					Bucket: aws.String(s.bucket),
+					Key:    object.Key,
+				})
+				if err != nil {
+					fmt.Printf("failed to delete object %s: %v", *object.Key, err)
+				} else {
+					fmt.Printf("Deleted object %s", *object.Key)
+				}
+			}
+		}
+		return !lastPage
+	})
+	if err != nil {
+		return fmt.Errorf("failed to list objects: %v", err)
+	}
+
+	return nil
+
+}
+
+// Name returns the storage name
+func (s s3Storage) Name() string {
+	return "s3"
+}

pkg/storage/ssh/ssh.go

@@ -0,0 +1,124 @@
+package ssh
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/bramvdbogaerde/go-scp"
+	"github.com/bramvdbogaerde/go-scp/auth"
+	pkg "github.com/jkaninda/pg-bkup/pkg/storage"
+	"golang.org/x/crypto/ssh"
+	"os"
+	"path/filepath"
+)
+
+type sshStorage struct {
+	*pkg.Backend
+	client scp.Client
+}
+
+// Config holds the SSH connection details
+type Config struct {
+	Host         string
+	User         string
+	Password     string
+	Port         string
+	IdentifyFile string
+	LocalPath    string
+	RemotePath   string
+}
+
+// createClient creates SSH Client
+func createClient(conf Config) (scp.Client, error) {
+	if _, err := os.Stat(conf.IdentifyFile); os.IsNotExist(err) {
+		clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
+		return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
+	} else {
+		if conf.Password == "" {
+			return scp.Client{}, errors.New("ssh password required")
+		}
+		clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
+		return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
+
+	}
+}
+
+// NewStorage creates new Storage
+func NewStorage(conf Config) (pkg.Storage, error) {
+	client, err := createClient(conf)
+	if err != nil {
+		return nil, err
+	}
+	return &sshStorage{
+		client: client,
+		Backend: &pkg.Backend{
+			RemotePath: conf.RemotePath,
+			LocalPath:  conf.LocalPath,
+		},
+	}, nil
+}
+
+// Copy copies file to the remote server
+func (s sshStorage) Copy(fileName string) error {
+	client := s.client
+	// Connect to the remote server
+	err := client.Connect()
+	if err != nil {
+		return errors.New("couldn't establish a connection to the remote server")
+	}
+	// Open the local file
+	filePath := filepath.Join(s.LocalPath, fileName)
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to open file %s: %w", filePath, err)
+	}
+	defer client.Close()
+	// Copy file to the remote server
+	err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655")
+	if err != nil {
+		return fmt.Errorf("failed to copy file to remote server: %w", err)
+	}
+
+	return nil
+}
+
+// CopyFrom copies a file from the remote server to local storage
+func (s sshStorage) CopyFrom(fileName string) error {
+	// Create a new SCP client
+	client := s.client
+	// Connect to the remote server
+	err := client.Connect()
+	if err != nil {
+		return errors.New("couldn't establish a connection to the remote server")
+	}
+	// Close client connection after the file has been copied
+	defer client.Close()
+	file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777)
+	if err != nil {
+		return errors.New("couldn't open the output file")
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+			return
+		}
+	}(file)
+
+	err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
+
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Prune deletes old backup created more than specified days
+func (s sshStorage) Prune(retentionDays int) error {
+	fmt.Println("Deleting old backup from a remote server is not implemented yet")
+	return nil
+}
+
+// Name returns the storage name
+func (s sshStorage) Name() string {
+	return "ssh"
+}

pkg/storage/storage.go

@@ -0,0 +1,14 @@
+package pkg
+
+type Storage interface {
+	Copy(fileName string) error
+	CopyFrom(fileName string) error
+	Prune(retentionDays int) error
+	Name() string
+}
+type Backend struct {
+	// Local Path
+	LocalPath string
+	// Remote path or Destination path
+	RemotePath string
+}