mirror of
https://github.com/jkaninda/mysql-bkup.git
synced 2025-12-06 21:49:40 +01:00
Add SSH storage, add database backup encrypt and decrypt
This commit is contained in:
171
utils/s3.go
Normal file
171
utils/s3.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"golang.org/x/exp/slog"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CreateSession creates a new AWS session
|
||||
func CreateSession() (*session.Session, error) {
|
||||
// AwsVars Required environment variables for AWS S3 storage
|
||||
var awsVars = []string{
|
||||
"AWS_S3_ENDPOINT",
|
||||
"AWS_S3_BUCKET_NAME",
|
||||
"AWS_ACCESS_KEY",
|
||||
"AWS_SECRET_KEY",
|
||||
"AWS_REGION",
|
||||
"AWS_REGION",
|
||||
"AWS_REGION",
|
||||
}
|
||||
|
||||
endPoint := GetEnvVariable("AWS_S3_ENDPOINT", "S3_ENDPOINT")
|
||||
accessKey := GetEnvVariable("AWS_ACCESS_KEY", "ACCESS_KEY")
|
||||
secretKey := GetEnvVariable("AWS_SECRET_KEY", "SECRET_KEY")
|
||||
_ = GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME")
|
||||
|
||||
region := os.Getenv("AWS_REGION")
|
||||
awsDisableSsl, err := strconv.ParseBool(os.Getenv("AWS_DISABLE_SSL"))
|
||||
if err != nil {
|
||||
Fatal("Unable to parse AWS_DISABLE_SSL env var: %s", err)
|
||||
}
|
||||
|
||||
err = CheckEnvVars(awsVars)
|
||||
if err != nil {
|
||||
slog.Error(fmt.Sprintf("Error checking environment variables\n: %s", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
// S3 Config
|
||||
s3Config := &aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
Endpoint: aws.String(endPoint),
|
||||
Region: aws.String(region),
|
||||
DisableSSL: aws.Bool(awsDisableSsl),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
}
|
||||
return session.NewSession(s3Config)
|
||||
|
||||
}
|
||||
|
||||
// UploadFileToS3 uploads a file to S3 with a given prefix
|
||||
func UploadFileToS3(filePath, key, bucket, prefix string) error {
|
||||
sess, err := CreateSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
svc := s3.New(sess)
|
||||
|
||||
file, err := os.Open(filepath.Join(filePath, key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
fileInfo, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
objectKey := filepath.Join(prefix, key)
|
||||
|
||||
buffer := make([]byte, fileInfo.Size())
|
||||
file.Read(buffer)
|
||||
fileBytes := bytes.NewReader(buffer)
|
||||
fileType := http.DetectContentType(buffer)
|
||||
|
||||
_, err = svc.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(objectKey),
|
||||
Body: fileBytes,
|
||||
ContentLength: aws.Int64(fileInfo.Size()),
|
||||
ContentType: aws.String(fileType),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
func DownloadFile(destinationPath, key, bucket, prefix string) error {
|
||||
|
||||
sess, err := CreateSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
Info("Download backup from S3 storage...")
|
||||
file, err := os.Create(filepath.Join(destinationPath, key))
|
||||
if err != nil {
|
||||
fmt.Println("Failed to create file", err)
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
objectKey := filepath.Join(prefix, key)
|
||||
|
||||
downloader := s3manager.NewDownloader(sess)
|
||||
numBytes, err := downloader.Download(file,
|
||||
&s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("Failed to download file", err)
|
||||
return err
|
||||
}
|
||||
Info(fmt.Sprintf("Backup downloaded: ", file.Name(), " bytes size ", numBytes))
|
||||
|
||||
return nil
|
||||
}
|
||||
func DeleteOldBackup(bucket, prefix string, retention int) error {
|
||||
sess, err := CreateSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
svc := s3.New(sess)
|
||||
|
||||
// Get the current time and the time threshold for 7 days ago
|
||||
now := time.Now()
|
||||
backupRetentionDays := now.AddDate(0, 0, -retention)
|
||||
|
||||
// List objects in the bucket
|
||||
listObjectsInput := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String(prefix),
|
||||
}
|
||||
err = svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
|
||||
for _, object := range page.Contents {
|
||||
if object.LastModified.Before(backupRetentionDays) {
|
||||
// Object is older than retention days, delete it
|
||||
_, err := svc.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: object.Key,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to delete object %s: %v", *object.Key, err)
|
||||
} else {
|
||||
fmt.Printf("Deleted object %s\n", *object.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return !lastPage
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to list objects: %v", err)
|
||||
}
|
||||
|
||||
fmt.Println("Finished deleting old files.")
|
||||
return nil
|
||||
}
|
||||
143
utils/utils.go
143
utils/utils.go
@@ -7,25 +7,51 @@ package utils
|
||||
* @link https://github.com/jkaninda/mysql-bkup
|
||||
**/
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/exp/slog"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
func Info(v ...any) {
|
||||
fmt.Println("⒤ ", fmt.Sprint(v...))
|
||||
func Info(msg string, args ...any) {
|
||||
if len(args) == 0 {
|
||||
slog.Info(msg)
|
||||
} else {
|
||||
slog.Info(fmt.Sprintf(msg, args...))
|
||||
}
|
||||
}
|
||||
func Done(v ...any) {
|
||||
fmt.Println("✔ ", fmt.Sprint(v...))
|
||||
func Worn(msg string, args ...any) {
|
||||
if len(args) == 0 {
|
||||
slog.Warn(msg)
|
||||
} else {
|
||||
slog.Warn(fmt.Sprintf(msg, args...))
|
||||
}
|
||||
}
|
||||
func Fatal(v ...any) {
|
||||
fmt.Println("✘ ", fmt.Sprint(v...))
|
||||
os.Exit(1)
|
||||
func Error(msg string, args ...any) {
|
||||
if len(args) == 0 {
|
||||
slog.Error(msg)
|
||||
} else {
|
||||
slog.Error(fmt.Sprintf(msg, args...))
|
||||
}
|
||||
}
|
||||
func Fatalf(msg string, v ...any) {
|
||||
fmt.Printf("✘ "+msg, v...)
|
||||
func Done(msg string, args ...any) {
|
||||
if len(args) == 0 {
|
||||
slog.Info(msg)
|
||||
} else {
|
||||
slog.Info(fmt.Sprintf(msg, args...))
|
||||
}
|
||||
}
|
||||
func Fatal(msg string, args ...any) {
|
||||
// Fatal logs an error message and exits the program.
|
||||
if len(args) == 0 {
|
||||
slog.Error(msg)
|
||||
} else {
|
||||
slog.Error(fmt.Sprintf(msg, args...))
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -47,9 +73,45 @@ func WriteToFile(filePath, content string) error {
|
||||
_, err = file.WriteString(content)
|
||||
return err
|
||||
}
|
||||
func DeleteFile(filePath string) error {
|
||||
err := os.Remove(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func CopyFile(src, dst string) error {
|
||||
// Open the source file for reading
|
||||
sourceFile, err := os.Open(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open source file: %v", err)
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
// Create the destination file
|
||||
destinationFile, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create destination file: %v", err)
|
||||
}
|
||||
defer destinationFile.Close()
|
||||
|
||||
// Copy the content from source to destination
|
||||
_, err = io.Copy(destinationFile, sourceFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy file: %v", err)
|
||||
}
|
||||
|
||||
// Flush the buffer to ensure all data is written
|
||||
err = destinationFile.Sync()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to sync destination file: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
func ChangePermission(filePath string, mod int) {
|
||||
if err := os.Chmod(filePath, fs.FileMode(mod)); err != nil {
|
||||
Fatalf("Error changing permissions of %s: %v\n", filePath, err)
|
||||
Fatal("Error changing permissions of %s: %v\n", filePath, err)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -69,15 +131,31 @@ func IsDirEmpty(name string) (bool, error) {
|
||||
|
||||
// TestDatabaseConnection tests the database connection
|
||||
func TestDatabaseConnection() {
|
||||
Info("Testing database connection...")
|
||||
// Test database connection
|
||||
cmd := exec.Command("mysql", "-h", os.Getenv("DB_HOST"), "-P", os.Getenv("DB_PORT"), "-u", os.Getenv("DB_USERNAME"), "--password="+os.Getenv("DB_PASSWORD"), os.Getenv("DB_NAME"), "-e", "quit")
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
Fatal("Error testing database connection:", err)
|
||||
dbHost := os.Getenv("DB_HOST")
|
||||
dbPassword := os.Getenv("DB_PASSWORD")
|
||||
dbUserName := os.Getenv("DB_USERNAME")
|
||||
dbName := os.Getenv("DB_NAME")
|
||||
dbPort := os.Getenv("DB_PORT")
|
||||
|
||||
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" {
|
||||
Fatal("Please make sure all required database environment variables are set")
|
||||
} else {
|
||||
Info("Connecting to database ...")
|
||||
|
||||
cmd := exec.Command("mysql", "-h", dbHost, "-P", dbPort, "-u", dbUserName, "--password="+dbPassword, dbName, "-e", "quit")
|
||||
|
||||
// Capture the output
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
slog.Error(fmt.Sprintf("Error testing database connection: %v\nOutput: %s\n", err, out.String()))
|
||||
os.Exit(1)
|
||||
|
||||
}
|
||||
Info("Successfully connected to database")
|
||||
}
|
||||
|
||||
}
|
||||
func GetEnv(cmd *cobra.Command, flagName, envName string) string {
|
||||
value, _ := cmd.Flags().GetString(flagName)
|
||||
@@ -109,6 +187,37 @@ func SetEnv(key, value string) {
|
||||
return
|
||||
}
|
||||
}
|
||||
func GetEnvVariable(envName, oldEnvName string) string {
|
||||
value := os.Getenv(envName)
|
||||
if value == "" {
|
||||
value = os.Getenv(oldEnvName)
|
||||
if value != "" {
|
||||
err := os.Setenv(envName, value)
|
||||
if err != nil {
|
||||
return value
|
||||
}
|
||||
Worn("%s is deprecated, please use %s instead!\n", oldEnvName, envName)
|
||||
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
func ShowHistory() {
|
||||
}
|
||||
|
||||
// CheckEnvVars checks if all the specified environment variables are set
|
||||
func CheckEnvVars(vars []string) error {
|
||||
missingVars := []string{}
|
||||
|
||||
for _, v := range vars {
|
||||
if os.Getenv(v) == "" {
|
||||
missingVars = append(missingVars, v)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missingVars) > 0 {
|
||||
return fmt.Errorf("missing environment variables: %v", missingVars)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user