jkaninda/pg-bkup
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

chore: rename environment variable for database migration operation

Browse Source
This commit is contained in:
Jonas Kaninda
2024-09-03 06:09:56 +02:00
parent 66d4278797
commit 27b6ff679e
7 changed files with 112 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
docker/Dockerfile
View File

@@ -30,13 +30,13 @@ ENV SSH_PASSWORD=""
ENV SSH_HOST_NAME="" ENV SSH_HOST_NAME=""
ENV SSH_IDENTIFY_FILE="" ENV SSH_IDENTIFY_FILE=""
ENV SSH_PORT="22" ENV SSH_PORT="22"
ENV SOURCE_DB_HOST="" ENV TARGET_DB_HOST=""
ENV SOURCE_DB_PORT=5432 ENV TARGET_DB_PORT=5432
ENV SOURCE_DB_NAME="" ENV TARGET_DB_NAME=""
ENV SOURCE_DB_USERNAME="" ENV TARGET_DB_USERNAME=""
ENV SOURCE_DB_PASSWORD="" ENV TARGET_DB_PASSWORD=""
ARG DEBIAN_FRONTEND=noninteractive ARG DEBIAN_FRONTEND=noninteractive
ENV VERSION="v1.2.3" ENV VERSION="v1.2.4"
ENV BACKUP_CRON_EXPRESSION="" ENV BACKUP_CRON_EXPRESSION=""
ARG WORKDIR="/config" ARG WORKDIR="/config"
ARG BACKUPDIR="/backup" ARG BACKUPDIR="/backup"

112
docs/how-tos/migrate.md
View File

@@ -10,11 +10,13 @@ nav_order: 9
To migrate the database, you need to add `migrate` command. To migrate the database, you need to add `migrate` command.
{: .note } {: .note }
The pg backup has another great feature: migrating your database from a source database to another. The PostgresQL backup has another great feature: migrating your database from a source database to a target.
As you know, to restore a database from a source to a target database, you need 2 operations: which is to start by backing up the source database and then restoring the source backed database to the target database. As you know, to restore a database from a source to a target database, you need 2 operations: which is to start by backing up the source database and then restoring the source backed database to the target database.
Instead of proceeding like that, you can use the integrated feature `(migrate)`, which will help you migrate your database by doing only one operation. Instead of proceeding like that, you can use the integrated feature `(migrate)`, which will help you migrate your database by doing only one operation.
{: .warning }
The `migrate` operation is irreversible, please backup your target database before this action.
### Docker compose ### Docker compose
```yml ```yml
@@ -30,49 +32,50 @@ services:
volumes: volumes:
- ./backup:/backup - ./backup:/backup
environment: environment:
## Target database ## Source database
- DB_PORT=5432 - DB_PORT=5432
- DB_HOST=postgres - DB_HOST=postgres
- DB_NAME=database - DB_NAME=database
- DB_USERNAME=username - DB_USERNAME=username
- DB_PASSWORD=password - DB_PASSWORD=password
## Source database ## Target database
- SOURCE_DB_HOST=postgres - TARGET_DB_HOST=target-postgres
- SOURCE_DB_PORT=5432 - TARGET_DB_PORT=5432
- SOURCE_DB_NAME=sourcedb - TARGET_DB_NAME=dbname
- SOURCE_DB_USERNAME=jonas - TARGET_DB_USERNAME=username
- SOURCE_DB_PASSWORD=password - TARGET_DB_PASSWORD=password
# pg-bkup container must be connected to the same network with your database # mysql-bkup container must be connected to the same network with your database
networks: networks:
- web - web
networks: networks:
web: web:
``` ```
### Migrate database using Docker CLI ### Migrate database using Docker CLI
``` ```
## Target database ## Source database
DB_PORT=5432
DB_HOST=postgres DB_HOST=postgres
DB_NAME=targetdb DB_PORT=5432
DB_USERNAME=targetuser DB_NAME=dbname
DB_USERNAME=username
DB_PASSWORD=password DB_PASSWORD=password
## Source database ## Taget database
SOURCE_DB_HOST=postgres TARGET_DB_HOST=target-postgres
SOURCE_DB_PORT=5432 TARGET_DB_PORT=5432
SOURCE_DB_NAME=sourcedb TARGET_DB_NAME=dbname
SOURCE_DB_USERNAME=sourceuser TARGET_DB_USERNAME=username
SOURCE_DB_PASSWORD=password TARGET_DB_PASSWORD=password
``` ```
```shell ```shell
docker run --rm --network your_network_name \ docker run --rm --network your_network_name \
--env-file your-env --env-file your-env
-v $PWD/backup:/backup/ \ -v $PWD/backup:/backup/ \
jkaninda/pg-bkup migrate -d database_name jkaninda/pg-bkup migrate
``` ```
## Kubernetes ## Kubernetes
@@ -87,37 +90,42 @@ spec:
template: template:
spec: spec:
containers: containers:
- name: pg-bkup - name: pg-bkup
# In production, it is advised to lock your image tag to a proper # In production, it is advised to lock your image tag to a proper
# release version instead of using `latest`. # release version instead of using `latest`.
# Check https://github.com/jkaninda/pg-bkup/releases # Check https://github.com/jkaninda/pg-bkup/releases
# for a list of available releases. # for a list of available releases.
image: jkaninda/pg-bkup image: jkaninda/pg-bkup
command: command:
- /bin/sh - /bin/sh
- -c - -c
- migrate -d targetdb - migrate
resources: resources:
limits: limits:
memory: "128Mi" memory: "128Mi"
cpu: "500m" cpu: "500m"
env: env:
## Target DB ## Source Database
- name: DB_HOST - name: DB_HOST
value: "postgres-target" value: "postgres"
- name: DB_USERNAME - name: DB_PORT
value: "postgres" value: "5432"
- name: DB_PASSWORD - name: DB_NAME
value: "password" value: "dbname"
## Source DB - name: DB_USERNAME
- name: SOURCE_DB_HOST value: "username"
value: "postgres-source" - name: DB_PASSWORD
- name: SOURCE_DB_NAME value: "password"
value: "sourcedb" ## Target Database
- name: SOURCE_DB_USERNAME - name: TARGET_DB_HOST
value: "postgres" value: "target-postgres"
# Please use secret! - name: TARGET_DB_PORT
- name: SOURCE_DB_PASSWORD value: "5432"
value: "password" - name: TARGET_DB_NAME
value: "dbname"
- name: TARGET_DB_USERNAME
value: "username"
- name: TARGET_DB_PASSWORD
value: "password"
restartPolicy: Never restartPolicy: Never
``` ```

10
docs/reference/index.md
View File

@@ -57,11 +57,11 @@ Backup, restore and migrate targets, schedule and retention are configured using
| SSH_IDENTIFY_FILE | Optional, required for SSH storage | ssh remote user's private key | | SSH_IDENTIFY_FILE | Optional, required for SSH storage | ssh remote user's private key |
| SSH_PORT | Optional, required for SSH storage | ssh remote server port | | SSH_PORT | Optional, required for SSH storage | ssh remote server port |
| SSH_REMOTE_PATH | Optional, required for SSH storage | ssh remote path (/home/toto/backup) | | SSH_REMOTE_PATH | Optional, required for SSH storage | ssh remote path (/home/toto/backup) |
| SOURCE_DB_HOST | Optional, required for database migration | Source database host | | TARGET_DB_HOST | Optional, required for database migration | Target database host |
| SOURCE_DB_PORT | Optional, required for database migration | Source database port | | TARGET_DB_PORT | Optional, required for database migration | Target database port |
| SOURCE_DB_NAME | Optional, required for database migration | Source database name | | TARGET_DB_NAME | Optional, required for database migration | Target database name |
| SOURCE_DB_USERNAME | Optional, required for database migration | Source database username | | TARGET_DB_USERNAME | Optional, required for database migration | Target database username |
| SOURCE_DB_PASSWORD | Optional, required for database migration | Source database password | | TARGET_DB_PASSWORD | Optional, required for database migration | Target database password |
--- ---
## Run in Scheduled mode ## Run in Scheduled mode

34
pkg/config.go
View File

@@ -16,12 +16,12 @@ type dbConfig struct {
dbUserName string dbUserName string
dbPassword string dbPassword string
} }
type dbSourceConfig struct { type targetDbConfig struct {
sourceDbHost string targetDbHost string
sourceDbPort string targetDbPort string
sourceDbUserName string targetDbUserName string
sourceDbPassword string targetDbPassword string
sourceDbName string targetDbName string
} }
func getDbConfig(cmd *cobra.Command) *dbConfig { func getDbConfig(cmd *cobra.Command) *dbConfig {
@@ -41,18 +41,18 @@ func getDbConfig(cmd *cobra.Command) *dbConfig {
} }
return &dConf return &dConf
} }
func getSourceDbConfig() *dbSourceConfig { func getTargetDbConfig() *targetDbConfig {
sdbConfig := dbSourceConfig{} tdbConfig := targetDbConfig{}
sdbConfig.sourceDbHost = os.Getenv("SOURCE_DB_HOST") tdbConfig.targetDbHost = os.Getenv("TARGET_DB_HOST")
sdbConfig.sourceDbPort = os.Getenv("SOURCE_DB_PORT") tdbConfig.targetDbPort = os.Getenv("TARGET_DB_PORT")
sdbConfig.sourceDbName = os.Getenv("SOURCE_DB_NAME") tdbConfig.targetDbName = os.Getenv("TARGET_DB_NAME")
sdbConfig.sourceDbUserName = os.Getenv("SOURCE_DB_USERNAME") tdbConfig.targetDbUserName = os.Getenv("TARGET_DB_USERNAME")
sdbConfig.sourceDbPassword = os.Getenv("SOURCE_DB_PASSWORD") tdbConfig.targetDbPassword = os.Getenv("TARGET_DB_PASSWORD")
err := utils.CheckEnvVars(sdbRVars) err := utils.CheckEnvVars(tdbRVars)
if err != nil { if err != nil {
utils.Error("Please make sure all required environment variables for source database are set") utils.Error("Please make sure all required environment variables for the target database are set")
utils.Fatal("Error checking environment variables: %s", err) utils.Fatal("Error checking target database environment variables: %s", err)
} }
return &sdbConfig return &tdbConfig
} }

30
pkg/migrate.go
View File

@@ -11,21 +11,23 @@ func StartMigration(cmd *cobra.Command) {
utils.Info("Starting database migration...") utils.Info("Starting database migration...")
//Get DB config //Get DB config
dbConf = getDbConfig(cmd) dbConf = getDbConfig(cmd)
sDbConf = getSourceDbConfig() targetDbConf = getTargetDbConfig()
//Defining the target database variables
newDbConfig := dbConfig{}
newDbConfig.dbHost = targetDbConf.targetDbHost
newDbConfig.dbPort = targetDbConf.targetDbPort
newDbConfig.dbName = targetDbConf.targetDbName
newDbConfig.dbUserName = targetDbConf.targetDbUserName
newDbConfig.dbPassword = targetDbConf.targetDbPassword
//Generate file name //Generate file name
backupFileName := fmt.Sprintf("%s_%s.sql", sDbConf.sourceDbName, time.Now().Format("20060102_150405")) backupFileName := fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20060102_150405"))
//Backup Source Database //Backup source Database
newDbConfig := dbConfig{} BackupDatabase(dbConf, backupFileName, true)
newDbConfig.dbHost = sDbConf.sourceDbHost
newDbConfig.dbPort = sDbConf.sourceDbPort
newDbConfig.dbName = sDbConf.sourceDbName
newDbConfig.dbUserName = sDbConf.sourceDbUserName
newDbConfig.dbPassword = sDbConf.sourceDbPassword
BackupDatabase(&newDbConfig, backupFileName, true)
//Restore source database into target database //Restore source database into target database
utils.Info("Restoring [%s] database into [%s] database...", sDbConf.sourceDbName, dbConf.dbName) utils.Info("Restoring [%s] database into [%s] database...", dbConf.dbName, targetDbConf.targetDbName)
RestoreDatabase(dbConf, backupFileName) RestoreDatabase(&newDbConfig, backupFileName)
utils.Info("[%s] database has been restored into [%s] database", sDbConf.sourceDbName, dbConf.dbName) utils.Info("[%s] database has been restored into [%s] database", dbConf.dbName, targetDbConf.targetDbName)
utils.Info("Database migration completed!") utils.Info("Database migration completed.")
} }

14
pkg/var.go
View File

@@ -23,16 +23,16 @@ var dbHVars = []string{
"DB_USERNAME", "DB_USERNAME",
"DB_NAME", "DB_NAME",
} }
var sdbRVars = []string{ var tdbRVars = []string{
"SOURCE_DB_HOST", "TARGET_DB_HOST",
"SOURCE_DB_PORT", "TARGET_DB_PORT",
"SOURCE_DB_NAME", "TARGET_DB_NAME",
"SOURCE_DB_USERNAME", "TARGET_DB_USERNAME",
"SOURCE_DB_PASSWORD", "TARGET_DB_PASSWORD",
} }
var dbConf *dbConfig var dbConf *dbConfig
var sDbConf *dbSourceConfig var targetDbConf *targetDbConfig
// sshVars Required environment variables for SSH remote server storage // sshVars Required environment variables for SSH remote server storage
var sshVars = []string{ var sshVars = []string{

2
utils/s3.go
View File

@@ -119,7 +119,7 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error {
Error("Failed to download file", err) Error("Failed to download file", err)
return err return err
} }
Info("Backup downloaded: ", file.Name(), " bytes size ", numBytes) Info("Backup downloaded: %s bytes size %s ", file.Name(), numBytes)
return nil return nil
} }