From 28de0d7b52b0a1c46f41c331983e3b5886a40fa6 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 28 Oct 2026 11:18:56 +0100 Subject: [PATCH] feat: mysql-bkup v2 on bkupkit MysQL backup and restore built on the shared bkupkit core: streaming end to end, with no dump held in memory or written to local disk. --- .dockerignore | 30 + .env.example | 44 +- .github/workflows/build.yml | 10 +- .github/workflows/deploy-docs.yml | 8 +- .github/workflows/lint.yml | 13 +- .github/workflows/release.yml | 33 +- .github/workflows/tests.yml | 309 ++------- .golangci.yml | 18 +- Dockerfile | 95 +-- README.md | 644 +++++++++++-------- cmd/backup.go | 57 -- cmd/migrate.go | 45 -- cmd/restore.go | 54 -- cmd/root.go | 58 -- cmd/version.go | 47 -- docs/how-tos/azure-blob.md | 2 +- docs/how-tos/backup-all.md | 4 +- docs/how-tos/backup-to-ftp.md | 4 +- docs/how-tos/backup-to-s3.md | 14 +- docs/how-tos/backup-to-ssh.md | 41 +- docs/how-tos/backup.md | 6 +- docs/how-tos/deploy-on-kubernetes.md | 54 +- docs/how-tos/encrypt-backup.md | 24 +- docs/how-tos/migrate.md | 17 +- docs/how-tos/mutli-backup.md | 8 +- docs/how-tos/receive-notification.md | 49 +- docs/how-tos/restore-from-s3.md | 4 +- docs/how-tos/restore-from-ssh.md | 17 +- docs/how-tos/restore.md | 4 +- docs/quickstart/index.md | 16 +- docs/reference/index.md | 307 +++++++-- docs/upgrade/index.md | 47 ++ examples/docker-compose.s3.yaml | 8 +- examples/docker-compose.scheduled.local.yaml | 3 +- examples/docker-compose.scheduled.s3.yaml | 6 +- examples/docker-compose.yaml | 3 +- examples/k8s-job.yaml | 2 +- go.mod | 111 +++- go.sum | 303 ++++++--- integration_test.go | 639 ++++++++++++++++++ internal/mysql/client.go | 279 ++++++++ internal/mysql/conn.go | 199 ++++++ internal/mysql/engine.go | 487 ++++++++++++++ internal/mysql/mysql_test.go | 407 ++++++++++++ internal/mysql/options.go | 101 +++ main.go | 42 +- migrations/test_config.yaml | 2 +- pkg/azure.go | 128 ---- pkg/backup.go | 438 ------------- pkg/config.go | 363 ----------- pkg/helper.go | 203 ------ pkg/migrate.go | 63 -- pkg/remote.go | 228 ------- pkg/restore.go | 153 ----- pkg/s3.go | 138 ---- pkg/var.go | 87 --- templates/email-error.tmpl | 69 -- templates/email.tmpl | 69 -- templates/telegram-error.tmpl | 11 - templates/telegram.tmpl | 15 - utils/config.go | 95 --- utils/constant.go | 35 - utils/logger.go | 103 --- utils/notification.go | 179 ------ utils/utils.go | 272 -------- 65 files changed, 3453 insertions(+), 3871 deletions(-) create mode 100644 .dockerignore delete mode 100644 cmd/backup.go delete mode 100644 cmd/migrate.go delete mode 100644 cmd/restore.go delete mode 100644 cmd/root.go delete mode 100644 cmd/version.go create mode 100644 docs/upgrade/index.md create mode 100644 integration_test.go create mode 100644 internal/mysql/client.go create mode 100644 internal/mysql/conn.go create mode 100644 internal/mysql/engine.go create mode 100644 internal/mysql/mysql_test.go create mode 100644 internal/mysql/options.go delete mode 100644 pkg/azure.go delete mode 100644 pkg/backup.go delete mode 100644 pkg/config.go delete mode 100644 pkg/helper.go delete mode 100644 pkg/migrate.go delete mode 100644 pkg/remote.go delete mode 100644 pkg/restore.go delete mode 100644 pkg/s3.go delete mode 100644 pkg/var.go delete mode 100644 templates/email-error.tmpl delete mode 100644 templates/email.tmpl delete mode 100644 templates/telegram-error.tmpl delete mode 100644 templates/telegram.tmpl delete mode 100644 utils/config.go delete mode 100644 utils/constant.go delete mode 100644 utils/logger.go delete mode 100644 utils/notification.go delete mode 100644 utils/utils.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b778ae5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +# Only the Go sources are needed to build the image. +.git +.github +.idea +.history +.DS_Store +.env +.env.* +*.env +backup/ +backups/ +data/ +bin/ +dist/ +vendor/ +docs/ +examples/ +migrations/ +test.md +compose.yaml +docker-compose*.yaml +# Binaries and dumps +mysql-bkup +mysql-bkup-* +*.sql +*.sql.gz +*.gpg +*.age +*.tar +*.gz diff --git a/.env.example b/.env.example index da4dece..07ccfa1 100644 --- a/.env.example +++ b/.env.example @@ -1,28 +1,30 @@ ### Database DB_HOST= DB_PORT=3306 -DB_USERNAME= +DB_USER= DB_PASSWORD= DB_NAME= +#DB_SSL_MODE=disabled TZ=Europe/Paris ### Database Migration #TARGET_DB_HOST= #TARGET_DB_PORT=3306 #TARGET_DB_NAME= -#TARGET_DB_USERNAME= +#DB_SSL_MODE=disabled +#TARGET_DB_USER= #TARGET_DB_PASSWORD= ### Backup restoration #FILE_NAME= ### AWS S3 Storage -#ACCESS_KEY= -#SECRET_KEY= +#AWS_ACCESS_KEY= +#AWS_SECRET_KEY= #AWS_S3_BUCKET_NAME= #AWS_S3_ENDPOINT= #AWS_REGION= -#AWS_S3_PATH= +#REMOTE_PATH= #AWS_DISABLE_SSL=false #AWS_FORCE_PATH_STYLE=true @@ -30,29 +32,33 @@ TZ=Europe/Paris #BACKUP_CRON_EXPRESSION=@midnight ##Delete old backup created more than specified days ago #BACKUP_RETENTION_DAYS=7 +#BACKUP_RETENTION_COUNT=10 ####SSH Storage -#SSH_HOST_NAME= -#SSH_PORT=22 -#SSH_USER= -#SSH_PASSWORD= -#SSH_IDENTIFY_FILE=/tmp/id_ed25519 +#SFTP_HOST= +#SFTP_PORT=22 +#SFTP_USER= +#SFTP_PASSWORD= +#SFTP_IDENTITY_FILE=/tmp/id_ed25519 +#SFTP_KNOWN_HOSTS=/config/known_hosts ####FTP Storage #FTP_PASSWORD= -#FTP_HOST_NAME= +#FTP_HOST= #FTP_USER= #FTP_PORT=21 #REMOTE_PATH= ## Azure Blob storage -AZURE_STORAGE_CONTAINER_NAME= -AZURE_STORAGE_ACCOUNT_NAME= -AZURE_STORAGE_ACCOUNT_KEY= +#AZURE_STORAGE_CONTAINER_NAME= +#AZURE_STORAGE_ACCOUNT_NAME= +#AZURE_STORAGE_ACCOUNT_KEY= #### Backup encryption -#GPG_PUBLIC_KEY=/config/public_key.asc -#GPG_PRIVATE_KEY=/config/private_key.asc +#GPG_PUBLIC_KEY=/config/gnupg/public_key.asc +#GPG_PRIVATE_KEY=/config/gnupg/private_key.asc +#AGE_RECIPIENTS=age1... +#AGE_IDENTITY_FILE=/config/age/identity.txt #GPG_PASSPHRASE=Your strong passphrase ## For multiple database backup on Docker or Docker in Swarm mode @@ -75,7 +81,7 @@ AZURE_STORAGE_ACCOUNT_KEY= #MAIL_PASSWORD= #MAIL_FROM=Backup Jobs #MAIL_TO=backup@example.com,me@example.com,team@example.com -#MAIL_SKIP_TLS=false - - +#MAIL_TLS=starttls +#MAIL_INSECURE_SKIP_VERIFY=false +## Full reference: mysql-bkup config --env-docs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f627da..f191a1e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,19 +10,19 @@ jobs: steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: push: true file: "./Dockerfile" @@ -30,5 +30,5 @@ jobs: build-args: | appVersion=nightly tags: | - "${{vars.BUILDKIT_IMAGE}}:nightly" + "${{ vars.BUILDKIT_IMAGE || 'jkaninda/mysql-bkup' }}:nightly" diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 7e0c018..efd57e6 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Ruby uses: ruby/setup-ruby@v1 with: @@ -32,14 +32,14 @@ jobs: working-directory: docs - name: Setup Pages id: pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@v6 - name: Build with Jekyll working-directory: docs run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}" env: JEKYLL_ENV: production - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@v5 with: path: 'docs/_site/' @@ -52,4 +52,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 \ No newline at end of file + uses: actions/deploy-pages@v5 \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a7fc8e4..34a5593 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,18 +6,19 @@ on: jobs: lint: - name: Run on Ubuntu + name: golangci-lint runs-on: ubuntu-latest steps: - name: Clone the code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v7 with: - go-version: '~1.26' + go-version: '1.26' - name: Run linter - uses: golangci/golangci-lint-action@v7 + uses: golangci/golangci-lint-action@v9 with: - version: v2.9.0 \ No newline at end of file + version: v2.15.0 + args: --build-tags integration diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ca21eed..03984ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,18 +14,18 @@ jobs: steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -34,18 +34,29 @@ jobs: name: Get the tag name id: get_tag_name run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + - + name: Image tags + id: meta + uses: docker/metadata-action@v6 + with: + images: | + ${{ vars.BUILDKIT_IMAGE || 'jkaninda/mysql-bkup' }} + ghcr.io/${{ vars.BUILDKIT_IMAGE || 'jkaninda/mysql-bkup' }} + # v2.1.3 -> v2.1.3, 2.1.3, 2.1, 2 and latest (immutable semver tags next to latest) + tags: | + type=raw,value=${{ env.TAG_NAME }} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=latest - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: push: true file: "./Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | appVersion=${{ env.TAG_NAME }} - tags: | - "${{vars.BUILDKIT_IMAGE}}:${{ env.TAG_NAME }}" - "${{vars.BUILDKIT_IMAGE}}:latest" - "ghcr.io/${{vars.BUILDKIT_IMAGE}}:${{ env.TAG_NAME }}" - "ghcr.io/${{vars.BUILDKIT_IMAGE}}:latest" - + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7b31daf..6519a4e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,285 +5,48 @@ on: branches: - main - nightly + - v2 pull_request: branches: - main -env: - IMAGE_NAME: mysql-bkup jobs: - test: + unit: + name: Vet and unit tests runs-on: ubuntu-latest - services: - mysql: - image: mysql:9 - env: - MYSQL_ROOT_PASSWORD: password - MYSQL_DATABASE: testdb - MYSQL_USER: user - MYSQL_PASSWORD: password - ports: - - 3306:3306 - options: >- - --health-cmd="mysqladmin ping -h 127.0.0.1 -uuser -ppassword" - --health-interval=10s - --health-timeout=5s - --health-retries=5 - mysql8: - image: mysql:8 - env: - MYSQL_ROOT_PASSWORD: password - MYSQL_DATABASE: testdb - MYSQL_USER: user - MYSQL_PASSWORD: password - ports: - - 3308:3306 - options: >- - --health-cmd="mysqladmin ping -h 127.0.0.1 -uuser -ppassword" - --health-interval=10s - --health-timeout=5s - --health-retries=5 - mysql5: - image: mysql:5 - env: - MYSQL_ROOT_PASSWORD: password - MYSQL_DATABASE: testdb - MYSQL_USER: user - MYSQL_PASSWORD: password - ports: - - 3305:3306 - options: >- - --health-cmd="mysqladmin ping -h 127.0.0.1 -uuser -ppassword" - --health-interval=10s - --health-timeout=5s - --health-retries=5 steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: '1.26' + - name: Vet + run: | + go vet ./... + go vet -tags integration ./... + - name: Unit tests + run: go test -race ./... + - name: Env reference is up to date + run: go run . config --env-docs > /dev/null - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Create Minio container + integration: + name: Integration tests (MySQL 8.4 / 8.0, MariaDB 11.8, MinIO) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: '1.26' + - uses: docker/setup-buildx-action@v4 + - name: Build the image + run: docker buildx build --build-arg appVersion=test -t mysql-bkup:test --load . + - name: Image smoke test run: | - docker run -d --rm --name minio \ - --network host \ - -p 9000:9000 \ - -e MINIO_ACCESS_KEY=minioadmin \ - -e MINIO_SECRET_KEY=minioadmin \ - -e MINIO_REGION_NAME="eu" \ - minio/minio server /data - echo "Create Minio container completed" - - name: Install MinIO Client (mc) - run: | - curl -O https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - sudo mv mc /usr/local/bin/ - - - name: Wait for MinIO to be ready - run: sleep 5 - - - name: Configure MinIO Client - run: | - mc alias set local http://localhost:9000 minioadmin minioadmin - mc alias list - - - name: Create MinIO Bucket - run: | - mc mb local/backups - echo "Bucket backups created successfully." - # Build the Docker image - - name: Build Docker Image - run: | - docker buildx build --build-arg appVersion=test -t ${{ env.IMAGE_NAME }}:latest --load . - - - name: Verify Docker images - run: | - docker images - - - name: Wait for MySQL to be ready - run: | - docker run --rm --network host mysql:9 mysqladmin ping -h 127.0.0.1 -uuser -ppassword --wait - - name: Test restore - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=root \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest restore -f init.sql - echo "Database restore completed" - - name: Test restore Mysql8 - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_PORT=3308 \ - -e DB_USERNAME=root \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest restore -f init.sql - echo "Test restore Mysql8 completed" - - name: Test restore Mysql5 - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_PORT=3305 \ - -e DB_USERNAME=root \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest restore -f init.sql - echo "Test restore Mysql5 completed" - - name: Test backup - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=user \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest backup - echo "Database backup completed" - - name: Test backup Mysql8 - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_PORT=3308 \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=user \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest backup - echo "Test backup Mysql8 completed" - - name: Test backup Mysql5 - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_PORT=3305 \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=user \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest backup - echo "Test backup Mysql5 completed" - - name: Test encrypted backup - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=user \ - -e DB_PASSWORD=password \ - -e GPG_PASSPHRASE=password \ - ${{ env.IMAGE_NAME }}:latest backup -d testdb --disable-compression --custom-name encrypted-bkup - echo "Database encrypted backup completed" - - name: Test restore encrypted backup | testdb -> testdb2 - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=root \ - -e DB_PASSWORD=password \ - -e GPG_PASSPHRASE=password \ - -e DB_NAME=testdb2 \ - ${{ env.IMAGE_NAME }}:latest restore -f /backup/encrypted-bkup.sql.gpg - echo "Test restore encrypted backup completed" - - name: Test migrate database testdb -> testdb3 - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=root \ - -e DB_PASSWORD=password \ - -e GPG_PASSPHRASE=password \ - -e DB_NAME=testdb \ - -e TARGET_DB_HOST=127.0.0.1 \ - -e TARGET_DB_PORT=3306 \ - -e TARGET_DB_NAME=testdb3 \ - -e TARGET_DB_USERNAME=root \ - -e TARGET_DB_PASSWORD=password \ - ${{ env.IMAGE_NAME }}:latest migrate - echo "Test migrate database testdb -> testdb3 completed" - - name: Test backup all databases - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=root \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest backup --all-databases - echo "Database backup completed" - - name: Test multiple backup - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e TESTDB2_DB_USERNAME=root \ - -e TESTDB2_DB_PASSWORD=password \ - -e TESTDB2_DB_HOST=127.0.0.1 \ - ${{ env.IMAGE_NAME }}:latest backup -c /backup/test_config.yaml - echo "Database backup completed" - - name: Test backup Minio (s3) - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=user \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - -e AWS_S3_ENDPOINT="http://127.0.0.1:9000" \ - -e AWS_S3_BUCKET_NAME=backups \ - -e AWS_ACCESS_KEY=minioadmin \ - -e AWS_SECRET_KEY=minioadmin \ - -e AWS_DISABLE_SSL="true" \ - -e AWS_REGION="eu" \ - -e AWS_FORCE_PATH_STYLE="true" ${{ env.IMAGE_NAME }}:latest backup -s s3 --custom-name minio-backup - echo "Test backup Minio (s3) completed" - - name: Test restore Minio (s3) - run: | - docker run --rm --name ${{ env.IMAGE_NAME }} \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=user \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - -e AWS_S3_ENDPOINT="http://127.0.0.1:9000" \ - -e AWS_S3_BUCKET_NAME=backups \ - -e AWS_ACCESS_KEY=minioadmin \ - -e AWS_SECRET_KEY=minioadmin \ - -e AWS_DISABLE_SSL="true" \ - -e AWS_REGION="eu" \ - -e AWS_FORCE_PATH_STYLE="true" ${{ env.IMAGE_NAME }}:latest restore -s s3 -f minio-backup.sql.gz - echo "Test backup Minio (s3) completed" - - name: Test scheduled backup - run: | - docker run -d --rm --name ${{ env.IMAGE_NAME }} \ - -v ./migrations:/backup/ \ - --network host \ - -e DB_HOST=127.0.0.1 \ - -e DB_USERNAME=user \ - -e DB_PASSWORD=password \ - -e DB_NAME=testdb \ - ${{ env.IMAGE_NAME }}:latest backup -e "@every 10s" - - echo "Waiting for backup to be done..." - sleep 25 - docker logs ${{ env.IMAGE_NAME }} - echo "Test scheduled backup completed" - # Cleanup: Stop and remove containers - - name: Clean up - run: | - docker stop ${{ env.IMAGE_NAME }} || true - docker rm ${{ env.IMAGE_NAME }} || true \ No newline at end of file + docker run --rm mysql-bkup:test version + docker run --rm --entrypoint mysqldump mysql-bkup:test --version + docker run --rm --entrypoint mariadb-dump mysql-bkup:test --version + # Miabi deletes local backups with /bin/sh + docker run --rm --entrypoint /bin/sh mysql-bkup:test -c "rm -f /backup/x.sql.gz" + - name: Integration tests (client tools from the image) + env: + IT_CLIENT_IMAGE: mysql-bkup:test + run: go test -tags integration -count=1 -timeout 40m -v ./... diff --git a/.golangci.yml b/.golangci.yml index abeccc5..03a3c29 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,13 +1,14 @@ version: "2" run: allow-parallel-runners: true + build-tags: + - integration linters: default: none enable: - copyloopvar - dupl - errcheck - - ginkgolinter - goconst - gocyclo - govet @@ -22,6 +23,10 @@ linters: - unparam - unused settings: + goconst: + min-occurrences: 4 + lll: + line-length: 160 revive: rules: - name: comment-spacings @@ -31,11 +36,10 @@ linters: - linters: - dupl - lll - path: pkg/* - - linters: - - dupl - - lll - path: cmd/* + - goconst + - errcheck + - prealloc + path: _test\.go paths: - third_party$ - builtin$ @@ -48,4 +52,4 @@ formatters: paths: - third_party$ - builtin$ - - examples$ \ No newline at end of file + - examples$ diff --git a/Dockerfile b/Dockerfile index 999203e..6a18bc5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,64 @@ -FROM golang:1.26.0 AS build +# syntax=docker/dockerfile:1 +ARG GO_IMAGE=golang:1.26.3-trixie +ARG RUNTIME_IMAGE=debian:13.7-slim +ARG MYSQL_IMAGE=mysql:8.4.13 + +FROM --platform=$BUILDPLATFORM ${GO_IMAGE} AS build WORKDIR /app -ARG appVersion="" - -# Copy the source code. -COPY . . -# Installs Go dependencies +ARG TARGETOS +ARG TARGETARCH +ARG TARGETVARIANT +ARG appVersion="development" +COPY go.mod go.sum ./ RUN go mod download +COPY . . +RUN GOARM="${TARGETVARIANT#v}" CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags="-s -w -X main.Version=${appVersion}" -o /out/mysql-bkup . -# Build -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-X 'github.com/jkaninda/mysql-bkup/utils.Version=${appVersion}'" -o /app/mysql-bkup +# Oracle MySQL client binaries, per architecture. +FROM ${MYSQL_IMAGE} AS mysqlclient-amd64 +RUN mkdir -p /out && cp /usr/bin/mysql /usr/bin/mysqldump /out/ +FROM ${MYSQL_IMAGE} AS mysqlclient-arm64 +RUN mkdir -p /out && cp /usr/bin/mysql /usr/bin/mysqldump /out/ +FROM ${RUNTIME_IMAGE} AS mysqlclient-arm +RUN mkdir -p /out +FROM mysqlclient-${TARGETARCH} AS mysqlclient -FROM alpine:3.23.3 +FROM ${RUNTIME_IMAGE} +ARG appVersion="development" ENV TZ=UTC -ARG WORKDIR="/config" -ARG BACKUPDIR="/backup" -ARG BACKUP_TMP_DIR="/tmp/backup" -ARG TEMPLATES_DIR="/config/templates" -ARG appVersion="" -ENV VERSION=${appVersion} -LABEL org.opencontainers.image.title="mysql-bkup" -LABEL org.opencontainers.image.description="A lightweight MySQL backup and restore tool" -LABEL org.opencontainers.image.licenses="MIT" -LABEL org.opencontainers.image.authors="Jonas Kaninda " -LABEL org.opencontainers.image.version=${appVersion} -LABEL org.opencontainers.image.source="https://github.com/jkaninda/mysql-bkup" +LABEL org.opencontainers.image.title="mysql-bkup" \ + org.opencontainers.image.description="Back up and restore MySQL and MariaDB databases to local storage, S3, GCS, Azure, SFTP or FTP, with GPG or age encryption" \ + org.opencontainers.image.licenses="MIT" \ + org.opencontainers.image.authors="Jonas Kaninda " \ + org.opencontainers.image.version=${appVersion} \ + org.opencontainers.image.source="https://github.com/jkaninda/mysql-bkup" -RUN apk --update add --no-cache mysql-client mariadb-connector-c tzdata ca-certificates -RUN mkdir -p $WORKDIR $BACKUPDIR $TEMPLATES_DIR $BACKUP_TMP_DIR && \ - chmod a+rw $WORKDIR $BACKUPDIR $BACKUP_TMP_DIR -COPY --from=build /app/mysql-bkup /usr/local/bin/mysql-bkup -COPY ./templates/* $TEMPLATES_DIR/ -RUN chmod +x /usr/local/bin/mysql-bkup && \ - ln -s /usr/local/bin/mysql-bkup /usr/local/bin/bkup -# Create backup script and make it executable -RUN printf '#!/bin/sh\n/usr/local/bin/mysql-bkup backup "$@"' > /usr/local/bin/backup && \ - chmod +x /usr/local/bin/backup -# Create restore script and make it executable -RUN printf '#!/bin/sh\n/usr/local/bin/mysql-bkup restore "$@"' > /usr/local/bin/restore && \ - chmod +x /usr/local/bin/restore -# Create migrate script and make it executable -RUN printf '#!/bin/sh\n/usr/local/bin/mysql-bkup migrate "$@"' > /usr/local/bin/migrate && \ - chmod +x /usr/local/bin/migrate +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + mariadb-client-core libssl3t64 zlib1g libncurses6 libtinfo6 \ + tini tzdata ca-certificates \ + && cd /tmp && apt-get download mariadb-client \ + && dpkg-deb --fsys-tarfile mariadb-client_*.deb | tar -x -C / ./usr/bin/mariadb-dump \ + && rm -f mariadb-client_*.deb \ + && rm -rf /var/lib/apt/lists/* -WORKDIR $WORKDIR -ENTRYPOINT ["/usr/local/bin/mysql-bkup"] +# /usr/local/bin comes first in PATH: mysql and mysqldump are Oracle's there, +# while Debian's /usr/bin/mysql* are links to the MariaDB tools. +COPY --from=mysqlclient /out/ /usr/local/bin/ +COPY --from=build /out/mysql-bkup /usr/local/bin/mysql-bkup + + +RUN mkdir -p /config /backup /tmp/backup \ + && chmod 1777 /backup /tmp/backup \ + && ln -s /usr/local/bin/mysql-bkup /usr/local/bin/bkup \ + && printf '#!/bin/sh\nexec /usr/local/bin/mysql-bkup backup "$@"\n' > /usr/local/bin/backup \ + && printf '#!/bin/sh\nexec /usr/local/bin/mysql-bkup restore "$@"\n' > /usr/local/bin/restore \ + && printf '#!/bin/sh\nexec /usr/local/bin/mysql-bkup migrate "$@"\n' > /usr/local/bin/migrate \ + && chmod +x /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/migrate \ + && (mysqldump --version 2>/dev/null || true) && mariadb-dump --version && mysql-bkup version + +WORKDIR /config +HEALTHCHECK --interval=1m --timeout=10s --start-period=1m CMD ["/usr/local/bin/mysql-bkup", "health"] +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/mysql-bkup"] diff --git a/README.md b/README.md index 17d4571..72c6ee5 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,6 @@ # MYSQL-BKUP -**MYSQL-BKUP** is a Docker container image designed to **backup, restore, and migrate MySQL databases**. -It supports a variety of storage options and ensures data security through GPG encryption. - -MYSQL-BKUP is designed for seamless deployment on **Docker** and **Kubernetes**, simplifying MySQL backup, restoration, and migration across environments. -It is a lightweight, multi-architecture solution compatible with **Docker**, **Docker Swarm**, **Kubernetes**, and other container orchestration platforms. - - - - +**MYSQL-BKUP** backs up, restores and migrates **MySQL and MariaDB** databases. It runs as a Docker image (one-shot, as a scheduled daemon, or as a Kubernetes Job/CronJob) and streams every backup end to end: dump, compression, encryption and upload happen in one pipe, without temporary dump files and with a fixed memory footprint, whatever the database size. [![Tests](https://github.com/jkaninda/mysql-bkup/actions/workflows/tests.yml/badge.svg)](https://github.com/jkaninda/mysql-bkup/actions/workflows/tests.yml) [![Build](https://github.com/jkaninda/mysql-bkup/actions/workflows/release.yml/badge.svg)](https://github.com/jkaninda/mysql-bkup/actions/workflows/release.yml) @@ -17,323 +9,439 @@ It is a lightweight, multi-architecture solution compatible with **Docker**, **D ![Docker Pulls](https://img.shields.io/docker/pulls/jkaninda/mysql-bkup?style=flat-square) buy ma a coffee +Documentation: + ## Features -- **Flexible Storage Backends:** - - Local filesystem - - Amazon S3 & S3-compatible storage (e.g., MinIO, Wasabi) - - FTP - - SSH-compatible storage - - Azure Blob storage +- **MySQL and MariaDB, each with its own tools.** The server flavour is detected (`SELECT VERSION()`): MariaDB is dumped with `mariadb-dump` and restored with `mariadb`, MySQL 8+ with Oracle's `mysqldump` and `mysql` (8.4 LTS client). Tested against MySQL 8.0, MySQL 8.4 and MariaDB 11.8. +- **Storage:** local, Amazon S3 and S3-compatible (MinIO, Wasabi, R2, ...), Google Cloud Storage, Azure Blob, SFTP, FTP/FTPS. Extra copies to more storages from the config file. +- **Encryption:** GPG (passphrase or public key, the default and compatible with 1.x) or [age](https://age-encryption.org) (recipients or passphrase). Encryption fails closed: a configured key that cannot be loaded is an error, never a silent plaintext upload. +- **Compression:** gzip (default, parallel) or zstd; restore detects the format. +- **Backup index and safe retention:** every storage folder has an index (`mysql-bkup-metadata.json`) and a `.meta.json` sidecar per backup with its sha256. Retention by age and/or count only deletes backups the index knows, and always keeps the newest one. +- **Commands:** `backup`, `restore`, `migrate`, `verify`, `list`, `delete`, `prune`, `reconcile`, `config`, `health`. +- **Scheduling:** cron expressions (5 or 6 fields, `@daily`, `@every 6h`), no overlapping runs, clean shutdown on SIGTERM, Prometheus metrics, Docker `HEALTHCHECK`. +- **Notifications:** email, Telegram, Slack, Discord, ntfy, generic webhooks (HMAC-signed), healthchecks.io-style pings. +- **Security:** no shell anywhere; database passwords go to the client tools through a 0600 option file in a private directory (never on the command line), and every secret can be read from `_FILE` (Docker/Kubernetes secrets). +- **Multi-arch image:** `amd64`, `arm64`, `arm/v7` (see [Client tools](#client-tools)). -- **Data Security:** - - Backups can be encrypted using **GPG** to ensure confidentiality. +## Quick start -- **Deployment Flexibility:** - - Available as the [jkaninda/mysql-bkup](https://hub.docker.com/r/jkaninda/mysql-bkup) Docker image. - - Deployable on **Docker**, **Docker Swarm**, and **Kubernetes**. - - Supports recurring backups of MySQL databases when deployed: - - On Docker for automated backup schedules. - - As a **Job** or **CronJob** on Kubernetes. - -- **Notifications:** - - Get real-time updates on backup success or failure via: - - **Telegram** - - **Email** - -## πŸ’‘Use Cases - -- **Scheduled Backups**: Automate recurring backups using Docker or Kubernetes. -- **Disaster Recovery:** Quickly restore backups to a clean MySQL instance. -- **Database Migration**: Seamlessly move data across environments using the built-in `migrate` feature. -- **Secure Archiving:** Keep backups encrypted and safely stored in the cloud or remote servers. - - -## βœ… Verified Platforms: -MYSQL-BKUP has been tested and runs successfully on: - -- Docker -- Docker Swarm -- Kubernetes -- OpenShift - -## Documentation is found at - - -## Links: - -- [Docker Hub](https://hub.docker.com/r/jkaninda/mysql-bkup) -- [Github](https://github.com/jkaninda/mysql-bkup) - -## PostgreSQL solution : - -- [PostgreSQL](https://github.com/jkaninda/pg-bkup) - - -## Quickstart - -### Simple Backup Using Docker CLI - -To perform a one-time backup, bind your local volume to `/backup` in the container and run the `backup` command: +Back up the database `shop` into `./backup`: ```shell -docker run --rm --network your_network_name \ - -v $PWD/backup:/backup/ \ - -e "DB_HOST=dbhost" \ - -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ - -e "DB_PASSWORD=password" \ - jkaninda/mysql-bkup backup -d database_name +docker run --rm --network your_network \ + -v $PWD/backup:/backup \ + -e DB_HOST=mysql -e DB_USER=root -e DB_PASSWORD=password \ + jkaninda/mysql-bkup backup -d shop ``` -Alternatively, use an environment file (`--env-file`) for configuration: +The last line of the output names the backup: + +``` +level=INFO msg="Backup completed" database=shop file=shop_20261027_031500.sql.gz size="1.2 MiB" duration=2.1s storage=local location=/backup +``` + +Restore it: ```shell -docker run --rm --network your_network_name \ - --env-file your-env-file \ - -v $PWD/backup:/backup/ \ - jkaninda/mysql-bkup backup -d database_name +docker run --rm --network your_network \ + -v $PWD/backup:/backup \ + -e DB_HOST=mysql -e DB_USER=root -e DB_PASSWORD=password \ + jkaninda/mysql-bkup restore -d shop -f shop_20261027_031500.sql.gz ``` -### Backup All Databases +Restore accepts mysql-bkup backups and any plain `.sql` or gzip `.sql.gz` dump (encrypted or not). A missing database is created first. -To back up all databases on the server, use the `--all-databases` or `-a` flag. By default, this creates individual backup files for each database. - -```shell -docker run --rm --network your_network_name \ - -v $PWD/backup:/backup/ \ - -e "DB_HOST=dbhost" \ - -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ - -e "DB_PASSWORD=password" \ - jkaninda/mysql-bkup backup --all-databases --disable-compression -``` - -> **Note:** Use the `--all-in-one` or `-A` flag to combine backups into a single file. - ---- - -### Simple Restore Using Docker CLI - -To restore a database, bind your local volume to `/backup` and run the `restore` command: - -```shell -docker run --rm --network your_network_name \ - -v $PWD/backup:/backup/ \ - -e "DB_HOST=dbhost" \ - -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ - -e "DB_PASSWORD=password" \ - jkaninda/mysql-bkup restore -d database_name -f backup_file.sql.gz -``` - ---- - -### Backup with Docker Compose - -Below is an example of a `docker-compose.yml` file for running a one-time backup: +### Docker Compose, scheduled ```yaml services: - pg-bkup: - # In production, pin your image tag to a specific release version instead of `latest`. - # See available releases: https://github.com/jkaninda/mysql-bkup/releases + mysql-bkup: image: jkaninda/mysql-bkup - container_name: mysql-bkup - command: backup + command: backup -d shop --cron-expression "@daily" volumes: - ./backup:/backup environment: - - DB_PORT=3306 - DB_HOST=mysql - - DB_NAME=foo - - DB_USERNAME=bar - - DB_PASSWORD=password - - TZ=Europe/Paris - networks: - - web - -networks: - web: + - DB_USER=backup + - DB_PASSWORD_FILE=/run/secrets/db_password + - BACKUP_RETENTION_DAYS=7 + - GPG_PASSPHRASE=change-me ``` ---- - -### Recurring Backups with Docker - -You can schedule recurring backups using the `--cron-expression` or `-e` flag: - -```shell -docker run --rm --network network_name \ - -v $PWD/backup:/backup/ \ - -e "DB_HOST=hostname" \ - -e "DB_USERNAME=user" \ - -e "DB_PASSWORD=password" \ - jkaninda/mysql-bkup backup -d dbName --cron-expression "@every 15m" -``` - -For predefined schedules, refer to the [documentation](https://jkaninda.github.io/mysql-bkup/reference/#predefined-schedules). - ---- - -## Deploy on Kubernetes - -For Kubernetes, you can deploy `mysql-bkup` as a Job or CronJob. Below are examples for both. - -### Kubernetes Backup Job - -This example defines a one-time backup job: - -```yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: backup-job -spec: - ttlSecondsAfterFinished: 100 - template: - spec: - containers: - - name: mysql-bkup - # Pin the image tag to a specific release version in production. - # See available releases: https://github.com/jkaninda/mysql-bkup/releases - image: jkaninda/mysql-bkup - command: ["backup", "-d", "dbname"] - resources: - limits: - memory: "128Mi" - cpu: "500m" - env: - - name: DB_HOST - value: "mysql" - - name: DB_USERNAME - value: "user" - - name: DB_PASSWORD - value: "password" - volumeMounts: - - mountPath: /backup - name: backup - volumes: - - name: backup - hostPath: - path: /home/toto/backup # Directory location on the host - type: Directory # Optional field - restartPolicy: Never -``` - -### Kubernetes CronJob for Scheduled Backups - -For scheduled backups, use a `CronJob`: +### Kubernetes CronJob ```yaml apiVersion: batch/v1 kind: CronJob metadata: - name: pg-bkup-cronjob + name: mysql-bkup spec: - schedule: "0 2 * * *" # Runs daily at 2 AM + schedule: "0 1 * * *" jobTemplate: spec: template: spec: + restartPolicy: Never containers: - - name: pg-bkup + - name: mysql-bkup image: jkaninda/mysql-bkup - command: ["backup", "-d", "dbname"] - env: - - name: DB_HOST - value: "mysql" - - name: DB_USERNAME - value: "user" - - name: DB_PASSWORD - value: "password" - volumeMounts: - - mountPath: /backup - name: backup - volumes: - - name: backup - hostPath: - path: /home/toto/backup - type: Directory - restartPolicy: OnFailure + args: ["backup", "-d", "shop", "--storage", "s3", "--path", "/mysql"] + envFrom: + - secretRef: + name: mysql-bkup # DB_HOST, DB_USER, DB_PASSWORD, AWS_* ... ``` ---- +## Commands -## πŸš€ Why Use MYSQL-BKUP? +| Command | Purpose | +|---|---| +| `backup` | Back up now, or as a daemon with `--cron-expression` / `BACKUP_CRON_EXPRESSION`. `-d DB`, `-a` (every database, one file each), `-A` (every database in one file), `--exclude-db`, `--custom-name`, `--disable-compression`, `-c config.yaml` | +| `restore` | Stream a backup back: `-f FILE [-d DB]`. Without `-d`, the dump's own `USE` statements decide (all-in-one backups) | +| `migrate` | Copy `DB_NAME` (or every database with `-a`) straight into `TARGET_DB_*`: `dump \| restore` through a pipe, no file | +| `verify` | Download, decrypt and decompress a backup and check its sha256, without restoring | +| `list` | Backup history from the index (`--json`, `--storage-scan`) | +| `delete` | Delete a backup, its sidecar and its index entry | +| `prune` | Apply the retention policy (`--dry-run`) | +| `reconcile` | Rebuild the index from the sidecars (`--from-index OLD` after renaming it) | +| `config` | Show the resolved configuration with sources, secrets redacted; `--env-docs` prints the reference below | +| `health` | Docker `HEALTHCHECK` for the daemon | +| `version` | Print the version | -**MYSQL-BKUP** isn't just another MySQL backup tool, it's a robust, production-ready solution purpose-built for modern DevOps workflows. +The image also has the `backup`, `restore` and `migrate` shortcuts (`docker run ... --entrypoint backup`, or in Kubernetes `command: [backup]`) and the `bkup` alias. -Here’s why developers, sysadmins, and DevOps choose **MYSQL-BKUP**: +### Exit codes -### βœ… All-in-One Backup, Restore & Migration +`0` success, `1` failure, `2` configuration error, `3` database connection or authentication, `4` storage, `5` encryption/decryption. -Whether you're backing up a single database, restoring critical data, or migrating across environments, MYSQL-BKUP handles it all with a **single, unified CLI** no scripting gymnastics required. +## Storage + +```shell +# S3 / MinIO +-e AWS_S3_ENDPOINT=https://s3.amazonaws.com -e AWS_S3_BUCKET_NAME=backups -e AWS_REGION=us-east-1 \ +-e AWS_ACCESS_KEY=... -e AWS_SECRET_KEY=... \ +jkaninda/mysql-bkup backup -d shop --storage s3 --path /mysql +# MinIO and other S3 alternatives: add AWS_FORCE_PATH_STYLE=true (and AWS_DISABLE_SSL=true for plain http). +# Without AWS_ACCESS_KEY, the AWS default credential chain is used (IAM role, IRSA, profile). + +# SFTP (host keys are verified: mount a known_hosts file) +-e SFTP_HOST=backup.example.com -e SFTP_USER=backup -e SFTP_IDENTITY_FILE=/config/id_ed25519 \ +-e SFTP_KNOWN_HOSTS=/config/known_hosts \ +jkaninda/mysql-bkup backup -d shop --storage sftp --path /home/backup/mysql + +# FTP / FTPS +-e FTP_HOST=ftp.example.com -e FTP_USER=backup -e FTP_PASSWORD=... -e FTP_TLS=true \ +jkaninda/mysql-bkup backup -d shop --storage ftp --path /mysql + +# Azure Blob +-e AZURE_STORAGE_ACCOUNT_NAME=... -e AZURE_STORAGE_ACCOUNT_KEY=... -e AZURE_STORAGE_CONTAINER_NAME=backups \ +jkaninda/mysql-bkup backup -d shop --storage azure --path mysql + +# Google Cloud Storage +-e GCS_BUCKET=backups -e GCS_CREDENTIALS_FILE=/config/gcs.json \ +jkaninda/mysql-bkup backup -d shop --storage gcs --path mysql +``` + +`STORAGE` (env) and `--storage` are equivalent; `ssh` and `remote` are accepted for `sftp`. The object key is exactly `/`. + +## Encryption + +| Mode | Backup | Restore | File | +|---|---|---|---| +| GPG passphrase | `GPG_PASSPHRASE` | `GPG_PASSPHRASE` | `.sql.gz.gpg` | +| GPG public key | `GPG_PUBLIC_KEY` (path or inline), or `/config/gnupg/public_key.asc` | `GPG_PRIVATE_KEY` (+ `GPG_PRIVATE_KEY_PASSPHRASE`) | `.sql.gz.gpg` | +| age recipients | `AGE_RECIPIENTS` (`age1...`, `ssh-ed25519 ...`) or `AGE_RECIPIENTS_FILE` | `AGE_IDENTITY` / `AGE_IDENTITY_FILE` (default `/config/age/identity.txt`) | `.sql.gz.age` | +| age passphrase | `AGE_PASSPHRASE` | `AGE_PASSPHRASE` | `.sql.gz.age` | + +`ENCRYPTION=auto` (default) picks the mode from the keys that are set; GPG wins when both GPG and age keys are set. Restore detects the format from the content. + +## Backup index and retention + +Each storage folder holds an index file (`BACKUP_INDEX_NAME`, default `{tool}-metadata.json`; placeholders `{tool}`, `{engine}` (always `mysql`), `{name}`, `{db}`) and one `.meta.json` sidecar per backup. `list`, `delete`, `prune` and retention work from the index, so mysql-bkup only ever deletes files it created itself, and several instances can share a folder with different index names. + +- `BACKUP_RETENTION_DAYS=N` deletes backups older than N days, `BACKUP_RETENTION_COUNT=N` keeps the newest N. The newest successful backup is always kept. +- Backups made before the index existed (1.x) are never deleted, unless `BACKUP_RETENTION_ADOPT_LEGACY=true` imports them. +- `BACKUP_INDEX=false` turns the index off; retention then only matches this tool's file names directly in the folder. + +## Multiple databases + +```shell +backup -a # every database, one file each (system databases are skipped) +backup -a --exclude-db test,tmp +backup -A # everything in one file: all_databases_.sql.gz +``` + +Or a YAML file (`-c /config/config.yaml`, also picked up automatically at `/config/config.yaml`); the 1.x format still works: + +```yaml +cronExpression: "@daily" +databases: + - name: shop + host: mysql1 + user: backup + password: ${SHOP_PASSWORD} + path: /shop + - name: blog # credentials from DB_USER_BLOG / BLOG_DB_USER ... or the global DB_* +``` + +## Client tools + +| Server | Dump | Restore | +|---|---|---| +| MariaDB | `mariadb-dump` (MariaDB 11.8 LTS client) | `mariadb` | +| MySQL 8.0 / 8.4 / 9.x | Oracle `mysqldump` 8.4 LTS | Oracle `mysql` 8.4 LTS | + +Dumps use `--single-transaction --routines --triggers --events --hex-blob --quick`, plus `--no-tablespaces --set-gtid-purged=OFF` on MySQL. A single-database dump has no `CREATE DATABASE`/`USE`, so it can be restored under another name. + +The image is based on Debian 13 (trixie) slim. Oracle publishes its client for `amd64` and `arm64` only: on `arm/v7` the image has the MariaDB tools only, and MySQL servers are dumped with `mariadb-dump` (a warning is logged). `MYSQL_CLIENT=mysql|mariadb` forces one family. + +TLS: `DB_SSL_MODE=disabled` (default, as in 1.x) `| preferred | required | verify_ca | verify_identity`, with `DB_SSL_CA` (used for the verify modes), `DB_SSL_CERT`, `DB_SSL_KEY`. 1.x values `0` and `1` still mean disabled and required. With the MariaDB client, `verify_ca` also checks the host name. + +Restores are not atomic: MySQL cannot roll back DDL, so a restore that fails halfway leaves the statements before the failure applied (the exit code is non-zero). Restore into a new database, or keep a backup of the target, when that matters. + +## Use with Miabi + +Miabi runs `backup -d `, `backup --storage s3 -d --path ` and `restore -d -f [--storage s3 --path ]` with `DB_*`, `AWS_*` and `GPG_PASSPHRASE`, and reads the artifact name from the last `file=` in the output. That contract is unchanged and covered by the integration tests. + +## Upgrading from 1.x + +Everything that worked in 1.x keeps working; a few behaviours changed on purpose. + +- **Image:** Debian 13 slim instead of Alpine, with Oracle's `mysqldump` for MySQL servers (correct MySQL 8 dumps) and `mariadb-dump` for MariaDB. `/bin/sh`, `/backup`, `/config`, the `backup`/`restore`/`migrate` shortcuts and the `bkup` alias are still there; `tini` is now PID 1. +- **Renamed variables** are still read, with a deprecation warning: `DB_USER` β†’ `DB_USER`, `S3_ENDPOINT` β†’ `AWS_S3_ENDPOINT`, `BUCKET_NAME`/`AWS_BUCKET_NAME` β†’ `AWS_S3_BUCKET_NAME`, `ACCESS_KEY`/`SECRET_KEY` β†’ `AWS_ACCESS_KEY`/`AWS_SECRET_KEY`, `AWS_S3_PATH`/`S3_PATH`/`REMOTE_PATH` β†’ `REMOTE_PATH`, `SFTP_HOST` β†’ `SFTP_HOST`, `SFTP_IDENTITY_FILE` β†’ `SFTP_IDENTITY_FILE`, `FTP_HOST_NAME` β†’ `FTP_HOST`, `BACKUP_TMP_DIR` β†’ `TMP_DIR`. +- **`DB_SSL_MODE`** takes MySQL's mode names; `0`/`1` still work. +- **`STORAGE` and `STORAGE_PATH` env are honoured** (1.x always used the `--storage` default and `/backup`). Unknown storage names are an error instead of a silent local backup. +- **SFTP variables are renamed, and the old names are no longer read:** `SSH_HOST`/`SSH_HOST_NAME` β†’ `SFTP_HOST`, `SSH_PORT` β†’ `SFTP_PORT`, `SSH_USER` β†’ `SFTP_USER`, `SSH_PASSWORD` β†’ `SFTP_PASSWORD`, `SSH_IDENTIFY_FILE`/`SSH_IDENTITY_FILE` β†’ `SFTP_IDENTITY_FILE`, `SSH_REMOTE_PATH` β†’ `REMOTE_PATH`. +- **SFTP host keys are verified by default.** Set `SFTP_KNOWN_HOSTS=/path/known_hosts`, or `SFTP_INSECURE_IGNORE_HOST_KEY=true` to keep the 1.x behaviour. +- **`MAIL_SKIP_TLS` is read with its documented meaning** (`true` skips certificate verification). 1.x had it inverted, so `MAIL_SKIP_TLS=false` (the value in `.env.example`) turned verification *off*. Use `MAIL_INSECURE_SKIP_VERIFY` from now on. +- **Retention only deletes indexed backups** (see above). 1.x deleted every file older than N days in the folder. Set `BACKUP_RETENTION_ADOPT_LEGACY=true` once to let retention handle 1.x backups, or `BACKUP_INDEX=false` for name-based retention. +- **Encryption fails closed**: a `GPG_PUBLIC_KEY` that cannot be read is an error (1.x uploaded plaintext). The env var now wins over the auto-detected `/config/gnupg` key. +- **Errors no longer kill the daemon**: a failed run is logged, notified and retried at the next schedule; other databases of the same run continue (`BACKUP_RESCUE_MODE` is no longer needed). +- **Restore** works from every storage (1.x had no SFTP restore), is case-insensitive about the storage name, and no longer goes through `sh -c`. +- **`migrate`** streams directly into the target (no intermediate file), creates the target database if needed, and `TARGET_DB_NAME` defaults to the source name; `-a` migrates every database. +- **Notification templates** are built in; custom ones still go in `/config/templates` (`TEMPLATES_DIR`). +- **Version** is injected with `-ldflags "-X main.Version=..."` (was `utils.Version`). + +## Building + +```shell +make build # bin/mysql-bkup +make test # unit tests +make test-integration # Docker: MySQL 8.4/8.0, MariaDB 11.8, MinIO +make docker-build VERSION=v2.0.0 +make env-docs # regenerate the reference below +``` + +## Environment variables + +Generated with `mysql-bkup config --env-docs`. Every secret also accepts `_FILE`. Resolution order: flag, then env, then deprecated names, then the YAML `env:` section, then the default. + +### storage + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `STORAGE` | `--storage` `-s` | `local` | | Storage backend: local, s3, gcs, azure, sftp (ssh, remote), ftp (local, s3, gcs, azure, sftp, ssh, ftp, remote) | +| `STORAGE_PATH` | | `/backup` | | Root directory of the local storage backend | +| `REMOTE_PATH` | `--path` `-P` | | `AWS_S3_PATH` (deprecated), `S3_PATH` (deprecated) | Folder inside the storage backend (without file name) | + +### general + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `BACKUP_NAME` | `--name` `-n` | | | Base name of the artifact; a timestamp is appended (default: the database or volume name) | +| `BACKUP_CUSTOM_NAME` | `--custom-name` | | | Fixed artifact name without timestamp (overwrites the previous one) | +| `FILE_NAME` | `--file` `-f` | | | Backup file to restore, verify or delete | +| `BACKUP_CONFIG_FILE` | `--config` `-c` | | | YAML configuration file (default /config/config.yaml if present) | +| `BACKUP_TIMEOUT` | | | | Abort a backup after this duration (0 = none) | +| `RESTORE_TIMEOUT` | | | | Abort a restore after this duration (0 = none) | +| `TMP_DIR` | | `/tmp/backup` | `BACKUP_TMP_DIR` (deprecated) | Scratch directory (only for secret files and disk-based modes) | +| `COMPRESSION` | `--compression` | `gzip` | | Compression: gzip, zstd or none (gzip, zstd, none) | +| `COMPRESSION_LEVEL` | | | | Compression level (0 = default) | +| `DISABLE_COMPRESSION` | `--disable-compression` | | | Same as COMPRESSION=none | +| `RESTORE_VERIFY_FIRST` | `--verify-first` | `auto` | | Verify the whole artifact before restoring (auto: on for GPG-encrypted volumes) (auto, true, false) | +| `LOG_LEVEL` | | `info` | | Log level (debug, info, warn, error) | +| `LOG_FORMAT` | | `text` | | Log format (text, json) | +| `TIME_FORMAT` | | `2006-01-02 15:04:05` | | Time layout used in notifications | +| `RESULT_FILE` | `--result-file` | | | Write the run result as JSON to this path | + +### scheduling + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `BACKUP_CRON_EXPRESSION` | `--cron-expression` `-e` | | | Run as a daemon on this cron schedule (5 or 6 fields, @daily, @every 6h) | +| `BACKUP_RUN_ON_START` | | | | In daemon mode, run one backup immediately at start | +| `BACKUP_RESCUE_MODE` | | | | Deprecated: runs always continue with the remaining targets after a failure | +| `METRICS_ADDR` | | | | Serve Prometheus /metrics and /healthz on this address in daemon mode (e.g. :9090) | +| `HEALTH_FILE` | | `/tmp/bkup-health.json` | | State file read by the health command | +| `HEALTH_MAX_AGE` | | | | health fails when the last success is older than this (default: 2x the schedule interval) | + +### retention + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `BACKUP_RETENTION_DAYS` | | `0` | | Delete backups older than N days (0 = keep) | +| `BACKUP_RETENTION_COUNT` | | `0` | | Keep only the newest N backups (0 = no limit) | +| `BACKUP_RETENTION_ALLOW_ROOT` | | | | Allow pattern-based retention at the storage root (index disabled) | +| `BACKUP_RETENTION_ADOPT_LEGACY` | | | | Import pre-index backups matching this tool's naming into the index | +| `BACKUP_INDEX` | | `true` | | Keep the backup index file and per-backup .meta.json sidecars | +| `BACKUP_INDEX_NAME` | `--index-name` | `{tool}-metadata.json` | | Index file name; placeholders {tool} {engine} {name} {db} | + +### storage-s3 + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `AWS_S3_ENDPOINT` | | | `S3_ENDPOINT` (deprecated), `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL` | S3 endpoint (empty = AWS) | +| `AWS_S3_BUCKET_NAME` | | | `BUCKET_NAME` (deprecated), `AWS_BUCKET_NAME` (deprecated) | S3 bucket | +| `AWS_REGION` | | `us-east-1` | `AWS_DEFAULT_REGION` | S3 region | +| `AWS_ACCESS_KEY` | | | `ACCESS_KEY` (deprecated), `AWS_ACCESS_KEY_ID` | S3 access key (empty = default AWS credential chain) | +| `AWS_SECRET_KEY` | | | `SECRET_KEY` (deprecated), `AWS_SECRET_ACCESS_KEY`, `AWS_SECRET_KEY_FILE` | S3 secret key | +| `AWS_SESSION_TOKEN` | | | `AWS_SESSION_TOKEN_FILE` | S3 session token | +| `AWS_PROFILE` | | | | AWS shared config profile | +| `AWS_DISABLE_SSL` | | | | Use http for the S3 endpoint | +| `AWS_FORCE_PATH_STYLE` | | | | Path-style S3 addressing (MinIO) | +| `S3_STORAGE_CLASS` | | | | S3 storage class | +| `S3_SSE` | | | | Server-side encryption: AES256 or aws:kms (aes256, aws:kms) | +| `S3_SSE_KMS_KEY_ID` | | | | KMS key id for aws:kms | +| `S3_PART_SIZE` | | `64MiB` | | Multipart part size; max object size is 10000 x this | +| `S3_UPLOAD_CONCURRENCY` | | `2` | | Parallel part uploads (memory = part size x this) | + +### storage-sftp + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `SFTP_HOST` | | | | SFTP host | +| `SFTP_PORT` | | `22` | | SFTP port | +| `SFTP_USER` | | | | SFTP user | +| `SFTP_PASSWORD` | | | `SFTP_PASSWORD_FILE` | SFTP password | +| `SFTP_IDENTITY_FILE` | | | | SFTP private key file (the SSH agent is also used when SSH_AUTH_SOCK is set) | +| `SFTP_IDENTITY_PASSPHRASE` | | | `SFTP_IDENTITY_PASSPHRASE_FILE` | Passphrase of the SFTP private key | +| `SFTP_KNOWN_HOSTS` | | | | known_hosts file used to verify the SFTP host key (default ~/.ssh/known_hosts) | +| `SFTP_INSECURE_IGNORE_HOST_KEY` | | | | Do not verify the SFTP host key (insecure) | + +### storage-ftp + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `FTP_HOST` | | | `FTP_HOST_NAME` (deprecated) | FTP host | +| `FTP_PORT` | | `21` | | FTP port | +| `FTP_USER` | | | | FTP user | +| `FTP_PASSWORD` | | | `FTP_PASSWORD_FILE` | FTP password | +| `FTP_TLS` | | | | Use explicit FTPS | + +### storage-azure + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `AZURE_STORAGE_ACCOUNT_NAME` | | | | Azure storage account | +| `AZURE_STORAGE_ACCOUNT_KEY` | | | `AZURE_STORAGE_ACCOUNT_KEY_FILE` | Azure storage account key | +| `AZURE_STORAGE_CONTAINER_NAME` | | | | Azure blob container | +| `AZURE_STORAGE_ENDPOINT` | | | | Azure blob endpoint (Azurite, sovereign clouds) | +| `AZURE_STORAGE_SAS_TOKEN` | | | `AZURE_STORAGE_SAS_TOKEN_FILE` | Azure SAS token (instead of the account key) | + +### storage-gcs + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `GCS_BUCKET` | | | | GCS bucket | +| `GCS_CREDENTIALS_FILE` | | | `GOOGLE_APPLICATION_CREDENTIALS` | GCS service account JSON file | +| `GCS_CREDENTIALS_JSON` | | | `GCS_CREDENTIALS_JSON_FILE` | GCS service account JSON content | +| `GCS_ENDPOINT` | | | | GCS endpoint (emulators) | + +### encryption + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `ENCRYPTION` | `--encryption` | `auto` | | auto (from the keys set), none, gpg or age (auto, none, gpg, age) | +| `ENCRYPTION_ARMOR` | | | | ASCII-armor encrypted output | +| `GPG_PASSPHRASE` | | | `GPG_PASSPHRASE_FILE` | GPG symmetric passphrase | +| `GPG_PUBLIC_KEY` | | | | GPG public key(s): file path or inline key (default /config/gnupg/public_key.{asc,gpg}) | +| `GPG_PRIVATE_KEY` | | | `GPG_PRIVATE_KEY_FILE` | GPG private key for restore: file path or inline key (default /config/gnupg/private_key.{asc,gpg}) | +| `GPG_PRIVATE_KEY_PASSPHRASE` | | | `GPG_PRIVATE_KEY_PASSPHRASE_FILE` | Passphrase of the GPG private key (default GPG_PASSPHRASE) | +| `AGE_RECIPIENTS` | | | | age recipients (age1..., ssh-ed25519 ...), comma or newline separated | +| `AGE_RECIPIENTS_FILE` | | | | File with age recipients | +| `AGE_PASSPHRASE` | | | `AGE_PASSPHRASE_FILE` | age passphrase (scrypt) | +| `AGE_IDENTITY` | | | `AGE_IDENTITY_FILE` | age identity (AGE-SECRET-KEY-...) for restore | +| `AGE_IDENTITY_FILE` | | | | age identity file for restore (default /config/age/identity.txt) | + +### notifications + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `MAIL_HOST` | | | | SMTP host | +| `MAIL_PORT` | | `587` | | SMTP port | +| `MAIL_USERNAME` | | | | SMTP user | +| `MAIL_PASSWORD` | | | `MAIL_PASSWORD_FILE` | SMTP password | +| `MAIL_FROM` | | | | Sender address | +| `MAIL_TO` | | | | Recipients, comma separated | +| `MAIL_SKIP_TLS` | | | | Deprecated: true skips certificate verification (use MAIL_INSECURE_SKIP_VERIFY) | +| `MAIL_INSECURE_SKIP_VERIFY` | | | | Skip SMTP certificate verification | +| `MAIL_TLS` | | `starttls` | | SMTP TLS mode (starttls, tls, none) | +| `TG_TOKEN` | | | `TELEGRAM_BOT_TOKEN`, `TG_TOKEN_FILE` | Telegram bot token | +| `TG_CHAT_ID` | | | `TELEGRAM_CHAT_ID` | Telegram chat id | +| `POSTA_URL` | | | | Posta base URL (https://posta.example.com) for email notifications through Posta | +| `POSTA_API_KEY` | | | `POSTA_API_KEY_FILE` | Posta API key with the send scope | +| `POSTA_WORKSPACE_ID` | | | | Posta workspace id (only for account-wide API keys) | +| `POSTA_FROM` | | | | Posta sender address (default MAIL_FROM) | +| `POSTA_TO` | | | | Posta recipients, comma separated (default MAIL_TO) | +| `POSTA_TEMPLATE` | | | | Posta template name for notifications; the event fields are passed as template data (default: built-in HTML) | +| `POSTA_TEMPLATE_ERROR` | | | | Posta template name for failures (default POSTA_TEMPLATE) | +| `POSTA_LANGUAGE` | | | | Posta template language | +| `WEBHOOK_URL` | | | | Generic JSON webhook URL | +| `WEBHOOK_SECRET` | | | `WEBHOOK_SECRET_FILE` | HMAC-SHA256 secret for the X-Bkup-Signature header | +| `SLACK_WEBHOOK_URL` | | | `SLACK_WEBHOOK_URL_FILE` | Slack incoming webhook URL | +| `DISCORD_WEBHOOK_URL` | | | `DISCORD_WEBHOOK_URL_FILE` | Discord webhook URL | +| `NTFY_URL` | | | | ntfy topic URL | +| `NTFY_TOKEN` | | | `NTFY_TOKEN_FILE` | ntfy access token | +| `NOTIFY_ON` | | `always` | | When to notify (always, failure, success, never) | +| `HEALTHCHECK_PING_URL` | | | | healthchecks.io-style URL pinged on start (/start), success and failure (/fail) | +| `BACKUP_REFERENCE` | | | | Free text included in notifications | +| `TEMPLATES_DIR` | | `/config/templates` | | Directory with custom notification templates | + +### database + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `DB_URL` | | | `DB_URL_FILE` | Connection URL mysql://user:password@host:port/database?ssl-mode=required (overrides host, port, user, password) | +| `DB_HOST` | | `localhost` | | Database host (a path starting with / is a Unix socket) | +| `DB_PORT` | | `3306` | | Database port | +| `DB_NAME` | `--dbname` `-d` | | | Database name | +| `DB_USER` | | | `DB_USERNAME` | Database user | +| `DB_PASSWORD` | | | `DB_PASSWORD_FILE` | Database password | +| `DB_SSL_MODE` | | `disabled` | | TLS mode: disabled, preferred, required, verify_ca, verify_identity (1.x values 0 and 1 mean disabled and required) | +| `DB_SSL_CA` | | `/etc/ssl/certs/ca-certificates.crt` | | CA certificate file, used with DB_SSL_MODE=verify_ca or verify_identity | +| `DB_SSL_CERT` | | | | Client certificate file | +| `DB_SSL_KEY` | | | | Client key file | +| `BACKUP_ALL_DATABASES` | `--all-databases` `-a` | | | Back up every database, one file each | +| `BACKUP_ALL_IN_ONE` | `--all-in-one` `-A` | | | Back up every database into a single file | +| `BACKUP_EXCLUDE_DATABASES` | `--exclude-db` | | | Databases to skip with --all-databases, comma separated | +| `TARGET_DB_URL` | | | `TARGET_DB_URL_FILE` | migrate: target connection URL | +| `TARGET_DB_HOST` | | | | migrate: target host | +| `TARGET_DB_PORT` | | `3306` | | migrate: target port | +| `TARGET_DB_NAME` | | | | migrate: target database (default: the source name) | +| `TARGET_DB_USER` | | | `TARGET_DB_USERNAME` | migrate: target user | +| `TARGET_DB_PASSWORD` | | | `TARGET_DB_PASSWORD_FILE` | migrate: target password | +| `TARGET_DB_SSL_MODE` | | `disabled` | | migrate: target TLS mode (same values as DB_SSL_MODE) | +| `TARGET_DB_SSL_CA` | | `/etc/ssl/certs/ca-certificates.crt` | | migrate: target CA certificate file | +| `MYSQL_CLIENT` | | `auto` | | Client tools: auto (mariadb-dump for MariaDB, Oracle mysqldump for MySQL), mysql or mariadb (auto, mysql, mariadb) | -### πŸ”„ Works Everywhere You Deploy - -Designed to be cloud-native: - -* **Runs seamlessly on Docker, Docker Swarm, and Kubernetes** -* Supports **CronJobs** for automated scheduled backups -* Compatible with GitOps and CI/CD workflows - -### ☁️ Flexible Storage Integrations - -Store your backups **anywhere**: - -* Local disks -* Amazon S3, MinIO, Wasabi, Azure Blob, FTP, SSH - -### πŸ”’ Enterprise-Grade Security - -* **GPG Encryption**: Protect sensitive data with optional encryption before storing backups locally or in the cloud. -* **Secure Storage** Options: Supports S3, Azure Blob, SFTP, and SSH with encrypted transfers, keeping backups safe from unauthorized access. - -### πŸ“¬ Instant Notifications - -Stay in the loop with real-time notifications via **Telegram** and **Email**. Know immediately when a backup succeedsβ€”or fails. - -### πŸƒβ€β™‚οΈ Lightweight and Fast - -Written in **Go**, MYSQL-BKUP is fast, multi-arch compatible (`amd64`, `arm64`, `arm/v7`), and optimized for minimal memory and CPU usage. Ideal for both cloud and edge deployments. - -### πŸ§ͺ Tested. Verified. Trusted. - -Actively maintained with **automated testing**, **Docker image size optimizations**, and verified support across major container platforms. - ---- ## Available image registries -This Docker image is published to both Docker Hub and the GitHub container registry. -Depending on your preferences and needs, you can reference both `jkaninda/mysql-bkup` as well as `ghcr.io/jkaninda/mysql-bkup`: - ``` docker pull jkaninda/mysql-bkup docker pull ghcr.io/jkaninda/mysql-bkup ``` -Documentation references Docker Hub, but all examples will work using ghcr.io just as well. +Tags: `latest`, and immutable release tags (`v2.0.0`, `2.0.0`, `2.0`, `2`). Pin a release tag in production. -## References - -We created this image as a simpler and more lightweight alternative to existing solutions. Here’s why: - -- **Lightweight:** Written in Go, the image is optimized for performance and minimal resource usage. -- **Multi-Architecture Support:** Supports `arm64` and `arm/v7` architectures. -- **Docker Swarm Support:** Fully compatible with Docker in Swarm mode. -- **Kubernetes Support:** Designed to work seamlessly with Kubernetes. +## PostgreSQL +Looking for PostgreSQL? See [pg-bkup](https://github.com/jkaninda/pg-bkup). ## License This project is licensed under the MIT License. See the LICENSE file for details. -## Authors - -**Jonas Kaninda** -- ## Copyright diff --git a/cmd/backup.go b/cmd/backup.go deleted file mode 100644 index 290ad2a..0000000 --- a/cmd/backup.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package cmd - -import ( - "github.com/jkaninda/mysql-bkup/pkg" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" -) - -var BackupCmd = &cobra.Command{ - Use: "backup ", - Short: "Backup database operation", - Example: utils.BackupExample, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - pkg.StartBackup(cmd) - } else { - utils.Fatal(`"backup" accepts no argument %q`, args) - } - }, -} - -func init() { - // Backup - BackupCmd.PersistentFlags().StringP("storage", "s", "local", "Define storage: local, s3, ssh, ftp, azure") - BackupCmd.PersistentFlags().StringP("path", "P", "", "Storage path without file name. e.g: /custom_path or ssh remote path `/home/foo/backup`") - BackupCmd.PersistentFlags().StringP("cron-expression", "e", "", "Backup cron expression (e.g., `0 0 * * *` or `@daily`)") - BackupCmd.PersistentFlags().StringP("config", "c", "", "Configuration file for multi database backup. (e.g: `/backup/config.yaml`)") - BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") - BackupCmd.PersistentFlags().BoolP("all-databases", "a", false, "Backup all databases") - BackupCmd.PersistentFlags().BoolP("all-in-one", "A", false, "Backup all databases in a single file") - BackupCmd.PersistentFlags().StringP("custom-name", "", "", "Custom backup name") - -} diff --git a/cmd/migrate.go b/cmd/migrate.go deleted file mode 100644 index dcfc6fa..0000000 --- a/cmd/migrate.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package cmd - -import ( - "github.com/jkaninda/mysql-bkup/pkg" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" -) - -var MigrateCmd = &cobra.Command{ - Use: "migrate", - Short: "Migrate database from a source database to a target database", - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - pkg.StartMigration(cmd) - } else { - utils.Fatal(`"migrate" accepts no argument %q`, args) - - } - - }, -} diff --git a/cmd/restore.go b/cmd/restore.go deleted file mode 100644 index e0fc68b..0000000 --- a/cmd/restore.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package cmd - -import ( - "github.com/jkaninda/mysql-bkup/pkg" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" -) - -var RestoreCmd = &cobra.Command{ - Use: "restore", - Short: "Restore database operation", - Example: utils.RestoreExample, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - pkg.StartRestore(cmd) - } else { - utils.Fatal(`"restore" accepts no argument %q`, args) - - } - - }, -} - -func init() { - // Restore - RestoreCmd.PersistentFlags().StringP("file", "f", "", "File name of database") - RestoreCmd.PersistentFlags().StringP("storage", "s", "local", "Define storage: local, s3, ssh, ftp") - RestoreCmd.PersistentFlags().StringP("path", "P", "", "AWS S3 path without file name. eg: /custom_path or ssh remote path `/home/foo/backup`") - -} diff --git a/cmd/root.go b/cmd/root.go deleted file mode 100644 index cd32128..0000000 --- a/cmd/root.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package cmd - -import ( - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" - "os" -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "mysql-bkup [Command]", - Short: "MySQL Backup tool, backup database to S3 or Object Storage", - Long: `MySQL Database backup and restoration tool. Backup database to AWS S3 storage or any S3 Alternatives for Object Storage.`, - Example: utils.MainExample, - Version: appVersion, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - err := rootCmd.Execute() - if err != nil { - os.Exit(1) - } -} - -func init() { - rootCmd.PersistentFlags().StringP("dbname", "d", "", "Database name") - rootCmd.AddCommand(VersionCmd) - rootCmd.AddCommand(BackupCmd) - rootCmd.AddCommand(RestoreCmd) - rootCmd.AddCommand(MigrateCmd) - -} diff --git a/cmd/version.go b/cmd/version.go deleted file mode 100644 index 2ed5e20..0000000 --- a/cmd/version.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package cmd - -import ( - "fmt" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" - "os" -) - -var appVersion = os.Getenv("VERSION") - -var VersionCmd = &cobra.Command{ - Use: "version", - Short: "Show version", - Run: func(cmd *cobra.Command, args []string) { - Version() - }, -} - -func Version() { - fmt.Printf("Version: %s \n", utils.Version) - fmt.Println() -} diff --git a/docs/how-tos/azure-blob.md b/docs/how-tos/azure-blob.md index 3ded41f..ee40fc0 100644 --- a/docs/how-tos/azure-blob.md +++ b/docs/how-tos/azure-blob.md @@ -48,7 +48,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## Azure Blob Configuration - AZURE_STORAGE_CONTAINER_NAME=backup-container diff --git a/docs/how-tos/backup-all.md b/docs/how-tos/backup-all.md index 1c75bba..f5b8b8f 100644 --- a/docs/how-tos/backup-all.md +++ b/docs/how-tos/backup-all.md @@ -30,7 +30,7 @@ docker run --rm --network your_network_name \ -v $PWD/backup:/backup/ \ -e "DB_HOST=dbhost" \ -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ + -e "DB_USER=username" \ -e "DB_PASSWORD=password" \ jkaninda/mysql-bkup backup --all-databases ``` @@ -50,7 +50,7 @@ docker run --rm --network your_network_name \ -v $PWD/backup:/backup/ \ -e "DB_HOST=dbhost" \ -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ + -e "DB_USER=username" \ -e "DB_PASSWORD=password" \ jkaninda/mysql-bkup backup --all-in-one ``` diff --git a/docs/how-tos/backup-to-ftp.md b/docs/how-tos/backup-to-ftp.md index a70a95c..f09da3a 100644 --- a/docs/how-tos/backup-to-ftp.md +++ b/docs/how-tos/backup-to-ftp.md @@ -50,10 +50,10 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## FTP Configuration - - FTP_HOST="hostname" + - FTP_HOST=hostname - FTP_PORT=21 - FTP_USER=user - FTP_PASSWORD=password diff --git a/docs/how-tos/backup-to-s3.md b/docs/how-tos/backup-to-s3.md index 93b4c9c..29347c8 100644 --- a/docs/how-tos/backup-to-s3.md +++ b/docs/how-tos/backup-to-s3.md @@ -42,14 +42,14 @@ services: # In production, lock your image tag to a specific release version # instead of using `latest`. Check https://github.com/jkaninda/pg-bkup/releases # for available releases. - image: jkaninda/pg-bkup + image: jkaninda/mysql-bkup container_name: pg-bkup command: backup --storage s3 -d database --path /my-custom-path environment: - DB_PORT=5432 - DB_HOST=postgres - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## AWS Configuration - AWS_S3_ENDPOINT=https://s3.amazonaws.com @@ -58,7 +58,7 @@ services: - AWS_ACCESS_KEY=xxxx - AWS_SECRET_KEY=xxxxx ## Optional: Disable SSL for S3 alternatives like Minio - - AWS_DISABLE_SSL="false" + - AWS_DISABLE_SSL=false ## Optional: Enable path-style access for S3 alternatives like Minio - AWS_FORCE_PATH_STYLE=false @@ -91,7 +91,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## AWS Configuration - AWS_S3_ENDPOINT=https://s3.amazonaws.com @@ -104,11 +104,11 @@ services: ## Optional: Delete old backups after a specified number of days #- BACKUP_RETENTION_DAYS=7 ## Optional: Disable SSL for S3 alternatives like Minio - - AWS_DISABLE_SSL="false" + - AWS_DISABLE_SSL=false ## Optional: Enable path-style access for S3 alternatives like Minio - AWS_FORCE_PATH_STYLE=false - # Ensure the pg-bkup container is connected to the same network as your database + # Ensure the mysql-bkup container is connected to the same network as your database networks: - web @@ -122,5 +122,5 @@ networks: - **Cron Expression**: Use the `--cron-expression` flag or `BACKUP_CRON_EXPRESSION` environment variable to define the backup schedule. For example, `0 1 * * *` runs the backup daily at 1:00 AM. - **Backup Retention**: Optionally, use the `BACKUP_RETENTION_DAYS` environment variable to automatically delete backups older than a specified number of days. -- **S3 Alternatives**: If using an S3 alternative like Minio, set `AWS_DISABLE_SSL="true"` and `AWS_FORCE_PATH_STYLE="true"` as needed. +- **S3 Alternatives**: If using an S3 alternative like Minio, set `AWS_DISABLE_SSL=true` and `AWS_FORCE_PATH_STYLE=true` as needed. In the Compose list syntax, do not quote values: the quotes would become part of the value. diff --git a/docs/how-tos/backup-to-ssh.md b/docs/how-tos/backup-to-ssh.md index b7a469f..6c3116e 100644 --- a/docs/how-tos/backup-to-ssh.md +++ b/docs/how-tos/backup-to-ssh.md @@ -23,15 +23,16 @@ This section explains how to set up and configure SSH-based backups. 3. **Required Environment Variables** The following environment variables are mandatory for SSH-based backups: - - `SSH_HOST`: The hostname or IP address of the remote server. - - `SSH_USER`: The username for SSH authentication. + - `SFTP_HOST`: The hostname or IP address of the remote server. + - `SFTP_USER`: The username for SSH authentication. - `REMOTE_PATH`: The directory on the remote server where backups will be stored. - - `SSH_IDENTIFY_FILE`: The path to the private key file for SSH authentication. - - `SSH_PORT`: The SSH port (default is `22`). - - `SSH_PASSWORD`: (Optional) Use this only if you are not using a private key for authentication. + - `SFTP_IDENTITY_FILE`: The path to the private key file for SSH authentication (`SFTP_IDENTITY_FILE` still works). + - `SFTP_KNOWN_HOSTS`: A `known_hosts` file used to verify the server's host key. Host keys are verified by default since v2; `SFTP_INSECURE_IGNORE_HOST_KEY=true` turns the check off (not recommended). + - `SFTP_PORT`: The SSH port (default is `22`). + - `SFTP_PASSWORD`: (Optional) Use this only if you are not using a private key for authentication. {: .note } - **Security Recommendation**: Using a private key (`SSH_IDENTIFY_FILE`) is strongly recommended over password-based authentication (`SSH_PASSWORD`) for better security. + **Security Recommendation**: Using a private key (`SFTP_IDENTITY_FILE`) is strongly recommended over password-based authentication (`SFTP_PASSWORD`) for better security. --- @@ -54,16 +55,17 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## SSH Configuration - - SSH_HOST="hostname" - - SSH_PORT=22 - - SSH_USER=user + - SFTP_HOST=hostname + - SFTP_PORT=22 + - SFTP_USER=user - REMOTE_PATH=/home/jkaninda/backups - - SSH_IDENTIFY_FILE=/tmp/id_ed25519 + - SFTP_IDENTITY_FILE=/tmp/id_ed25519 + - SFTP_KNOWN_HOSTS=/config/known_hosts ## Optional: Use password instead of private key (not recommended) - #- SSH_PASSWORD=password + #- SFTP_PASSWORD=password # Ensure the mysql-bkup container is connected to the same network as your database networks: @@ -97,18 +99,19 @@ services: - DB_PORT=3306 - DB_HOST=postgres - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## SSH Configuration - - SSH_HOST="hostname" - - SSH_PORT=22 - - SSH_USER=user + - SFTP_HOST=hostname + - SFTP_PORT=22 + - SFTP_USER=user - REMOTE_PATH=/home/jkaninda/backups - - SSH_IDENTIFY_FILE=/tmp/id_ed25519 + - SFTP_IDENTITY_FILE=/tmp/id_ed25519 + - SFTP_KNOWN_HOSTS=/config/known_hosts ## Optional: Delete old backups after a specified number of days #- BACKUP_RETENTION_DAYS=7 ## Optional: Use password instead of private key (not recommended) - #- SSH_PASSWORD=password + #- SFTP_PASSWORD=password # Ensure the mysql-bkup container is connected to the same network as your database networks: @@ -124,6 +127,6 @@ networks: - **Cron Expression**: Use the `--cron-expression` flag or `BACKUP_CRON_EXPRESSION` environment variable to define the backup schedule. For example, `0 1 * * *` runs the backup daily at 1:00 AM. - **Backup Retention**: Optionally, use the `BACKUP_RETENTION_DAYS` environment variable to automatically delete backups older than a specified number of days. -- **Security**: Always prefer private key authentication (`SSH_IDENTIFY_FILE`) over password-based authentication (`SSH_PASSWORD`) for enhanced security. +- **Security**: Always prefer private key authentication (`SFTP_IDENTITY_FILE`) over password-based authentication (`SFTP_PASSWORD`) for enhanced security. --- \ No newline at end of file diff --git a/docs/how-tos/backup.md b/docs/how-tos/backup.md index d4ff933..3228e9e 100644 --- a/docs/how-tos/backup.md +++ b/docs/how-tos/backup.md @@ -43,7 +43,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password # Ensure the mysql-bkup container is connected to the same network as your database @@ -64,7 +64,7 @@ You can also run backups directly using the Docker CLI: docker run --rm --network your_network_name \ -v $PWD/backup:/backup/ \ -e "DB_HOST=dbhost" \ - -e "DB_USERNAME=username" \ + -e "DB_USER=username" \ -e "DB_PASSWORD=password" \ jkaninda/pg-bkup backup -d database_name ``` @@ -92,7 +92,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## Optional: Define a cron schedule for recurring backups - BACKUP_CRON_EXPRESSION=@midnight diff --git a/docs/how-tos/deploy-on-kubernetes.md b/docs/how-tos/deploy-on-kubernetes.md index 290f971..7d12b03 100644 --- a/docs/how-tos/deploy-on-kubernetes.md +++ b/docs/how-tos/deploy-on-kubernetes.md @@ -46,7 +46,7 @@ spec: value: "" - name: DB_NAME value: "" - - name: DB_USERNAME + - name: DB_USER value: "" # Use Kubernetes Secrets for sensitive data like passwords - name: DB_PASSWORD @@ -104,20 +104,20 @@ spec: value: "" - name: DB_NAME value: "dbname" - - name: DB_USERNAME + - name: DB_USER value: "postgres" # Use Kubernetes Secrets for sensitive data like passwords - name: DB_PASSWORD value: "" - - name: SSH_HOST_NAME + - name: SFTP_HOST value: "xxx" - - name: SSH_PORT + - name: SFTP_PORT value: "22" - - name: SSH_USER + - name: SFTP_USER value: "xxx" - - name: SSH_PASSWORD + - name: SFTP_PASSWORD value: "xxxx" - - name: SSH_REMOTE_PATH + - name: REMOTE_PATH value: "/home/toto/backup" # Optional: Required if you want to encrypt your backup - name: GPG_PASSPHRASE @@ -161,20 +161,20 @@ spec: value: "" - name: DB_NAME value: "dbname" - - name: DB_USERNAME + - name: DB_USER value: "postgres" # Use Kubernetes Secrets for sensitive data like passwords - name: DB_PASSWORD value: "" - - name: SSH_HOST_NAME + - name: SFTP_HOST value: "xxx" - - name: SSH_PORT + - name: SFTP_PORT value: "22" - - name: SSH_USER + - name: SFTP_USER value: "xxx" - - name: SSH_PASSWORD + - name: SFTP_PASSWORD value: "xxxx" - - name: SSH_REMOTE_PATH + - name: REMOTE_PATH value: "/home/toto/backup" # Optional: Required if your backup was encrypted #- name: GPG_PASSPHRASE @@ -220,20 +220,20 @@ spec: value: "" - name: DB_NAME value: "test" - - name: DB_USERNAME + - name: DB_USER value: "postgres" # Use Kubernetes Secrets for sensitive data like passwords - name: DB_PASSWORD value: "" - - name: SSH_HOST_NAME + - name: SFTP_HOST value: "192.168.1.16" - - name: SSH_PORT + - name: SFTP_PORT value: "2222" - - name: SSH_USER + - name: SFTP_USER value: "jkaninda" - - name: SSH_REMOTE_PATH + - name: REMOTE_PATH value: "/config/backup" - - name: SSH_PASSWORD + - name: SFTP_PASSWORD value: "password" # Optional: Required if you want to encrypt your backup #- name: GPG_PASSPHRASE @@ -283,20 +283,20 @@ spec: value: "" - name: DB_NAME value: "test" - - name: DB_USERNAME + - name: DB_USER value: "postgres" # Use Kubernetes Secrets for sensitive data like passwords - name: DB_PASSWORD value: "" - - name: SSH_HOST_NAME + - name: SFTP_HOST value: "192.168.1.16" - - name: SSH_PORT + - name: SFTP_PORT value: "2222" - - name: SSH_USER + - name: SFTP_USER value: "jkaninda" - - name: SSH_REMOTE_PATH + - name: REMOTE_PATH value: "/config/backup" - - name: SSH_PASSWORD + - name: SFTP_PASSWORD value: "password" # Optional: Required if you want to encrypt your backup #- name: GPG_PASSPHRASE @@ -341,7 +341,7 @@ spec: value: "3306" - name: DB_NAME value: "dbname" - - name: DB_USERNAME + - name: DB_USER value: "username" - name: DB_PASSWORD value: "password" @@ -352,7 +352,7 @@ spec: value: "3306" - name: TARGET_DB_NAME value: "dbname" - - name: TARGET_DB_USERNAME + - name: TARGET_DB_USER value: "username" - name: TARGET_DB_PASSWORD value: "password" diff --git a/docs/how-tos/encrypt-backup.md b/docs/how-tos/encrypt-backup.md index 71199a6..23f7c27 100644 --- a/docs/how-tos/encrypt-backup.md +++ b/docs/how-tos/encrypt-backup.md @@ -42,7 +42,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## Required to encrypt backup - GPG_PASSPHRASE=my-secure-passphrase @@ -58,7 +58,7 @@ networks: ## Using GPG Public Key -To encrypt backups using a GPG public key, set the `GPG_PUBLIC_KEY` environment variable to the path of your public key file. Backups encrypted with a public key require manual decryption before restoration. +To encrypt backups using a GPG public key, set the `GPG_PUBLIC_KEY` environment variable to the path of your public key file. To restore such backups, set `GPG_PRIVATE_KEY` (a file path or the inline armored key; `/config/gnupg/private_key.asc` is picked up automatically) and, if the key is protected, `GPG_PRIVATE_KEY_PASSPHRASE` (defaults to `GPG_PASSPHRASE`). ### Example Configuration @@ -78,7 +78,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## Required to encrypt backup - GPG_PUBLIC_KEY=/config/public_key.asc @@ -116,5 +116,21 @@ gpg --output database_20240730_044201.sql.gz \ ## Key Notes - **Automatic Restoration**: Backups encrypted with a GPG passphrase can be restored directly without manual decryption. -- **Manual Decryption**: Backups encrypted with a GPG public key require manual decryption using the corresponding private key. +- **Private key restore**: Backups encrypted with a GPG public key restore directly when `GPG_PRIVATE_KEY` is set; manual decryption also works. +- **Fail closed**: if `GPG_PUBLIC_KEY` (or any configured key) cannot be read, the backup fails instead of being stored unencrypted. - **Security**: Always keep your GPG passphrase and private key secure. Use Kubernetes Secrets or other secure methods to manage sensitive data. + +--- + +## Using age + +[age](https://age-encryption.org) is a modern alternative to GPG. Set `AGE_RECIPIENTS` (one or more `age1...` or `ssh-ed25519 ...` public keys, comma or newline separated, or `AGE_RECIPIENTS_FILE`) and backups are saved as `.sql.gz.age`. Restore with the matching identity in `AGE_IDENTITY` (`AGE-SECRET-KEY-...`) or `AGE_IDENTITY_FILE` (default `/config/age/identity.txt`). + +```yaml + environment: + - AGE_RECIPIENTS=age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p + # restore: + # - AGE_IDENTITY_FILE=/run/secrets/age_identity +``` + +`AGE_PASSPHRASE` encrypts with a passphrase instead (it needs about 256 MiB of memory for the key derivation). `ENCRYPTION=auto|none|gpg|age` chooses explicitly; with `auto` (default) GPG wins when both GPG and age keys are set. Decrypt by hand with `age -d -i key.txt file.sql.gz.age | gunzip`. diff --git a/docs/how-tos/migrate.md b/docs/how-tos/migrate.md index b4e316c..c6e936c 100644 --- a/docs/how-tos/migrate.md +++ b/docs/how-tos/migrate.md @@ -45,14 +45,14 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## Target Database - - TARGET_DB_HOST=target-postgres + - TARGET_DB_HOST=target-mysql - TARGET_DB_PORT=3306 - TARGET_DB_NAME=dbname - - TARGET_DB_USERNAME=username + - TARGET_DB_USER=username - TARGET_DB_PASSWORD=password # Ensure the mysql-bkup container is connected to the same network as your database @@ -78,14 +78,14 @@ Save your source and target database connection details in an environment file ( DB_HOST=postgres DB_PORT=3306 DB_NAME=dbname -DB_USERNAME=username +DB_USER=username DB_PASSWORD=password ## Target Database -TARGET_DB_HOST=target-postgres +TARGET_DB_HOST=target-mysql TARGET_DB_PORT=3306 TARGET_DB_NAME=dbname -TARGET_DB_USERNAME=username +TARGET_DB_USER=username TARGET_DB_PASSWORD=password ``` @@ -95,12 +95,15 @@ TARGET_DB_PASSWORD=password docker run --rm --network your_network_name \ --env-file your-env \ -v $PWD/backup:/backup/ \ - jkaninda/pg-bkup migrate + jkaninda/mysql-bkup migrate ``` --- ## Key Notes +- **Streaming**: the dump is piped straight into the target server; no file is written. The target database is created if it does not exist, and `TARGET_DB_NAME` defaults to the source name. +- **All databases**: `migrate --all-databases` copies every database (system databases excluded, `--exclude-db` / `BACKUP_EXCLUDE_DATABASES` honoured) under the same names. +- **MySQL and MariaDB**: source and target may be different flavours; each side uses its own client. Objects that only exist in one flavour (for example MySQL-only collations on old MariaDB versions) can make a cross-flavour migration fail. - **Irreversible Operation**: The `migrate` command directly transfers data from the source to the target database. Ensure you have a backup of the target database before proceeding. - **Network Configuration**: Ensure the `mysql-bkup` container is connected to the same network as your source and target databases. diff --git a/docs/how-tos/mutli-backup.md b/docs/how-tos/mutli-backup.md index 5fb2593..11ddfae 100644 --- a/docs/how-tos/mutli-backup.md +++ b/docs/how-tos/mutli-backup.md @@ -40,28 +40,28 @@ databases: - host: mysql1 # Optional: Overrides DB_HOST or uses DB_HOST_DATABASE1. port: 3306 # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_DATABASE1. name: database1 # Required: Database name. - user: database1 # Optional: Overrides DB_USERNAME or uses DB_USERNAME_DATABASE1. + user: database1 # Optional: Overrides DB_USER or uses DB_USER_DATABASE1. password: password # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_DATABASE1. path: /s3-path/database1 # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/). - host: mysql2 # Optional: Overrides DB_HOST or uses DB_HOST_LLAP. port: 3306 # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_LLAP. name: lldap # Required: Database name. - user: lldap # Optional: Overrides DB_USERNAME or uses DB_USERNAME_LLAP. + user: lldap # Optional: Overrides DB_USER or uses DB_USER_LLAP. password: password # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_LLAP. path: /s3-path/lldap # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/). - host: mysql3 # Optional: Overrides DB_HOST or uses DB_HOST_KEYCLOAK. port: 3306 # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_KEYCLOAK. name: keycloak # Required: Database name. - user: keycloak # Optional: Overrides DB_USERNAME or uses DB_USERNAME_KEYCLOAK. + user: keycloak # Optional: Overrides DB_USER or uses DB_USER_KEYCLOAK. password: password # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_KEYCLOAK. path: /s3-path/keycloak # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/). - host: mysql4 # Optional: Overrides DB_HOST or uses DB_HOST_JOPLIN. port: 3306 # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_JOPLIN. name: joplin # Required: Database name. - user: joplin # Optional: Overrides DB_USERNAME or uses DB_USERNAME_JOPLIN. + user: joplin # Optional: Overrides DB_USER or uses DB_USER_JOPLIN. password: password # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_JOPLIN. path: /s3-path/joplin # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/). ``` diff --git a/docs/how-tos/receive-notification.md b/docs/how-tos/receive-notification.md index 43fa17f..e1ead8e 100644 --- a/docs/how-tos/receive-notification.md +++ b/docs/how-tos/receive-notification.md @@ -7,7 +7,7 @@ nav_order: 13 # Receive Notifications -You can configure the system to send email or Telegram notifications when a backup succeeds or fails. +You can configure the system to send email (SMTP or Posta) or Telegram notifications when a backup succeeds or fails. This section explains how to set up and customize notifications. @@ -31,7 +31,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## SMTP Configuration - MAIL_HOST=smtp.example.com @@ -41,7 +41,8 @@ services: - MAIL_FROM=Backup Jobs ## Multiple recipients separated by a comma - MAIL_TO=me@example.com,team@example.com,manager@example.com - - MAIL_SKIP_TLS=false + ## Only for SMTP servers with a self-signed certificate + #- MAIL_INSECURE_SKIP_VERIFY=true ## Time format for notifications - TIME_FORMAT=2006-01-02 at 15:04:05 ## Backup reference (e.g., database/cluster name or server name) @@ -55,6 +56,44 @@ networks: --- +## Posta Notifications + +[Posta](https://github.com/goposta/posta) can deliver the notification emails instead of an SMTP server. Set the Posta URL, an API key with the `send` scope and the recipients. By default the same HTML email is sent (with a plain-text version). To use a template stored in Posta instead, set `POSTA_TEMPLATE` (and optionally `POSTA_TEMPLATE_ERROR` for failures); the notification fields (`database`, `file`, `duration`, `storage`, `location`, `size`, `backup_size`, `sha256`, `encryption`, `error`, `time`, `success`, `tool`...) are passed as template data. + +### Example: Posta Notification Configuration + +```yaml +services: + mysql-bkup: + image: jkaninda/mysql-bkup + container_name: mysql-bkup + command: backup + volumes: + - ./backup:/backup + environment: + - DB_PORT=3306 + - DB_HOST=db + - DB_NAME=database + - DB_USER=username + - DB_PASSWORD=password + ## Posta + - POSTA_URL=https://posta.example.com + - POSTA_API_KEY=psk_your_api_key + - POSTA_FROM=Backup Jobs + ## Multiple recipients separated by a comma (default MAIL_TO) + - POSTA_TO=me@example.com,team@example.com + ## Optional: templates stored in Posta + #- POSTA_TEMPLATE=backup-success + #- POSTA_TEMPLATE_ERROR=backup-failure + #- POSTA_LANGUAGE=en + ## Only for account-wide API keys + #- POSTA_WORKSPACE_ID=42 + ## Only notify failures + #- NOTIFY_ON=failure +``` + +--- + ## Telegram Notifications To send Telegram notifications, provide your bot token and chat ID. Notifications will be sent for both successful and failed backup runs. @@ -73,7 +112,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## Telegram Configuration - TG_TOKEN=[BOT ID]:[BOT TOKEN] @@ -191,7 +230,7 @@ Backup Reference: {{.BackupReference}} ## Key Notes -- **SMTP Configuration**: Ensure your SMTP server supports TLS unless `MAIL_SKIP_TLS` is set to `true`. +- **SMTP Configuration**: Ensure your SMTP server supports TLS unless `MAIL_INSECURE_SKIP_VERIFY` is set to `true`. `MAIL_TLS=starttls|tls|none` selects the TLS mode. The 1.x variable `MAIL_SKIP_TLS` is still read with its documented meaning (`true` = skip certificate verification); 1.x had it inverted. Slack (`SLACK_WEBHOOK_URL`), Discord (`DISCORD_WEBHOOK_URL`), ntfy (`NTFY_URL`), generic webhooks (`WEBHOOK_URL`, `WEBHOOK_SECRET`) and healthchecks.io pings (`HEALTHCHECK_PING_URL`) are also available; `NOTIFY_ON=always|failure|success` filters them. - **Telegram Configuration**: Obtain your bot token and chat ID from Telegram. - **Custom Templates**: Mount custom templates to `/config/templates` to override default notifications. - **Time Format**: Use the `TIME_FORMAT` environment variable to customize the timestamp format in notifications. \ No newline at end of file diff --git a/docs/how-tos/restore-from-s3.md b/docs/how-tos/restore-from-s3.md index 947104b..77ee903 100644 --- a/docs/how-tos/restore-from-s3.md +++ b/docs/how-tos/restore-from-s3.md @@ -44,7 +44,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## AWS S3 Configuration - AWS_S3_ENDPOINT=https://s3.amazonaws.com @@ -56,7 +56,7 @@ services: - AWS_DISABLE_SSL=false ## Optional: Enable path-style access for S3 alternatives like Minio - AWS_FORCE_PATH_STYLE=false - # Ensure the pg-bkup container is connected to the same network as your database + # Ensure the mysql-bkup container is connected to the same network as your database networks: - web diff --git a/docs/how-tos/restore-from-ssh.md b/docs/how-tos/restore-from-ssh.md index 880bdba..9551613 100644 --- a/docs/how-tos/restore-from-ssh.md +++ b/docs/how-tos/restore-from-ssh.md @@ -45,16 +45,17 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## SSH Configuration - - SSH_HOST_NAME=hostname - - SSH_PORT=22 - - SSH_USER=user - - SSH_REMOTE_PATH=/home/jkaninda/backups - - SSH_IDENTIFY_FILE=/tmp/id_ed25519 + - SFTP_HOST=hostname + - SFTP_PORT=22 + - SFTP_USER=user + - REMOTE_PATH=/home/jkaninda/backups + - SFTP_IDENTITY_FILE=/tmp/id_ed25519 + - SFTP_KNOWN_HOSTS=/config/known_hosts ## Optional: Use password instead of private key (not recommended) - #- SSH_PASSWORD=password + #- SFTP_PASSWORD=password # Ensure the mysql-bkup container is connected to the same network as your database networks: - web @@ -70,5 +71,5 @@ networks: - **Supported File Formats**: The restore process supports `.sql`, `.sql.gz`, `.sql.gpg`, and `.sql.gz.gpg` files. - **SSH Path**: Use the `--path` flag to specify the folder on the SSH remote server where the backup file is located. - **Encrypted Backups**: If the backup is encrypted with GPG, ensure the `GPG_PASSPHRASE` environment variable is set for automatic decryption. -- **SSH Authentication**: Use a private key (`SSH_IDENTIFY_FILE`) for SSH authentication instead of a password for better security. +- **SSH Authentication**: Use a private key (`SFTP_IDENTITY_FILE`) for SSH authentication instead of a password for better security. Host keys are verified against `SFTP_KNOWN_HOSTS` (or set `SFTP_INSECURE_IGNORE_HOST_KEY=true`). - **Network Configuration**: Ensure the `mysql-bkup` container is connected to the same network as your database. \ No newline at end of file diff --git a/docs/how-tos/restore.md b/docs/how-tos/restore.md index 5a4e9cf..b4d74dc 100644 --- a/docs/how-tos/restore.md +++ b/docs/how-tos/restore.md @@ -45,9 +45,9 @@ services: - DB_PORT=3306 - DB_HOST=postgres - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password - # Ensure the pg-bkup container is connected to the same network as your database + # Ensure the mysql-bkup container is connected to the same network as your database networks: - web diff --git a/docs/quickstart/index.md b/docs/quickstart/index.md index 7638aad..2ae46f7 100644 --- a/docs/quickstart/index.md +++ b/docs/quickstart/index.md @@ -19,7 +19,7 @@ docker run --rm --network your_network_name \ -v $PWD/backup:/backup/ \ -e "DB_HOST=dbhost" \ -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ + -e "DB_USER=username" \ -e "DB_PASSWORD=password" \ jkaninda/mysql-bkup backup -d database_name ``` @@ -42,7 +42,7 @@ docker run --rm --network your_network_name \ -v $PWD/backup:/backup/ \ -e "DB_HOST=dbhost" \ -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ + -e "DB_USER=username" \ -e "DB_PASSWORD=password" \ jkaninda/mysql-bkup backup --all-databases --disable-compression ``` @@ -60,7 +60,7 @@ docker run --rm --network your_network_name \ -v $PWD/backup:/backup/ \ -e "DB_HOST=dbhost" \ -e "DB_PORT=3306" \ - -e "DB_USERNAME=username" \ + -e "DB_USER=username" \ -e "DB_PASSWORD=password" \ jkaninda/mysql-bkup restore -d database_name -f backup_file.sql.gz ``` @@ -73,7 +73,7 @@ Below is an example of a `docker-compose.yml` file for running a one-time backup ```yaml services: - pg-bkup: + mysql-bkup: # In production, pin your image tag to a specific release version instead of `latest`. # See available releases: https://github.com/jkaninda/mysql-bkup/releases image: jkaninda/mysql-bkup @@ -85,7 +85,7 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=foo - - DB_USERNAME=bar + - DB_USER=bar - DB_PASSWORD=password - TZ=Europe/Paris networks: @@ -105,7 +105,7 @@ You can schedule recurring backups using the `--cron-expression` or `-e` flag: docker run --rm --network network_name \ -v $PWD/backup:/backup/ \ -e "DB_HOST=hostname" \ - -e "DB_USERNAME=user" \ + -e "DB_USER=user" \ -e "DB_PASSWORD=password" \ jkaninda/mysql-bkup backup -d dbName --cron-expression "@every 15m" ``` @@ -147,7 +147,7 @@ spec: env: - name: DB_HOST value: "mysql" - - name: DB_USERNAME + - name: DB_USER value: "user" - name: DB_PASSWORD value: "password" @@ -187,7 +187,7 @@ spec: env: - name: DB_HOST value: "mysql" - - name: DB_USERNAME + - name: DB_USER value: "user" - name: DB_PASSWORD value: "password" diff --git a/docs/reference/index.md b/docs/reference/index.md index 480d4c5..c2f4d24 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -6,81 +6,250 @@ nav_order: 3 # Configuration Reference -MySQL backup, restore, and migration processes can be configured using **environment variables** or **CLI flags**. +mysql-bkup is configured with **environment variables**, **CLI flags** and an optional **YAML file**. For every setting the order is: flag, environment variable, deprecated name, YAML `env:` section, default. Every secret (`DB_PASSWORD`, `AWS_SECRET_KEY`, `GPG_PASSPHRASE`, ...) can also be read from a file with `_FILE`. -## CLI Utility Usage +Run `mysql-bkup config` to see the resolved configuration and where each value comes from (secrets redacted). -The `mysql-bkup` CLI provides commands and options to manage MySQL backups efficiently. +## Commands -| Option | Short Flag | Description | -|-------------------------|------------|-----------------------------------------------------------------------------------------| -| `mysql-bkup` | `bkup` | CLI tool for managing MySQL backups, restoration, and migration. | -| `backup` | | Executes a backup operation. | -| `restore` | | Restores a database from a backup file. | -| `migrate` | | Migrates a database from one instance to another. | -| `--storage` | `-s` | Specifies the storage type (`local`, `s3`, `ssh`, etc.). Default: `local`. | -| `--file` | `-f` | Defines the backup file name for restoration. | -| `--path` | | Sets the storage path (e.g., `/custom_path` for S3 or `/home/foo/backup` for SSH). | -| `--config` | `-c` | Provides a configuration file for multi-database backups (e.g., `/backup/config.yaml`). | -| `--dbname` | `-d` | Specifies the database name to back up or restore. | -| `--port` | `-p` | Defines the database port. Default: `3306`. | -| `--disable-compression` | | Disables compression for database backups. | -| `--cron-expression` | `-e` | Schedules backups using a cron expression (e.g., `0 0 * * *` or `@daily`). | -| `--all-databases` | `-a` | Backs up all databases separately (e.g., `backup --all-databases`). | -| `--all-in-one` | `-A` | Backs up all databases in a single file (e.g., `backup --all-databases --single-file`). | -| `--custom-name` | `` | Sets custom backup name for one time backup | -| `--help` | `-h` | Displays the help message and exits. | -| `--version` | `-V` | Shows version information and exits. | +| Command | Description | +|---|---| +| `backup` | Back up now, or as a daemon with `--cron-expression` | +| `restore` | Restore a backup: `-f FILE [-d DB]` | +| `migrate` | Copy a database (or every database with `-a`) into `TARGET_DB_*`, streamed | +| `verify` | Download, decrypt and decompress a backup and check its sha256 | +| `list` | Backup history from the index (`--json`, `--storage-scan`) | +| `delete` | Delete a backup, its sidecar and its index entry (`-f FILE`) | +| `prune` | Apply the retention policy (`--dry-run`) | +| `reconcile` | Rebuild the backup index (`--from-index OLD`, `--dry-run`) | +| `config` | Show the resolved configuration; `--env-docs` prints the tables below | +| `health` | Exit 0 when the daemon is healthy (Docker `HEALTHCHECK`) | +| `version` | Print the version | ---- +## Flags -## Environment Variables +| Flag | Short | Commands | Variable | +|---|---|---|---| +| `--dbname` | `-d` | all | `DB_NAME` | +| `--storage` | `-s` | all | `STORAGE` (`local`, `s3`, `gcs`, `azure`, `sftp`/`ssh`/`remote`, `ftp`) | +| `--path` (`--remote-path`) | `-P` | all | `REMOTE_PATH` | +| `--file` | `-f` | restore, verify, delete | `FILE_NAME` | +| `--config` | `-c` | backup, list, prune, config | `BACKUP_CONFIG_FILE` | +| `--cron-expression` | `-e` | backup | `BACKUP_CRON_EXPRESSION` | +| `--all-databases` | `-a` | backup, migrate | `BACKUP_ALL_DATABASES` | +| `--all-in-one` | `-A` | backup | `BACKUP_ALL_IN_ONE` | +| `--exclude-db` | | backup | `BACKUP_EXCLUDE_DATABASES` | +| `--custom-name` | | backup | `BACKUP_CUSTOM_NAME` (fixed name, no timestamp) | +| `--name` | `-n` | backup, list, prune | `BACKUP_NAME` | +| `--disable-compression` | | backup | `DISABLE_COMPRESSION` | +| `--compression` | | backup | `COMPRESSION` (`gzip`, `zstd`, `none`) | +| `--encryption` | | backup | `ENCRYPTION` (`auto`, `none`, `gpg`, `age`) | +| `--index-name` | | all | `BACKUP_INDEX_NAME` | +| `--result-file` | | all | `RESULT_FILE` | +| `--help` | `-h` | all | | +| `--version` | `-v` | root | | -| Name | Requirement | Description | -|--------------------------------|--------------------------------------|----------------------------------------------------------------------------| -| `DB_PORT` | Optional (default: `3306`) | Database port number. | -| `DB_HOST` | Required | Database host. | -| `DB_NAME` | Optional (if provided via `-d` flag) | Database name. | -| `DB_USERNAME` | Required | Database username. | -| `DB_PASSWORD` | Required | Database password. | -| `DB_SSL_CA` | Optional | Database client CA certificate file | -| `DB_SSL_MODE` | Optional(`0 or 1`) default: `0` | Database client Enable CA validation | -| `AWS_ACCESS_KEY` | Required for S3 storage | AWS S3 Access Key. | -| `AWS_SECRET_KEY` | Required for S3 storage | AWS S3 Secret Key. | -| `AWS_BUCKET_NAME` | Required for S3 storage | AWS S3 Bucket Name. | -| `AWS_REGION` | Required for S3 storage | AWS Region. | -| `AWS_DISABLE_SSL` | Optional | Disable SSL for S3 storage. | -| `AWS_FORCE_PATH_STYLE` | Optional | Force path-style access for S3 storage. | -| `FILE_NAME` | Optional (if provided via `--file`) | File name for restoration (e.g., `.sql`, `.sql.gz`). | -| `GPG_PASSPHRASE` | Optional | GPG passphrase for encrypting/decrypting backups. | -| `GPG_PUBLIC_KEY` | Optional | GPG public key for encrypting backups (e.g., `/config/public_key.asc`). | -| `BACKUP_CRON_EXPRESSION` | Optional (flag `-e`) | Cron expression for scheduled backups. | -| `BACKUP_RETENTION_DAYS` | Optional | Delete backups older than the specified number of days. | -| `BACKUP_CONFIG_FILE` | Optional (flag `-c`) | Configuration file for multi database backup. (e.g: `/backup/config.yaml`) | -| `SSH_HOST` | Required for SSH storage | SSH remote hostname or IP. | -| `SSH_USER` | Required for SSH storage | SSH remote username. | -| `SSH_PASSWORD` | Optional | SSH remote user's password. | -| `SSH_IDENTIFY_FILE` | Optional | SSH remote user's private key. | -| `SSH_PORT` | Optional (default: `22`) | SSH remote server port. | -| `REMOTE_PATH` | Required for SSH/FTP storage | Remote path (e.g., `/home/toto/backup`). | -| `FTP_HOST` | Required for FTP storage | FTP hostname. | -| `FTP_PORT` | Optional (default: `21`) | FTP server port. | -| `FTP_USER` | Required for FTP storage | FTP username. | -| `FTP_PASSWORD` | Required for FTP storage | FTP user password. | -| `TARGET_DB_HOST` | Required for migration | Target database host. | -| `TARGET_DB_PORT` | Optional (default: `5432`) | Target database port. | -| `TARGET_DB_NAME` | Required for migration | Target database name. | -| `TARGET_DB_USERNAME` | Required for migration | Target database username. | -| `TARGET_DB_PASSWORD` | Required for migration | Target database password. | -| `TARGET_DB_URL` | Optional | Target database URL in JDBC URI format. | -| `TG_TOKEN` | Required for Telegram notifications | Telegram token (`BOT-ID:BOT-TOKEN`). | -| `TG_CHAT_ID` | Required for Telegram notifications | Telegram Chat ID. | -| `TZ` | Optional | Time zone for scheduling. | -| `AZURE_STORAGE_CONTAINER_NAME` | Required for Azure Blob Storage | Azure storage container name. | -| `AZURE_STORAGE_ACCOUNT_NAME` | Required for Azure Blob Storage | Azure storage account name. | -| `AZURE_STORAGE_ACCOUNT_KEY` | Required for Azure Blob Storage | Azure storage account key. | +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | success | +| `1` | failure | +| `2` | configuration error | +| `3` | database connection or authentication | +| `4` | storage | +| `5` | encryption or decryption | + +## Environment variables + +### storage + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `STORAGE` | `--storage` `-s` | `local` | | Storage backend: local, s3, gcs, azure, sftp (ssh, remote), ftp (local, s3, gcs, azure, sftp, ssh, ftp, remote) | +| `STORAGE_PATH` | | `/backup` | | Root directory of the local storage backend | +| `REMOTE_PATH` | `--path` `-P` | | `AWS_S3_PATH` (deprecated), `S3_PATH` (deprecated) | Folder inside the storage backend (without file name) | + +### general + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `BACKUP_NAME` | `--name` `-n` | | | Base name of the artifact; a timestamp is appended (default: the database or volume name) | +| `BACKUP_CUSTOM_NAME` | `--custom-name` | | | Fixed artifact name without timestamp (overwrites the previous one) | +| `FILE_NAME` | `--file` `-f` | | | Backup file to restore, verify or delete | +| `BACKUP_CONFIG_FILE` | `--config` `-c` | | | YAML configuration file (default /config/config.yaml if present) | +| `BACKUP_TIMEOUT` | | | | Abort a backup after this duration (0 = none) | +| `RESTORE_TIMEOUT` | | | | Abort a restore after this duration (0 = none) | +| `TMP_DIR` | | `/tmp/backup` | `BACKUP_TMP_DIR` (deprecated) | Scratch directory (only for secret files and disk-based modes) | +| `COMPRESSION` | `--compression` | `gzip` | | Compression: gzip, zstd or none (gzip, zstd, none) | +| `COMPRESSION_LEVEL` | | | | Compression level (0 = default) | +| `DISABLE_COMPRESSION` | `--disable-compression` | | | Same as COMPRESSION=none | +| `RESTORE_VERIFY_FIRST` | `--verify-first` | `auto` | | Verify the whole artifact before restoring (auto: on for GPG-encrypted volumes) (auto, true, false) | +| `LOG_LEVEL` | | `info` | | Log level (debug, info, warn, error) | +| `LOG_FORMAT` | | `text` | | Log format (text, json) | +| `TIME_FORMAT` | | `2006-01-02 15:04:05` | | Time layout used in notifications | +| `RESULT_FILE` | `--result-file` | | | Write the run result as JSON to this path | + +### scheduling + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `BACKUP_CRON_EXPRESSION` | `--cron-expression` `-e` | | | Run as a daemon on this cron schedule (5 or 6 fields, @daily, @every 6h) | +| `BACKUP_RUN_ON_START` | | | | In daemon mode, run one backup immediately at start | +| `BACKUP_RESCUE_MODE` | | | | Deprecated: runs always continue with the remaining targets after a failure | +| `METRICS_ADDR` | | | | Serve Prometheus /metrics and /healthz on this address in daemon mode (e.g. :9090) | +| `HEALTH_FILE` | | `/tmp/bkup-health.json` | | State file read by the health command | +| `HEALTH_MAX_AGE` | | | | health fails when the last success is older than this (default: 2x the schedule interval) | + +### retention + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `BACKUP_RETENTION_DAYS` | | `0` | | Delete backups older than N days (0 = keep) | +| `BACKUP_RETENTION_COUNT` | | `0` | | Keep only the newest N backups (0 = no limit) | +| `BACKUP_RETENTION_ALLOW_ROOT` | | | | Allow pattern-based retention at the storage root (index disabled) | +| `BACKUP_RETENTION_ADOPT_LEGACY` | | | | Import pre-index backups matching this tool's naming into the index | +| `BACKUP_INDEX` | | `true` | | Keep the backup index file and per-backup .meta.json sidecars | +| `BACKUP_INDEX_NAME` | `--index-name` | `{tool}-metadata.json` | | Index file name; placeholders {tool} {engine} {name} {db} | + +### storage-s3 + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `AWS_S3_ENDPOINT` | | | `S3_ENDPOINT` (deprecated), `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL` | S3 endpoint (empty = AWS) | +| `AWS_S3_BUCKET_NAME` | | | `BUCKET_NAME` (deprecated), `AWS_BUCKET_NAME` (deprecated) | S3 bucket | +| `AWS_REGION` | | `us-east-1` | `AWS_DEFAULT_REGION` | S3 region | +| `AWS_ACCESS_KEY` | | | `ACCESS_KEY` (deprecated), `AWS_ACCESS_KEY_ID` | S3 access key (empty = default AWS credential chain) | +| `AWS_SECRET_KEY` | | | `SECRET_KEY` (deprecated), `AWS_SECRET_ACCESS_KEY`, `AWS_SECRET_KEY_FILE` | S3 secret key | +| `AWS_SESSION_TOKEN` | | | `AWS_SESSION_TOKEN_FILE` | S3 session token | +| `AWS_PROFILE` | | | | AWS shared config profile | +| `AWS_DISABLE_SSL` | | | | Use http for the S3 endpoint | +| `AWS_FORCE_PATH_STYLE` | | | | Path-style S3 addressing (MinIO) | +| `S3_STORAGE_CLASS` | | | | S3 storage class | +| `S3_SSE` | | | | Server-side encryption: AES256 or aws:kms (aes256, aws:kms) | +| `S3_SSE_KMS_KEY_ID` | | | | KMS key id for aws:kms | +| `S3_PART_SIZE` | | `64MiB` | | Multipart part size; max object size is 10000 x this | +| `S3_UPLOAD_CONCURRENCY` | | `2` | | Parallel part uploads (memory = part size x this) | + +### storage-sftp + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `SFTP_HOST` | | | | SFTP host | +| `SFTP_PORT` | | `22` | | SFTP port | +| `SFTP_USER` | | | | SFTP user | +| `SFTP_PASSWORD` | | | `SFTP_PASSWORD_FILE` | SFTP password | +| `SFTP_IDENTITY_FILE` | | | | SFTP private key file (the SSH agent is also used when SSH_AUTH_SOCK is set) | +| `SFTP_IDENTITY_PASSPHRASE` | | | `SFTP_IDENTITY_PASSPHRASE_FILE` | Passphrase of the SFTP private key | +| `SFTP_KNOWN_HOSTS` | | | | known_hosts file used to verify the SFTP host key (default ~/.ssh/known_hosts) | +| `SFTP_INSECURE_IGNORE_HOST_KEY` | | | | Do not verify the SFTP host key (insecure) | + +### storage-ftp + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `FTP_HOST` | | | `FTP_HOST_NAME` (deprecated) | FTP host | +| `FTP_PORT` | | `21` | | FTP port | +| `FTP_USER` | | | | FTP user | +| `FTP_PASSWORD` | | | `FTP_PASSWORD_FILE` | FTP password | +| `FTP_TLS` | | | | Use explicit FTPS | + +### storage-azure + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `AZURE_STORAGE_ACCOUNT_NAME` | | | | Azure storage account | +| `AZURE_STORAGE_ACCOUNT_KEY` | | | `AZURE_STORAGE_ACCOUNT_KEY_FILE` | Azure storage account key | +| `AZURE_STORAGE_CONTAINER_NAME` | | | | Azure blob container | +| `AZURE_STORAGE_ENDPOINT` | | | | Azure blob endpoint (Azurite, sovereign clouds) | +| `AZURE_STORAGE_SAS_TOKEN` | | | `AZURE_STORAGE_SAS_TOKEN_FILE` | Azure SAS token (instead of the account key) | + +### storage-gcs + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `GCS_BUCKET` | | | | GCS bucket | +| `GCS_CREDENTIALS_FILE` | | | `GOOGLE_APPLICATION_CREDENTIALS` | GCS service account JSON file | +| `GCS_CREDENTIALS_JSON` | | | `GCS_CREDENTIALS_JSON_FILE` | GCS service account JSON content | +| `GCS_ENDPOINT` | | | | GCS endpoint (emulators) | + +### encryption + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `ENCRYPTION` | `--encryption` | `auto` | | auto (from the keys set), none, gpg or age (auto, none, gpg, age) | +| `ENCRYPTION_ARMOR` | | | | ASCII-armor encrypted output | +| `GPG_PASSPHRASE` | | | `GPG_PASSPHRASE_FILE` | GPG symmetric passphrase | +| `GPG_PUBLIC_KEY` | | | | GPG public key(s): file path or inline key (default /config/gnupg/public_key.{asc,gpg}) | +| `GPG_PRIVATE_KEY` | | | `GPG_PRIVATE_KEY_FILE` | GPG private key for restore: file path or inline key (default /config/gnupg/private_key.{asc,gpg}) | +| `GPG_PRIVATE_KEY_PASSPHRASE` | | | `GPG_PRIVATE_KEY_PASSPHRASE_FILE` | Passphrase of the GPG private key (default GPG_PASSPHRASE) | +| `AGE_RECIPIENTS` | | | | age recipients (age1..., ssh-ed25519 ...), comma or newline separated | +| `AGE_RECIPIENTS_FILE` | | | | File with age recipients | +| `AGE_PASSPHRASE` | | | `AGE_PASSPHRASE_FILE` | age passphrase (scrypt) | +| `AGE_IDENTITY` | | | `AGE_IDENTITY_FILE` | age identity (AGE-SECRET-KEY-...) for restore | +| `AGE_IDENTITY_FILE` | | | | age identity file for restore (default /config/age/identity.txt) | + +### notifications + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `MAIL_HOST` | | | | SMTP host | +| `MAIL_PORT` | | `587` | | SMTP port | +| `MAIL_USERNAME` | | | | SMTP user | +| `MAIL_PASSWORD` | | | `MAIL_PASSWORD_FILE` | SMTP password | +| `MAIL_FROM` | | | | Sender address | +| `MAIL_TO` | | | | Recipients, comma separated | +| `MAIL_SKIP_TLS` | | | | Deprecated: true skips certificate verification (use MAIL_INSECURE_SKIP_VERIFY) | +| `MAIL_INSECURE_SKIP_VERIFY` | | | | Skip SMTP certificate verification | +| `MAIL_TLS` | | `starttls` | | SMTP TLS mode (starttls, tls, none) | +| `TG_TOKEN` | | | `TELEGRAM_BOT_TOKEN`, `TG_TOKEN_FILE` | Telegram bot token | +| `TG_CHAT_ID` | | | `TELEGRAM_CHAT_ID` | Telegram chat id | +| `POSTA_URL` | | | | Posta base URL (https://posta.example.com) for email notifications through Posta | +| `POSTA_API_KEY` | | | `POSTA_API_KEY_FILE` | Posta API key with the send scope | +| `POSTA_WORKSPACE_ID` | | | | Posta workspace id (only for account-wide API keys) | +| `POSTA_FROM` | | | | Posta sender address (default MAIL_FROM) | +| `POSTA_TO` | | | | Posta recipients, comma separated (default MAIL_TO) | +| `POSTA_TEMPLATE` | | | | Posta template name for notifications; the event fields are passed as template data (default: built-in HTML) | +| `POSTA_TEMPLATE_ERROR` | | | | Posta template name for failures (default POSTA_TEMPLATE) | +| `POSTA_LANGUAGE` | | | | Posta template language | +| `WEBHOOK_URL` | | | | Generic JSON webhook URL | +| `WEBHOOK_SECRET` | | | `WEBHOOK_SECRET_FILE` | HMAC-SHA256 secret for the X-Bkup-Signature header | +| `SLACK_WEBHOOK_URL` | | | `SLACK_WEBHOOK_URL_FILE` | Slack incoming webhook URL | +| `DISCORD_WEBHOOK_URL` | | | `DISCORD_WEBHOOK_URL_FILE` | Discord webhook URL | +| `NTFY_URL` | | | | ntfy topic URL | +| `NTFY_TOKEN` | | | `NTFY_TOKEN_FILE` | ntfy access token | +| `NOTIFY_ON` | | `always` | | When to notify (always, failure, success, never) | +| `HEALTHCHECK_PING_URL` | | | | healthchecks.io-style URL pinged on start (/start), success and failure (/fail) | +| `BACKUP_REFERENCE` | | | | Free text included in notifications | +| `TEMPLATES_DIR` | | `/config/templates` | | Directory with custom notification templates | + +### database + +| Variable | Flag | Default | Fallbacks | Description | +|---|---|---|---|---| +| `DB_URL` | | | `DB_URL_FILE` | Connection URL mysql://user:password@host:port/database?ssl-mode=required (overrides host, port, user, password) | +| `DB_HOST` | | `localhost` | | Database host (a path starting with / is a Unix socket) | +| `DB_PORT` | | `3306` | | Database port | +| `DB_NAME` | `--dbname` `-d` | | | Database name | +| `DB_USER` | | | `DB_USERNAME` | Database user | +| `DB_PASSWORD` | | | `DB_PASSWORD_FILE` | Database password | +| `DB_SSL_MODE` | | `disabled` | | TLS mode: disabled, preferred, required, verify_ca, verify_identity (1.x values 0 and 1 mean disabled and required) | +| `DB_SSL_CA` | | `/etc/ssl/certs/ca-certificates.crt` | | CA certificate file, used with DB_SSL_MODE=verify_ca or verify_identity | +| `DB_SSL_CERT` | | | | Client certificate file | +| `DB_SSL_KEY` | | | | Client key file | +| `BACKUP_ALL_DATABASES` | `--all-databases` `-a` | | | Back up every database, one file each | +| `BACKUP_ALL_IN_ONE` | `--all-in-one` `-A` | | | Back up every database into a single file | +| `BACKUP_EXCLUDE_DATABASES` | `--exclude-db` | | | Databases to skip with --all-databases, comma separated | +| `TARGET_DB_URL` | | | `TARGET_DB_URL_FILE` | migrate: target connection URL | +| `TARGET_DB_HOST` | | | | migrate: target host | +| `TARGET_DB_PORT` | | `3306` | | migrate: target port | +| `TARGET_DB_NAME` | | | | migrate: target database (default: the source name) | +| `TARGET_DB_USER` | | | `TARGET_DB_USERNAME` | migrate: target user | +| `TARGET_DB_PASSWORD` | | | `TARGET_DB_PASSWORD_FILE` | migrate: target password | +| `TARGET_DB_SSL_MODE` | | `disabled` | | migrate: target TLS mode (same values as DB_SSL_MODE) | +| `TARGET_DB_SSL_CA` | | `/etc/ssl/certs/ca-certificates.crt` | | migrate: target CA certificate file | +| `MYSQL_CLIENT` | | `auto` | | Client tools: auto (mariadb-dump for MariaDB, Oracle mysqldump for MySQL), mysql or mariadb (auto, mysql, mariadb) | ---- ## Scheduled Backups @@ -91,7 +260,7 @@ The `mysql-bkup` CLI provides commands and options to manage MySQL backups effic ### Cron Syntax -The cron syntax consists of five fields: +The cron syntax consists of five fields (a sixth, leading seconds field is also accepted): ```conf * * * * * command diff --git a/docs/upgrade/index.md b/docs/upgrade/index.md new file mode 100644 index 0000000..ad2652a --- /dev/null +++ b/docs/upgrade/index.md @@ -0,0 +1,47 @@ +--- +title: Upgrading to v2 +layout: default +nav_order: 5 +--- + +# Upgrading from 1.x to v2 + +v2 rebuilds mysql-bkup on a shared streaming core: dump, compression, encryption and upload run in one pipe, without temporary files, whatever the size of the database. Existing commands, flags and environment variables keep working. + + +Everything that worked in 1.x keeps working; a few behaviours changed on purpose. + +- **Image:** Debian 13 slim instead of Alpine, with Oracle's `mysqldump` for MySQL servers (correct MySQL 8 dumps) and `mariadb-dump` for MariaDB. `/bin/sh`, `/backup`, `/config`, the `backup`/`restore`/`migrate` shortcuts and the `bkup` alias are still there; `tini` is now PID 1. +- **Renamed variables** are still read, with a deprecation warning: `S3_ENDPOINT` β†’ `AWS_S3_ENDPOINT`, `BUCKET_NAME`/`AWS_BUCKET_NAME` β†’ `AWS_S3_BUCKET_NAME`, `ACCESS_KEY`/`SECRET_KEY` β†’ `AWS_ACCESS_KEY`/`AWS_SECRET_KEY`, `AWS_S3_PATH`/`S3_PATH` β†’ `REMOTE_PATH`, `FTP_HOST_NAME` β†’ `FTP_HOST`, `BACKUP_TMP_DIR` β†’ `TMP_DIR`. +- **`DB_USER` is the database user variable** (also `TARGET_DB_USER`). `DB_USERNAME` and `TARGET_DB_USERNAME` are still read without a warning, including the per-database forms (`DB_USERNAME_`, `_DB_USERNAME`). +- **`DB_SSL_MODE`** takes MySQL's mode names; `0`/`1` still work. +- **`STORAGE` and `STORAGE_PATH` env are honoured** (1.x always used the `--storage` default and `/backup`). Unknown storage names are an error instead of a silent local backup. +- **SFTP variables are renamed, and the old names are no longer read:** `SSH_HOST`/`SSH_HOST_NAME` β†’ `SFTP_HOST`, `SSH_PORT` β†’ `SFTP_PORT`, `SSH_USER` β†’ `SFTP_USER`, `SSH_PASSWORD` β†’ `SFTP_PASSWORD`, `SSH_IDENTIFY_FILE`/`SSH_IDENTITY_FILE` β†’ `SFTP_IDENTITY_FILE`, `SSH_REMOTE_PATH` β†’ `REMOTE_PATH`. +- **SFTP host keys are verified by default.** Set `SFTP_KNOWN_HOSTS=/path/known_hosts`, or `SFTP_INSECURE_IGNORE_HOST_KEY=true` to keep the 1.x behaviour. +- **`MAIL_SKIP_TLS` is read with its documented meaning** (`true` skips certificate verification). 1.x had it inverted, so `MAIL_SKIP_TLS=false` (the value in `.env.example`) turned verification *off*. Use `MAIL_INSECURE_SKIP_VERIFY` from now on. +- **Retention only deletes indexed backups** (see the [configuration reference](../reference/)). 1.x deleted every file older than N days in the folder. Set `BACKUP_RETENTION_ADOPT_LEGACY=true` once to let retention handle 1.x backups, or `BACKUP_INDEX=false` for name-based retention. +- **Encryption fails closed**: a `GPG_PUBLIC_KEY` that cannot be read is an error (1.x uploaded plaintext). The env var now wins over the auto-detected `/config/gnupg` key. +- **Errors no longer kill the daemon**: a failed run is logged, notified and retried at the next schedule; other databases of the same run continue (`BACKUP_RESCUE_MODE` is no longer needed). +- **Restore** works from every storage (1.x had no SFTP restore), is case-insensitive about the storage name, and no longer goes through `sh -c`. +- **`migrate`** streams directly into the target (no intermediate file), creates the target database if needed, and `TARGET_DB_NAME` defaults to the source name; `-a` migrates every database. +- **Notification templates** are built in; custom ones still go in `/config/templates` (`TEMPLATES_DIR`). +- **Version** is injected with `-ldflags "-X main.Version=..."` (was `utils.Version`). + + +## Checklist + +1. **SFTP users:** mount a `known_hosts` file and set `SFTP_KNOWN_HOSTS`, or set `SFTP_INSECURE_IGNORE_HOST_KEY=true`. +2. **Email notifications with `MAIL_SKIP_TLS=false`:** certificate verification is now on, as the name always promised. Remove the variable, or set `MAIL_INSECURE_SKIP_VERIFY=true` if your SMTP server has a self-signed certificate. +3. **Retention:** set `BACKUP_RETENTION_ADOPT_LEGACY=true` for one run if retention should also delete the backups made by 1.x. +4. **Quoted values in Compose files:** in the list syntax (`- AWS_DISABLE_SSL="false"`) the quotes are part of the value. v2 validates booleans and rejects `"false"`; write `- AWS_DISABLE_SSL=false`. +5. **`--user` runs:** `/backup` and `/tmp/backup` are writable by any user (sticky, like `/tmp`); mounted volumes need the usual permissions. +6. **Restoring 1.x backups:** nothing to do. `.sql`, `.sql.gz`, `.sql.gpg` and `.sql.gz.gpg` files from 1.x restore as before. + +## New in v2 + +- MySQL servers are dumped with Oracle `mysqldump` 8.4, MariaDB servers with `mariadb-dump` 11.8 (flavour detected automatically). +- `DB_URL` (`mysql://user:password@host:3306/db?ssl-mode=required`), TLS modes, client certificates. +- age encryption, zstd compression, Google Cloud Storage, FTPS, extra storage copies. +- `verify`, `list`, `delete`, `prune`, `reconcile`, `config`, `health` commands; backup index with sha256 per backup. +- Slack, Discord, ntfy, webhook and healthcheck notifications; Prometheus metrics in daemon mode. +- `migrate -a`, and migrate without an intermediate file. diff --git a/examples/docker-compose.s3.yaml b/examples/docker-compose.s3.yaml index 1142c19..afc845e 100644 --- a/examples/docker-compose.s3.yaml +++ b/examples/docker-compose.s3.yaml @@ -6,21 +6,21 @@ services: # for a list of available releases. image: jkaninda/mysql-bkup container_name: mysql-bkup - command: backup --storage s3 -d my-database" + command: backup --storage s3 -d my-database environment: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## AWS configurations - AWS_S3_ENDPOINT=https://s3.amazonaws.com - AWS_S3_BUCKET_NAME=backup - - AWS_REGION="us-west-2" + - AWS_REGION=us-west-2 - AWS_ACCESS_KEY=xxxx - AWS_SECRET_KEY=xxxxx ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true - - AWS_DISABLE_SSL="false" + - AWS_DISABLE_SSL=false - AWS_FORCE_PATH_STYLE=true # true for S3 alternative such as Minio # mysql-bkup container must be connected to the same network with your database networks: diff --git a/examples/docker-compose.scheduled.local.yaml b/examples/docker-compose.scheduled.local.yaml index a7f6b42..cf25d7e 100644 --- a/examples/docker-compose.scheduled.local.yaml +++ b/examples/docker-compose.scheduled.local.yaml @@ -1,4 +1,3 @@ -version: "3" services: mysql-bkup: # In production, it is advised to lock your image tag to a proper @@ -11,7 +10,7 @@ services: environment: - DB_PORT=3306 - DB_HOST=mysql - - DB_USERNAME=userName + - DB_USER=userName - DB_PASSWORD=${DB_PASSWORD} # See: https://jkaninda.github.io/mysql-bkup/reference/#predefined-schedules - BACKUP_CRON_EXPRESSION=@daily #@every 5m|@weekly | @monthly |0 1 * * * \ No newline at end of file diff --git a/examples/docker-compose.scheduled.s3.yaml b/examples/docker-compose.scheduled.s3.yaml index 95f8ca7..5b45251 100644 --- a/examples/docker-compose.scheduled.s3.yaml +++ b/examples/docker-compose.scheduled.s3.yaml @@ -11,16 +11,16 @@ services: - DB_PORT=3306 - DB_HOST=mysql - DB_NAME=database - - DB_USERNAME=username + - DB_USER=username - DB_PASSWORD=password ## AWS configurations - AWS_S3_ENDPOINT=https://s3.amazonaws.com - AWS_S3_BUCKET_NAME=backup - - AWS_REGION="us-west-2" + - AWS_REGION=us-west-2 - AWS_ACCESS_KEY=xxxx - AWS_SECRET_KEY=xxxxx ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true - - AWS_DISABLE_SSL="false" + - AWS_DISABLE_SSL=false - AWS_FORCE_PATH_STYLE=true # true for S3 alternative such as Minio # See: https://jkaninda.github.io/mysql-bkup/reference/#predefined-schedules - BACKUP_CRON_EXPRESSION=@daily #@every 5m|@weekly | @monthly |0 1 * * * diff --git a/examples/docker-compose.yaml b/examples/docker-compose.yaml index d74b018..1c24cd1 100644 --- a/examples/docker-compose.yaml +++ b/examples/docker-compose.yaml @@ -1,4 +1,3 @@ -version: "3" services: mysql-bkup: image: jkaninda/mysql-bkup @@ -9,5 +8,5 @@ services: environment: - DB_PORT=3306 - DB_HOST=mysql - - DB_USERNAME=userName + - DB_USER=userName - DB_PASSWORD=${DB_PASSWORD} \ No newline at end of file diff --git a/examples/k8s-job.yaml b/examples/k8s-job.yaml index f31f487..8e72f7b 100644 --- a/examples/k8s-job.yaml +++ b/examples/k8s-job.yaml @@ -27,7 +27,7 @@ spec: value: "" - name: DB_NAME value: "dbname" - - name: DB_USERNAME + - name: DB_USER value: "username" # Please use secret! - name: DB_PASSWORD diff --git a/go.mod b/go.mod index 559449e..4a06b1e 100644 --- a/go.mod +++ b/go.mod @@ -2,38 +2,101 @@ module github.com/jkaninda/mysql-bkup go 1.26.0 -require github.com/spf13/pflag v1.0.10 // indirect - require ( - github.com/go-mail/mail v2.3.1+incompatible - github.com/jkaninda/encryptor v0.0.0-20241111100652-926393c9437e - github.com/jkaninda/go-storage v0.1.3 - github.com/jkaninda/go-utils v0.1.4 - github.com/robfig/cron/v3 v3.0.1 - github.com/spf13/cobra v1.10.2 - gopkg.in/yaml.v3 v3.0.1 + filippo.io/age v1.3.2 + github.com/jkaninda/bkupkit v0.0.2 ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 // indirect - github.com/ProtonMail/go-crypto v1.1.0 // indirect - github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect - github.com/ProtonMail/gopenpgp/v2 v2.8.0 // indirect - github.com/aws/aws-sdk-go v1.55.8 // indirect - github.com/bramvdbogaerde/go-scp v1.5.0 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/storage v1.60.0 // indirect + filippo.io/edwards25519 v1.2.0 // indirect + filippo.io/hpke v0.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/ProtonMail/go-crypto v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect + github.com/aws/smithy-go v1.28.5 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect + github.com/goposta/posta-go v0.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jkaninda/encryptor v0.3.0 // indirect + github.com/jkaninda/go-storage v0.2.0 // indirect + github.com/jkaninda/okapi v0.11.0 // indirect github.com/jlaffaye/ftp v0.2.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect + github.com/klauspost/compress v1.20.3 // indirect + github.com/klauspost/pgzip v1.3.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/pkg/sftp v1.13.10 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/api v0.267.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/mail.v2 v2.3.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 85c3b45..0b86f6f 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,137 @@ -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/ProtonMail/go-crypto v1.1.0 h1:OnlSGxXflfrWJESDsGQOmACNQRM9IflG3q8XTrOqvbE= -github.com/ProtonMail/go-crypto v1.1.0/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= -github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= -github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= -github.com/ProtonMail/gopenpgp/v2 v2.8.0 h1:WvMv3CMcFsqKSM4/Qf8sf3tgyQkzDqQmoSE49bnBuP4= -github.com/ProtonMail/gopenpgp/v2 v2.8.0/go.mod h1:qb2GUSnmA9ipBW5GVtCtEhkummSlqs2A8Ar3S0HBgSY= -github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= -github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM= -github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +c2sp.org/CCTV/age v0.0.0-20260829155415-4448f2097b2d h1:Blprhc2SbChNZtWcU+BLTM4YdoqYAS9V7cJgOwJKyAs= +c2sp.org/CCTV/age v0.0.0-20260829155415-4448f2097b2d/go.mod h1:SrHC2C7r5GkDk8R+NFVzYy/sdj0Ypg9htaPXQq5Cqeo= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY= +cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8= +cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +filippo.io/age v1.3.2 h1:r6RSZLFSMm6rzKepZ7ZAYkKCu14f3/Me8c7uKYh7C8c= +filippo.io/age v1.3.2/go.mod h1:TH/Yr2sSRhCKbaH4XPxpUV0Us8Gv6txYUpiZQWz8Evk= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A= +filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= +github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0= +github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43 h1:Fhx4uwBshQF0+jmhV/FKDM4LPj7L9iUy5n3MvLxLy00= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.43/go.mod h1:4Tyfc1eIPEyrAMysLCVkcYL1Hm/hkGWbElLo8pejHPo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-mail/mail v2.3.1+incompatible h1:UzNOn0k5lpfVtO31cK3hn6I4VEVGhe3lX8AJBAxXExM= -github.com/go-mail/mail v2.3.1+incompatible/go.mod h1:VPWjmmNyRsWXQZHVHT3g0YbIINUkSmuKOiLIDkWbL6M= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= +github.com/goposta/posta-go v0.1.0 h1:UkMRZ2FAxUbjIZRiT9GIdrm4ypFtngKI/hZeC9hYB9Y= +github.com/goposta/posta-go v0.1.0/go.mod h1:IjYW+tr9L+3NAzPWScOKvxdi41x6JBL1ju63usJLgRM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -39,18 +139,22 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jkaninda/encryptor v0.0.0-20241111100652-926393c9437e h1:jtFKZHt/PLGQWXNgjEFTEwVbxiQQRMoJ7m37trbkJGw= -github.com/jkaninda/encryptor v0.0.0-20241111100652-926393c9437e/go.mod h1:Y1EXpPWQ9PNd7y7E6ez3xgnzZc8fuDWXwX/1/dXNCE4= -github.com/jkaninda/go-storage v0.1.3 h1:lEpHVgFLKSvjsi/6tAek96Y07za3vxmsXF2/+jiCMZU= -github.com/jkaninda/go-storage v0.1.3/go.mod h1:zVRnLprBk/9AUz2+za6Y03MgoNYrqKLy3edVtjqMaps= -github.com/jkaninda/go-utils v0.1.4 h1:ZdNlI+yLWc4/S0qKcCNQIPj+6lHSdJcGaxtRADAifAU= -github.com/jkaninda/go-utils v0.1.4/go.mod h1:Aa54jEAcDykc3CnOdreqZG80UfSZOvrYecyusu+oPb4= +github.com/jkaninda/bkupkit v0.0.2 h1:3a5J/8qwXX7TWYBwIX57OQvUQCqo3er/JZZwvgU8I1g= +github.com/jkaninda/bkupkit v0.0.2/go.mod h1:TB2HVP8jRau/TdwfsO4QOgJY4Vi4h6HX+BlxJzXdcIg= +github.com/jkaninda/encryptor v0.3.0 h1:m7r7ROyjXtVXpFTtdGAhHdcPLDD+6kb0eaTF9cY4KM4= +github.com/jkaninda/encryptor v0.3.0/go.mod h1:ktF+sv4yLWa97xQt6Quhr1AWKwTnE7/HHEb6JJ3O7QU= +github.com/jkaninda/go-storage v0.2.0 h1:EjhIcu+zFkqTqnCleJqyS2TJfIC8U+V+Imw/qN3OVJM= +github.com/jkaninda/go-storage v0.2.0/go.mod h1:uJOFRtwwfq9yhj/yG3we5OTTg3JGeBv24/SrgmsSPKQ= +github.com/jkaninda/okapi v0.11.0 h1:73Ke2BJF5TTYG0DLKkFnSKrAP87bzErRQ6tOxF+DHF4= +github.com/jkaninda/okapi v0.11.0/go.mod h1:/Bt3U6MpUaGdb7iPKjyRpqFhv9/AIj/SReNbRjGQ1mw= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/pgzip v1.3.0 h1:8t+KjjzD/0n/8vNlJGGZmTeK4njuwffOQvJm8tOSc8o= +github.com/klauspost/pgzip v1.3.0/go.mod h1:Xx7tAv5DH8ObLHUDQKXoE/nd8i/FYdNDTVMvi3SLTKI= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -59,66 +163,79 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= +github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= +github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE= +google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -126,7 +243,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk= gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integration_test.go b/integration_test.go new file mode 100644 index 0000000..642a100 --- /dev/null +++ b/integration_test.go @@ -0,0 +1,639 @@ +//go:build integration + +// Integration tests: real MySQL / MariaDB servers in Docker, the real client +// tools, the whole bkupkit pipeline through cli.App.Run. +// +// The client tools run in containers too: shim scripts named mysql, mysqldump, +// mariadb and mariadb-dump are put first in PATH and `docker run -i` the +// official images (or IT_CLIENT_IMAGE, e.g. the mysql-bkup image itself). The +// private TMP_DIR holding the option files is bind-mounted at the same path. +// +// go test -tags integration -v -run . ./... +// IT_CLIENT_IMAGE=jkaninda/mysql-bkup:test go test -tags integration ./... +// IT_SKIP_MYSQL80=1 ... skip the MySQL 8.0 server +package main + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "filippo.io/age" + "github.com/jkaninda/bkupkit/cli" + "github.com/jkaninda/mysql-bkup/internal/mysql" +) + +const ( + rootPW = "r00t-pw" + // appPW exercises option-file quoting. + appPW = `p"a#s s\w'd$x` +) + +var ( + itNetwork string + itTmp string // TMP_DIR shared with the client containers + itShims string + itSkip string + servers []*dbServer + minioPort string +) + +type dbServer struct { + name string // container name and DB_HOST + image string + client string // client binary inside the server image + mariadb bool +} + +// Miabi takes the LAST match of this regex in the output as the artifact. +var miabiDB = regexp.MustCompile(`[\w.\-]+\.(?:sql|archive)\.gz(?:\.gpg)?`) + +func docker(args ...string) (string, error) { + // Stdout only on success: "docker run" prints image pull progress on stderr, + // which would otherwise end up in the container id on a runner without the image. + cmd := exec.Command("docker", args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + out = append(out, stderr.Bytes()...) + } + return strings.TrimSpace(string(out)), err +} + +func TestMain(m *testing.M) { + if _, err := exec.LookPath("docker"); err != nil { + itSkip = "docker not installed" + os.Exit(m.Run()) + } + if _, err := docker("info"); err != nil { + itSkip = "docker not running" + os.Exit(m.Run()) + } + code := func() int { + suffix := fmt.Sprintf("%d", time.Now().UnixNano()%1e9) + itNetwork = "mysqlbkup-it-" + suffix + if out, err := docker("network", "create", itNetwork); err != nil { + fmt.Fprintln(os.Stderr, out) + return 1 + } + defer docker("network", "rm", itNetwork) + var err error + if itTmp, err = os.MkdirTemp("", "mysqlbkup-it-tmp-"); err != nil { + return 1 + } + defer os.RemoveAll(itTmp) + // macOS: /var/folders is a symlink to /private/var/folders; mount the real path. + if p, err := filepath.EvalSymlinks(itTmp); err == nil { + itTmp = p + } + if itShims, err = os.MkdirTemp("", "mysqlbkup-it-bin-"); err != nil { + return 1 + } + defer os.RemoveAll(itShims) + writeShims() + + servers = []*dbServer{ + {name: "it-mysql84-" + suffix, image: "mysql:8.4", client: "mysql"}, + {name: "it-mariadb118-" + suffix, image: "mariadb:11.8", client: "mariadb", mariadb: true}, + } + if os.Getenv("IT_SKIP_MYSQL80") == "" { + servers = append(servers, &dbServer{name: "it-mysql80-" + suffix, image: "mysql:8.0", client: "mysql"}) + } + for _, s := range servers { + args := []string{"run", "-d", "--rm", "--name", s.name, "--network", itNetwork} + if s.mariadb { + args = append(args, "-e", "MARIADB_ROOT_PASSWORD="+rootPW) + } else { + args = append(args, "-e", "MYSQL_ROOT_PASSWORD="+rootPW) + } + if out, err := docker(append(args, s.image)...); err != nil { + fmt.Fprintln(os.Stderr, out) + return 1 + } + defer docker("rm", "-f", s.name) + } + minio := "it-minio-" + suffix + if out, err := docker("run", "-d", "--rm", "--name", minio, "-p", "127.0.0.1::9000", + "-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin", + "pgsty/minio:latest", "server", "/data"); err != nil { + fmt.Fprintln(os.Stderr, out) + return 1 + } + defer docker("rm", "-f", minio) + for _, s := range servers { + if err := s.wait(); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + } + out, err := docker("port", minio, "9000/tcp") + if err != nil { + fmt.Fprintln(os.Stderr, out) + return 1 + } + minioPort = out[strings.LastIndex(out, ":")+1:] + deadline := time.Now().Add(60 * time.Second) + for { + out, err = docker("exec", minio, "sh", "-c", "mc alias set l http://127.0.0.1:9000 minioadmin minioadmin && mc mb --ignore-existing l/backups") + if err == nil { + break + } + if time.Now().After(deadline) { + fmt.Fprintln(os.Stderr, "minio:", out) + return 1 + } + time.Sleep(time.Second) + } + minioContainer = minio + return m.Run() + }() + os.Exit(code) +} + +var minioContainer string + +func writeShims() { + imgs := map[string]string{"mysql": "mysql:8.4", "mysqldump": "mysql:8.4", "mariadb": "mariadb:11.8", "mariadb-dump": "mariadb:11.8"} + for bin, img := range imgs { + if ci := os.Getenv("IT_CLIENT_IMAGE"); ci != "" { + img = ci + } + sh := fmt.Sprintf("#!/bin/sh\nexec docker run --rm -i --network %s -v %s:%s --entrypoint %s %s \"$@\"\n", itNetwork, itTmp, itTmp, bin, img) + if err := os.WriteFile(filepath.Join(itShims, bin), []byte(sh), 0o755); err != nil { + panic(err) + } + } +} + +// sql runs statements as root inside the server container. +func (s *dbServer) sql(stmts string) (string, error) { + cmd := exec.Command("docker", "exec", "-i", "-e", "MYSQL_PWD="+rootPW, s.name, s.client, "-uroot", "-h127.0.0.1", "-N", "-B") + cmd.Stdin = strings.NewReader(stmts) + out, err := cmd.CombinedOutput() + return strings.TrimSpace(string(out)), err +} + +func (s *dbServer) wait() error { + deadline := time.Now().Add(3 * time.Minute) + for { + // TCP only answers once the image's init server is gone. + if _, err := s.sql("SELECT 1"); err == nil { + return nil + } + if time.Now().After(deadline) { + logs, _ := docker("logs", s.name) + return fmt.Errorf("%s not ready:\n%s", s.name, logs) + } + time.Sleep(time.Second) + } +} + +func (s *dbServer) mustSQL(t *testing.T, stmts string) string { + t.Helper() + out, err := s.sql(stmts) + if err != nil { + t.Fatalf("%s: %v\n%s", s.name, err, out) + } + return out +} + +const fixture = ` +DROP DATABASE IF EXISTS %[1]s; +CREATE DATABASE %[1]s CHARACTER SET utf8mb4; +USE %[1]s; +CREATE TABLE products (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(100), price DECIMAL(10,2), data BLOB, created DATETIME); +CREATE TABLE audit (id INT PRIMARY KEY AUTO_INCREMENT, product_id INT, note TEXT); +CREATE TRIGGER trg_products AFTER INSERT ON products FOR EACH ROW INSERT INTO audit (product_id, note) VALUES (NEW.id, 'created'); +CREATE PROCEDURE p_count() SELECT COUNT(*) FROM products; +CREATE VIEW v_products AS SELECT id, name FROM products; +CREATE EVENT ev_cleanup ON SCHEDULE EVERY 1 DAY DO DELETE FROM audit WHERE id < 0; +INSERT INTO products (name, price, data, created) VALUES + ('cafΓ© πŸ˜€', 9.99, X'00FF0A0D27225C', '2026-01-02 03:04:05'), + ('it''s "quoted" \\ back', 0.01, NULL, NULL), + ('tab\tnew\nline', 12345678.90, REPEAT(X'AB', 60000), '1999-12-31 23:59:59'); +INSERT INTO products (name, price) SELECT CONCAT('bulk-', a.n * 10 + b.n), a.n + b.n FROM + (SELECT 0 n UNION SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 4 UNION SELECT 5 UNION SELECT 6 UNION SELECT 7 UNION SELECT 8 UNION SELECT 9) a, + (SELECT 0 n UNION SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 4 UNION SELECT 5 UNION SELECT 6 UNION SELECT 7 UNION SELECT 8 UNION SELECT 9) b; +` + +func (s *dbServer) loadFixture(t *testing.T, db string) { + t.Helper() + s.mustSQL(t, fmt.Sprintf(fixture, db)) +} + +// fingerprint summarises the data and objects of db. +func (s *dbServer) fingerprint(t *testing.T, db string) string { + t.Helper() + q := fmt.Sprintf(`SET SESSION group_concat_max_len = 100000000; +SELECT COUNT(*), MD5(GROUP_CONCAT(CONCAT_WS('|', id, name, price, HEX(data), created) ORDER BY id SEPARATOR ';')) FROM %[1]s.products; +SELECT COUNT(*) FROM %[1]s.audit; +SELECT COUNT(*) FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = '%[1]s'; +SELECT COUNT(*) FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = '%[1]s'; +SELECT COUNT(*) FROM information_schema.VIEWS WHERE TABLE_SCHEMA = '%[1]s'; +SELECT COUNT(*) FROM information_schema.EVENTS WHERE EVENT_SCHEMA = '%[1]s';`, db) + return s.mustSQL(t, q) +} + +func needDocker(t *testing.T) { + t.Helper() + if itSkip != "" { + t.Skip(itSkip) + } +} + +// run executes the CLI in-process and returns its combined output. +func run(t *testing.T, args ...string) (string, int) { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + oldOut, oldErr := os.Stdout, os.Stderr + os.Stdout, os.Stderr = w, w + var buf bytes.Buffer + done := make(chan struct{}) + go func() { _, _ = io.Copy(&buf, r); close(done) }() + code := (&cli.App{Engine: mysql.New(), Version: "it"}).Run(args) + _ = w.Close() + <-done + os.Stdout, os.Stderr = oldOut, oldErr + return buf.String(), code +} + +func mustRun(t *testing.T, args ...string) string { + t.Helper() + out, code := run(t, args...) + if code != 0 { + t.Fatalf("%v: exit %d\n%s", args, code, out) + } + return out +} + +// setEnv sets the common environment for s and returns the local storage dir. +func setEnv(t *testing.T, s *dbServer, user, pw string) string { + t.Helper() + store := t.TempDir() + t.Setenv("PATH", itShims+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("TMP_DIR", itTmp) + t.Setenv("STORAGE_PATH", store) + t.Setenv("HEALTH_FILE", filepath.Join(t.TempDir(), "health.json")) + t.Setenv("TEMPLATES_DIR", "") + t.Setenv("DB_HOST", s.name) + t.Setenv("DB_PORT", "3306") + t.Setenv("DB_USERNAME", user) + t.Setenv("DB_PASSWORD", pw) + return store +} + +// artifact returns the last Miabi regex match and checks the file exists in dir. +func artifact(t *testing.T, out, dir string) string { + t.Helper() + m := miabiDB.FindAllString(out, -1) + if len(m) == 0 { + t.Fatalf("no artifact name in output:\n%s", out) + } + last := m[len(m)-1] + if dir != "" { + if _, err := os.Stat(filepath.Join(dir, last)); err != nil { + t.Fatalf("last match %q is not the stored artifact: %v\n%s", last, err, out) + } + } + return last +} + +func assertTmpClean(t *testing.T) { + t.Helper() + if ents, _ := os.ReadDir(itTmp); len(ents) != 0 { + var names []string + for _, e := range ents { + names = append(names, e.Name()) + } + t.Fatalf("TMP_DIR not empty: %v", names) + } +} + +func forEachServer(t *testing.T, fn func(t *testing.T, s *dbServer)) { + needDocker(t) + for _, s := range servers { + t.Run(strings.SplitN(strings.TrimPrefix(s.name, "it-"), "-", 2)[0], func(t *testing.T) { fn(t, s) }) + } +} + +// TestSingleDatabase: Miabi's argv (backup -d db, restore -d db -f file), +// gzip default, restore after data loss, the client chosen per flavour. +func TestSingleDatabase(t *testing.T) { + forEachServer(t, func(t *testing.T, s *dbServer) { + s.loadFixture(t, "shop") + want := s.fingerprint(t, "shop") + store := setEnv(t, s, "root", rootPW) + + out := mustRun(t, "backup", "-d", "shop") + file := artifact(t, out, store) + if !strings.HasSuffix(file, ".sql.gz") || !strings.HasPrefix(file, "shop_") { + t.Fatalf("artifact %q", file) + } + wantClient := "client=mysqldump" + if s.mariadb { + wantClient = "client=mariadb-dump" + } + if !strings.Contains(out, wantClient) { + t.Fatalf("expected %s:\n%s", wantClient, out) + } + assertTmpClean(t) + + s.mustSQL(t, "DROP DATABASE shop") + mustRun(t, "restore", "-d", "shop", "-f", file) + if got := s.fingerprint(t, "shop"); got != want { + t.Fatalf("restored data differs:\n got %s\nwant %s", got, want) + } + assertTmpClean(t) + + // verify and list work without a database connection. + mustRun(t, "verify", "-f", file) + if out := mustRun(t, "list"); !strings.Contains(out, file) { + t.Fatalf("list:\n%s", out) + } + }) +} + +// TestNonRootUser: a user with only database privileges (no PROCESS, no +// global SELECT) and a password that needs option-file quoting. +func TestNonRootUser(t *testing.T) { + forEachServer(t, func(t *testing.T, s *dbServer) { + s.mustSQL(t, fmt.Sprintf(`DROP USER IF EXISTS 'app'@'%%'; CREATE USER 'app'@'%%' IDENTIFIED BY '%s'; +DROP DATABASE IF EXISTS appdb; CREATE DATABASE appdb; GRANT ALL ON appdb.* TO 'app'@'%%'; +CREATE TABLE appdb.items (id INT PRIMARY KEY, v VARCHAR(20)); INSERT INTO appdb.items VALUES (1,'a'),(2,'b');`, + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(appPW))) + store := setEnv(t, s, "app", appPW) + out := mustRun(t, "backup", "-d", "appdb") + file := artifact(t, out, store) + s.mustSQL(t, "DELETE FROM appdb.items") + mustRun(t, "restore", "-d", "appdb", "-f", file) + if got := s.mustSQL(t, "SELECT COUNT(*) FROM appdb.items"); got != "2" { + t.Fatalf("rows after restore: %s", got) + } + }) +} + +// TestEncryption: GPG passphrase and age recipients, restored under another name. +func TestEncryption(t *testing.T) { + forEachServer(t, func(t *testing.T, s *dbServer) { + s.loadFixture(t, "shop") + want := s.fingerprint(t, "shop") + store := setEnv(t, s, "root", rootPW) + + t.Run("gpg", func(t *testing.T) { + t.Setenv("GPG_PASSPHRASE", "correct horse") + out := mustRun(t, "backup", "-d", "shop") + file := artifact(t, out, store) + if !strings.HasSuffix(file, ".sql.gz.gpg") { + t.Fatalf("artifact %q", file) + } + s.mustSQL(t, "DROP DATABASE IF EXISTS shop_gpg") + mustRun(t, "restore", "-d", "shop_gpg", "-f", file) + if got := s.fingerprint(t, "shop_gpg"); got != want { + t.Fatalf("restored data differs:\n got %s\nwant %s", got, want) + } + t.Setenv("GPG_PASSPHRASE", "wrong") + if out, code := run(t, "restore", "-d", "shop_gpg", "-f", file); code == 0 { + t.Fatalf("restore with a wrong passphrase succeeded:\n%s", out) + } + }) + t.Run("age", func(t *testing.T) { + id, err := age.GenerateX25519Identity() + if err != nil { + t.Fatal(err) + } + t.Setenv("AGE_RECIPIENTS", id.Recipient().String()) + out := mustRun(t, "backup", "-d", "shop") + m := regexp.MustCompile(`shop_[\w.\-]+\.sql\.gz\.age`).FindAllString(out, -1) + if len(m) == 0 { + t.Fatalf("no age artifact:\n%s", out) + } + file := m[len(m)-1] + s.mustSQL(t, "DROP DATABASE IF EXISTS shop_age") + t.Setenv("AGE_IDENTITY", id.String()) + mustRun(t, "restore", "-d", "shop_age", "-f", file) + if got := s.fingerprint(t, "shop_age"); got != want { + t.Fatalf("restored data differs:\n got %s\nwant %s", got, want) + } + }) + assertTmpClean(t) + }) +} + +// TestAllDatabases: one file per database (-a, --exclude-db) and one file for +// everything (-A), restored without -d. +func TestAllDatabases(t *testing.T) { + forEachServer(t, func(t *testing.T, s *dbServer) { + s.mustSQL(t, "DROP DATABASE IF EXISTS shop_gpg; DROP DATABASE IF EXISTS shop_age; DROP DATABASE IF EXISTS appdb; DROP DATABASE IF EXISTS skipme; CREATE DATABASE skipme;") + s.loadFixture(t, "shop") + s.loadFixture(t, "blog") + wantShop, wantBlog := s.fingerprint(t, "shop"), s.fingerprint(t, "blog") + store := setEnv(t, s, "root", rootPW) + + out := mustRun(t, "backup", "-a", "--exclude-db", "skipme") + ents, _ := os.ReadDir(store) + seen := map[string]bool{} + for _, e := range ents { + if strings.HasSuffix(e.Name(), ".sql.gz") { + seen[e.Name()[:strings.Index(e.Name(), "_2")]] = true + } + } + if !seen["shop"] || !seen["blog"] || seen["skipme"] || seen["mysql"] || seen["sys"] { + t.Fatalf("artifacts %v\n%s", seen, out) + } + artifact(t, out, store) + + out = mustRun(t, "backup", "-A") + file := artifact(t, out, store) + if !strings.HasPrefix(file, mysql.AllInOneName+"_") { + t.Fatalf("all-in-one artifact %q", file) + } + s.mustSQL(t, "DROP DATABASE shop; DROP DATABASE blog") + t.Setenv("DB_NAME", "") + mustRun(t, "restore", "-f", file) + if got := s.fingerprint(t, "shop"); got != wantShop { + t.Fatalf("shop differs:\n got %s\nwant %s", got, wantShop) + } + if got := s.fingerprint(t, "blog"); got != wantBlog { + t.Fatalf("blog differs:\n got %s\nwant %s", got, wantBlog) + } + }) +} + +// TestUploadsAndCorruption: Miabi uploads of plain .sql and gzip .sql.gz +// restore; corrupted inputs fail with a non-zero exit, and damaged or tampered +// archives are caught before the client runs, so the database is untouched. +func TestUploadsAndCorruption(t *testing.T) { + forEachServer(t, func(t *testing.T, s *dbServer) { + s.loadFixture(t, "shop") + want := s.fingerprint(t, "shop") + store := setEnv(t, s, "root", rootPW) + out := mustRun(t, "backup", "-d", "shop", "--disable-compression") + m := regexp.MustCompile(`shop_\d{8}_\d{6}(?:_\d+)?\.sql\b`).FindAllString(out, -1) + if len(m) == 0 { + t.Fatalf("no .sql artifact:\n%s", out) + } + plain, err := os.ReadFile(filepath.Join(store, m[len(m)-1])) + if err != nil { + t.Fatal(err) + } + write := func(name string, b []byte) { + if err := os.WriteFile(filepath.Join(store, name), b, 0o644); err != nil { + t.Fatal(err) + } + } + write("upload.sql", plain) + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + _, _ = zw.Write(plain) + _ = zw.Close() + write("upload.sql.gz", gz.Bytes()) + write("truncated.sql.gz", gz.Bytes()[:gz.Len()/2]) + write("bad.sql", append(append([]byte{}, plain...), []byte("\nTHIS IS NOT SQL;\n")...)) + + // 1.x style absolute path: restore -f /backup/upload.sql.gz + for _, f := range []string{"upload.sql", "upload.sql.gz", filepath.Join(store, "upload.sql.gz")} { + s.mustSQL(t, "DROP DATABASE IF EXISTS shop_up") + mustRun(t, "restore", "-d", "shop_up", "-f", f) + if got := s.fingerprint(t, "shop_up"); got != want { + t.Fatalf("%s: restored data differs", f) + } + } + // A valid file with a bad statement cannot be caught before it runs. + for _, f := range []string{"bad.sql", "missing.sql.gz"} { + if out, code := run(t, "restore", "-d", "shop_bad", "-f", f); code == 0 { + t.Fatalf("%s: restore succeeded:\n%s", f, out) + } + } + // A damaged archive is verified first and never reaches the database. + if out, code := run(t, "restore", "-d", "shop", "-f", "truncated.sql.gz"); code == 0 { + t.Fatalf("truncated.sql.gz: restore succeeded:\n%s", out) + } + if got := s.fingerprint(t, "shop"); got != want { + t.Fatal("truncated.sql.gz changed the database") + } + // Tampered GPG artifact. + t.Setenv("GPG_PASSPHRASE", "pw") + out = mustRun(t, "backup", "-d", "shop") + file := artifact(t, out, store) + b, _ := os.ReadFile(filepath.Join(store, file)) + b[len(b)/2] ^= 0xFF + write("tampered.sql.gz.gpg", b) + if out, code := run(t, "restore", "-d", "shop", "-f", "tampered.sql.gz.gpg"); code == 0 { + t.Fatalf("tampered restore succeeded:\n%s", out) + } + if got := s.fingerprint(t, "shop"); got != want { + t.Fatal("tampered.sql.gz.gpg changed the database") + } + assertTmpClean(t) + }) +} + +// TestMiabiS3: Miabi's exact S3 argv and env against MinIO. +func TestMiabiS3(t *testing.T) { + forEachServer(t, func(t *testing.T, s *dbServer) { + s.loadFixture(t, "shop") + want := s.fingerprint(t, "shop") + setEnv(t, s, "root", rootPW) + t.Setenv("AWS_S3_ENDPOINT", "http://127.0.0.1:"+minioPort) + t.Setenv("AWS_S3_BUCKET_NAME", "backups") + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("AWS_ACCESS_KEY", "minioadmin") + t.Setenv("AWS_SECRET_KEY", "minioadmin") + t.Setenv("AWS_DISABLE_SSL", "true") + t.Setenv("AWS_FORCE_PATH_STYLE", "true") + t.Setenv("GPG_PASSPHRASE", "miabi") + path := "miabi/" + strings.SplitN(s.name, "-", 3)[1] + + var out string + for i := 0; i < 2; i++ { // two runs: older names are printed too + out = mustRun(t, "backup", "--storage", "s3", "-d", "shop", "--path", path) + } + file := artifact(t, out, "") + if !strings.HasSuffix(file, ".sql.gz.gpg") { + t.Fatalf("artifact %q", file) + } + if o, err := docker("exec", minioContainer, "mc", "stat", "l/backups/"+path+"/"+file); err != nil { + t.Fatalf("object %s/%s not in the bucket: %s", path, file, o) + } + s.mustSQL(t, "DROP DATABASE shop") + mustRun(t, "restore", "--storage", "s3", "-d", "shop", "--path", path, "-f", file) + if got := s.fingerprint(t, "shop"); got != want { + t.Fatalf("restored data differs") + } + }) +} + +// TestMigrate: dump | restore between two servers, single database and -a. +func TestMigrate(t *testing.T) { + needDocker(t) + src := servers[0] // MySQL 8.4 + dst := servers[len(servers)-1] + if dst.mariadb || dst == src { + t.Skip("needs a second MySQL server (IT_SKIP_MYSQL80 is set)") + } + src.loadFixture(t, "shop") + src.loadFixture(t, "blog") + want := src.fingerprint(t, "shop") + setEnv(t, src, "root", rootPW) + t.Setenv("TARGET_DB_HOST", dst.name) + t.Setenv("TARGET_DB_USERNAME", "root") + t.Setenv("TARGET_DB_PASSWORD", rootPW) + + dst.mustSQL(t, "DROP DATABASE IF EXISTS shop_copy") + t.Setenv("TARGET_DB_NAME", "shop_copy") + mustRun(t, "migrate", "-d", "shop") + if got := dst.fingerprint(t, "shop_copy"); got != want { + t.Fatalf("migrated data differs:\n got %s\nwant %s", got, want) + } + + t.Setenv("TARGET_DB_NAME", "") + dst.mustSQL(t, "DROP DATABASE IF EXISTS shop; DROP DATABASE IF EXISTS blog") + src.mustSQL(t, "DROP DATABASE IF EXISTS shop_gpg; DROP DATABASE IF EXISTS shop_age; DROP DATABASE IF EXISTS shop_up; DROP DATABASE IF EXISTS appdb; DROP DATABASE IF EXISTS skipme; DROP DATABASE IF EXISTS shop_bad") + mustRun(t, "migrate", "-a") + if got := dst.fingerprint(t, "blog"); got != src.fingerprint(t, "blog") { + t.Fatalf("blog differs") + } + + // A failing target aborts the migration with a non-zero exit. + t.Setenv("TARGET_DB_PASSWORD", "wrong") + if out, code := run(t, "migrate", "-d", "shop"); code == 0 { + t.Fatalf("migrate with bad target credentials succeeded:\n%s", out) + } + assertTmpClean(t) +} + +// TestMigrateMySQLToMariaDB: cross-flavour migrate (Oracle mysqldump | mariadb). +func TestMigrateMySQLToMariaDB(t *testing.T) { + needDocker(t) + var src, dst *dbServer + for _, s := range servers { + if s.mariadb { + dst = s + } else if src == nil { + src = s + } + } + src.mustSQL(t, "DROP DATABASE IF EXISTS plain; CREATE DATABASE plain; CREATE TABLE plain.t (id INT PRIMARY KEY, v VARCHAR(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci); INSERT INTO plain.t VALUES (1,'x'),(2,'y');") + setEnv(t, src, "root", rootPW) + t.Setenv("TARGET_DB_HOST", dst.name) + t.Setenv("TARGET_DB_USERNAME", "root") + t.Setenv("TARGET_DB_PASSWORD", rootPW) + t.Setenv("TARGET_DB_NAME", "plain") + dst.mustSQL(t, "DROP DATABASE IF EXISTS plain") + mustRun(t, "migrate", "-d", "plain") + if got := dst.mustSQL(t, "SELECT COUNT(*) FROM plain.t"); got != "2" { + t.Fatalf("rows: %s", got) + } +} diff --git a/internal/mysql/client.go b/internal/mysql/client.go new file mode 100644 index 0000000..b650309 --- /dev/null +++ b/internal/mysql/client.go @@ -0,0 +1,279 @@ +package mysql + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "os/exec" + "strings" + + "github.com/jkaninda/bkupkit/config" + "github.com/jkaninda/bkupkit/execx" +) + +// Client binaries. Debian links mysql/mysqldump to the MariaDB tools, so the +// Oracle ones are recognized by their --version output, not by name. +const ( + mariadbCLI = "mariadb" + mariadbDump = "mariadb-dump" + mysqlCLI = "mysql" + mysqlDump = "mysqldump" +) + +// Client preferences (MYSQL_CLIENT). +const ( + ClientAuto = "auto" + ClientMySQL = "mysql" + ClientMariaDB = "mariadb" +) + +// tool is a resolved client binary. +type tool struct { + name string // logical name (mysqldump, mariadb-dump, ...) + path string + mariadb bool // MariaDB client (option names differ) +} + +// server is what Ping learned about a server. +type server struct { + version string + mariadb bool +} + +// IsMariaDB reports whether a SELECT VERSION() result is a MariaDB server. +func IsMariaDB(version string) bool { + return strings.Contains(strings.ToLower(version), "mariadb") +} + +// lookTool resolves a binary and its client family (cached). +func (e *Engine) lookTool(ctx context.Context, name string) (tool, bool) { + e.mu.Lock() + t, ok := e.tools[name] + e.mu.Unlock() + if ok { + return t, t.path != "" + } + t = tool{name: name} + if p, err := e.lookPath(name); err == nil { + t.path = p + switch name { + case mariadbCLI, mariadbDump: + t.mariadb = true + default: + out, err := execx.Output(ctx, p, []string{"--version"}, nil) + if err != nil { + t.path = "" + } else { + t.mariadb = IsMariaDB(out) + } + } + } + e.mu.Lock() + e.tools[name] = t + e.mu.Unlock() + return t, t.path != "" +} + +func (e *Engine) lookPath(name string) (string, error) { + if e.LookPath != nil { + return e.LookPath(name) + } + return exec.LookPath(name) +} + +// oracle returns the Oracle MySQL binary called name, if installed. +func (e *Engine) oracle(ctx context.Context, name string) (tool, bool) { + t, ok := e.lookTool(ctx, name) + return t, ok && !t.mariadb +} + +// maria returns the MariaDB binary (mariadb-dump, or a mysqldump that is MariaDB's). +func (e *Engine) maria(ctx context.Context, name, legacy string) (tool, bool) { + if t, ok := e.lookTool(ctx, name); ok { + return t, true + } + t, ok := e.lookTool(ctx, legacy) + return t, ok && t.mariadb +} + +// pick chooses the dump (dump=true) or command-line client for a server: +// mariadb-dump/mariadb for MariaDB, Oracle mysqldump/mysql for MySQL. +func (e *Engine) pick(ctx context.Context, pref string, srv server, dump bool) (tool, error) { + oName, mName := mysqlCLI, mariadbCLI + if dump { + oName, mName = mysqlDump, mariadbDump + } + switch pref { + case ClientMySQL: + if t, ok := e.oracle(ctx, oName); ok { + return t, nil + } + return tool{}, fmt.Errorf("MYSQL_CLIENT=mysql but the Oracle MySQL %s client is not installed", oName) + case ClientMariaDB: + if t, ok := e.maria(ctx, mName, oName); ok { + return t, nil + } + return tool{}, fmt.Errorf("MYSQL_CLIENT=mariadb but %s is not installed", mName) + } + if srv.mariadb { + if t, ok := e.maria(ctx, mName, oName); ok { + return t, nil + } + if t, ok := e.oracle(ctx, oName); ok { + e.warnOnce("oracle-for-mariadb", "MariaDB client not installed; using the Oracle MySQL client against a MariaDB server", "client", t.name) + return t, nil + } + return tool{}, fmt.Errorf("no %s or %s client installed", mName, oName) + } + if t, ok := e.oracle(ctx, oName); ok { + return t, nil + } + if t, ok := e.maria(ctx, mName, oName); ok { + e.warnOnce("mariadb-for-mysql", + "Oracle MySQL client not available on this platform; using the MariaDB client against a MySQL server "+ + "(dumps of MySQL-only features may be incomplete)", + "client", t.name, "server_version", srv.version) + return t, nil + } + return tool{}, fmt.Errorf("no %s or %s client installed", oName, mName) +} + +func (e *Engine) warnOnce(key, msg string, args ...any) { + e.mu.Lock() + seen := e.warned[key] + e.warned[key] = true + e.mu.Unlock() + if !seen { + slog.Default().Warn(msg, args...) + } +} + +// withOptionFile writes c's option file (0600, private directory) and calls fn +// with its path. The file is removed when fn returns. +func withOptionFile(v *config.Values, c Conn, t tool, fn func(path string) error) error { + dir, cleanup, err := execx.PrivateDir(v.String(config.TmpDir)) + if err != nil { + return fmt.Errorf("create private directory: %w", err) + } + defer cleanup() + p, err := execx.WriteSecret(dir, "my.cnf", optionFile(c, t.mariadb)) + if err != nil { + return fmt.Errorf("write client option file: %w", err) + } + return fn(p) +} + +// query runs one SQL statement with the command-line client and returns the +// raw tab-separated output without headers. +func query(ctx context.Context, v *config.Values, c Conn, t tool, sql string) (string, error) { + var out string + err := withOptionFile(v, c, t, func(cnf string) error { + var err error + out, err = execx.Output(ctx, t.path, []string{"--defaults-extra-file=" + cnf, "-N", "-B", "-r", "-e", sql}, nil) + return err + }) + return out, err +} + +// server detects the server version and flavour (cached per address and user). +// It tries the MariaDB client first (it speaks every MySQL authentication +// method), then the Oracle client. +func (e *Engine) server(ctx context.Context, v *config.Values, c Conn) (server, error) { + key := c.Addr() + "\x00" + c.User + e.mu.Lock() + s, ok := e.servers[key] + e.mu.Unlock() + if ok { + return s, nil + } + var cands []tool + if t, ok := e.maria(ctx, mariadbCLI, mysqlCLI); ok { + cands = append(cands, t) + } + if t, ok := e.oracle(ctx, mysqlCLI); ok { + cands = append(cands, t) + } + if len(cands) == 0 { + return s, errors.New("no mariadb or mysql client installed") + } + var firstErr error + for _, t := range cands { + out, err := query(ctx, v, c, t, "SELECT VERSION()") + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + s = server{version: strings.TrimSpace(out), mariadb: IsMariaDB(out)} + e.mu.Lock() + e.servers[key] = s + e.mu.Unlock() + return s, nil + } + return s, fmt.Errorf("failed to connect to database server %s: %w", c.Addr(), firstErr) +} + +// systemDatabases are never backed up one by one (mysql-bkup 1.x list). +var systemDatabases = map[string]bool{ + "information_schema": true, "performance_schema": true, "mysql": true, "sys": true, "innodb": true, +} + +// listDatabases returns the user databases, minus exclude. +func (e *Engine) listDatabases(ctx context.Context, v *config.Values, c Conn, exclude []string) ([]string, error) { + srv, err := e.server(ctx, v, c) + if err != nil { + return nil, err + } + t, err := e.pick(ctx, v.String(SettingClient), srv, false) + if err != nil { + return nil, err + } + out, err := query(ctx, v, c, t, "SHOW DATABASES") + if err != nil { + return nil, fmt.Errorf("failed to list databases: %w", err) + } + return filterDatabases(strings.Split(out, "\n"), exclude), nil +} + +func filterDatabases(names, exclude []string) []string { + skip := map[string]bool{} + for _, x := range exclude { + skip[x] = true + } + var dbs []string + for _, n := range names { + n = strings.TrimRight(n, "\r") + if n == "" || systemDatabases[strings.ToLower(n)] || skip[n] { + continue + } + dbs = append(dbs, n) + } + return dbs +} + +// quoteIdent quotes a MySQL identifier. +func quoteIdent(s string) string { + return "`" + strings.ReplaceAll(s, "`", "``") + "`" +} + +// ensureDatabase creates db when it does not exist yet. The existence check +// runs first so that users without CREATE privilege can restore into an +// existing database. +func ensureDatabase(ctx context.Context, v *config.Values, c Conn, t tool, db string) error { + lit := "CONVERT(X'" + hex.EncodeToString([]byte(db)) + "' USING utf8mb4)" + out, err := query(ctx, v, c, t, "SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = "+lit) + if err != nil { + return fmt.Errorf("check database %s: %w", db, err) + } + if strings.TrimSpace(out) != "0" { + return nil + } + slog.Default().Info("Creating database", "database", db) + if _, err := query(ctx, v, c, t, "CREATE DATABASE IF NOT EXISTS "+quoteIdent(db)); err != nil { + return fmt.Errorf("create database %s: %w", db, err) + } + return nil +} diff --git a/internal/mysql/conn.go b/internal/mysql/conn.go new file mode 100644 index 0000000..fca3713 --- /dev/null +++ b/internal/mysql/conn.go @@ -0,0 +1,199 @@ +package mysql + +import ( + "errors" + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/jkaninda/bkupkit/config" +) + +// TLS modes accepted in DB_SSL_MODE (MySQL --ssl-mode names, lower-case). +const ( + SSLDisabled = "disabled" + SSLPreferred = "preferred" + SSLRequired = "required" + SSLVerifyCA = "verify_ca" + SSLVerifyIdentity = "verify_identity" +) + +// ParseSSLMode normalizes DB_SSL_MODE. mysql-bkup 1.x wrote the value into +// my.cnf as "ssl=", so "0"/"1" (and booleans) are still accepted. +// An empty value means disabled, the 1.x default. +func ParseSSLMode(s string) (string, error) { + m := strings.ToLower(strings.TrimSpace(s)) + m = strings.ReplaceAll(m, "-", "_") + switch m { + case "", "0", "false", "off", "no", SSLDisabled, "disable": + return SSLDisabled, nil + case "1", "true", "on", "yes", SSLRequired, "require": + return SSLRequired, nil + case SSLPreferred, "prefer": + return SSLPreferred, nil + case SSLVerifyCA: + return SSLVerifyCA, nil + case SSLVerifyIdentity, "verify_full": + return SSLVerifyIdentity, nil + } + return "", fmt.Errorf("invalid TLS mode %q: use disabled, preferred, required, verify_ca or verify_identity", s) +} + +// Conn is a resolved connection. +type Conn struct { + Host string + Port int + User string + Password string + DB string + SSLMode string + SSLCA string + SSLCert string + SSLKey string +} + +// Addr is host:port (or the socket path), for logs and cache keys. +func (c Conn) Addr() string { + if strings.HasPrefix(c.Host, "/") { + return c.Host + } + return c.Host + ":" + strconv.Itoa(c.Port) +} + +// ParseURL parses mysql://user:pass@host:port/db?ssl-mode=required (also +// mariadb://). Query parameters: ssl-mode (sslmode, tls), ssl-ca, ssl-cert, ssl-key. +func ParseURL(raw string) (Conn, error) { + var c Conn + u, err := url.Parse(raw) + if err != nil { + return c, errors.New("invalid database URL") + } + switch strings.ToLower(u.Scheme) { + case "mysql", "mariadb", "mysqlx": + default: + return c, fmt.Errorf("unsupported database URL scheme %q (use mysql://user:password@host:port/database)", u.Scheme) + } + c.Host = u.Hostname() + if p := u.Port(); p != "" { + n, err := strconv.Atoi(p) + if err != nil || n <= 0 || n > 65535 { + return c, fmt.Errorf("invalid port %q in database URL", p) + } + c.Port = n + } + if u.User != nil { + c.User = u.User.Username() + c.Password, _ = u.User.Password() + } + c.DB = strings.TrimPrefix(u.Path, "/") + q := u.Query() + for _, k := range []string{"ssl-mode", "sslmode", "ssl_mode", "tls"} { + if v := q.Get(k); v != "" { + if c.SSLMode, err = ParseSSLMode(v); err != nil { + return c, err + } + break + } + } + c.SSLCA = first(q.Get("ssl-ca"), q.Get("sslrootcert")) + c.SSLCert = first(q.Get("ssl-cert"), q.Get("sslcert")) + c.SSLKey = first(q.Get("ssl-key"), q.Get("sslkey")) + return c, nil +} + +func first(s ...string) string { + for _, v := range s { + if v != "" { + return v + } + } + return "" +} + +// connKeys names the settings of one side (source or migrate target). +type connKeys struct { + URL, Host, Port, Name, User, Pass, SSLMode, SSLCA, SSLCert, SSLKey string +} + +var ( + sourceKeys = connKeys{config.DBURL, config.DBHost, config.DBPort, config.DBName, config.DBUser, config.DBPassword, + config.DBSSLMode, config.DBSSLCA, config.DBSSLCert, config.DBSSLKey} + targetKeys = connKeys{config.TargetDBURL, config.TargetDBHost, config.TargetDBPort, config.TargetDBName, config.TargetDBUser, + config.TargetDBPass, config.TargetDBSSL, config.TargetDBSSLCA, "", ""} +) + +// connFrom resolves a connection: DB_URL first, then the individual settings +// (which also fill in what the URL leaves out). +func connFrom(v *config.Values, k connKeys) (Conn, error) { + var c Conn + if raw := v.String(k.URL); raw != "" { + var err error + if c, err = ParseURL(raw); err != nil { + return c, fmt.Errorf("%s: %w", k.URL, err) + } + } + str := func(dst *string, key string) { + if *dst == "" && key != "" { + *dst = v.String(key) + } + } + str(&c.Host, k.Host) + str(&c.User, k.User) + str(&c.Password, k.Pass) + str(&c.DB, k.Name) + str(&c.SSLCA, k.SSLCA) + str(&c.SSLCert, k.SSLCert) + str(&c.SSLKey, k.SSLKey) + if c.Port == 0 { + c.Port = v.Int(k.Port) + } + if c.Port == 0 { + c.Port = 3306 + } + if c.SSLMode == "" { + m, err := ParseSSLMode(v.String(k.SSLMode)) + if err != nil { + return c, fmt.Errorf("%s: %w", k.SSLMode, err) + } + c.SSLMode = m + } + return c, nil +} + +// SourceConn returns the DB_* connection of v. +func SourceConn(v *config.Values) (Conn, error) { return connFrom(v, sourceKeys) } + +// TargetConn returns the TARGET_DB_* connection of v (migrate). +func TargetConn(v *config.Values) (Conn, error) { + c, err := connFrom(v, targetKeys) + if err != nil { + return c, err + } + if v.String(config.TargetDBURL) == "" && v.String(config.TargetDBHost) == "" { + return c, errors.New("missing required configuration: TARGET_DB_HOST (or TARGET_DB_URL)") + } + return c, nil +} + +func (c Conn) validate(prefix string) error { + var missing []string + if c.Host == "" { + missing = append(missing, prefix+"DB_HOST") + } + if c.User == "" { + missing = append(missing, prefix+"DB_USER") + } + if len(missing) > 0 { + return fmt.Errorf("missing required configuration: %s", strings.Join(missing, ", ")) + } + return nil +} + +// checkDBName rejects names that a client would parse as an option. +func checkDBName(name string) error { + if strings.HasPrefix(name, "-") { + return fmt.Errorf("invalid database name %q", name) + } + return nil +} diff --git a/internal/mysql/engine.go b/internal/mysql/engine.go new file mode 100644 index 0000000..6546c17 --- /dev/null +++ b/internal/mysql/engine.go @@ -0,0 +1,487 @@ +// Package mysql is the mysql-bkup engine for bkupkit: MySQL and MariaDB +// logical backups with mysqldump / mariadb-dump, restored with mysql / mariadb. +package mysql + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "path/filepath" + "strings" + "sync" + + "github.com/jkaninda/bkupkit/cli" + "github.com/jkaninda/bkupkit/config" + "github.com/jkaninda/bkupkit/engine" + "github.com/jkaninda/bkupkit/execx" +) + +// Engine-specific settings. +const ( + SettingClient = "MYSQL_CLIENT" +) + +// AllInOneName is the target (and default artifact) name of --all-in-one backups. +const AllInOneName = "all_databases" + +// DefaultCA is the 1.x default of DB_SSL_CA. +const DefaultCA = "/etc/ssl/certs/ca-certificates.crt" + +// Engine implements engine.Engine and engine.Migrator. +type Engine struct { + // LookPath finds client binaries (default exec.LookPath); tests replace it. + LookPath func(string) (string, error) + + mu sync.Mutex + tools map[string]tool + servers map[string]server + warned map[string]bool +} + +var ( + _ engine.Engine = (*Engine)(nil) + _ engine.Migrator = (*Engine)(nil) +) + +// New returns the engine. +func New() *Engine { + return &Engine{tools: map[string]tool{}, servers: map[string]server{}, warned: map[string]bool{}} +} + +// Tool implements engine.Engine. +func (e *Engine) Tool() string { return "mysql-bkup" } + +// Kind implements engine.Engine. It is "mysql" for MySQL and MariaDB alike so +// that {engine} in the index name does not depend on the server. +func (e *Engine) Kind() string { return "mysql" } + +// Settings implements engine.Engine. +func (e *Engine) Settings(r *config.Registry) { + config.RegisterDatabase(r, "3306") + r.Override(config.DBURL, func(s *config.Setting) { + s.Usage = "Connection URL mysql://user:password@host:port/database?ssl-mode=required (overrides host, port, user, password)" + }) + r.Override(config.DBHost, func(s *config.Setting) { + s.Usage = "Database host (a path starting with / is a Unix socket)" + }) + r.Override(config.DBSSLMode, func(s *config.Setting) { + s.Default = SSLDisabled + s.Usage = "TLS mode: disabled, preferred, required, verify_ca, verify_identity (1.x values 0 and 1 mean disabled and required)" + }) + r.Override(config.DBSSLCA, func(s *config.Setting) { + s.Default = DefaultCA + s.Usage = "CA certificate file, used with DB_SSL_MODE=verify_ca or verify_identity" + }) + r.Override(config.TargetDBSSL, func(s *config.Setting) { + s.Default = SSLDisabled + s.Usage = "migrate: target TLS mode (same values as DB_SSL_MODE)" + }) + r.Override(config.TargetDBSSLCA, func(s *config.Setting) { + s.Default = DefaultCA + }) + r.Override(config.TargetDBName, func(s *config.Setting) { + s.Usage = "migrate: target database (default: the source name)" + }) + // 1.x accepted --storage remote for SFTP. + r.Override(config.Storage, func(s *config.Setting) { + s.Enum = append(s.Enum, "remote") + s.Usage = "Storage backend: local, s3, gcs, azure, sftp (ssh, remote), ftp" + }) + r.Add(config.Setting{ + Key: SettingClient, Default: ClientAuto, Enum: []string{ClientAuto, ClientMySQL, ClientMariaDB}, + Group: config.GroupDatabase, + Usage: "Client tools: auto (mariadb-dump for MariaDB, Oracle mysqldump for MySQL), mysql or mariadb", + }) +} + +// Configure implements engine.Engine. It only validates values: commands that +// do not connect (list, prune, verify) must work without credentials. +func (e *Engine) Configure(_ context.Context, v *config.Values) error { + normalizeStorage(v) + var errs []error + if _, err := SourceConn(v); err != nil { + errs = append(errs, err) + } + if v.Has(config.TargetDBURL) || v.Has(config.TargetDBHost) { + if _, err := TargetConn(v); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// normalizeStorage maps the 1.x storage name "remote" to sftp, and an +// absolute local FILE_NAME (1.x: restore -f /backup/x.sql.gz) to a file name +// relative to STORAGE_PATH (or to its own directory, as 1.x did). +func normalizeStorage(v *config.Values) { + if v.String(config.Storage) == "remote" { + v.Set(config.Storage, "sftp") + } + f := v.String(config.FileName) + if v.String(config.Storage) != "local" || !filepath.IsAbs(f) || v.Has(config.RemotePath) { + return + } + root := filepath.Clean(v.String(config.StoragePath)) + if rel, err := filepath.Rel(root, filepath.Clean(f)); err == nil && !strings.HasPrefix(rel, "..") { + v.Set(config.FileName, filepath.ToSlash(rel)) + return + } + v.Set(config.StoragePath, filepath.Dir(f)) + v.Set(config.FileName, filepath.Base(f)) +} + +// restoreContext reports whether v belongs to a restore/verify/delete run, +// where one database at most is involved. Outside the CLI (no command), a +// FILE_NAME means the same. +func restoreContext(v *config.Values) bool { + switch v.Command { + case "restore", "verify", "delete": + return true + case "": + return v.Has(config.FileName) && !v.Bool(config.AllDatabases) && !v.Bool(config.AllInOne) + } + return false +} + +// Targets implements engine.Engine. +func (e *Engine) Targets(ctx context.Context, v *config.Values) ([]engine.Target, error) { + normalizeStorage(v) // per-job values of a config file are resolved again + c, err := SourceConn(v) + if err != nil { + return nil, err + } + if restoreContext(v) { + // restore -f FILE [-d DB]: without a database, the dump's own USE + // statements decide (--all-in-one dumps). + if err := checkDBName(c.DB); err != nil { + return nil, err + } + return []engine.Target{{Name: c.DB, Values: v}}, nil + } + switch { + case v.Bool(config.AllInOne): + return []engine.Target{{Name: AllInOneName, Values: v}}, nil + case v.Bool(config.AllDatabases): + if err := c.validate(""); err != nil { + return nil, err + } + dbs, err := e.listDatabases(ctx, v, c, v.List(config.ExcludeDBs)) + if err != nil { + return nil, err + } + if len(dbs) == 0 { + return nil, errors.New("no databases found") + } + ts := make([]engine.Target, 0, len(dbs)) + for _, db := range dbs { + ts = append(ts, engine.Target{Name: db, Values: v}) + } + return ts, nil + } + // No database: fine for list, prune, verify and delete; Ping rejects it + // for a backup. + if err := checkDBName(c.DB); err != nil { + return nil, err + } + return []engine.Target{{Name: c.DB, Values: v}}, nil +} + +// errNoDatabase is a configuration error (exit code 2). +var errNoDatabase = &cli.ExitError{Code: cli.ExitConfig, Err: errors.New("database name is required, use DB_NAME environment variable or -d flag")} + +// Ext implements engine.Engine. +func (e *Engine) Ext(engine.Target) string { return ".sql" } + +// NativeCompression implements engine.Engine. +func (e *Engine) NativeCompression(engine.Target) bool { return false } + +// VerifyBeforeRestore makes RESTORE_VERIFY_FIRST=auto read and check the whole +// dump before the client runs: MySQL cannot roll back the dump's DROP/CREATE +// TABLE statements, so a backup that turns out to be damaged halfway would +// otherwise leave the database half-restored. +func (e *Engine) VerifyBeforeRestore(engine.Target, engine.ArtifactInfo) bool { return true } + +var _ engine.RestoreVerifier = (*Engine)(nil) + +// Ping implements engine.Engine: it connects, and detects the server flavour. +func (e *Engine) Ping(ctx context.Context, t engine.Target) error { + c, err := SourceConn(t.Values) + if err != nil { + return err + } + if t.Name == "" && !restoreContext(t.Values) { + return errNoDatabase + } + if err := c.validate(""); err != nil { + return &cli.ExitError{Code: cli.ExitConfig, Err: err} + } + srv, err := e.server(ctx, t.Values, c) + if err != nil { + return err + } + flavour := "MySQL" + if srv.mariadb { + flavour = "MariaDB" + } + slog.Default().Info("Connected to database server", "server", c.Addr(), "flavour", flavour, "version", srv.version) + return nil +} + +// ServerVersion implements engine.Engine. +func (e *Engine) ServerVersion(ctx context.Context, t engine.Target) string { + c, err := SourceConn(t.Values) + if err != nil { + return "" + } + srv, err := e.server(ctx, t.Values, c) + if err != nil { + return "" + } + return srv.version +} + +// dumpSpec describes what a dump covers. +type dumpSpec struct { + db string // one database (no CREATE DATABASE / USE: restorable under another name) + all bool // --all-databases + databases []string // --databases list (all-in-one with exclusions) +} + +// DumpArgs returns the dump arguments after --defaults-extra-file. +// mariadbTool is the client family, mariadbServer the server flavour. +func DumpArgs(mariadbTool, mariadbServer bool, s dumpSpec) []string { + args := []string{"--single-transaction", "--routines", "--triggers", "--events", "--hex-blob", "--quick"} + if !mariadbServer { + // MySQL: no tablespace statements (they need the PROCESS privilege + // and are useless in logical backups). + args = append(args, "--no-tablespaces") + if !mariadbTool { + args = append(args, "--set-gtid-purged=OFF") + } + } else if !mariadbTool { + // Oracle mysqldump against MariaDB: no column_statistics table there. + args = append(args, "--column-statistics=0") + } + switch { + case s.all: + args = append(args, "--all-databases") + case len(s.databases) > 0: + args = append(args, "--databases") + args = append(args, s.databases...) + default: + args = append(args, s.db) + } + return args +} + +func (e *Engine) dumpCmd(ctx context.Context, t engine.Target, c Conn) (tool, []string, error) { + v := t.Values + srv, err := e.server(ctx, v, c) + if err != nil { + return tool{}, nil, err + } + tl, err := e.pick(ctx, v.String(SettingClient), srv, true) + if err != nil { + return tool{}, nil, err + } + spec := dumpSpec{db: t.Name} + if t.Name == AllInOneName && v.Bool(config.AllInOne) { + spec = dumpSpec{all: true} + if ex := v.List(config.ExcludeDBs); len(ex) > 0 { + dbs, err := e.listDatabases(ctx, v, c, ex) + if err != nil { + return tool{}, nil, err + } + if len(dbs) == 0 { + return tool{}, nil, errors.New("no databases left after BACKUP_EXCLUDE_DATABASES") + } + spec = dumpSpec{databases: dbs} + } + } else if err := checkDBName(t.Name); err != nil || t.Name == "" { + return tool{}, nil, fmt.Errorf("invalid database name %q", t.Name) + } + return tl, DumpArgs(tl.mariadb, srv.mariadb, spec), nil +} + +// Dump implements engine.Engine: the dump goes straight to w. +func (e *Engine) Dump(ctx context.Context, t engine.Target, w io.Writer) error { + c, err := SourceConn(t.Values) + if err != nil { + return err + } + tl, args, err := e.dumpCmd(ctx, t, c) + if err != nil { + return err + } + slog.Default().Info("Dumping database", "database", t.Name, "client", tl.name) + return withOptionFile(t.Values, c, tl, func(cnf string) error { + return execx.Run(ctx, execx.Cmd{ + Name: tl.path, Args: append([]string{"--defaults-extra-file=" + cnf}, args...), + Stdout: w, Log: slog.Default(), StderrLevel: slog.LevelWarn, + }) + }) +} + +// restoreDB returns the database a dump is replayed into ("" = none: the +// dump's USE statements decide). +func restoreDB(t engine.Target) string { + if t.Name == AllInOneName && t.Values.Bool(config.AllInOne) { + return "" + } + return t.Name +} + +func (e *Engine) restoreCmd(ctx context.Context, v *config.Values, c Conn, db string) (tool, []string, error) { + if err := checkDBName(db); err != nil { + return tool{}, nil, err + } + srv, err := e.server(ctx, v, c) + if err != nil { + return tool{}, nil, err + } + tl, err := e.pick(ctx, v.String(SettingClient), srv, false) + if err != nil { + return tool{}, nil, err + } + if db != "" { + if err := ensureDatabase(ctx, v, c, tl, db); err != nil { + return tool{}, nil, err + } + } + // --binary-mode: dumps may contain \0 and \r in string literals. + args := []string{"--binary-mode"} + if db != "" { + args = append(args, db) + } + return tl, args, nil +} + +// Restore implements engine.Engine. The client reads the dump from stdin and +// stops at the first failing statement (non-zero exit). If r fails, execx +// kills the client before it sees EOF. MySQL cannot roll back DDL, so a +// failed restore can leave the statements before the failure applied. +func (e *Engine) Restore(ctx context.Context, t engine.Target, r io.Reader, info engine.ArtifactInfo) error { + c, err := SourceConn(t.Values) + if err != nil { + return err + } + db := restoreDB(t) + tl, args, err := e.restoreCmd(ctx, t.Values, c, db) + if err != nil { + return err + } + slog.Default().Info("Restoring database", "database", db, "file", info.File, "client", tl.name) + return withOptionFile(t.Values, c, tl, func(cnf string) error { + return execx.Run(ctx, execx.Cmd{ + Name: tl.path, Args: append([]string{"--defaults-extra-file=" + cnf}, args...), + Stdin: r, Log: slog.Default(), StderrLevel: slog.LevelWarn, + }) + }) +} + +// MigrateTargets implements engine.Migrator: DB_NAME β†’ TARGET_DB_NAME (default +// the same name), or every database with --all-databases. +func (e *Engine) MigrateTargets(ctx context.Context, v *config.Values) ([][2]engine.Target, error) { + tc, err := TargetConn(v) + if err != nil { + return nil, err + } + if err := tc.validate("TARGET_"); err != nil { + return nil, err + } + sc, err := SourceConn(v) + if err != nil { + return nil, err + } + if err := sc.validate(""); err != nil { + return nil, err + } + if v.Bool(config.AllDatabases) || v.Bool(config.AllInOne) { + dbs, err := e.listDatabases(ctx, v, sc, v.List(config.ExcludeDBs)) + if err != nil { + return nil, err + } + pairs := make([][2]engine.Target, 0, len(dbs)) + for _, db := range dbs { + pairs = append(pairs, [2]engine.Target{{Name: db, Values: v}, {Name: db, Values: v}}) + } + return pairs, nil + } + if sc.DB == "" { + return nil, errNoDatabase + } + name := tc.DB + if name == "" { + name = sc.DB + } + for _, n := range []string{sc.DB, name} { + if err := checkDBName(n); err != nil { + return nil, err + } + } + return [][2]engine.Target{{{Name: sc.DB, Values: v}, {Name: name, Values: v}}}, nil +} + +// Migrate implements engine.Migrator: dump | restore through a pipe, no file. +func (e *Engine) Migrate(ctx context.Context, source, target engine.Target) error { + sc, err := SourceConn(source.Values) + if err != nil { + return err + } + tc, err := TargetConn(target.Values) + if err != nil { + return err + } + dumpTool, dumpArgs, err := e.dumpCmd(ctx, source, sc) + if err != nil { + return fmt.Errorf("source: %w", err) + } + restTool, restArgs, err := e.restoreCmd(ctx, target.Values, tc, target.Name) + if err != nil { + return fmt.Errorf("target: %w", err) + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // The first failure is the cause; the other side then fails because of it. + var ( + once sync.Once + cause error + ) + fail := func(err error) { + once.Do(func() { cause = err }) + cancel() + } + pr, pw := io.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + err := withOptionFile(source.Values, sc, dumpTool, func(cnf string) error { + return execx.Run(ctx, execx.Cmd{ + Name: dumpTool.path, Args: append([]string{"--defaults-extra-file=" + cnf}, dumpArgs...), + Stdout: pw, Log: slog.Default(), StderrLevel: slog.LevelWarn, + }) + }) + if err != nil { + err = fmt.Errorf("dump %s: %w", source.Name, err) + fail(err) + // The restore side must never see a clean EOF. + _ = pw.CloseWithError(err) + return + } + _ = pw.Close() + }() + err = withOptionFile(target.Values, tc, restTool, func(cnf string) error { + return execx.Run(ctx, execx.Cmd{ + Name: restTool.path, Args: append([]string{"--defaults-extra-file=" + cnf}, restArgs...), + Stdin: pr, Log: slog.Default(), StderrLevel: slog.LevelWarn, + }) + }) + if err != nil { + fail(fmt.Errorf("restore into %s: %w", target.Name, err)) + _ = pr.CloseWithError(err) + } + <-done + return cause +} diff --git a/internal/mysql/mysql_test.go b/internal/mysql/mysql_test.go new file mode 100644 index 0000000..6451bc4 --- /dev/null +++ b/internal/mysql/mysql_test.go @@ -0,0 +1,407 @@ +package mysql + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/jkaninda/bkupkit/cli" + "github.com/jkaninda/bkupkit/config" + "github.com/jkaninda/bkupkit/engine" +) + +func resolve(t *testing.T, env config.MapEnv) *config.Values { + t.Helper() + r := config.NewRegistry() + config.RegisterCommon(r) + New().Settings(r) + v, err := r.Resolve(config.ResolveOptions{Env: env}) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestParseSSLMode(t *testing.T) { + cases := map[string]string{ + "": SSLDisabled, "0": SSLDisabled, "false": SSLDisabled, "DISABLED": SSLDisabled, + "1": SSLRequired, "true": SSLRequired, "Required": SSLRequired, + "preferred": SSLPreferred, "VERIFY_CA": SSLVerifyCA, "verify-identity": SSLVerifyIdentity, + } + for in, want := range cases { + got, err := ParseSSLMode(in) + if err != nil || got != want { + t.Errorf("ParseSSLMode(%q) = %q, %v; want %q", in, got, err, want) + } + } + if _, err := ParseSSLMode("2"); err == nil { + t.Error("expected an error for 2") + } +} + +func TestParseURL(t *testing.T) { + c, err := ParseURL("mysql://bob:p%40ss%3Aw%2Frd@db.example:3307/shop?ssl-mode=verify_identity&ssl-ca=/ca.pem") + if err != nil { + t.Fatal(err) + } + want := Conn{Host: "db.example", Port: 3307, User: "bob", Password: "p@ss:w/rd", DB: "shop", SSLMode: SSLVerifyIdentity, SSLCA: "/ca.pem"} + if c != want { + t.Fatalf("got %+v\nwant %+v", c, want) + } + if _, err := ParseURL("postgres://a@b/c"); err == nil { + t.Fatal("expected scheme error") + } + if _, err := ParseURL("mysql://a@b:99999/c"); err == nil { + t.Fatal("expected port error") + } +} + +func TestSourceConnPrecedence(t *testing.T) { + v := resolve(t, config.MapEnv{ + "DB_URL": "mariadb://u:pw@h1/db1", "DB_HOST": "ignored", "DB_PORT": "3310", "DB_NAME": "ignored", + "DB_SSL_MODE": "1", + }) + c, err := SourceConn(v) + if err != nil { + t.Fatal(err) + } + if c.Host != "h1" || c.Port != 3310 || c.User != "u" || c.Password != "pw" || c.DB != "db1" || c.SSLMode != SSLRequired { + t.Fatalf("%+v", c) + } + if c.SSLCA != DefaultCA { + t.Fatalf("default CA = %q", c.SSLCA) + } + + v = resolve(t, config.MapEnv{"DB_HOST": "h", "DB_USER": "legacy", "DB_PASSWORD": "x", "DB_NAME": "d"}) + c, _ = SourceConn(v) + if c.User != "legacy" || c.Port != 3306 || c.SSLMode != SSLDisabled { + t.Fatalf("%+v", c) + } + + v = resolve(t, config.MapEnv{"DB_SSL_MODE": "sometimes"}) + if err := New().Configure(context.Background(), v); err == nil { + t.Fatal("invalid DB_SSL_MODE accepted") + } +} + +func TestTargetConn(t *testing.T) { + v := resolve(t, config.MapEnv{"DB_NAME": "a"}) + if _, err := TargetConn(v); err == nil { + t.Fatal("expected missing TARGET_DB_HOST") + } + v = resolve(t, config.MapEnv{"TARGET_DB_HOST": "t", "TARGET_DB_USERNAME": "tu", "TARGET_DB_PASSWORD": "tp", "TARGET_DB_SSL_MODE": "verify_ca"}) + c, err := TargetConn(v) + if err != nil { + t.Fatal(err) + } + if c.Host != "t" || c.User != "tu" || c.Password != "tp" || c.Port != 3306 || c.SSLMode != SSLVerifyCA { + t.Fatalf("%+v", c) + } +} + +func TestQuoteOption(t *testing.T) { + cases := map[string]string{ + `plain`: `"plain"`, + `a"b`: `"a\"b"`, + `back\slash`: `"back\\slash"`, + " # spaced ": `" # spaced "`, + "new\nline\t": `"new\nline\t"`, + `it's $x`: `"it's $x"`, + } + for in, want := range cases { + if got := quoteOption(in); got != want { + t.Errorf("quoteOption(%q) = %s, want %s", in, got, want) + } + } +} + +func TestOptionFile(t *testing.T) { + c := Conn{Host: "db", Port: 3306, User: "root", Password: `p"w#`, SSLMode: SSLDisabled, SSLCA: DefaultCA} + got := optionFile(c, false) + for _, want := range []string{"[client]\n", "protocol=TCP\n", `host="db"` + "\n", "port=3306\n", `user="root"` + "\n", + `password="p\"w#"` + "\n", "ssl-mode=DISABLED\n", "get-server-public-key=1\n"} { + if !strings.Contains(got, want) { + t.Errorf("mysql option file lacks %q:\n%s", want, got) + } + } + if strings.Contains(got, "ssl-ca") { + t.Errorf("CA written without verification:\n%s", got) + } + got = optionFile(c, true) + if !strings.Contains(got, "ssl=0\n") || strings.Contains(got, "ssl-mode") || strings.Contains(got, "get-server-public-key") { + t.Errorf("mariadb option file:\n%s", got) + } + + c.SSLMode = SSLVerifyIdentity + if got := optionFile(c, false); !strings.Contains(got, "ssl-mode=VERIFY_IDENTITY\n") || !strings.Contains(got, `ssl-ca="`+DefaultCA+`"`) { + t.Errorf("verify_identity:\n%s", got) + } + if got := optionFile(c, true); !strings.Contains(got, "ssl=1\nssl-verify-server-cert=1\n") || !strings.Contains(got, "ssl-ca=") { + t.Errorf("mariadb verify:\n%s", got) + } + c.SSLMode = SSLRequired + if got := optionFile(c, true); !strings.Contains(got, "ssl=1\nssl-verify-server-cert=0\n") || strings.Contains(got, "ssl-ca=") { + t.Errorf("mariadb required:\n%s", got) + } + + c.Host = "/run/mysqld/mysqld.sock" + if got := optionFile(c, true); !strings.Contains(got, "protocol=SOCKET\n") || !strings.Contains(got, `socket="/run/mysqld/mysqld.sock"`) || strings.Contains(got, "host=") { + t.Errorf("socket:\n%s", got) + } +} + +func TestDumpArgs(t *testing.T) { + base := []string{"--single-transaction", "--routines", "--triggers", "--events", "--hex-blob", "--quick"} + cases := []struct { + name string + toolMaria, srvMaria bool + spec dumpSpec + want []string + }{ + {"oracle-mysql", false, false, dumpSpec{db: "shop"}, append(append([]string{}, base...), "--no-tablespaces", "--set-gtid-purged=OFF", "shop")}, + {"maria-maria", true, true, dumpSpec{db: "shop"}, append(append([]string{}, base...), "shop")}, + {"maria-mysql", true, false, dumpSpec{all: true}, append(append([]string{}, base...), "--no-tablespaces", "--all-databases")}, + {"oracle-maria", false, true, dumpSpec{databases: []string{"a", "b"}}, append(append([]string{}, base...), "--column-statistics=0", "--databases", "a", "b")}, + } + for _, c := range cases { + if got := DumpArgs(c.toolMaria, c.srvMaria, c.spec); !reflect.DeepEqual(got, c.want) { + t.Errorf("%s:\n got %v\nwant %v", c.name, got, c.want) + } + } +} + +func TestFilterDatabases(t *testing.T) { + got := filterDatabases([]string{"information_schema", "shop", "mysql", "Performance_Schema", "sys", "blog", "", "tmp\r"}, []string{"tmp"}) + if !reflect.DeepEqual(got, []string{"shop", "blog"}) { + t.Fatal(got) + } +} + +func TestIsMariaDB(t *testing.T) { + if !IsMariaDB("11.8.3-MariaDB-ubu2404") || IsMariaDB("8.4.6") { + t.Fatal("flavour detection") + } +} + +func TestQuoteIdent(t *testing.T) { + if got := quoteIdent("we`ird"); got != "`we``ird`" { + t.Fatal(got) + } +} + +func TestTargets(t *testing.T) { + e := New() + ctx := context.Background() + for _, c := range []struct{ storagePath, file, wantRoot, wantFile string }{ + {"/backup", "/backup/x.sql.gz", "/backup", "x.sql.gz"}, + {"/backup", "/backup/sub/x.sql.gz", "/backup", "sub/x.sql.gz"}, + {"/backup", "/data/x.sql", "/data", "x.sql"}, + {"/backup", "x.sql", "/backup", "x.sql"}, + } { + v := resolve(t, config.MapEnv{"STORAGE_PATH": c.storagePath, "FILE_NAME": c.file, "DB_NAME": "x"}) + if _, err := e.Targets(ctx, v); err != nil || v.String(config.StoragePath) != c.wantRoot || v.String(config.FileName) != c.wantFile { + t.Errorf("%s: %q %q %v", c.file, v.String(config.StoragePath), v.String(config.FileName), err) + } + } + v0 := resolve(t, config.MapEnv{"STORAGE": "Remote", "DB_NAME": "x"}) + if _, err := e.Targets(ctx, v0); err != nil || v0.String(config.Storage) != "sftp" { + t.Fatalf("storage remote: %q %v", v0.String(config.Storage), err) + } + v := resolve(t, config.MapEnv{"DB_NAME": "shop"}) + ts, err := e.Targets(ctx, v) + if err != nil || len(ts) != 1 || ts[0].Name != "shop" { + t.Fatalf("%v %v", ts, err) + } + v = resolve(t, config.MapEnv{}) + ts, err = e.Targets(ctx, v) + if err != nil || len(ts) != 1 || ts[0].Name != "" { + t.Fatalf("no name: %v %v", ts, err) + } + var ee *cli.ExitError + if err := e.Ping(ctx, ts[0]); !errors.As(err, &ee) || ee.Code != cli.ExitConfig || !strings.Contains(err.Error(), "DB_NAME") { + t.Fatalf("backup without a database: %v", err) + } + // restore without a database (all-in-one dumps carry their own USE). + v = resolve(t, config.MapEnv{"FILE_NAME": "all_databases_20260101_000000.sql.gz"}) + if ts, err := e.Targets(ctx, v); err != nil || ts[0].Name != "" { + t.Fatalf("%v %v", ts, err) + } + v = resolve(t, config.MapEnv{"BACKUP_ALL_IN_ONE": "true"}) + if ts, err := e.Targets(ctx, v); err != nil || ts[0].Name != AllInOneName || restoreDB(ts[0]) != "" { + t.Fatalf("%v %v", ts, err) + } + v = resolve(t, config.MapEnv{"DB_NAME": "-x"}) + if _, err := e.Targets(ctx, v); err == nil { + t.Fatal("option-like database name accepted") + } +} + +// fakeClients installs shell scripts standing in for the client binaries. +// They log their argv and the option file, and answer the queries the engine runs. +func fakeClients(t *testing.T, serverVersion string, oracle bool) (dir, logFile string) { + t.Helper() + dir = t.TempDir() + logFile = filepath.Join(dir, "calls.log") + script := `#!/bin/sh +log="` + logFile + `" +echo "ARGV $(basename "$0") $*" >> "$log" +case "$1" in --defaults-extra-file=*) f="${1#--defaults-extra-file=}"; echo "MODE $(ls -ln "$f" | cut -c1-10)" >> "$log"; sed 's/^/CNF /' "$f" >> "$log";; esac +for a in "$@"; do + case "$a" in + --version) echo "$(basename "$0") Ver 8.4.6 for Linux (MySQL Community Server - GPL)"; exit 0;; + "SELECT VERSION()") echo "` + serverVersion + `"; exit 0;; + "SHOW DATABASES") printf 'information_schema\nmysql\nshop\nblog\nsys\n'; exit 0;; + SELECT\ COUNT*) echo 0; exit 0;; + CREATE\ DATABASE*) exit 0;; + esac +done +case "$(basename "$0")" in + *dump) echo "-- dump of $*";; + *) cat > "` + filepath.Join(dir, "restored.sql") + `";; +esac +` + names := []string{"mariadb", "mariadb-dump"} + if oracle { + names = append(names, "mysql", "mysqldump") + } + for _, n := range names { + if err := os.WriteFile(filepath.Join(dir, n), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + } + return dir, logFile +} + +func engineWith(dir string) *Engine { + e := New() + e.LookPath = func(name string) (string, error) { + p := filepath.Join(dir, name) + if _, err := os.Stat(p); err != nil { + return "", err + } + return p, nil + } + return e +} + +func TestDumpAndRestoreUseOptionFile(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name, version string + oracle bool + wantDump, wantCL string + }{ + {"mysql", "8.4.6", true, "mysqldump", "mysql"}, + {"mariadb", "11.8.3-MariaDB", true, "mariadb-dump", "mariadb"}, + {"mysql-without-oracle-client", "8.4.6", false, "mariadb-dump", "mariadb"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir, logFile := fakeClients(t, tc.version, tc.oracle) + tmp := t.TempDir() + v := resolve(t, config.MapEnv{"DB_HOST": "db", "DB_USERNAME": "root", "DB_PASSWORD": "s3cr3t pw", "DB_NAME": "shop", "TMP_DIR": tmp}) + e := engineWith(dir) + tg := engine.Target{Name: "shop", Values: v} + if err := e.Ping(ctx, tg); err != nil { + t.Fatal(err) + } + if got := e.ServerVersion(ctx, tg); got != tc.version { + t.Fatalf("version %q", got) + } + var out bytes.Buffer + if err := e.Dump(ctx, tg, &out); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(out.String(), "-- dump of --defaults-extra-file=") || !strings.HasSuffix(strings.TrimSpace(out.String()), " shop") { + t.Fatalf("dump output %q", out.String()) + } + if err := e.Restore(ctx, tg, strings.NewReader("CREATE TABLE t (id int);\n"), engine.ArtifactInfo{File: "x.sql"}); err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(logFile) + calls := string(b) + if strings.Contains(strings.Join(argvLines(calls), "\n"), "s3cr3t") { + t.Fatalf("password in argv:\n%s", calls) + } + if !strings.Contains(calls, `CNF password="s3cr3t pw"`) { + t.Fatalf("password not in option file:\n%s", calls) + } + if !strings.Contains(calls, "MODE -rw-------") || strings.Contains(calls, "MODE -rw-r") { + t.Fatalf("option file mode:\n%s", calls) + } + for _, l := range argvLines(calls) { + f := strings.Fields(l) + if len(f) < 3 || !strings.HasPrefix(f[2], "--defaults-extra-file=") { + t.Fatalf("--defaults-extra-file is not the first argument: %q", l) + } + } + if !strings.Contains(calls, "ARGV "+tc.wantDump+" ") || !strings.Contains(calls, "ARGV "+tc.wantCL+" --defaults-extra-file=") { + t.Fatalf("clients:\n%s", calls) + } + if !strings.Contains(calls, "--binary-mode shop") { + t.Fatalf("restore args:\n%s", calls) + } + if r, _ := os.ReadFile(filepath.Join(dir, "restored.sql")); string(r) != "CREATE TABLE t (id int);\n" { + t.Fatalf("restored %q", r) + } + if ents, _ := os.ReadDir(tmp); len(ents) != 0 { + t.Fatalf("TMP_DIR not cleaned: %v", ents) + } + }) + } +} + +func argvLines(log string) []string { + var out []string + for _, l := range strings.Split(log, "\n") { + if strings.HasPrefix(l, "ARGV ") && !strings.Contains(l, "--version") { + out = append(out, l) + } + } + return out +} + +func TestAllDatabasesTargets(t *testing.T) { + dir, _ := fakeClients(t, "8.4.6", true) + v := resolve(t, config.MapEnv{"DB_HOST": "db", "DB_USERNAME": "root", "BACKUP_ALL_DATABASES": "1", "BACKUP_EXCLUDE_DATABASES": "blog", "TMP_DIR": t.TempDir()}) + ts, err := engineWith(dir).Targets(context.Background(), v) + if err != nil { + t.Fatal(err) + } + if len(ts) != 1 || ts[0].Name != "shop" { + t.Fatalf("%v", ts) + } +} + +func TestMigratePipes(t *testing.T) { + dir, logFile := fakeClients(t, "11.8.3-MariaDB", true) + v := resolve(t, config.MapEnv{"DB_HOST": "src", "DB_USERNAME": "root", "DB_PASSWORD": "a", "DB_NAME": "shop", + "TARGET_DB_HOST": "dst", "TARGET_DB_USERNAME": "root", "TARGET_DB_PASSWORD": "b", "TARGET_DB_NAME": "shop2", "TMP_DIR": t.TempDir()}) + e := engineWith(dir) + pairs, err := e.MigrateTargets(context.Background(), v) + if err != nil || len(pairs) != 1 || pairs[0][0].Name != "shop" || pairs[0][1].Name != "shop2" { + t.Fatalf("%v %v", pairs, err) + } + if err := e.Migrate(context.Background(), pairs[0][0], pairs[0][1]); err != nil { + t.Fatal(err) + } + r, _ := os.ReadFile(filepath.Join(dir, "restored.sql")) + if !strings.HasPrefix(string(r), "-- dump of --defaults-extra-file=") { + t.Fatalf("restored %q", r) + } + b, _ := os.ReadFile(logFile) + if !strings.Contains(string(b), `CNF host="dst"`) || !strings.Contains(string(b), "--binary-mode shop2") { + t.Fatalf("calls:\n%s", b) + } +} + +func TestVerifiesBeforeRestore(t *testing.T) { + // MySQL cannot roll back DDL: a damaged dump must be caught first. + if !New().VerifyBeforeRestore(engine.Target{}, engine.ArtifactInfo{}) { + t.Fatal("restores must verify the dump first") + } +} diff --git a/internal/mysql/options.go b/internal/mysql/options.go new file mode 100644 index 0000000..a6ed9da --- /dev/null +++ b/internal/mysql/options.go @@ -0,0 +1,101 @@ +package mysql + +import ( + "strconv" + "strings" +) + +// quoteOption quotes a value for a MySQL/MariaDB option file. Both client +// families parse option files with the same mysys code: a value wrapped in +// double quotes keeps leading/trailing spaces and '#', and the escapes +// \\ \" \n \r \t are decoded inside it. +func quoteOption(s string) string { + var b strings.Builder + b.Grow(len(s) + 2) + b.WriteByte('"') + for _, r := range s { + switch r { + case '\\': + b.WriteString(`\\`) + case '"': + b.WriteString(`\"`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + b.WriteRune(r) + } + } + b.WriteByte('"') + return b.String() +} + +// optionFile returns the [client] option file for c. mariadbClient selects +// the MariaDB client option names (ssl, ssl-verify-server-cert) instead of +// the Oracle MySQL ones (ssl-mode, get-server-public-key). +// +// The file holds the password; it is written 0600 in a private directory and +// passed as --defaults-extra-file, never on the command line. +func optionFile(c Conn, mariadbClient bool) string { + var b strings.Builder + line := func(k, v string) { + b.WriteString(k) + b.WriteByte('=') + b.WriteString(v) + b.WriteByte('\n') + } + b.WriteString("[client]\n") + if strings.HasPrefix(c.Host, "/") { + line("protocol", "SOCKET") + line("socket", quoteOption(c.Host)) + } else { + // TCP even for "localhost" (a client would otherwise look for a socket). + line("protocol", "TCP") + line("host", quoteOption(c.Host)) + line("port", strconv.Itoa(c.Port)) + } + line("user", quoteOption(c.User)) + line("password", quoteOption(c.Password)) + line("max-allowed-packet", "1G") + + verify := c.SSLMode == SSLVerifyCA || c.SSLMode == SSLVerifyIdentity + if mariadbClient { + switch c.SSLMode { + case SSLDisabled: + line("ssl", "0") + case SSLPreferred: + // Leave TLS to the client default, without certificate checks. + line("ssl-verify-server-cert", "0") + case SSLRequired: + line("ssl", "1") + line("ssl-verify-server-cert", "0") + default: // verify_ca, verify_identity: MariaDB always checks the host name too + line("ssl", "1") + line("ssl-verify-server-cert", "1") + } + } else { + line("ssl-mode", strings.ToUpper(c.SSLMode)) + if c.SSLMode != SSLVerifyIdentity && c.SSLMode != SSLVerifyCA { + // caching_sha2_password over a non-TLS connection needs the + // server's RSA key (what the MariaDB client does by itself). + line("get-server-public-key", "1") + } + } + if c.SSLMode != SSLDisabled { + // The CA is only used when certificates are verified: with the MySQL + // client, a CA would otherwise silently turn "required" into "verify_ca". + if verify && c.SSLCA != "" { + line("ssl-ca", quoteOption(c.SSLCA)) + } + if c.SSLCert != "" { + line("ssl-cert", quoteOption(c.SSLCert)) + } + if c.SSLKey != "" { + line("ssl-key", quoteOption(c.SSLKey)) + } + } + return b.String() +} diff --git a/main.go b/main.go index e92e569..9094af3 100644 --- a/main.go +++ b/main.go @@ -1,33 +1,19 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - +// Command mysql-bkup backs up and restores MySQL and MariaDB databases to +// local storage, S3, GCS, Azure Blob, SFTP or FTP, with optional GPG or age +// encryption. See https://github.com/jkaninda/mysql-bkup. package main -import "github.com/jkaninda/mysql-bkup/cmd" +import ( + "github.com/jkaninda/bkupkit/cli" + "github.com/jkaninda/mysql-bkup/internal/mysql" +) + +var Version = "development" func main() { - - cmd.Execute() - + (&cli.App{ + Engine: mysql.New(), + Version: Version, + Short: "Back up and restore MySQL and MariaDB databases (local, S3, GCS, Azure, SFTP, FTP; GPG or age encryption)", + }).Main() } diff --git a/migrations/test_config.yaml b/migrations/test_config.yaml index 97a8b56..875c4d1 100644 --- a/migrations/test_config.yaml +++ b/migrations/test_config.yaml @@ -8,6 +8,6 @@ databases: password: password - name: testdb2 # database credentials from environment variables - #TESTDB2_DB_USERNAME + #TESTDB2_DB_USER #TESTDB2_DB_PASSWORD #TESTDB2_DB_HOST \ No newline at end of file diff --git a/pkg/azure.go b/pkg/azure.go deleted file mode 100644 index b16edcc..0000000 --- a/pkg/azure.go +++ /dev/null @@ -1,128 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "fmt" - "github.com/jkaninda/go-storage/pkg/azure" - goutils "github.com/jkaninda/go-utils" - "github.com/jkaninda/mysql-bkup/utils" - - "os" - "path/filepath" - "time" -) - -func azureBackup(db *dbConfig, config *BackupConfig) { - utils.Info("Backup database to Azure Blob Storage") - - // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) - if err != nil { - recoverMode(err, "Error backing up database") - return - } - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") - } - utils.Info("Uploading backup archive to Azure Blob storage ...") - utils.Info("Backup name is %s", finalFileName) - azureConfig := loadAzureConfig() - azureStorage, err := azure.NewStorage(azure.Config{ - ContainerName: azureConfig.containerName, - AccountName: azureConfig.accountName, - AccountKey: azureConfig.accountKey, - RemotePath: config.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating Azure storage: %s", err) - } - err = azureStorage.Copy(finalFileName) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) - // Get backup info - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - // Delete backup file from tmp folder - err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error deleting file: %v", err) - - } - if config.prune { - err := azureStorage.Prune(config.backupRetention) - if err != nil { - utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - - } - - utils.Info("Backup name is %s", finalFileName) - utils.Info("Backup size: %s", utils.ConvertBytes(uint64(backupSize))) - utils.Info("Uploading backup archive to Azure Blob storage ... done ") - - duration := goutils.FormatDuration(time.Since(startTime), 0) - - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: utils.ConvertBytes(uint64(backupSize)), - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(config.remotePath, finalFileName), - Duration: duration, - }) - // Delete temp - deleteTemp() - utils.Info("The backup of the %s database has been completed in %s", db.dbName, duration) -} -func azureRestore(db *dbConfig, conf *RestoreConfig) { - utils.Info("Restore database from Azure Blob storage") - azureConfig := loadAzureConfig() - azureStorage, err := azure.NewStorage(azure.Config{ - ContainerName: azureConfig.containerName, - AccountName: azureConfig.accountName, - AccountKey: azureConfig.accountKey, - RemotePath: conf.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating SSH storage: %s", err) - } - - err = azureStorage.CopyFrom(conf.file) - if err != nil { - utils.Fatal("Error downloading backup file: %s", err) - } - RestoreDatabase(db, conf) -} diff --git a/pkg/backup.go b/pkg/backup.go deleted file mode 100644 index 68034ef..0000000 --- a/pkg/backup.go +++ /dev/null @@ -1,438 +0,0 @@ -// Package internal / -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "bytes" - "errors" - "fmt" - "github.com/jkaninda/encryptor" - "github.com/jkaninda/go-storage/pkg/local" - goutils "github.com/jkaninda/go-utils" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/robfig/cron/v3" - "github.com/spf13/cobra" - "os" - "os/exec" - "path/filepath" - "strings" - "time" -) - -func StartBackup(cmd *cobra.Command) { - intro() - // Initialize backup configs - config := initBackupConfig(cmd) - // Load backup configuration file - configFile, err := loadConfigFile() - if err != nil { - dbConf = initDbConfig(cmd) - if config.cronExpression == "" { - config.allowCustomName = true - createBackupTask(dbConf, config) - } else { - if utils.IsValidCronExpression(config.cronExpression) { - scheduledMode(dbConf, config) - } else { - utils.Fatal("Cron expression is not valid: %s", config.cronExpression) - } - } - } else { - startMultiBackup(config, configFile) - } - -} - -// scheduledMode Runs backup in scheduled mode -func scheduledMode(db *dbConfig, config *BackupConfig) { - utils.Info("Running in Scheduled mode") - utils.Info("Backup cron expression: %s", config.cronExpression) - utils.Info("The next scheduled time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat)) - utils.Info("Storage type %s ", config.storage) - - // Test backup - utils.Info("Testing backup configurations...") - err := testDatabaseConnection(db) - if err != nil { - utils.Error("Error connecting to database: %s", db.dbName) - utils.Fatal("Error: %s", err) - } - utils.Info("Testing backup configurations...done") - utils.Info("Creating backup job...") - // Create a new cron instance - c := cron.New() - - _, err = c.AddFunc(config.cronExpression, func() { - createBackupTask(db, config) - utils.Info("Next backup time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat)) - - }) - if err != nil { - return - } - // Start the cron scheduler - c.Start() - utils.Info("Creating backup job...done") - utils.Info("Backup job started") - defer c.Stop() - select {} -} - -// multiBackupTask backup multi database -func multiBackupTask(databases []Database, bkConfig *BackupConfig) { - for _, db := range databases { - // Check if path is defined in config file - if db.Path != "" { - bkConfig.remotePath = db.Path - } - createBackupTask(getDatabase(db), bkConfig) - } -} - -// createBackupTask backup task -func createBackupTask(db *dbConfig, config *BackupConfig) { - if config.all && !config.allInOne { - backupAll(db, config) - } else { - if db.dbName == "" && !config.all { - utils.Fatal("Database name is required, use DB_NAME environment variable or -d flag") - } - backupTask(db, config) - } -} - -// backupAll backup all databases -func backupAll(db *dbConfig, config *BackupConfig) { - databases, err := listDatabases(*db) - if err != nil { - utils.Fatal("Error listing databases: %s", err) - } - for _, dbName := range databases { - if dbName == "information_schema" || dbName == "performance_schema" || dbName == "mysql" || dbName == "sys" || dbName == "innodb" || dbName == "Database" { - continue - } - db.dbName = dbName - config.backupFileName = fmt.Sprintf("%s_%s.sql.gz", dbName, time.Now().Format("20060102_150405")) - backupTask(db, config) - } - -} - -// backupTask backup task -func backupTask(db *dbConfig, config *BackupConfig) { - utils.Info("Starting backup task...") - startTime = time.Now() - prefix := db.dbName - if config.all && config.allInOne { - prefix = "all_databases" - } - - // Generate file name - backupFileName := fmt.Sprintf("%s_%s.sql.gz", prefix, time.Now().Format("20060102_150405")) - if config.disableCompression { - backupFileName = fmt.Sprintf("%s_%s.sql", prefix, time.Now().Format("20060102_150405")) - } - if config.customName != "" && config.allowCustomName && !config.all { - backupFileName = fmt.Sprintf("%s.sql.gz", config.customName) - if config.disableCompression { - backupFileName = fmt.Sprintf("%s.sql", config.customName) - } - } - config.backupFileName = backupFileName - s := strings.ToLower(config.storage) - switch s { - case "local": - localBackup(db, config) - case "s3": - s3Backup(db, config) - case "ssh", "remote", "sftp": - sshBackup(db, config) - case "ftp": - ftpBackup(db, config) - case "azure": - azureBackup(db, config) - default: - localBackup(db, config) - } -} - -// startMultiBackup start multi backup -func startMultiBackup(bkConfig *BackupConfig, configFile string) { - utils.Info("Starting Multi backup task...") - conf, err := readConf(configFile) - if err != nil { - utils.Fatal("Error reading config file: %s", err) - } - // Check if cronExpression is defined in config file - if conf.CronExpression != "" { - bkConfig.cronExpression = conf.CronExpression - } - if len(conf.Databases) == 0 { - utils.Fatal("No databases found") - } - // Check if cronExpression is defined - if bkConfig.cronExpression == "" { - multiBackupTask(conf.Databases, bkConfig) - } else { - backupRescueMode = conf.BackupRescueMode - // Check if cronExpression is valid - if utils.IsValidCronExpression(bkConfig.cronExpression) { - utils.Info("Running backup in Scheduled mode") - utils.Info("Backup cron expression: %s", bkConfig.cronExpression) - utils.Info("The next scheduled time is: %v", utils.CronNextTime(bkConfig.cronExpression).Format(timeFormat)) - utils.Info("Storage type %s ", bkConfig.storage) - - // Test backup - utils.Info("Testing backup configurations...") - for _, db := range conf.Databases { - err = testDatabaseConnection(getDatabase(db)) - if err != nil { - recoverMode(err, fmt.Sprintf("Error connecting to database: %s", db.Name)) - continue - } - } - utils.Info("Testing backup configurations...done") - utils.Info("Creating backup job...") - // Create a new cron instance - c := cron.New() - - _, err := c.AddFunc(bkConfig.cronExpression, func() { - multiBackupTask(conf.Databases, bkConfig) - utils.Info("Next backup time is: %v", utils.CronNextTime(bkConfig.cronExpression).Format(timeFormat)) - - }) - if err != nil { - return - } - // Start the cron scheduler - c.Start() - utils.Info("Creating backup job...done") - utils.Info("Backup job started") - defer c.Stop() - select {} - - } else { - utils.Fatal("Cron expression is not valid: %s", bkConfig.cronExpression) - } - } - -} - -// BackupDatabase backup database -func BackupDatabase(db *dbConfig, backupFileName string, disableCompression, all, singleFile bool) error { - storagePath = os.Getenv("STORAGE_PATH") - utils.Info("Starting database backup...") - - if err := testDatabaseConnection(db); err != nil { - return fmt.Errorf("database connection failed: %w", err) - } - - dumpArgs := []string{fmt.Sprintf("--defaults-file=%s", mysqlClientConfig)} - if all && singleFile { - utils.Info("Backing up all databases...") - dumpArgs = append(dumpArgs, "--all-databases", "--single-transaction", "--routines", "--triggers") - } else { - utils.Info("Backing up %s database...", db.dbName) - dumpArgs = append(dumpArgs, db.dbName) - } - - backupPath := filepath.Join(tmpPath, backupFileName) - if disableCompression { - return runCommandAndSaveOutput("mysqldump", dumpArgs, backupPath) - } - return runCommandWithCompression("mysqldump", dumpArgs, backupPath) -} - -// runCommandAndSaveOutput runs a command and saves the output to a file -func runCommandAndSaveOutput(command string, args []string, outputPath string) error { - cmd := exec.Command(command, args...) - output, err := cmd.Output() - if err != nil { - return fmt.Errorf("failed to execute %s: %v, output: %s", command, err, string(output)) - } - - return os.WriteFile(outputPath, output, 0644) -} - -// runCommandWithCompression runs a command and compresses the output -func runCommandWithCompression(command string, args []string, outputPath string) error { - cmd := exec.Command(command, args...) - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("failed to create stdout pipe: %w", err) - } - - gzipCmd := exec.Command("gzip") - gzipCmd.Stdin = stdout - gzipFile, err := os.Create(outputPath) - if err != nil { - return fmt.Errorf("failed to create gzip file: %w", err) - } - defer func(gzipFile *os.File) { - err := gzipFile.Close() - if err != nil { - utils.Error("Error closing gzip file: %v", err) - } - }(gzipFile) - gzipCmd.Stdout = gzipFile - - if err := gzipCmd.Start(); err != nil { - return fmt.Errorf("failed to start gzip: %w", err) - } - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to execute %s: %w", command, err) - } - if err := gzipCmd.Wait(); err != nil { - return fmt.Errorf("failed to wait for gzip completion: %w", err) - } - - utils.Info("Database has been backed up") - return nil -} - -// localBackup backup database to local storage -func localBackup(db *dbConfig, config *BackupConfig) { - utils.Info("Backup database to local storage") - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) - if err != nil { - recoverMode(err, "Error backing up database") - return - } - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, gpgExtension) - } - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - localStorage := local.NewStorage(local.Config{ - LocalPath: tmpPath, - RemotePath: storagePath, - }) - err = localStorage.Copy(finalFileName) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - utils.Info("Backup name is %s", finalFileName) - utils.Info("Backup size: %s", utils.ConvertBytes(uint64(backupSize))) - utils.Info("Backup saved in %s", filepath.Join(storagePath, finalFileName)) - duration := goutils.FormatDuration(time.Since(startTime), 0) - - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: utils.ConvertBytes(uint64(backupSize)), - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(storagePath, finalFileName), - Duration: duration, - }) - // Delete old backup - if config.prune { - err = localStorage.Prune(config.backupRetention) - if err != nil { - utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - - } - // Delete temp - deleteTemp() - utils.Info("The backup of the %s database has been completed in %s", db.dbName, duration) -} - -// encryptBackup encrypt backup -func encryptBackup(config *BackupConfig) { - backupFile, err := os.ReadFile(filepath.Join(tmpPath, config.backupFileName)) - outputFile := fmt.Sprintf("%s.%s", filepath.Join(tmpPath, config.backupFileName), gpgExtension) - if err != nil { - utils.Fatal("Error reading backup file: %s ", err) - } - if config.usingKey { - utils.Info("Encrypting backup using public key...") - pubKey, err := os.ReadFile(config.publicKey) - if err != nil { - utils.Fatal("Error reading public key: %s ", err) - } - err = encryptor.EncryptWithPublicKey(backupFile, fmt.Sprintf("%s.%s", filepath.Join(tmpPath, config.backupFileName), gpgExtension), pubKey) - if err != nil { - utils.Fatal("Error encrypting backup file: %v ", err) - } - utils.Info("Encrypting backup using public key...done") - - } else if config.passphrase != "" { - utils.Info("Encrypting backup using passphrase...") - err := encryptor.Encrypt(backupFile, outputFile, config.passphrase) - if err != nil { - utils.Fatal("error during encrypting backup %v", err) - } - utils.Info("Encrypting backup using passphrase...done") - - } - -} - -// listDatabases list all databases -func listDatabases(db dbConfig) ([]string, error) { - databases := []string{} - // Create the mysql client config file - if err := createMysqlClientConfigFile(db); err != nil { - return databases, errors.New(err.Error()) - } - utils.Info("Listing databases...") - // Step 1: List all databases - cmd := exec.Command("mariadb", fmt.Sprintf("--defaults-file=%s", mysqlClientConfig), "-e", "SHOW DATABASES;") - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - return databases, fmt.Errorf("failed to list databases: %s", err) - } - // Step 2: Parse the output - for _, _db := range strings.Split(out.String(), "\n") { - if _db != "" { - databases = append(databases, _db) - } - } - return databases, nil -} -func recoverMode(err error, msg string) { - if err != nil { - if backupRescueMode { - utils.NotifyError(fmt.Sprintf("%s : %v", msg, err)) - utils.Error("Error: %s", msg) - utils.Error("Backup rescue mode is enabled") - utils.Error("Backup will continue") - } else { - utils.Error("Error: %s", msg) - utils.Fatal("Error: %v", err) - return - } - } - -} diff --git a/pkg/config.go b/pkg/config.go deleted file mode 100644 index 64ccda3..0000000 --- a/pkg/config.go +++ /dev/null @@ -1,363 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "fmt" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" - "os" - "strconv" - "strings" -) - -type Database struct { - Host string `yaml:"host"` - Port string `yaml:"port"` - Name string `yaml:"name"` - User string `yaml:"user"` - Password string `yaml:"password"` - Path string `yaml:"path"` -} -type Config struct { - CronExpression string `yaml:"cronExpression"` - BackupRescueMode bool `yaml:"backupRescueMode"` - Databases []Database `yaml:"databases"` -} - -type dbConfig struct { - dbHost string - dbPort string - dbName string - dbUserName string - dbPassword string -} -type targetDbConfig struct { - targetDbHost string - targetDbPort string - targetDbUserName string - targetDbPassword string - targetDbName string -} -type TgConfig struct { - Token string - ChatId string -} -type BackupConfig struct { - backupFileName string - backupRetention int - disableCompression bool - prune bool - remotePath string - encryption bool - usingKey bool - passphrase string - publicKey string - storage string - cronExpression string - all bool - allInOne bool - customName string - allowCustomName bool -} -type FTPConfig struct { - host string - user string - password string - port int - remotePath string -} -type AzureConfig struct { - accountName string - accountKey string - containerName string -} - -// SSHConfig holds the SSH connection details -type SSHConfig struct { - user string - password string - hostName string - port int - identifyFile string -} -type AWSConfig struct { - endpoint string - bucket string - accessKey string - secretKey string - region string - remotePath string - disableSsl bool - forcePathStyle bool -} - -func initDbConfig(cmd *cobra.Command) *dbConfig { - // Set env - utils.GetEnv(cmd, "dbname", "DB_NAME") - dConf := dbConfig{} - dConf.dbHost = os.Getenv("DB_HOST") - dConf.dbPort = utils.EnvWithDefault("DB_PORT", "3306") - dConf.dbName = os.Getenv("DB_NAME") - dConf.dbUserName = os.Getenv("DB_USERNAME") - dConf.dbPassword = os.Getenv("DB_PASSWORD") - - err := utils.CheckEnvVars(dbHVars) - if err != nil { - utils.Error("Please make sure all required environment variables for database are set") - utils.Fatal("Error checking environment variables: %s", err) - } - return &dConf -} - -func getDatabase(database Database) *dbConfig { - // Set default values from environment variables if not provided - database.User = getEnvOrDefault(database.User, "DB_USERNAME", database.Name, "") - database.Password = getEnvOrDefault(database.Password, "DB_PASSWORD", database.Name, "") - database.Host = getEnvOrDefault(database.Host, "DB_HOST", database.Name, "") - database.Port = getEnvOrDefault(database.Port, "DB_PORT", database.Name, "3306") - return &dbConfig{ - dbHost: database.Host, - dbPort: database.Port, - dbName: database.Name, - dbUserName: database.User, - dbPassword: database.Password, - } -} - -// Helper function to get environment variable or use a default value -func getEnvOrDefault(currentValue, envKey, suffix, defaultValue string) string { - // Return the current value if it's already set - if currentValue != "" { - return currentValue - } - - // Check for suffixed or prefixed environment variables if a suffix is provided - if suffix != "" { - suffixUpper := strings.ToUpper(suffix) - envSuffix := os.Getenv(fmt.Sprintf("%s_%s", envKey, suffixUpper)) - if envSuffix != "" { - return envSuffix - } - - envPrefix := os.Getenv(fmt.Sprintf("%s_%s", suffixUpper, envKey)) - if envPrefix != "" { - return envPrefix - } - } - - // Fall back to the default value using a helper function - return utils.EnvWithDefault(envKey, defaultValue) -} - -// loadSSHConfig loads the SSH configuration from environment variables -func loadSSHConfig() (*SSHConfig, error) { - utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME") - sshVars := []string{"SSH_USER", "SSH_HOST", "SSH_PORT", "REMOTE_PATH"} - err := utils.CheckEnvVars(sshVars) - if err != nil { - return nil, fmt.Errorf("error missing environment variables: %w", err) - } - - return &SSHConfig{ - user: os.Getenv("SSH_USER"), - password: os.Getenv("SSH_PASSWORD"), - hostName: os.Getenv("SSH_HOST"), - port: utils.GetIntEnv("SSH_PORT"), - identifyFile: os.Getenv("SSH_IDENTIFY_FILE"), - }, nil -} -func loadFtpConfig() *FTPConfig { - // Initialize data configs - fConfig := FTPConfig{} - fConfig.host = utils.GetEnvVariable("FTP_HOST", "FTP_HOST_NAME") - fConfig.user = os.Getenv("FTP_USER") - fConfig.password = os.Getenv("FTP_PASSWORD") - fConfig.port = utils.GetIntEnv("FTP_PORT") - fConfig.remotePath = os.Getenv("REMOTE_PATH") - err := utils.CheckEnvVars(ftpVars) - if err != nil { - utils.Error("Please make sure all required environment variables for FTP are set") - utils.Fatal("Error missing environment variables: %s", err) - } - return &fConfig -} -func loadAzureConfig() *AzureConfig { - // Initialize data configs - aConfig := AzureConfig{} - aConfig.containerName = os.Getenv("AZURE_STORAGE_CONTAINER_NAME") - aConfig.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT_NAME") - aConfig.accountKey = os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") - - err := utils.CheckEnvVars(azureVars) - if err != nil { - utils.Error("Please make sure all required environment variables for Azure Blob storage are set") - utils.Fatal("Error missing environment variables: %s", err) - } - return &aConfig -} - -func initAWSConfig() *AWSConfig { - // Initialize AWS configs - aConfig := AWSConfig{} - aConfig.endpoint = utils.GetEnvVariable("AWS_S3_ENDPOINT", "S3_ENDPOINT") - aConfig.accessKey = utils.GetEnvVariable("AWS_ACCESS_KEY", "ACCESS_KEY") - aConfig.secretKey = utils.GetEnvVariable("AWS_SECRET_KEY", "SECRET_KEY") - aConfig.bucket = utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME") - aConfig.remotePath = utils.GetEnvVariable("AWS_S3_PATH", "S3_PATH") - - aConfig.region = os.Getenv("AWS_REGION") - disableSsl, err := strconv.ParseBool(os.Getenv("AWS_DISABLE_SSL")) - if err != nil { - disableSsl = false - } - forcePathStyle, err := strconv.ParseBool(os.Getenv("AWS_FORCE_PATH_STYLE")) - if err != nil { - forcePathStyle = false - } - aConfig.disableSsl = disableSsl - aConfig.forcePathStyle = forcePathStyle - err = utils.CheckEnvVars(awsVars) - if err != nil { - utils.Error("Please make sure all required environment variables for AWS S3 are set") - utils.Fatal("Error checking environment variables: %s", err) - } - return &aConfig -} -func initBackupConfig(cmd *cobra.Command) *BackupConfig { - utils.SetEnv("STORAGE_PATH", storagePath) - utils.GetEnv(cmd, "cron-expression", "BACKUP_CRON_EXPRESSION") - utils.GetEnv(cmd, "path", "REMOTE_PATH") - utils.GetEnv(cmd, "config", "BACKUP_CONFIG_FILE") - // Get flag value and set env - remotePath := utils.GetEnvVariable("REMOTE_PATH", "SSH_REMOTE_PATH") - storage = utils.GetEnv(cmd, "storage", "STORAGE") - prune := false - backupRetention := utils.GetIntEnv("BACKUP_RETENTION_DAYS") - if backupRetention > 0 { - prune = true - } - disableCompression, _ = cmd.Flags().GetBool("disable-compression") - customName, _ := cmd.Flags().GetString("custom-name") - all, _ := cmd.Flags().GetBool("all-databases") - allInOne, _ := cmd.Flags().GetBool("all-in-one") - if allInOne { - all = true - } - _, _ = cmd.Flags().GetString("mode") - passphrase := os.Getenv("GPG_PASSPHRASE") - _ = utils.GetEnv(cmd, "path", "AWS_S3_PATH") - cronExpression := os.Getenv("BACKUP_CRON_EXPRESSION") - - publicKeyFile, err := checkPubKeyFile(os.Getenv("GPG_PUBLIC_KEY")) - if err == nil { - encryption = true - usingKey = true - } else if passphrase != "" { - encryption = true - usingKey = false - } - // Initialize backup configs - config := BackupConfig{} - config.backupRetention = backupRetention - config.disableCompression = disableCompression - config.prune = prune - config.storage = storage - config.encryption = encryption - config.remotePath = remotePath - config.passphrase = passphrase - config.publicKey = publicKeyFile - config.usingKey = usingKey - config.cronExpression = cronExpression - config.all = all - config.allInOne = allInOne - config.customName = customName - return &config -} - -type RestoreConfig struct { - s3Path string - remotePath string - storage string - file string - bucket string - usingKey bool - passphrase string - privateKey string -} - -func initRestoreConfig(cmd *cobra.Command) *RestoreConfig { - utils.SetEnv("STORAGE_PATH", storagePath) - utils.GetEnv(cmd, "path", "REMOTE_PATH") - - // Get flag value and set env - s3Path := utils.GetEnv(cmd, "path", "AWS_S3_PATH") - remotePath := utils.GetEnvVariable("REMOTE_PATH", "SSH_REMOTE_PATH") - storage = utils.GetEnv(cmd, "storage", "STORAGE") - file = utils.GetEnv(cmd, "file", "FILE_NAME") - bucket := utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME") - passphrase := os.Getenv("GPG_PASSPHRASE") - privateKeyFile, err := checkPrKeyFile(os.Getenv("GPG_PRIVATE_KEY")) - if err == nil { - usingKey = true - } else if passphrase != "" { - usingKey = false - } - - // Initialize restore configs - rConfig := RestoreConfig{} - rConfig.s3Path = s3Path - rConfig.remotePath = remotePath - rConfig.storage = storage - rConfig.bucket = bucket - rConfig.file = file - rConfig.storage = storage - rConfig.passphrase = passphrase - rConfig.usingKey = usingKey - rConfig.privateKey = privateKeyFile - return &rConfig -} -func initTargetDbConfig() *targetDbConfig { - tdbConfig := targetDbConfig{} - tdbConfig.targetDbHost = os.Getenv("TARGET_DB_HOST") - tdbConfig.targetDbPort = utils.EnvWithDefault("TARGET_DB_PORT", "3306") - tdbConfig.targetDbName = os.Getenv("TARGET_DB_NAME") - tdbConfig.targetDbUserName = os.Getenv("TARGET_DB_USERNAME") - tdbConfig.targetDbPassword = os.Getenv("TARGET_DB_PASSWORD") - - err := utils.CheckEnvVars(tdbRVars) - if err != nil { - utils.Error("Please make sure all required environment variables for the target database are set") - utils.Fatal("Error checking target database environment variables: %s", err) - } - return &tdbConfig -} -func loadConfigFile() (string, error) { - backupConfigFile, err := checkConfigFile(os.Getenv("BACKUP_CONFIG_FILE")) - if err == nil { - return backupConfigFile, nil - } - return "", fmt.Errorf("backup config file not found") -} diff --git a/pkg/helper.go b/pkg/helper.go deleted file mode 100644 index 298dd1e..0000000 --- a/pkg/helper.go +++ /dev/null @@ -1,203 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "bytes" - "errors" - "fmt" - goutils "github.com/jkaninda/go-utils" - "github.com/jkaninda/mysql-bkup/utils" - "gopkg.in/yaml.v3" - "os" - "os/exec" - "path/filepath" - "strings" -) - -func intro() { - fmt.Println("Starting MYSQL-BKUP...") - fmt.Printf("Version: %s\n", utils.Version) - fmt.Println("Copyright (c) 2024 Jonas Kaninda") -} - -// copyToTmp copy file to temporary directory -func deleteTemp() { - utils.Info("Deleting %s ...", tmpPath) - err := filepath.Walk(tmpPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // Check if the current item is a file - if !info.IsDir() { - // Delete the file - err = os.Remove(path) - if err != nil { - return err - } - } - return nil - }) - if err != nil { - utils.Error("Error deleting files: %v", err) - } else { - utils.Info("Deleting %s ... done", tmpPath) - } -} - -// TestDatabaseConnection tests the database connection -func testDatabaseConnection(db *dbConfig) error { - // Create the mysql client config file - if err := createMysqlClientConfigFile(*db); err != nil { - return errors.New(err.Error()) - } - utils.Info("Connecting to %s database ...", db.dbName) - // Set database name for notification error - utils.DatabaseName = db.dbName - - // Prepare the command to test the database connection - cmd := exec.Command("mariadb", fmt.Sprintf("--defaults-file=%s", mysqlClientConfig), db.dbName, "-e", "quit") - // Capture the output - var out bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &out - - // Run the command - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to connect to database %s: %v, output: %s", db.dbName, err, out.String()) - } - - utils.Info("Successfully connected to %s database", db.dbName) - return nil -} - -// checkPubKeyFile checks gpg public key -func checkPubKeyFile(pubKey string) (string, error) { - // Define possible key file names - keyFiles := []string{filepath.Join(gpgHome, "public_key.asc"), filepath.Join(gpgHome, "public_key.gpg"), pubKey} - - // Loop through key file names and check if they exist - for _, keyFile := range keyFiles { - if _, err := os.Stat(keyFile); err == nil { - // File exists - return keyFile, nil - } else if os.IsNotExist(err) { - // File does not exist, continue to the next one - continue - } else { - // An unexpected error occurred - return "", err - } - } - - // Return an error if neither file exists - return "", fmt.Errorf("no public key file found") -} - -// checkPrKeyFile checks private key -func checkPrKeyFile(prKey string) (string, error) { - // Define possible key file names - keyFiles := []string{filepath.Join(gpgHome, "private_key.asc"), filepath.Join(gpgHome, "private_key.gpg"), prKey} - - // Loop through key file names and check if they exist - for _, keyFile := range keyFiles { - if _, err := os.Stat(keyFile); err == nil { - // File exists - return keyFile, nil - } else if os.IsNotExist(err) { - // File does not exist, continue to the next one - continue - } else { - // An unexpected error occurred - return "", err - } - } - - // Return an error if neither file exists - return "", fmt.Errorf("no public key file found") -} - -// readConf reads config file and returns Config -func readConf(configFile string) (*Config, error) { - if utils.FileExists(configFile) { - buf, err := os.ReadFile(configFile) - if err != nil { - return nil, err - } - - c := &Config{} - err = yaml.Unmarshal(buf, c) - if err != nil { - return nil, fmt.Errorf("in file %q: %w", configFile, err) - } - - return c, err - } - return nil, fmt.Errorf("config file %q not found", configFile) -} - -// checkConfigFile checks config files and returns one config file -func checkConfigFile(filePath string) (string, error) { - // Remove the quotes - filePath = strings.Trim(filePath, `"`) - // Define possible config file names - configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath} - - // Loop through config file names and check if they exist - for _, configFile := range configFiles { - if _, err := os.Stat(configFile); err == nil { - // File exists - return configFile, nil - } else if os.IsNotExist(err) { - // File does not exist, continue to the next one - continue - } else { - // An unexpected error occurred - return "", err - } - } - - // Return an error if neither file exists - return "", fmt.Errorf("no config file found") -} -func RemoveLastExtension(filename string) string { - if idx := strings.LastIndex(filename, "."); idx != -1 { - return filename[:idx] - } - return filename -} - -// Create mysql client config file -func createMysqlClientConfigFile(db dbConfig) error { - caCertPath := goutils.GetStringEnvWithDefault("DB_SSL_CA", "/etc/ssl/certs/ca-certificates.crt") - sslMode := goutils.GetStringEnvWithDefault("DB_SSL_MODE", "0") - // Create the mysql client config file - mysqlClientConfigFile := filepath.Join(tmpPath, "my.cnf") - mysqlCl := fmt.Sprintf("[client]\nhost=%s\nport=%s\nuser=%s\npassword=%s\nssl-ca=%s\nssl=%s\n", db.dbHost, db.dbPort, db.dbUserName, db.dbPassword, caCertPath, sslMode) - if err := os.WriteFile(mysqlClientConfigFile, []byte(mysqlCl), 0644); err != nil { - return fmt.Errorf("failed to create mysql client config file: %v", err) - } - return nil -} diff --git a/pkg/migrate.go b/pkg/migrate.go deleted file mode 100644 index e6d7fb7..0000000 --- a/pkg/migrate.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "fmt" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" - "time" -) - -func StartMigration(cmd *cobra.Command) { - intro() - utils.Info("Starting database migration...") - // Get DB config - dbConf = initDbConfig(cmd) - targetDbConf = initTargetDbConfig() - - // Defining the target database variables - newDbConfig := dbConfig{} - newDbConfig.dbHost = targetDbConf.targetDbHost - newDbConfig.dbPort = targetDbConf.targetDbPort - newDbConfig.dbName = targetDbConf.targetDbName - newDbConfig.dbUserName = targetDbConf.targetDbUserName - newDbConfig.dbPassword = targetDbConf.targetDbPassword - - // Generate file name - backupFileName := fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20060102_150405")) - conf := &RestoreConfig{} - conf.file = backupFileName - // Backup source Database - err := BackupDatabase(dbConf, backupFileName, true, false, false) - if err != nil { - utils.Fatal("Error backing up database: %s", err) - } - // Restore source database into target database - utils.Info("Restoring [%s] database into [%s] database...", dbConf.dbName, targetDbConf.targetDbName) - RestoreDatabase(&newDbConfig, conf) - utils.Info("[%s] database has been restored into [%s] database", dbConf.dbName, targetDbConf.targetDbName) - utils.Info("Database migration completed.") -} diff --git a/pkg/remote.go b/pkg/remote.go deleted file mode 100644 index 2989b21..0000000 --- a/pkg/remote.go +++ /dev/null @@ -1,228 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "fmt" - "github.com/jkaninda/go-storage/pkg/ftp" - "github.com/jkaninda/go-storage/pkg/ssh" - goutils "github.com/jkaninda/go-utils" - "github.com/jkaninda/mysql-bkup/utils" - - "os" - "path/filepath" - "time" -) - -func sshBackup(db *dbConfig, config *BackupConfig) { - utils.Info("Backup database to Remote server") - // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) - if err != nil { - recoverMode(err, "Error backing up database") - return - } - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") - } - utils.Info("Uploading backup archive to remote storage ... ") - sshConfig, err := loadSSHConfig() - if err != nil { - utils.Fatal("Error loading ssh config: %s", err) - } - - sshStorage, err := ssh.NewStorage(ssh.Config{ - Host: sshConfig.hostName, - Port: sshConfig.port, - User: sshConfig.user, - Password: sshConfig.password, - IdentifyFile: sshConfig.identifyFile, - RemotePath: config.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating SSH storage: %s", err) - } - err = sshStorage.Copy(finalFileName) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - // Get backup info - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - utils.Info("Backup name is %s", finalFileName) - utils.Info("Backup size: %s", utils.ConvertBytes(uint64(backupSize))) - utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) - - // Delete backup file from tmp folder - err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error deleting file: %v", err) - - } - if config.prune { - err := sshStorage.Prune(config.backupRetention) - if err != nil { - utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - - } - utils.Info("Uploading backup archive to remote storage ... done ") - duration := goutils.FormatDuration(time.Since(startTime), 0) - - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: utils.ConvertBytes(uint64(backupSize)), - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(config.remotePath, finalFileName), - Duration: duration, - }) - // Delete temp - deleteTemp() - utils.Info("The backup of the %s database has been completed in %s", db.dbName, duration) - -} -func remoteRestore(db *dbConfig, conf *RestoreConfig) { - utils.Info("Restore database from remote server") - sshConfig, err := loadSSHConfig() - if err != nil { - utils.Fatal("Error loading ssh config: %s", err) - } - - sshStorage, err := ssh.NewStorage(ssh.Config{ - Host: sshConfig.hostName, - Port: sshConfig.port, - User: sshConfig.user, - Password: sshConfig.password, - IdentifyFile: sshConfig.identifyFile, - RemotePath: conf.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating SSH storage: %s", err) - } - err = sshStorage.CopyFrom(conf.file) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - RestoreDatabase(db, conf) -} -func ftpRestore(db *dbConfig, conf *RestoreConfig) { - utils.Info("Restore database from FTP server") - ftpConfig := loadFtpConfig() - ftpStorage, err := ftp.NewStorage(ftp.Config{ - Host: ftpConfig.host, - Port: ftpConfig.port, - User: ftpConfig.user, - Password: ftpConfig.password, - RemotePath: conf.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating SSH storage: %s", err) - } - err = ftpStorage.CopyFrom(conf.file) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - RestoreDatabase(db, conf) -} -func ftpBackup(db *dbConfig, config *BackupConfig) { - utils.Info("Backup database to the remote FTP server") - - // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) - if err != nil { - recoverMode(err, "Error backing up database") - return - } - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") - } - utils.Info("Uploading backup archive to the remote FTP server ... ") - utils.Info("Backup name is %s", finalFileName) - ftpConfig := loadFtpConfig() - ftpStorage, err := ftp.NewStorage(ftp.Config{ - Host: ftpConfig.host, - Port: ftpConfig.port, - User: ftpConfig.user, - Password: ftpConfig.password, - RemotePath: config.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating SSH storage: %s", err) - } - err = ftpStorage.Copy(finalFileName) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) - // Get backup info - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - // Delete backup file from tmp folder - err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error deleting file: %v", err) - - } - if config.prune { - err := ftpStorage.Prune(config.backupRetention) - if err != nil { - utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - - } - utils.Info("Backup name is %s", finalFileName) - utils.Info("Backup size: %s", utils.ConvertBytes(uint64(backupSize))) - utils.Info("Uploading backup archive to the remote FTP server ... done ") - duration := goutils.FormatDuration(time.Since(startTime), 0) - - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: utils.ConvertBytes(uint64(backupSize)), - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(config.remotePath, finalFileName), - Duration: duration, - }) - // Delete temp - deleteTemp() - utils.Info("The backup of the %s database has been completed in %s", db.dbName, duration) -} diff --git a/pkg/restore.go b/pkg/restore.go deleted file mode 100644 index 42633fc..0000000 --- a/pkg/restore.go +++ /dev/null @@ -1,153 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "fmt" - "github.com/jkaninda/encryptor" - "github.com/jkaninda/go-storage/pkg/local" - "github.com/jkaninda/mysql-bkup/utils" - "github.com/spf13/cobra" - "os" - "os/exec" - "path/filepath" -) - -func StartRestore(cmd *cobra.Command) { - intro() - dbConf = initDbConfig(cmd) - restoreConf := initRestoreConfig(cmd) - - switch restoreConf.storage { - case "local": - localRestore(dbConf, restoreConf) - case "s3", "S3": - s3Restore(dbConf, restoreConf) - case "ssh", "SSH", "remote": - remoteRestore(dbConf, restoreConf) - case "ftp", "FTP": - ftpRestore(dbConf, restoreConf) - case "azure": - azureRestore(dbConf, restoreConf) - default: - localRestore(dbConf, restoreConf) - } -} -func localRestore(dbConf *dbConfig, restoreConf *RestoreConfig) { - utils.Info("Restore database from local") - basePath := filepath.Dir(restoreConf.file) - fileName := filepath.Base(restoreConf.file) - restoreConf.file = fileName - if basePath == "" || basePath == "." { - basePath = storagePath - } - localStorage := local.NewStorage(local.Config{ - RemotePath: basePath, - LocalPath: tmpPath, - }) - err := localStorage.CopyFrom(fileName) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - RestoreDatabase(dbConf, restoreConf) - -} - -// RestoreDatabase restores the database from a backup file -func RestoreDatabase(db *dbConfig, conf *RestoreConfig) { - if conf.file == "" { - utils.Fatal("Error, file required") - } - - filePath := filepath.Join(tmpPath, conf.file) - rFile, err := os.ReadFile(filePath) - if err != nil { - utils.Fatal("Error reading backup file: %v", err) - } - - extension := filepath.Ext(filePath) - outputFile := RemoveLastExtension(filePath) - - if extension == ".gpg" { - decryptBackup(conf, rFile, outputFile) - } - - restorationFile := filepath.Join(tmpPath, conf.file) - if !utils.FileExists(restorationFile) { - utils.Fatal("File not found: %s", restorationFile) - } - - if err := testDatabaseConnection(db); err != nil { - utils.Fatal("Error connecting to the database: %v", err) - } - - utils.Info("Restoring database...") - restoreDatabaseFile(db, restorationFile) -} - -func decryptBackup(conf *RestoreConfig, rFile []byte, outputFile string) { - if conf.usingKey { - utils.Info("Decrypting backup using private key...") - prKey, err := os.ReadFile(conf.privateKey) - if err != nil { - utils.Fatal("Error reading private key: %v", err) - } - if err := encryptor.DecryptWithPrivateKey(rFile, outputFile, prKey, conf.passphrase); err != nil { - utils.Fatal("Error decrypting backup: %v", err) - } - } else { - if conf.passphrase == "" { - utils.Fatal("Passphrase or private key required for GPG file.") - } - utils.Info("Decrypting backup using passphrase...") - if err := encryptor.Decrypt(rFile, outputFile, conf.passphrase); err != nil { - utils.Fatal("Error decrypting file: %v", err) - } - conf.file = RemoveLastExtension(conf.file) - } -} - -func restoreDatabaseFile(db *dbConfig, restorationFile string) { - extension := filepath.Ext(restorationFile) - var cmdStr string - - switch extension { - case ".gz": - cmdStr = fmt.Sprintf("zcat %s | mariadb --defaults-file=%s %s", restorationFile, mysqlClientConfig, db.dbName) - case ".sql": - cmdStr = fmt.Sprintf("cat %s | mariadb --defaults-file=%s %s", restorationFile, mysqlClientConfig, db.dbName) - default: - utils.Fatal("Unknown file extension: %s", extension) - } - - cmd := exec.Command("sh", "-c", cmdStr) - output, err := cmd.CombinedOutput() - if err != nil { - utils.Fatal("Error restoring database: %v\nOutput: %s", err, string(output)) - } - - utils.Info("Database has been restored successfully.") - deleteTemp() -} diff --git a/pkg/s3.go b/pkg/s3.go deleted file mode 100644 index be80481..0000000 --- a/pkg/s3.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "fmt" - "github.com/jkaninda/go-storage/pkg/s3" - goutils "github.com/jkaninda/go-utils" - "github.com/jkaninda/mysql-bkup/utils" - - "os" - "path/filepath" - "time" -) - -func s3Backup(db *dbConfig, config *BackupConfig) { - - utils.Info("Backup database to s3 storage") - // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) - if err != nil { - recoverMode(err, "Error backing up database") - return - } - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") - } - utils.Info("Uploading backup archive to remote storage S3 ... ") - awsConfig := initAWSConfig() - if config.remotePath == "" { - config.remotePath = awsConfig.remotePath - } - utils.Info("Backup name is %s", finalFileName) - s3Storage, err := s3.NewStorage(s3.Config{ - Endpoint: awsConfig.endpoint, - Bucket: awsConfig.bucket, - AccessKey: awsConfig.accessKey, - SecretKey: awsConfig.secretKey, - Region: awsConfig.region, - DisableSsl: awsConfig.disableSsl, - ForcePathStyle: awsConfig.forcePathStyle, - RemotePath: config.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating s3 storage: %s", err) - } - err = s3Storage.Copy(finalFileName) - if err != nil { - utils.Fatal("Error copying backup file: %s", err) - } - // Get backup info - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - utils.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - - // Delete backup file from tmp folder - err = utils.DeleteFile(filepath.Join(tmpPath, config.backupFileName)) - if err != nil { - fmt.Println("Error deleting file: ", err) - - } - // Delete old backup - if config.prune { - err := s3Storage.Prune(config.backupRetention) - if err != nil { - utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - } - utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) - utils.Info("Uploading backup archive to remote storage S3 ... done ") - duration := goutils.FormatDuration(time.Since(startTime), 0) - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: utils.ConvertBytes(uint64(backupSize)), - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(config.remotePath, finalFileName), - Duration: duration, - }) - // Delete temp - deleteTemp() - utils.Info("The backup of the %s database has been completed in %s", db.dbName, duration) - -} -func s3Restore(db *dbConfig, conf *RestoreConfig) { - utils.Info("Restore database from s3") - awsConfig := initAWSConfig() - if conf.remotePath == "" { - conf.remotePath = awsConfig.remotePath - } - s3Storage, err := s3.NewStorage(s3.Config{ - Endpoint: awsConfig.endpoint, - Bucket: awsConfig.bucket, - AccessKey: awsConfig.accessKey, - SecretKey: awsConfig.secretKey, - Region: awsConfig.region, - DisableSsl: awsConfig.disableSsl, - ForcePathStyle: awsConfig.forcePathStyle, - RemotePath: conf.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - utils.Fatal("Error creating s3 storage: %s", err) - } - err = s3Storage.CopyFrom(conf.file) - if err != nil { - utils.Fatal("Error download file from S3 storage: %s", err) - } - RestoreDatabase(db, conf) -} diff --git a/pkg/var.go b/pkg/var.go deleted file mode 100644 index a00ecd7..0000000 --- a/pkg/var.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package pkg - -import ( - "path/filepath" - "time" -) - -const tmpPath = "/tmp/backup" -const gpgHome = "/config/gnupg" -const gpgExtension = "gpg" -const timeFormat = "2006-01-02 at 15:04:05" - -var ( - storage = "local" - file = "" - - storagePath = "/backup" - workingDir = "/config" - disableCompression = false - encryption = false - usingKey = false - backupSize int64 = 0 - startTime = time.Now() - backupRescueMode = false - mysqlClientConfig = filepath.Join(tmpPath, "my.cnf") -) - -// dbHVars Required environment variables for database -var dbHVars = []string{ - "DB_HOST", - "DB_PASSWORD", - "DB_USERNAME", -} -var tdbRVars = []string{ - "TARGET_DB_HOST", - "TARGET_DB_NAME", - "TARGET_DB_USERNAME", - "TARGET_DB_PASSWORD", -} - -var dbConf *dbConfig -var targetDbConf *targetDbConfig - -var ftpVars = []string{ - "FTP_HOST_NAME", - "FTP_USER", - "FTP_PASSWORD", - "FTP_PORT", -} -var azureVars = []string{ - "AZURE_STORAGE_CONTAINER_NAME", - "AZURE_STORAGE_ACCOUNT_NAME", - "AZURE_STORAGE_ACCOUNT_KEY", -} - -// AwsVars Required environment variables for AWS S3 storage -var awsVars = []string{ - "AWS_S3_ENDPOINT", - "AWS_S3_BUCKET_NAME", - "AWS_ACCESS_KEY", - "AWS_SECRET_KEY", - "AWS_REGION", -} diff --git a/templates/email-error.tmpl b/templates/email-error.tmpl deleted file mode 100644 index 7dc2fbb..0000000 --- a/templates/email-error.tmpl +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - πŸ”΄ Urgent: Database Backup Failure - - - -

πŸ”΄ Urgent: Database Backup Failure Notification

-

Hi,

-

An error occurred during the database backup process. Please review the details below and take the necessary actions:

- -
-

Failure Details:

-
    -
  • Database Name: {{.DatabaseName}}
    • -
  • Date: {{.EndTime}}
    • -
  • Backup Reference: {{.BackupReference}}
    • -
  • Error Message: {{.Error}}
    • -
-
- -

We recommend investigating the issue as soon as possible to prevent potential data loss or service disruptions.

- -

For more information, visit the mysql-bkup documentation.

- -
- © 2024 mysql-bkup | Automated Backup System -
- - diff --git a/templates/email.tmpl b/templates/email.tmpl deleted file mode 100644 index d27aa43..0000000 --- a/templates/email.tmpl +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - βœ… Database Backup Successful – {{.Database}} - - - -

βœ… Database Backup Successful

-

Hi,

-

The backup process for the {{.Database}} database was successfully completed. Please find the details below:

- -
-

Backup Details:

-
    -
  • Database Name: {{.Database}}
    • -
  • Backup Duration: {{.Duration}}
    • -
  • Backup Storage: {{.Storage}}
    • -
  • Backup Location: {{.BackupLocation}}
    • -
  • Backup Size: {{.BackupSize}}
    • -
  • Backup Reference: {{.BackupReference}}
    • -
-
- -

You can access the backup at the specified location if needed. Thank you for using mysql-bkup.

- - - - diff --git a/templates/telegram-error.tmpl b/templates/telegram-error.tmpl deleted file mode 100644 index b9f5a39..0000000 --- a/templates/telegram-error.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -πŸ”΄ Urgent: Database Backup Failure Notification - -Hi, -An error occurred during the database backup process. -Please review the details below and take the necessary actions: -Failure Details: -- Database Name: {{.DatabaseName}} -- Date: {{.EndTime}} -- Backup Reference: {{.BackupReference}} -- Error Message: {{.Error}} -We recommend investigating the issue as soon as possible to prevent potential data loss or service disruptions. \ No newline at end of file diff --git a/templates/telegram.tmpl b/templates/telegram.tmpl deleted file mode 100644 index 5ddf990..0000000 --- a/templates/telegram.tmpl +++ /dev/null @@ -1,15 +0,0 @@ -βœ… Database Backup Successful - -Hi, -The backup process for the {{.Database}} database was successfully completed. -Please find the details below: - -Backup Details: -- Database Name: {{.Database}} -- Backup Duration: {{.Duration}} -- Backup Storage: {{.Storage}} -- Backup Location: {{.BackupLocation}} -- Backup Size: {{.BackupSize}} -- Backup Reference: {{.BackupReference}} - -You can access the backup at the specified location if needed. \ No newline at end of file diff --git a/utils/config.go b/utils/config.go deleted file mode 100644 index cb09ea3..0000000 --- a/utils/config.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package utils - -import "os" - -type MailConfig struct { - MailHost string - MailPort int - MailUserName string - MailPassword string - MailTo string - MailFrom string - SkipTls bool -} -type NotificationData struct { - File string - BackupSize string - Database string - Duration string - Storage string - BackupLocation string - BackupReference string -} -type ErrorMessage struct { - Database string - EndTime string - Error string - BackupReference string - DatabaseName string -} - -// loadMailConfig gets mail environment variables and returns MailConfig -func loadMailConfig() *MailConfig { - return &MailConfig{ - MailHost: os.Getenv("MAIL_HOST"), - MailPort: GetIntEnv("MAIL_PORT"), - MailUserName: os.Getenv("MAIL_USERNAME"), - MailPassword: os.Getenv("MAIL_PASSWORD"), - MailTo: os.Getenv("MAIL_TO"), - MailFrom: os.Getenv("MAIL_FROM"), - SkipTls: os.Getenv("MAIL_SKIP_TLS") == "false", - } - -} - -// TimeFormat returns the format of the time -func TimeFormat() string { - format := os.Getenv("TIME_FORMAT") - if format == "" { - return "2006-01-02 at 15:04:05" - - } - return format -} - -func backupReference() string { - return os.Getenv("BACKUP_REFERENCE") -} - -const templatePath = "/config/templates" - -var DatabaseName = "" -var vars = []string{ - "TG_TOKEN", - "TG_CHAT_ID", -} -var mailVars = []string{ - "MAIL_HOST", - "MAIL_PORT", - "MAIL_FROM", - "MAIL_TO", -} diff --git a/utils/constant.go b/utils/constant.go deleted file mode 100644 index 6fa2b51..0000000 --- a/utils/constant.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package utils - -const RestoreExample = "restore --dbname database --file db_20231219_022941.sql.gz\n" + - "restore --dbname database --storage s3 --path /custom-path --file db_20231219_022941.sql.gz" -const BackupExample = "backup --dbname database --disable-compression\n" + - "backup --dbname database --storage s3 --path /custom-path --disable-compression" - -const MainExample = "mysql-bkup backup --dbname database --disable-compression\n" + - "backup --dbname database --storage s3 --path /custom-path\n" + - "restore --dbname database --file db_20231219_022941.sql.gz" -const traceLog = "trace" diff --git a/utils/logger.go b/utils/logger.go deleted file mode 100644 index cfc08c0..0000000 --- a/utils/logger.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package utils - -import ( - "fmt" - "log" - "os" - "runtime" - "strings" -) - -// Info returns info log -func Info(msg string, args ...interface{}) { - log.SetOutput(getStd("/dev/stdout")) - logWithCaller("INFO", msg, args...) - -} - -// Warn returns warning log -func Warn(msg string, args ...interface{}) { - log.SetOutput(getStd("/dev/stdout")) - logWithCaller("WARN", msg, args...) - -} - -// Error logs error messages -func Error(msg string, args ...interface{}) { - log.SetOutput(getStd("/dev/stderr")) - logWithCaller("ERROR", msg, args...) -} - -func Fatal(msg string, args ...interface{}) { - log.SetOutput(os.Stdout) - // Format message if there are additional arguments - formattedMessage := msg - if len(args) > 0 { - formattedMessage = fmt.Sprintf(msg, args...) - } - logWithCaller("ERROR", msg, args...) - NotifyError(formattedMessage) - os.Exit(1) -} - -// Helper function to format and log messages with file and line number -func logWithCaller(level, msg string, args ...interface{}) { - // Format message if there are additional arguments - formattedMessage := msg - if len(args) > 0 { - formattedMessage = fmt.Sprintf(msg, args...) - } - - // Get the caller's file and line number (skip 2 frames) - _, file, line, ok := runtime.Caller(2) - if !ok { - file = "unknown" - line = 0 - } - // Log message with caller information if GOMA_LOG_LEVEL is trace - if strings.ToLower(level) != "off" { - if strings.ToLower(level) == traceLog { - log.Printf("%s: %s (File: %s, Line: %d)\n", level, formattedMessage, file, line) - } else { - log.Printf("%s: %s\n", level, formattedMessage) - } - } -} - -func getStd(out string) *os.File { - switch out { - case "/dev/stdout": - return os.Stdout - case "/dev/stderr": - return os.Stderr - case "/dev/stdin": - return os.Stdin - default: - return os.Stdout - - } -} diff --git a/utils/notification.go b/utils/notification.go deleted file mode 100644 index 35a0f54..0000000 --- a/utils/notification.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package utils - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "fmt" - "github.com/go-mail/mail" - "html/template" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "time" -) - -func parseTemplate[T any](data T, fileName string) (string, error) { - // Open the file - tmpl, err := template.ParseFiles(filepath.Join(templatePath, fileName)) - if err != nil { - return "", err - } - - var buf bytes.Buffer - if err = tmpl.Execute(&buf, data); err != nil { - return "", err - } - - return buf.String(), nil -} - -func SendEmail(subject, body string) error { - Info("Start sending email notification....") - config := loadMailConfig() - emails := strings.Split(config.MailTo, ",") - m := mail.NewMessage() - m.SetHeader("From", config.MailFrom) - m.SetHeader("To", emails...) - m.SetHeader("Subject", subject) - m.SetBody("text/html", body) - d := mail.NewDialer(config.MailHost, config.MailPort, config.MailUserName, config.MailPassword) - d.TLSConfig = &tls.Config{InsecureSkipVerify: config.SkipTls} - - if err := d.DialAndSend(m); err != nil { - Error("Error could not send email : %v", err) - return err - } - Info("Email notification has been sent") - return nil - -} -func sendMessage(msg string) error { - - Info("Sending Telegram notification... ") - chatId := os.Getenv("TG_CHAT_ID") - body, _ := json.Marshal(map[string]string{ - "chat_id": chatId, - "text": msg, - }) - url := fmt.Sprintf("%s/sendMessage", getTgUrl()) - // Create an HTTP post request - request, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) - if err != nil { - panic(err) - } - request.Header.Add("Content-Type", "application/json") - client := &http.Client{} - response, err := client.Do(request) - if err != nil { - return err - } - code := response.StatusCode - if code == 200 { - Info("Telegram notification has been sent") - return nil - } else { - body, _ := io.ReadAll(response.Body) - Error("Error could not send message, error: %s", string(body)) - return fmt.Errorf("error could not send message %s", string(body)) - } - -} -func NotifySuccess(notificationData *NotificationData) { - notificationData.BackupReference = backupReference() - // Email notification - err := CheckEnvVars(mailVars) - if err == nil { - body, err := parseTemplate(*notificationData, "email.tmpl") - if err != nil { - Error("Could not parse email template: %v", err) - } - err = SendEmail(fmt.Sprintf("βœ… Database Backup Notification – %s", notificationData.Database), body) - if err != nil { - Error("Could not send email: %v", err) - } - } - // Telegram notification - err = CheckEnvVars(vars) - if err == nil { - message, err := parseTemplate(*notificationData, "telegram.tmpl") - if err != nil { - Error("Could not parse telegram template: %v", err) - } - - err = sendMessage(message) - if err != nil { - Error("Could not send Telegram message: %v", err) - } - } -} -func NotifyError(error string) { - - // Email notification - err := CheckEnvVars(mailVars) - if err == nil { - body, err := parseTemplate(ErrorMessage{ - Error: error, - EndTime: time.Now().Format(TimeFormat()), - BackupReference: os.Getenv("BACKUP_REFERENCE"), - DatabaseName: DatabaseName, - }, "email-error.tmpl") - if err != nil { - Error("Could not parse error template: %v", err) - } - err = SendEmail("πŸ”΄ Urgent: Database Backup Failure Notification", body) - if err != nil { - Error("Could not send email: %v", err) - } - } - // Telegram notification - err = CheckEnvVars(vars) - if err == nil { - message, err := parseTemplate(ErrorMessage{ - Error: error, - EndTime: time.Now().Format(TimeFormat()), - BackupReference: os.Getenv("BACKUP_REFERENCE"), - DatabaseName: DatabaseName, - }, "telegram-error.tmpl") - if err != nil { - Error("Could not parse error template: %v", err) - - } - - err = sendMessage(message) - if err != nil { - Error("Could not send telegram message: %v", err) - } - } -} - -func getTgUrl() string { - return fmt.Sprintf("https://api.telegram.org/bot%s", os.Getenv("TG_TOKEN")) - -} diff --git a/utils/utils.go b/utils/utils.go deleted file mode 100644 index d873b79..0000000 --- a/utils/utils.go +++ /dev/null @@ -1,272 +0,0 @@ -/* -MIT License - -Copyright (c) 2023 Jonas Kaninda - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -package utils - -import ( - "fmt" - "github.com/robfig/cron/v3" - "github.com/spf13/cobra" - "io" - "io/fs" - "os" - "strconv" - "time" -) - -var Version = "" - -// FileExists checks if the file does exist -func FileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -func WriteToFile(filePath, content string) error { - file, err := os.Create(filePath) - if err != nil { - return err - } - defer func(file *os.File) { - err := file.Close() - if err != nil { - return - - } - }(file) - - _, err = file.WriteString(content) - return err -} -func DeleteFile(filePath string) error { - err := os.Remove(filePath) - if err != nil { - return fmt.Errorf("failed to delete file: %v", err) - } - return nil -} -func CopyFile(src, dst string) error { - // Open the source file for reading - sourceFile, err := os.Open(src) - if err != nil { - return fmt.Errorf("failed to open source file: %v", err) - } - defer func(sourceFile *os.File) { - err := sourceFile.Close() - if err != nil { - return - } - }(sourceFile) - - // Create the destination file - destinationFile, err := os.Create(dst) - if err != nil { - return fmt.Errorf("failed to create destination file: %v", err) - } - defer func(destinationFile *os.File) { - err := destinationFile.Close() - if err != nil { - return - - } - }(destinationFile) - - // Copy the content from source to destination - _, err = io.Copy(destinationFile, sourceFile) - if err != nil { - return fmt.Errorf("failed to copy file: %v", err) - } - - // Flush the buffer to ensure all data is written - err = destinationFile.Sync() - if err != nil { - return fmt.Errorf("failed to sync destination file: %v", err) - } - - return nil -} -func ChangePermission(filePath string, mod int) { - if err := os.Chmod(filePath, fs.FileMode(mod)); err != nil { - Fatal("Error changing permissions of %s: %v\n", filePath, err) - } - -} -func IsDirEmpty(name string) (bool, error) { - f, err := os.Open(name) - if err != nil { - return false, err - } - defer func(f *os.File) { - err := f.Close() - if err != nil { - return - } - }(f) - - _, err = f.Readdirnames(1) - if err == nil { - return false, nil - } - return true, nil -} - -func GetEnv(cmd *cobra.Command, flagName, envName string) string { - value, _ := cmd.Flags().GetString(flagName) - if value != "" { - err := os.Setenv(envName, value) - if err != nil { - return value - } - } - return os.Getenv(envName) -} -func FlagGetString(cmd *cobra.Command, flagName string) string { - value, _ := cmd.Flags().GetString(flagName) - if value != "" { - return value - - } - return "" -} -func FlagGetBool(cmd *cobra.Command, flagName string) bool { - value, _ := cmd.Flags().GetBool(flagName) - return value -} - -func SetEnv(key, value string) { - - err := os.Setenv(key, value) - if err != nil { - return - } -} -func GetEnvVariable(envName, oldEnvName string) string { - value := os.Getenv(envName) - if value == "" { - value = os.Getenv(oldEnvName) - if value != "" { - err := os.Setenv(envName, value) - if err != nil { - return value - } - Warn("%s is deprecated, please use %s instead! ", oldEnvName, envName) - } - } - return value -} - -// CheckEnvVars checks if all the specified environment variables are set -func CheckEnvVars(vars []string) error { - missingVars := []string{} - - for _, v := range vars { - if os.Getenv(v) == "" { - missingVars = append(missingVars, v) - } - } - - if len(missingVars) > 0 { - return fmt.Errorf("missing environment variables: %v", missingVars) - } - - return nil -} - -// MakeDir create directory -func MakeDir(dirPath string) error { - err := os.Mkdir(dirPath, 0700) - if err != nil { - return err - } - return nil -} - -// MakeDirAll create directory -func MakeDirAll(dirPath string) error { - err := os.MkdirAll(dirPath, 0700) - if err != nil { - return err - } - return nil -} -func GetIntEnv(envName string) int { - val := os.Getenv(envName) - if val == "" { - return 0 - } - ret, err := strconv.Atoi(val) - if err != nil { - Error("Error: %v", err) - } - return ret -} - -func EnvWithDefault(envName string, defaultValue string) string { - value := os.Getenv(envName) - if value == "" { - return defaultValue - } - return value -} - -// IsValidCronExpression verify cronExpression and returns boolean -func IsValidCronExpression(cronExpr string) bool { - // Parse the cron expression - _, err := cron.ParseStandard(cronExpr) - return err == nil -} - -// CronNextTime returns cronExpression next time -func CronNextTime(cronExpr string) time.Time { - // Parse the cron expression - schedule, err := cron.ParseStandard(cronExpr) - if err != nil { - Error("Error parsing cron expression: %s", err) - return time.Time{} - } - // Get the current time - now := time.Now() - // Get the next scheduled time - next := schedule.Next(now) - return next -} - -// ConvertBytes converts bytes to a human-readable string with the appropriate unit (bytes, MiB, or GiB). -func ConvertBytes(bytes uint64) string { - const ( - MiB = 1024 * 1024 - GiB = MiB * 1024 - ) - switch { - case bytes >= GiB: - return fmt.Sprintf("%.2f GiB", float64(bytes)/float64(GiB)) - case bytes >= MiB: - return fmt.Sprintf("%.2f MiB", float64(bytes)/float64(MiB)) - default: - return fmt.Sprintf("%d bytes", bytes) - } -}