mirror of
https://github.com/jkaninda/mysql-bkup.git
synced 2026-11-13 08:37:15 +00:00
feat: mysql-bkup v2 on bkupkit
MysQL backup and restore built on the shared bkupkit core: streaming end to end, with no dump held in memory or written to local disk.
This commit is contained in:
jkaninda committed
1 parent
ccb3fcdf26
commit
28de0d7b52
65 files changed
+3453
-3871
No files matched your search
+36
-273
@@ -5,285 +5,48 @@ on:
|
||||
branches:
|
||||
- main
|
||||
- nightly
|
||||
- v2
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
env:
|
||||
IMAGE_NAME: mysql-bkup
|
||||
|
||||
jobs:
|
||||
test:
|
||||
unit:
|
||||
name: Vet and unit tests
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:9
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
MYSQL_DATABASE: testdb
|
||||
MYSQL_USER: user
|
||||
MYSQL_PASSWORD: password
|
||||
ports:
|
||||
- 3306:3306
|
||||
options: >-
|
||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uuser -ppassword"
|
||||
--health-interval=10s
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
mysql8:
|
||||
image: mysql:8
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
MYSQL_DATABASE: testdb
|
||||
MYSQL_USER: user
|
||||
MYSQL_PASSWORD: password
|
||||
ports:
|
||||
- 3308:3306
|
||||
options: >-
|
||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uuser -ppassword"
|
||||
--health-interval=10s
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
mysql5:
|
||||
image: mysql:5
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
MYSQL_DATABASE: testdb
|
||||
MYSQL_USER: user
|
||||
MYSQL_PASSWORD: password
|
||||
ports:
|
||||
- 3305:3306
|
||||
options: >-
|
||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uuser -ppassword"
|
||||
--health-interval=10s
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.26'
|
||||
- name: Vet
|
||||
run: |
|
||||
go vet ./...
|
||||
go vet -tags integration ./...
|
||||
- name: Unit tests
|
||||
run: go test -race ./...
|
||||
- name: Env reference is up to date
|
||||
run: go run . config --env-docs > /dev/null
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Create Minio container
|
||||
integration:
|
||||
name: Integration tests (MySQL 8.4 / 8.0, MariaDB 11.8, MinIO)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.26'
|
||||
- uses: docker/setup-buildx-action@v4
|
||||
- name: Build the image
|
||||
run: docker buildx build --build-arg appVersion=test -t mysql-bkup:test --load .
|
||||
- name: Image smoke test
|
||||
run: |
|
||||
docker run -d --rm --name minio \
|
||||
--network host \
|
||||
-p 9000:9000 \
|
||||
-e MINIO_ACCESS_KEY=minioadmin \
|
||||
-e MINIO_SECRET_KEY=minioadmin \
|
||||
-e MINIO_REGION_NAME="eu" \
|
||||
minio/minio server /data
|
||||
echo "Create Minio container completed"
|
||||
- name: Install MinIO Client (mc)
|
||||
run: |
|
||||
curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
|
||||
chmod +x mc
|
||||
sudo mv mc /usr/local/bin/
|
||||
|
||||
- name: Wait for MinIO to be ready
|
||||
run: sleep 5
|
||||
|
||||
- name: Configure MinIO Client
|
||||
run: |
|
||||
mc alias set local http://localhost:9000 minioadmin minioadmin
|
||||
mc alias list
|
||||
|
||||
- name: Create MinIO Bucket
|
||||
run: |
|
||||
mc mb local/backups
|
||||
echo "Bucket backups created successfully."
|
||||
# Build the Docker image
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker buildx build --build-arg appVersion=test -t ${{ env.IMAGE_NAME }}:latest --load .
|
||||
|
||||
- name: Verify Docker images
|
||||
run: |
|
||||
docker images
|
||||
|
||||
- name: Wait for MySQL to be ready
|
||||
run: |
|
||||
docker run --rm --network host mysql:9 mysqladmin ping -h 127.0.0.1 -uuser -ppassword --wait
|
||||
- name: Test restore
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=root \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest restore -f init.sql
|
||||
echo "Database restore completed"
|
||||
- name: Test restore Mysql8
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_PORT=3308 \
|
||||
-e DB_USERNAME=root \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest restore -f init.sql
|
||||
echo "Test restore Mysql8 completed"
|
||||
- name: Test restore Mysql5
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_PORT=3305 \
|
||||
-e DB_USERNAME=root \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest restore -f init.sql
|
||||
echo "Test restore Mysql5 completed"
|
||||
- name: Test backup
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=user \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest backup
|
||||
echo "Database backup completed"
|
||||
- name: Test backup Mysql8
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_PORT=3308 \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=user \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest backup
|
||||
echo "Test backup Mysql8 completed"
|
||||
- name: Test backup Mysql5
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_PORT=3305 \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=user \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest backup
|
||||
echo "Test backup Mysql5 completed"
|
||||
- name: Test encrypted backup
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=user \
|
||||
-e DB_PASSWORD=password \
|
||||
-e GPG_PASSPHRASE=password \
|
||||
${{ env.IMAGE_NAME }}:latest backup -d testdb --disable-compression --custom-name encrypted-bkup
|
||||
echo "Database encrypted backup completed"
|
||||
- name: Test restore encrypted backup | testdb -> testdb2
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=root \
|
||||
-e DB_PASSWORD=password \
|
||||
-e GPG_PASSPHRASE=password \
|
||||
-e DB_NAME=testdb2 \
|
||||
${{ env.IMAGE_NAME }}:latest restore -f /backup/encrypted-bkup.sql.gpg
|
||||
echo "Test restore encrypted backup completed"
|
||||
- name: Test migrate database testdb -> testdb3
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=root \
|
||||
-e DB_PASSWORD=password \
|
||||
-e GPG_PASSPHRASE=password \
|
||||
-e DB_NAME=testdb \
|
||||
-e TARGET_DB_HOST=127.0.0.1 \
|
||||
-e TARGET_DB_PORT=3306 \
|
||||
-e TARGET_DB_NAME=testdb3 \
|
||||
-e TARGET_DB_USERNAME=root \
|
||||
-e TARGET_DB_PASSWORD=password \
|
||||
${{ env.IMAGE_NAME }}:latest migrate
|
||||
echo "Test migrate database testdb -> testdb3 completed"
|
||||
- name: Test backup all databases
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=root \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest backup --all-databases
|
||||
echo "Database backup completed"
|
||||
- name: Test multiple backup
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e TESTDB2_DB_USERNAME=root \
|
||||
-e TESTDB2_DB_PASSWORD=password \
|
||||
-e TESTDB2_DB_HOST=127.0.0.1 \
|
||||
${{ env.IMAGE_NAME }}:latest backup -c /backup/test_config.yaml
|
||||
echo "Database backup completed"
|
||||
- name: Test backup Minio (s3)
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=user \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
-e AWS_S3_ENDPOINT="http://127.0.0.1:9000" \
|
||||
-e AWS_S3_BUCKET_NAME=backups \
|
||||
-e AWS_ACCESS_KEY=minioadmin \
|
||||
-e AWS_SECRET_KEY=minioadmin \
|
||||
-e AWS_DISABLE_SSL="true" \
|
||||
-e AWS_REGION="eu" \
|
||||
-e AWS_FORCE_PATH_STYLE="true" ${{ env.IMAGE_NAME }}:latest backup -s s3 --custom-name minio-backup
|
||||
echo "Test backup Minio (s3) completed"
|
||||
- name: Test restore Minio (s3)
|
||||
run: |
|
||||
docker run --rm --name ${{ env.IMAGE_NAME }} \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=user \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
-e AWS_S3_ENDPOINT="http://127.0.0.1:9000" \
|
||||
-e AWS_S3_BUCKET_NAME=backups \
|
||||
-e AWS_ACCESS_KEY=minioadmin \
|
||||
-e AWS_SECRET_KEY=minioadmin \
|
||||
-e AWS_DISABLE_SSL="true" \
|
||||
-e AWS_REGION="eu" \
|
||||
-e AWS_FORCE_PATH_STYLE="true" ${{ env.IMAGE_NAME }}:latest restore -s s3 -f minio-backup.sql.gz
|
||||
echo "Test backup Minio (s3) completed"
|
||||
- name: Test scheduled backup
|
||||
run: |
|
||||
docker run -d --rm --name ${{ env.IMAGE_NAME }} \
|
||||
-v ./migrations:/backup/ \
|
||||
--network host \
|
||||
-e DB_HOST=127.0.0.1 \
|
||||
-e DB_USERNAME=user \
|
||||
-e DB_PASSWORD=password \
|
||||
-e DB_NAME=testdb \
|
||||
${{ env.IMAGE_NAME }}:latest backup -e "@every 10s"
|
||||
|
||||
echo "Waiting for backup to be done..."
|
||||
sleep 25
|
||||
docker logs ${{ env.IMAGE_NAME }}
|
||||
echo "Test scheduled backup completed"
|
||||
# Cleanup: Stop and remove containers
|
||||
- name: Clean up
|
||||
run: |
|
||||
docker stop ${{ env.IMAGE_NAME }} || true
|
||||
docker rm ${{ env.IMAGE_NAME }} || true
|
||||
docker run --rm mysql-bkup:test version
|
||||
docker run --rm --entrypoint mysqldump mysql-bkup:test --version
|
||||
docker run --rm --entrypoint mariadb-dump mysql-bkup:test --version
|
||||
# Miabi deletes local backups with /bin/sh
|
||||
docker run --rm --entrypoint /bin/sh mysql-bkup:test -c "rm -f /backup/x.sql.gz"
|
||||
- name: Integration tests (client tools from the image)
|
||||
env:
|
||||
IT_CLIENT_IMAGE: mysql-bkup:test
|
||||
run: go test -tags integration -count=1 -timeout 40m -v ./...
|
||||
Reference in new issue
Block a user