diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee8ad36..d975206 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,7 @@ name: Build on: push: - branches: [ "main" ] + branches: [ "main","v1.0"] workflow_dispatch: inputs: docker_tag: @@ -11,9 +11,13 @@ on: type: string env: BUILDKIT_IMAGE: jkaninda/pg-bkup + TAG: v1.0 jobs: docker: runs-on: ubuntu-latest + permissions: + packages: write + contents: read steps: - name: Set up QEMU @@ -27,13 +31,21 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Log in to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v3 with: push: true file: "./docker/Dockerfile" - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | - "${{env.BUILDKIT_IMAGE}}:v0.7" - "${{env.BUILDKIT_IMAGE}}:latest" + "${{env.BUILDKIT_IMAGE}}:${{env.TAG}}" + # "${{env.BUILDKIT_IMAGE}}:latest" + "ghcr.io/${{env.BUILDKIT_IMAGE}}:${{TAG}}" + # "ghcr.io/${{env.BUILDKIT_IMAGE}}:latest" diff --git a/Makefile b/Makefile index 65e4c19..e2823c3 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ docker-build: docker build -f docker/Dockerfile -t jkaninda/pg-bkup:latest . docker-run: docker-build - docker run --rm --network internal --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --prune --keep-last 2 + docker run --rm --network internal --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup backup --prune --keep-last 2 docker-restore: docker-build docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup restore -f ${FILE_NAME} @@ -30,14 +30,17 @@ docker-run-scheduled-s3: docker-build docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage s3 --mode scheduled --path /custom-path --period "* * * * *" docker-run-s3: docker-build - docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage s3 #--path /custom-path + docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage s3 #--path /custom-path docker-restore-s3: docker-build docker run --rm --network internal --privileged --device /dev/fuse --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup restore --storage s3 -f ${FILE_NAME} #--path /custom-path docker-run-ssh: docker-build - docker run --rm --network internal --name pg-bkup -v "/Users/jonas/.ssh/id_ed25519:/tmp/id_ed25519" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage ssh + docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage ssh docker-restore-ssh: docker-build - docker run --rm --network internal --name pg-bkup -v "/Users/jonas/.ssh/id_ed25519:/tmp/id_ed25519" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" jkaninda/pg-bkup bkup restore --storage ssh -f uzaraka_20240731_200104.sql.gz.gpg \ No newline at end of file + docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" jkaninda/pg-bkup bkup restore --storage ssh -f data_20240731_200104.sql.gz.gpg + +run-docs: + cd docs && bundle exec jekyll serve -H 0.0.0.0 -t \ No newline at end of file diff --git a/cmd/backup.go b/cmd/backup.go index 3f5aa7c..5d48819 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -21,8 +21,8 @@ var BackupCmd = &cobra.Command{ func init() { //Backup - BackupCmd.PersistentFlags().StringP("mode", "m", "default", "Set execution mode. default or scheduled") - BackupCmd.PersistentFlags().StringP("period", "", "0 1 * * *", "Set schedule period time") + BackupCmd.PersistentFlags().StringP("mode", "m", "default", "Execution mode. default or scheduled") + BackupCmd.PersistentFlags().StringP("period", "", "0 1 * * *", "Schedule period time") BackupCmd.PersistentFlags().BoolP("prune", "", false, "Delete old backup, default disabled") BackupCmd.PersistentFlags().IntP("keep-last", "", 7, "Delete files created more than specified days ago, default 7 days") BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") diff --git a/cmd/history.go b/cmd/history.go deleted file mode 100644 index cf232e7..0000000 --- a/cmd/history.go +++ /dev/null @@ -1,14 +0,0 @@ -package cmd - -import ( - "github.com/jkaninda/pg-bkup/utils" - "github.com/spf13/cobra" -) - -var HistoryCmd = &cobra.Command{ - Use: "history", - Short: "Show the history of backup", - Run: func(cmd *cobra.Command, args []string) { - utils.ShowHistory() - }, -} diff --git a/cmd/root.go b/cmd/root.go index edb9c10..d3688ec 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,8 +13,8 @@ import ( // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "pg-bkup [Command]", - Short: "PostgreSQL Backup tool, backup database to S3 or Object Storage", - Long: `PostgreSQL Database backup and restoration tool. Backup database to AWS S3 storage or any S3 Alternatives for Object Storage.`, + Short: "PostgreSQL Backup tool, backup database to AWS S3 or SSH Remote Server", + Long: `PostgreSQL Database backup and restoration tool. Backup database to AWS S3 storage, any S3 Alternatives for Object Storage or SSH remote server.`, Example: utils.MainExample, Version: appVersion, } @@ -30,16 +30,13 @@ func Execute() { } func init() { - rootCmd.PersistentFlags().StringP("storage", "s", "local", "Set storage. local or s3") - rootCmd.PersistentFlags().StringP("path", "P", "", "Set s3 path, without file name. for S3 storage only") - rootCmd.PersistentFlags().StringP("dbname", "d", "", "Set database name") - rootCmd.PersistentFlags().IntP("timeout", "t", 30, "Set timeout") - rootCmd.PersistentFlags().IntP("port", "p", 5432, "Set database port") + rootCmd.PersistentFlags().StringP("storage", "s", "local", "Storage. local or s3") + rootCmd.PersistentFlags().StringP("path", "P", "", "AWS S3 path without file name. eg: /custom_path or ssh remote path `/home/foo/backup`") + rootCmd.PersistentFlags().StringP("dbname", "d", "", "Database name") + rootCmd.PersistentFlags().IntP("port", "p", 5432, "Database port") rootCmd.PersistentFlags().StringVarP(&operation, "operation", "o", "", "Set operation, for old version only") rootCmd.AddCommand(VersionCmd) rootCmd.AddCommand(BackupCmd) rootCmd.AddCommand(RestoreCmd) - rootCmd.AddCommand(S3MountCmd) - rootCmd.AddCommand(HistoryCmd) } diff --git a/cmd/s3mount.go b/cmd/s3mount.go deleted file mode 100644 index ab56e0d..0000000 --- a/cmd/s3mount.go +++ /dev/null @@ -1,13 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" -) - -var S3MountCmd = &cobra.Command{ - Use: "s3mount", - Short: "Mount AWS S3 storage", - Run: func(cmd *cobra.Command, args []string) { - //pkg.S3Mount() - }, -} diff --git a/docker/Dockerfile b/docker/Dockerfile index 4e3f512..2bd9779 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,7 +40,7 @@ LABEL author="Jonas Kaninda" RUN apt-get update -qq -RUN apt install postgresql-client postgresql-client-common supervisor cron openssh-client gnupg -y +RUN apt install postgresql-client postgresql-client-common supervisor cron gnupg -y # Clear cache RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..45c1505 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +_site +.sass-cache +.jekyll-metadata diff --git a/docs/404.html b/docs/404.html new file mode 100644 index 0000000..c472b4e --- /dev/null +++ b/docs/404.html @@ -0,0 +1,24 @@ +--- +layout: default +--- + + + +
+

404

+ +

Page not found :(

+

The requested page could not be found.

+
diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 0000000..5e1108f --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,12 @@ +FROM ruby:3.3.4 + +ENV LC_ALL C.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +WORKDIR /usr/src/app + +COPY . ./ +RUN gem install bundler && bundle install + +EXPOSE 4000 \ No newline at end of file diff --git a/docs/Gemfile b/docs/Gemfile new file mode 100644 index 0000000..3347de5 --- /dev/null +++ b/docs/Gemfile @@ -0,0 +1,43 @@ +source "https://rubygems.org" + +# Hello! This is where you manage which Jekyll version is used to run. +# When you want to use a different version, change it below, save the +# file and run `bundle install`. Run Jekyll with `bundle exec`, like so: +# +# bundle exec jekyll serve +# +# This will help ensure the proper Jekyll version is running. +# Happy Jekylling! +gem "jekyll", "~> 3.10.0" + +# This is the default theme for new Jekyll sites. You may change this to anything you like. +gem "minima", "~> 2.0" + +# If you want to use GitHub Pages, remove the "gem "jekyll"" above and +# uncomment the line below. To upgrade, run `bundle update github-pages`. +# gem "github-pages", group: :jekyll_plugins + +# If you have any plugins, put them here! +group :jekyll_plugins do + gem "jekyll-feed", "~> 0.6" +end + +# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem +# and associated library. +platforms :mingw, :x64_mingw, :mswin, :jruby do + gem "tzinfo", ">= 1", "< 3" + gem "tzinfo-data" +end + +# Performance-booster for watching directories on Windows +gem "wdm", "~> 0.1.0", :install_if => Gem.win_platform? + +# kramdown v2 ships without the gfm parser by default. If you're using +# kramdown v1, comment out this line. +gem "kramdown-parser-gfm" + +# Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem +# do not have a Java counterpart. +gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] +gem "just-the-docs" + diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock new file mode 100644 index 0000000..1bf9a5d --- /dev/null +++ b/docs/Gemfile.lock @@ -0,0 +1,116 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + colorator (1.1.0) + concurrent-ruby (1.3.3) + csv (3.3.0) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + eventmachine (1.2.7) + ffi (1.17.0) + ffi (1.17.0-aarch64-linux-gnu) + ffi (1.17.0-aarch64-linux-musl) + ffi (1.17.0-arm-linux-gnu) + ffi (1.17.0-arm-linux-musl) + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86-linux-gnu) + ffi (1.17.0-x86-linux-musl) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) + ffi (1.17.0-x86_64-linux-musl) + forwardable-extended (2.6.0) + http_parser.rb (0.8.0) + i18n (1.14.5) + concurrent-ruby (~> 1.0) + jekyll (3.10.0) + addressable (~> 2.4) + colorator (~> 1.0) + csv (~> 3.0) + em-websocket (~> 0.5) + i18n (>= 0.7, < 2) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + webrick (>= 1.0) + jekyll-feed (0.17.0) + jekyll (>= 3.7, < 5.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + just-the-docs (0.8.2) + jekyll (>= 3.8.5) + jekyll-include-cache + jekyll-seo-tag (>= 2.0) + rake (>= 12.3.1) + kramdown (2.4.0) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.9.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.3.6) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (6.0.1) + rake (13.2.1) + rb-fsevent (0.11.2) + rb-inotify (0.11.1) + ffi (~> 1.0) + rexml (3.3.2) + strscan + rouge (3.30.0) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + strscan (3.1.0) + wdm (0.1.1) + webrick (1.8.1) + +PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + ruby + x86-linux-gnu + x86-linux-musl + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + http_parser.rb (~> 0.6.0) + jekyll (~> 3.10.0) + jekyll-feed (~> 0.6) + just-the-docs + kramdown-parser-gfm + minima (~> 2.0) + tzinfo (>= 1, < 3) + tzinfo-data + wdm (~> 0.1.0) + +BUNDLED WITH + 2.5.16 diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..40ee934 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,69 @@ +# Welcome to Jekyll! +# +# This config file is meant for settings that affect your whole blog, values +# which you are expected to set up once and rarely edit after that. If you find +# yourself editing this file very often, consider using Jekyll's data files +# feature for the data you need to update frequently. +# +# For technical reasons, this file is *NOT* reloaded automatically when you use +# 'bundle exec jekyll serve'. If you change this file, please restart the server process. + +# Site settings +# These are used to personalize your new site. If you look in the HTML files, +# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. +# You can create any custom variable you would like, and they will be accessible +# in the templates via {{ site.myvariable }}. +title: Postgres Backup +email: hi@jonaskaninda.com +description: >- # this means to ignore newlines until "baseurl:" + PostgreSQL Backup and Restoration tool. Backup database to AWS S3 storage or any S3 Alternatives for Object Storage. +baseurl: "" # the subpath of your site, e.g. /blog +url: "" # the base hostname & protocol for your site, e.g. http://example.com +twitter_username: jonaskaninda +github_username: jkaninda + +callouts_level: quiet +callouts: + highlight: + color: yellow + important: + title: Important + color: blue + new: + title: New + color: green + note: + title: Note + color: purple + warning: + title: Warning + color: red +# Build settings +markdown: kramdown +theme: just-the-docs +plugins: + - jekyll-feed +aux_links: + 'GitHub Repository': + - https://github.com/jkaninda/pg-bkup + +nav_external_links: + - title: GitHub Repository + url: https://github.com/jkaninda/pg-bkup + +footer_content: >- + Copyright © 2024 Jonas Kaninda. + Distributed under the MIT License.
+ Something missing, unclear or not working? Open an issue. + +# Exclude from processing. +# The following items will not be processed, by default. Create a custom list +# to override the default setting. +# exclude: +# - Gemfile +# - Gemfile2.lock +# - node_modules +# - vendor/bundle/ +# - vendor/cache/ +# - vendor/gems/ +# - vendor/ruby/ diff --git a/docs/_posts/2024-07-29-welcome-to-jekyll.markdown b/docs/_posts/2024-07-29-welcome-to-jekyll.markdown new file mode 100644 index 0000000..6c2334f --- /dev/null +++ b/docs/_posts/2024-07-29-welcome-to-jekyll.markdown @@ -0,0 +1,25 @@ +--- +layout: post +title: "Welcome to Jekyll!" +date: 2024-07-29 03:36:13 +0200 +categories: jekyll update +--- +You’ll find this post in your `_posts` directory. Go ahead and edit it and re-build the site to see your changes. You can rebuild the site in many different ways, but the most common way is to run `jekyll serve`, which launches a web server and auto-regenerates your site when a file is updated. + +To add new posts, simply add a file in the `_posts` directory that follows the convention `YYYY-MM-DD-name-of-post.ext` and includes the necessary front matter. Take a look at the source for this post to get an idea about how it works. + +Jekyll also offers powerful support for code snippets: + +{% highlight ruby %} +def print_hi(name) + puts "Hi, #{name}" +end +print_hi('Tom') +#=> prints 'Hi, Tom' to STDOUT. +{% endhighlight %} + +Check out the [Jekyll docs][jekyll-docs] for more info on how to get the most out of Jekyll. File all bugs/feature requests at [Jekyll’s GitHub repo][jekyll-gh]. If you have questions, you can ask them on [Jekyll Talk][jekyll-talk]. + +[jekyll-docs]: https://jekyllrb.com/docs/home +[jekyll-gh]: https://github.com/jekyll/jekyll +[jekyll-talk]: https://talk.jekyllrb.com/ diff --git a/docs/docker-compose.yml b/docs/docker-compose.yml new file mode 100644 index 0000000..5ceb7d5 --- /dev/null +++ b/docs/docker-compose.yml @@ -0,0 +1,13 @@ +services: + jekyll: + build: + context: ./ + ports: + - 4000:4000 + environment: + - JEKYLL_ENV=development + volumes: + - .:/usr/src/app + stdin_open: true + tty: true + command: bundle exec jekyll serve -H 0.0.0.0 -t \ No newline at end of file diff --git a/docs/how-tos/backup-to-s3.md b/docs/how-tos/backup-to-s3.md new file mode 100644 index 0000000..c802290 --- /dev/null +++ b/docs/how-tos/backup-to-s3.md @@ -0,0 +1,141 @@ +--- +title: Backup to AWS S3 +layout: default +parent: How Tos +nav_order: 2 +--- +# Backup to AWS S3 + +{: .note } +As described on local backup section, to change the storage of you backup and use S3 as storage. You need to add `--storage s3` (-s s3). +You can also specify a specify folder where you want to save you data by adding `--path /my-custom-path` flag. + + +## Backup to S3 + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup --storage s3 -d database --path /my-custom-path + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## AWS configurations + - AWS_S3_ENDPOINT=https://s3.amazonaws.com + - AWS_S3_BUCKET_NAME=backup + - AWS_REGION="us-west-2" + - AWS_ACCESS_KEY=xxxx + - AWS_SECRET_KEY=xxxxx + ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true + - AWS_DISABLE_SSL="false" + + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + +### Recurring backups to S3 + +As explained above, you need just to add AWS environment variables and specify the storage type `--storage s3`. +In case you need to use recurring backups, you can use `--mode scheduled` and specify the periodical backup time by adding `--period "0 1 * * *"` flag as described below. + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup --storage s3 -d my-database --mode scheduled --period "0 1 * * *" + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## AWS configurations + - AWS_S3_ENDPOINT=https://s3.amazonaws.com + - AWS_S3_BUCKET_NAME=backup + - AWS_REGION="us-west-2" + - AWS_ACCESS_KEY=xxxx + - AWS_SECRET_KEY=xxxxx + ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true + - AWS_DISABLE_SSL="false" + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + +## Deploy on Kubernetes + +For Kubernetes, you don't need to run it in scheduled mode. You can deploy it as CronJob. + +### Simple Kubernetes CronJob usage: + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: bkup-job +spec: + schedule: "0 1 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-bkup + image: jkaninda/pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup -s s3 --path /custom_path + env: + - name: DB_PORT + value: "5432" + - name: DB_HOST + value: "" + - name: DB_NAME + value: "" + - name: DB_USERNAME + value: "" + # Please use secret! + - name: DB_PASSWORD + value: "" + - name: ACCESS_KEY + value: "" + - name: AWS_S3_ENDPOINT + value: "https://s3.amazonaws.com" + - name: AWS_S3_BUCKET_NAME + value: "xxx" + - name: AWS_REGION + value: "us-west-2" + - name: AWS_ACCESS_KEY + value: "xxxx" + - name: AWS_SECRET_KEY + value: "xxxx" + - name: AWS_DISABLE_SSL + value: "false" + restartPolicy: OnFailure +``` \ No newline at end of file diff --git a/docs/how-tos/backup-to-ssh.md b/docs/how-tos/backup-to-ssh.md new file mode 100644 index 0000000..4d75030 --- /dev/null +++ b/docs/how-tos/backup-to-ssh.md @@ -0,0 +1,146 @@ +--- +title: Backup to SSH +layout: default +parent: How Tos +nav_order: 3 +--- +# Backup to SSH remote server + + +As described for s3 backup section, to change the storage of you backup and use S3 as storage. You need to add `--storage ssh` or `--storage remote`. +You need to add the full remote path by adding `--path /home/jkaninda/backups` flag or using `SSH_REMOTE_PATH` environment variable. + +{: .note } +These environment variables are required for SSH backup `SSH_HOST_NAME`, `SSH_USER`, `SSH_REMOTE_PATH`, `SSH_IDENTIFY_FILE`, `SSH_PORT` or `SSH_PASSWORD` if you dont use a private key to access to your server. +Accessing the remote server using password is not recommended, use private key instead. + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup --storage remote -d database + volumes: + - ./id_ed25519:/tmp/id_ed25519" + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## SSH config + - SSH_HOST_NAME="hostname" + - SSH_PORT=22 + - SSH_USER=user + - SSH_REMOTE_PATH=/home/jkaninda/backups + - SSH_IDENTIFY_FILE=/tmp/id_ed25519 + ## We advise you to use a private jey instead of password + #- SSH_PASSWORD=password + + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + + +### Recurring backups to SSH remote server + +As explained above, you need just to add required environment variables and specify the storage type `--storage ssh`. +You can use `--mode scheduled` and specify the periodical backup time by adding `--period "0 1 * * *"` flag as described below. + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup -d database --storage s3 --mode scheduled --period "0 1 * * *" + volumes: + - ./id_ed25519:/tmp/id_ed25519" + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## SSH config + - SSH_HOST_NAME="hostname" + - SSH_PORT=22 + - SSH_USER=user + - SSH_REMOTE_PATH=/home/jkaninda/backups + - SSH_IDENTIFY_FILE=/tmp/id_ed25519 + ## We advise you to use a private jey instead of password + #- SSH_PASSWORD=password + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + +## Deploy on Kubernetes + +For Kubernetes, you don't need to run it in scheduled mode. +You can deploy it as CronJob. + +Simple Kubernetes CronJob usage: + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: bkup-job +spec: + schedule: "0 1 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-bkup + image: jkaninda/pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup -s s3 --path /custom_path + env: + - name: DB_PORT + value: "5432" + - name: DB_HOST + value: "" + - name: DB_NAME + value: "" + - name: DB_USERNAME + value: "" + # Please use secret! + - name: DB_PASSWORD + value: "" + - name: SSH_HOST_NAME + value: "" + - name: SSH_PORT + value: "22" + - name: SSH_USER + value: "xxx" + - name: SSH_REMOTE_PATH + value: "/home/jkaninda/backups" + - name: AWS_ACCESS_KEY + value: "xxxx" + - name: SSH_IDENTIFY_FILE + value: "/home/jkaninda/backups" + restartPolicy: OnFailure +``` \ No newline at end of file diff --git a/docs/how-tos/backup.md b/docs/how-tos/backup.md new file mode 100644 index 0000000..bf43d42 --- /dev/null +++ b/docs/how-tos/backup.md @@ -0,0 +1,89 @@ +--- +title: Backup +layout: default +parent: How Tos +nav_order: 1 +--- + +# Backup database + +To backup the database, you need to add `backup` subcommand to `pg-bkup` or `bkup`. + +{: .note } +The default storage is local storage mounted to __/backup__. The backup is compressed by default using gzip. The flag __`disable-compression`__ can be used when you need to disable backup compression. + +{: .warning } +Creating a user for backup tasks who has read-only access is recommended! + +The backup process can be run in scheduled mode for the recurring backups. +It handles __recurring__ backups of postgres database on Docker and can be deployed as __CronJob on Kubernetes__ using local, AWS S3 or SSH compatible storage. + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup -d database + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + +### Backup using Docker CLI + +```shell + docker run --rm --network your_network_name \ + -v $PWD/backup:/backup/ \ + -e "DB_HOST=dbhost" \ + -e "DB_USERNAME=username" \ + -e "DB_PASSWORD=password" \ + jkaninda/pg-bkup pg-bkup backup -d database_name +``` + +In case you need to use recurring backups, you can use `--mode scheduled` and specify the periodical backup time by adding `--period "0 1 * * *"` flag as described below. + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup -d database --mode scheduled --period "0 1 * * *" + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + diff --git a/docs/how-tos/encrypt-backup.md b/docs/how-tos/encrypt-backup.md new file mode 100644 index 0000000..2065dca --- /dev/null +++ b/docs/how-tos/encrypt-backup.md @@ -0,0 +1,44 @@ +--- +title: Encrypt backups using GPG +layout: default +parent: How Tos +nav_order: 7 +--- +# Encrypt backup + +The image supports encrypting backups using GPG out of the box. In case a `GPG_PASSPHRASE` environment variable is set, the backup archive will be encrypted using the given key and saved as a sql.gpg file instead or sql.gz.gpg. + +{: .warning } +To restore an encrypted backup, you need to provide the same GPG passphrase used during backup process. + +### Backup + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup -d database + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## Required to encrypt backup + - GPG_PASSPHRASE=my-secure-passphrase + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` \ No newline at end of file diff --git a/docs/how-tos/index.md b/docs/how-tos/index.md new file mode 100644 index 0000000..e869ec7 --- /dev/null +++ b/docs/how-tos/index.md @@ -0,0 +1,8 @@ +--- +title: How Tos +layout: default +nav_order: 3 +has_children: true +--- + +## How Tos \ No newline at end of file diff --git a/docs/how-tos/restore-from-s3.md b/docs/how-tos/restore-from-s3.md new file mode 100644 index 0000000..9f671d5 --- /dev/null +++ b/docs/how-tos/restore-from-s3.md @@ -0,0 +1,51 @@ +--- +title: Restore database from AWS S3 +layout: default +parent: How Tos +nav_order: 5 +--- + +# Restore database from S3 storage + +To restore the database, you need to add `restore` subcommand to `pg-bkup` or `bkup` and specify the file to restore by adding `--file store_20231219_022941.sql.gz`. + +{: .note } +It supports __.sql__ and __.sql.gz__ compressed file. + +### Restore + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup restore --storage s3 -d my-database -f store_20231219_022941.sql.gz --path /my-custom-path + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## AWS configurations + - AWS_S3_ENDPOINT=https://s3.amazonaws.com + - AWS_S3_BUCKET_NAME=backup + - AWS_REGION="us-west-2" + - AWS_ACCESS_KEY=xxxx + - AWS_SECRET_KEY=xxxxx + ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true + - AWS_DISABLE_SSL="false" + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` \ No newline at end of file diff --git a/docs/how-tos/restore-from-ssh.md b/docs/how-tos/restore-from-ssh.md new file mode 100644 index 0000000..1ec6845 --- /dev/null +++ b/docs/how-tos/restore-from-ssh.md @@ -0,0 +1,50 @@ +--- +title: Restore database from SSH +layout: default +parent: How Tos +nav_order: 6 +--- +# Restore database from SSH remote server + +To restore the database from your remote server, you need to add `restore` subcommand to `pg-bkup` or `bkup` and specify the file to restore by adding `--file store_20231219_022941.sql.gz`. + +{: .note } +It supports __.sql__ and __.sql.gz__ compressed file. + +### Restore + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup restore --storage ssh -d my-database -f store_20231219_022941.sql.gz --path /home/jkaninda/backups + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## SSH config + - SSH_HOST_NAME="hostname" + - SSH_PORT=22 + - SSH_USER=user + - SSH_REMOTE_PATH=/home/jkaninda/backups + - SSH_IDENTIFY_FILE=/tmp/id_ed25519 + ## We advise you to use a private jey instead of password + #- SSH_PASSWORD=password + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` \ No newline at end of file diff --git a/docs/how-tos/restore.md b/docs/how-tos/restore.md new file mode 100644 index 0000000..8acc16f --- /dev/null +++ b/docs/how-tos/restore.md @@ -0,0 +1,43 @@ +--- +title: Restore database +layout: default +parent: How Tos +nav_order: 4 +--- + +# Restore database + +To restore the database, you need to add `restore` subcommand to `pg-bkup` or `bkup` and specify the file to restore by adding `--file store_20231219_022941.sql.gz`. + +{: .note } +It supports __.sql__ and __.sql.gz__ compressed file. + +### Restore + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup restore -d database -f store_20231219_022941.sql.gz + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..0e4f20e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,107 @@ +--- +title: Overview +layout: home +nav_order: 1 +--- + +# About pg-bkup +{:.no_toc} +pg-bkup it's a Docker container image that can be used to backup and restore Postgres database. It supports local storage, AWS S3 or any S3 Alternatives for Object Storage, and SSH compatible storage. +It also supports __encrypting__ your backups using GPG. + +We are open to receiving stars, PRs, and issues! + + +{: .fs-6 .fw-300 } + +--- + +The [jkaninda/pg-bkup](https://hub.docker.com/r/jkaninda/pg-bkup) Docker image can be deployed on Docker, Docker Swarm and Kubernetes. +It handles __recurring__ backups of postgres database on Docker and can be deployed as __CronJob on Kubernetes__ using local, AWS S3 or SSH compatible storage. + +It also supports __encrypting__ your backups using GPG. + +{: .note } +Code and documentation for `v1` version on [this branch][v1-branch]. + +[v1-branch]: https://github.com/jkaninda/pg-bkup + +--- + +## Quickstart + +### Simple backup using Docker CLI + +To run a one time backup, bind your local volume to `/backup` in the container and run the `pg-bkup backup` command: + +```shell + docker run --rm --network your_network_name \ + -v $PWD/backup:/backup/ \ + -e "DB_HOST=dbhost" \ + -e "DB_USERNAME=username" \ + -e "DB_PASSWORD=password" \ + jkaninda/pg-bkup pg-bkup backup -d database_name +``` + +Alternatively, pass a `--env-file` in order to use a full config as described below. + + + +Add a `backup` service to your compose setup and mount the volumes you would like to see backed up: + +### Simple backup in docker compose file + +```yaml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=foo + - DB_USERNAME=bar + - DB_PASSWORD=password + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + +## Available image registries + +This Docker image is published to both Docker Hub and the GitHub container registry. +Depending on your preferences and needs, you can reference both `jkaninda/pg-bkup` as well as `ghcr.io/jkaninda/pg-bkup`: + +``` +docker pull jkaninda/pg-bkup:v1.0 +docker pull ghcr.io/jkaninda/pg-bkup:v1.0 +``` + +Documentation references Docker Hub, but all examples will work using ghcr.io just as well. + +## Supported Engines + +This image is developed and tested against the Docker CE engine and Kubernetes exclusively. +While it may work against different implementations, there are no guarantees about support for non-Docker engines. + +## References + +We decided to publish this image as a simpler and more lightweight alternative because of the following requirements: + +- The original image is based on `ubuntu` and requires additional tools, making it heavy. +- This image is written in Go. +- `arm64` and `arm/v7` architectures are supported. +- Docker in Swarm mode is supported. +- Kubernetes is supported. diff --git a/docs/old-version/index.md b/docs/old-version/index.md new file mode 100644 index 0000000..f9b0798 --- /dev/null +++ b/docs/old-version/index.md @@ -0,0 +1,358 @@ +--- +layout: page +title: Old version +permalink: /old-version/ +--- + +This is the documentation of pg-backup for all old versions bellow `v1.0`. +In the old version, S3 storage was mounted using s3fs, so we decided to migrate to the official AWS SDK. + +## Storage: +- local +- s3 +- Object storage + +## Volumes: + +- /s3mnt => S3 mounting path +- /backup => local storage mounting path + +### Usage + +| Options | Shorts | Usage | +|-----------------------|--------|------------------------------------------------------------------------| +| pg-bkup | bkup | CLI utility | +| backup | | Backup database operation | +| restore | | Restore database operation | +| history | | Show the history of backup | +| --storage | -s | Storage. local or s3 (default: local) | +| --file | -f | File name to restore | +| --path | | S3 path without file name. eg: /custom_path | +| --dbname | -d | Database name | +| --port | -p | Database port (default: 5432) | +| --mode | -m | Execution mode. default or scheduled (default: default) | +| --disable-compression | | Disable database backup compression | +| --prune | | Delete old backup, default disabled | +| --keep-last | | Delete old backup created more than specified days ago, default 7 days | +| --period | | Crontab period for scheduled mode only. (default: "0 1 * * *") | +| --help | -h | Print this help message and exit | +| --version | -V | Print version information and exit | + + +## Environment variables + +| Name | Requirement | Description | +|-------------|--------------------------------------------------|------------------------------------------------------| +| DB_PORT | Optional, default 5432 | Database port number | +| DB_HOST | Required | Database host | +| DB_NAME | Optional if it was provided from the -d flag | Database name | +| DB_USERNAME | Required | Database user name | +| DB_PASSWORD | Required | Database password | +| ACCESS_KEY | Optional, required for S3 storage | AWS S3 Access Key | +| SECRET_KEY | Optional, required for S3 storage | AWS S3 Secret Key | +| BUCKET_NAME | Optional, required for S3 storage | AWS S3 Bucket Name | +| S3_ENDPOINT | Optional, required for S3 storage | AWS S3 Endpoint | +| FILE_NAME | Optional if it was provided from the --file flag | Database file to restore (extensions: .sql, .sql.gz) | + + +## Note: + +Creating a user for backup tasks who has read-only access is recommended! + +> create read-only user + + +## Backup database : + +Simple backup usage + +```sh +bkup backup +``` + +### S3 + +```sh +pg-bkup backup --storage s3 +``` +## Docker run: + +```sh +docker run --rm --network your_network_name \ +--name pg-bkup -v $PWD/backup:/backup/ \ +-e "DB_HOST=database_host_name" \ +-e "DB_USERNAME=username" \ +-e "DB_PASSWORD=password" jkaninda/pg-bkup:v0.7 pg-bkup backup -d database_name +``` + +## Docker compose file: +```yaml +version: '3' +services: + postgres: + image: postgres:14.5 + container_name: postgres + restart: unless-stopped + volumes: + - ./postgres:/var/lib/postgresql/data + environment: + POSTGRES_DB: bkup + POSTGRES_PASSWORD: password + POSTGRES_USER: bkup + pg-bkup: + image: jkaninda/pg-bkup:v0.7 + container_name: pg-bkup + depends_on: + - postgres + command: + - /bin/sh + - -c + - pg-bkup backup -d bkup + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=bkup + - DB_USERNAME=bkup + - DB_PASSWORD=password +``` +## Restore database : + +Simple database restore operation usage + +```sh +pg-bkup restore --file database_20231217_115621.sql --dbname database_name +``` + +```sh +pg-bkup restore -f database_20231217_115621.sql -d database_name +``` +### S3 + +```sh +pg-bkup restore --storage s3 --file database_20231217_115621.sql --dbname database_name +``` + +## Docker run: + +```sh +docker run --rm --network your_network_name \ +--name pg-bkup \ +-v $PWD/backup:/backup/ \ +-e "DB_HOST=database_host_name" \ +-e "DB_USERNAME=username" \ +-e "DB_PASSWORD=password" \ +jkaninda/pg-bkup:v0.7 pg-bkup restore -d database_name -f store_20231219_022941.sql.gz +``` + +## Docker compose file: + +```yaml +version: '3' +services: + pg-bkup: + image: jkaninda/pg-bkup:v0.7 + container_name: pg-bkup + command: + - /bin/sh + - -c + - pg-bkup restore --file database_20231217_115621.sql -d database_name + volumes: + - ./backup:/backup + environment: + #- FILE_NAME=database_20231217_040238.sql.gz # Optional if file name is set from command + - DB_PORT=5432 + - DB_HOST=postgres + - DB_USERNAME=user_name + - DB_PASSWORD=password +``` +## Run + +```sh +docker-compose up -d +``` +## Backup to S3 + +```sh +docker run --rm --privileged \ +--device /dev/fuse --name pg-bkup \ +-e "DB_HOST=db_hostname" \ +-e "DB_USERNAME=username" \ +-e "DB_PASSWORD=password" \ +-e "ACCESS_KEY=your_access_key" \ +-e "SECRET_KEY=your_secret_key" \ +-e "BUCKETNAME=your_bucket_name" \ +-e "S3_ENDPOINT=https://s3.us-west-2.amazonaws.com" \ +jkaninda/pg-bkup:v0.7 pg-bkup backup -s s3 -d database_name +``` +> To change s3 backup path add this flag : --path /my_customPath . default path is /pg-bkup + +Simple S3 backup usage + +```sh +pg-bkup backup --storage s3 --dbname mydatabase +``` +```yaml + pg-bkup: + image: jkaninda/pg-bkup:v0.7 + container_name: pg-bkup + privileged: true + devices: + - "/dev/fuse" + command: + - /bin/sh + - -c + - pg-bkup restore --storage s3 -f database_20231217_115621.sql.gz --dbname database_name + environment: + - DB_PORT=5432 + - DB_HOST=postgress + - DB_USERNAME=user_name + - DB_PASSWORD=password + - ACCESS_KEY=${ACCESS_KEY} + - SECRET_KEY=${SECRET_KEY} + - BUCKET_NAME=${BUCKET_NAME} + - S3_ENDPOINT=${S3_ENDPOINT} + +``` +## Run in Scheduled mode + +This tool can be run as CronJob in Kubernetes for a regular backup which makes deployment on Kubernetes easy as Kubernetes has CronJob resources. +For Docker, you need to run it in scheduled mode by adding `--mode scheduled` flag and specify the periodical backup time by adding `--period "0 1 * * *"` flag. + +Make an automated backup on Docker + +## Syntax of crontab (field description) + +The syntax is: + +- 1: Minute (0-59) +- 2: Hours (0-23) +- 3: Day (0-31) +- 4: Month (0-12 [12 == December]) +- 5: Day of the week(0-7 [7 or 0 == sunday]) + +Easy to remember format: + +```conf +* * * * * command to be executed +``` + +```conf +- - - - - +| | | | | +| | | | ----- Day of week (0 - 7) (Sunday=0 or 7) +| | | ------- Month (1 - 12) +| | --------- Day of month (1 - 31) +| ----------- Hour (0 - 23) +------------- Minute (0 - 59) +``` + +> At every 30th minute + +```conf +*/30 * * * * +``` +> “At minute 0.” every hour +```conf +0 * * * * +``` + +> “At 01:00.” every day + +```conf +0 1 * * * +``` + +## Example of scheduled mode + +> Docker run : + +```sh +docker run --rm --name pg-bkup \ +-v $BACKUP_DIR:/backup/ \ +-e "DB_HOST=$DB_HOST" \ +-e "DB_USERNAME=$DB_USERNAME" \ +-e "DB_PASSWORD=$DB_PASSWORD" jkaninda/pg-bkup:v0.7 pg-bkup backup --dbname $DB_NAME --mode scheduled --period "0 1 * * *" +``` + +> With Docker compose + +```yaml +version: "3" +services: + pg-bkup: + image: jkaninda/pg-bkup:v0.7 + container_name: pg-bkup + privileged: true + devices: + - "/dev/fuse" + command: + - /bin/sh + - -c + - pg-bkup backup --storage s3 --path /mys3_custom_path --dbname database_name --mode scheduled --period "*/30 * * * *" + environment: + - DB_PORT=5432 + - DB_HOST=postgreshost + - DB_USERNAME=userName + - DB_PASSWORD=${DB_PASSWORD} + - ACCESS_KEY=${ACCESS_KEY} + - SECRET_KEY=${SECRET_KEY} + - BUCKET_NAME=${BUCKET_NAME} + - S3_ENDPOINT=${S3_ENDPOINT} +``` + +## Kubernetes CronJob + +For Kubernetes, you don't need to run it in scheduled mode. + +Simple Kubernetes CronJob usage: + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: bkup-job +spec: + schedule: "0 1 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-bkup + image: jkaninda/pg-bkup:v0.7 + securityContext: + privileged: true + command: + - /bin/sh + - -c + - pg-bkup backup -s s3 --path /custom_path + env: + - name: DB_PORT + value: "5432" + - name: DB_HOST + value: "" + - name: DB_NAME + value: "" + - name: DB_USERNAME + value: "" + # Please use secret! + - name: DB_PASSWORD + value: "" + - name: ACCESS_KEY + value: "" + - name: SECRET_KEY + value: "" + - name: BUCKET_NAME + value: "" + - name: S3_ENDPOINT + value: "https://s3.us-west-2.amazonaws.com" + restartPolicy: Never +``` + +## Authors + +**Jonas Kaninda** +- + diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 0000000..38e931f --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,105 @@ +--- +title: Configuration Reference +layout: default +nav_order: 2 +--- + +# Configuration reference + +Backup and restore targets, schedule and retention are configured using environment variables or flags. + + + + + +### CLI utility Usage + +| Options | Shorts | Usage | +|-----------------------|--------|----------------------------------------------------------------------------------------| +| pg-bkup | bkup | CLI utility | +| backup | | Backup database operation | +| restore | | Restore database operation | +| --storage | -s | Storage. local or s3 (default: local) | +| --file | -f | File name for restoration | +| --path | | AWS S3 path without file name. eg: /custom_path or ssh remote path `/home/foo/backup` | +| --dbname | -d | Database name | +| --port | -p | Database port (default: 5432) | +| --mode | -m | Execution mode. default or scheduled (default: default) | +| --disable-compression | | Disable database backup compression | +| --prune | | Delete old backup, default disabled | +| --keep-last | | Delete old backup created more than specified days ago, default 7 days | +| --period | | Crontab period for scheduled mode only. (default: "0 1 * * *") | +| --help | -h | Print this help message and exit | +| --version | -V | Print version information and exit | + +## Environment variables + +| Name | Requirement | Description | +|-------------------|--------------------------------------------------|------------------------------------------------------| +| DB_PORT | Optional, default 5432 | Database port number | +| DB_HOST | Required | Database host | +| DB_NAME | Optional if it was provided from the -d flag | Database name | +| DB_USERNAME | Required | Database user name | +| DB_PASSWORD | Required | Database password | +| AWS_ACCESS_KEY | Optional, required for S3 storage | AWS S3 Access Key | +| AWS_SECRET_KEY | Optional, required for S3 storage | AWS S3 Secret Key | +| AWS_BUCKET_NAME | Optional, required for S3 storage | AWS S3 Bucket Name | +| AWS_BUCKET_NAME | Optional, required for S3 storage | AWS S3 Bucket Name | +| AWS_REGION | Optional, required for S3 storage | AWS Region | +| AWS_DISABLE_SSL | Optional, required for S3 storage | Disable SSL | +| FILE_NAME | Optional if it was provided from the --file flag | Database file to restore (extensions: .sql, .sql.gz) | +| GPG_PASSPHRASE | Optional, required to encrypt and restore backup | GPG passphrase | +| SSH_HOST_NAME | Optional, required for SSH storage | ssh remote hostname or ip | +| SSH_USER | Optional, required for SSH storage | ssh remote user | +| SSH_PASSWORD | Optional, required for SSH storage | ssh remote user's password | +| SSH_IDENTIFY_FILE | Optional, required for SSH storage | ssh remote user's private key | +| SSH_PORT | Optional, required for SSH storage | ssh remote server port | +| SSH_REMOTE_PATH | Optional, required for SSH storage | ssh remote path (/home/toto/backup) | + +--- +## Run in Scheduled mode + +This image can be run as CronJob in Kubernetes for a regular backup which makes deployment on Kubernetes easy as Kubernetes has CronJob resources. +For Docker, you need to run it in scheduled mode by adding `--mode scheduled` flag and specify the periodical backup time by adding `--period "0 1 * * *"` flag. + +## Syntax of crontab (field description) + +The syntax is: + +- 1: Minute (0-59) +- 2: Hours (0-23) +- 3: Day (0-31) +- 4: Month (0-12 [12 == December]) +- 5: Day of the week(0-7 [7 or 0 == sunday]) + +Easy to remember format: + +```conf +* * * * * command to be executed +``` + +```conf +- - - - - +| | | | | +| | | | ----- Day of week (0 - 7) (Sunday=0 or 7) +| | | ------- Month (1 - 12) +| | --------- Day of month (1 - 31) +| ----------- Hour (0 - 23) +------------- Minute (0 - 59) +``` + +> At every 30th minute + +```conf +*/30 * * * * +``` +> “At minute 0.” every hour +```conf +0 * * * * +``` + +> “At 01:00.” every day + +```conf +0 1 * * * +``` \ No newline at end of file diff --git a/examples/docker-compose.s3.yaml b/examples/docker-compose.s3.yaml index 4fc2566..15ac3d1 100644 --- a/examples/docker-compose.s3.yaml +++ b/examples/docker-compose.s3.yaml @@ -1,21 +1,31 @@ -version: "3" services: pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. image: jkaninda/pg-bkup container_name: pg-bkup - privileged: true - devices: - - "/dev/fuse" command: - /bin/sh - -c - - pg-bkup backup --storage s3 --path /mys3_custom_path --dbname database_name + - pg-bkup backup --storage s3 -d my-database" environment: - DB_PORT=5432 - - DB_HOST=postgress - - DB_USERNAME=userName - - DB_PASSWORD=${DB_PASSWORD} - - ACCESS_KEY=${ACCESS_KEY} - - SECRET_KEY=${SECRET_KEY} - - BUCKET_NAME=${BUCKET_NAME} - - S3_ENDPOINT=https://s3.us-west-2.amazonaws.com \ No newline at end of file + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## AWS configurations + - AWS_S3_ENDPOINT=https://s3.amazonaws.com + - AWS_S3_BUCKET_NAME=backup + - AWS_REGION="us-west-2" + - AWS_ACCESS_KEY=xxxx + - AWS_SECRET_KEY=xxxxx + ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true + - AWS_DISABLE_SSL="false" + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: \ No newline at end of file diff --git a/examples/docker-compose.scheduled.s3.yaml b/examples/docker-compose.scheduled.s3.yaml index 1389018..b69e689 100644 --- a/examples/docker-compose.scheduled.s3.yaml +++ b/examples/docker-compose.scheduled.s3.yaml @@ -1,21 +1,31 @@ -version: "3" services: pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. image: jkaninda/pg-bkup container_name: pg-bkup - privileged: true - devices: - - "/dev/fuse" command: - /bin/sh - -c - - pg-bkup backup --storage s3 --path /mys3_custom_path --dbname database_name --mode scheduled --period "0 1 * * *" + - pg-bkup backup --storage s3 -d my-database --mode scheduled --period "0 1 * * *" environment: - DB_PORT=5432 - - DB_HOST=postgress - - DB_USERNAME=userName - - DB_PASSWORD=${DB_PASSWORD} - - ACCESS_KEY=${ACCESS_KEY} - - SECRET_KEY=${SECRET_KEY} - - BUCKET_NAME=${BUCKET_NAME} - - S3_ENDPOINT=https://s3.us-west-2.amazonaws.com \ No newline at end of file + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## AWS configurations + - AWS_S3_ENDPOINT=https://s3.amazonaws.com + - AWS_S3_BUCKET_NAME=backup + - AWS_REGION="us-west-2" + - AWS_ACCESS_KEY=xxxx + - AWS_SECRET_KEY=xxxxx + ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true + - AWS_DISABLE_SSL="false" + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: \ No newline at end of file diff --git a/examples/k8s-job.yaml b/examples/k8s-job.yaml index 43cba37..5aa6704 100644 --- a/examples/k8s-job.yaml +++ b/examples/k8s-job.yaml @@ -1,7 +1,7 @@ -apiVersion: batch/v1 +piVersion: batch/v1 kind: CronJob metadata: - name: pg-bkup-job + name: bkup-job spec: schedule: "0 1 * * *" jobTemplate: @@ -9,32 +9,36 @@ spec: template: spec: containers: - - name: pg-bkup - image: jkaninda/pg-bkup - securityContext: - privileged: true - command: - - /bin/sh - - -c - - pg-bkup backup --storage s3 --path /custom_path - env: - - name: DB_PORT - value: "5432" - - name: DB_HOST - value: "" - - name: DB_NAME - value: "" - - name: DB_USERNAME - value: "" - # Please use secret! - - name: DB_PASSWORD - value: "password" - - name: ACCESS_KEY - value: "" - - name: SECRET_KEY - value: "" - - name: BUCKETNAME - value: "" - - name: S3_ENDPOINT - value: "https://s3.us-west-2.amazonaws.com" - restartPolicy: Never \ No newline at end of file + - name: pg-bkup + image: jkaninda/pg-bkup + command: + - /bin/sh + - -c + - pg-bkup backup -s s3 --path /custom_path + env: + - name: DB_PORT + value: "5432" + - name: DB_HOST + value: "" + - name: DB_NAME + value: "" + - name: DB_USERNAME + value: "" + # Please use secret! + - name: DB_PASSWORD + value: "" + - name: ACCESS_KEY + value: "" + - name: AWS_S3_ENDPOINT + value: "https://s3.amazonaws.com" + - name: AWS_S3_BUCKET_NAME + value: "xxx" + - name: AWS_REGION + value: "us-west-2" + - name: AWS_ACCESS_KEY + value: "xxxx" + - name: AWS_SECRET_KEY + value: "xxxx" + - name: AWS_DISABLE_SSL + value: "false" + restartPolicy: OnFailure \ No newline at end of file diff --git a/go.mod b/go.mod index f59aa94..5333b0e 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/hpcloud/tail v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect golang.org/x/crypto v0.18.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/sys v0.22.0 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect diff --git a/go.sum b/go.sum index 7512d7f..8671785 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/scp.go b/pkg/scp.go index 242cc9d..ce73e12 100644 --- a/pkg/scp.go +++ b/pkg/scp.go @@ -8,6 +8,7 @@ import ( "github.com/bramvdbogaerde/go-scp/auth" "github.com/jkaninda/pg-bkup/utils" "golang.org/x/crypto/ssh" + "golang.org/x/exp/slog" "os" "path/filepath" ) @@ -27,7 +28,7 @@ func CopyToRemote(fileName, remotePath string) error { if sshPassword == "" { return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n") } - utils.Info("Accessing the remote server using password, private key is recommended\n") + slog.Warn("Accessing the remote server using password, password is not recommended\n") clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey()) } @@ -71,7 +72,7 @@ func CopyFromRemote(fileName, remotePath string) error { if sshPassword == "" { return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n") } - utils.Info("Accessing the remote server using password, private key is recommended\n") + slog.Warn("Accessing the remote server using password, password is not recommended\n") clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey()) } diff --git a/utils/utils.go b/utils/utils.go index da69d4e..ecae7cd 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -10,6 +10,7 @@ import ( "bytes" "fmt" "github.com/spf13/cobra" + "golang.org/x/exp/slog" "io" "io/fs" "os" @@ -19,6 +20,9 @@ import ( func Info(v ...any) { fmt.Println("⒤ ", fmt.Sprint(v...)) } +func Worn(msg string, v ...any) { + slog.Warn(fmt.Sprintf(msg, v)) +} func Done(v ...any) { fmt.Println("✔ ", fmt.Sprint(v...)) } @@ -183,7 +187,8 @@ func GetEnvVariable(envName, oldEnvName string) string { if value == "" { value = os.Getenv(oldEnvName) if value != "" { - fmt.Printf("%s is deprecated, please use %s instead!\n", oldEnvName, envName) + slog.Warn(fmt.Sprintf("%s is deprecated, please use %s instead!\n", oldEnvName, envName)) + } } return value