diff --git a/README.md b/README.md index 0a4cb22..98801c8 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ DevOps Portable toolkit - kustomize - jq - k6 +- s3fs ```yaml version: '3.7' @@ -25,4 +26,23 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ~/.kube:/root/.kube - ~/.ssh:/root/.ssh # If you use private CVS +``` +## Mount S3 + +> Command : s3-mount + +```yaml +version: '3.7' +services: + toolkit: + image: jkaninda/toolkit:latest + container_name: toolkit + command: ["/bin/bash", "sh"] + volumes: + - ./custome-volume:/custome-volume + environment: + - ACCESS_KEY=${ACCESS_KEY} + - SECRET_KEY=${SECRET_KEY} + - BUCKETNAME=${BUCKETNAME} + - S3_ENDPOINT=https://s3.us-west-2.amazonaws.com ``` \ No newline at end of file diff --git a/build.sh b/build.sh index 60f201d..48ea6c4 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #!/bin/bash - docker build -t jkaninda/toolkit:latest . + docker build -f src/Dockerfile -t jkaninda/toolkit:latest . docker compose up -d diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..fdd867a --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module toolkit + +go 1.20 diff --git a/Dockerfile b/src/Dockerfile similarity index 51% rename from Dockerfile rename to src/Dockerfile index d1c6f4c..cc48c21 100644 --- a/Dockerfile +++ b/src/Dockerfile @@ -1,9 +1,21 @@ -FROM ubuntu:23.10 +FROM ubuntu:22.04 +ENV BUCKETNAME="" +ENV ACCESS_KEY="" +ENV SECRET_KEY="" +ENV S3_ENDPOINT=https://s3.amazonaws.com +ARG DEBIAN_FRONTEND=noninteractive +ENV VERSION="0.1" +ARG TEMP_DIR=/temp RUN apt-get update -qq \ - && apt-get install -qqy apt-transport-https ca-certificates curl gnupg2 software-properties-common jq git + && apt-get install -qqy apt-transport-https ca-certificates curl gnupg2 software-properties-common jq git build-essential libcurl4-openssl-dev libxml2-dev mime-support s3fs -y RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - + RUN curl https://get.docker.com/ > dockerinstall && chmod 777 dockerinstall && ./dockerinstall + +# Clear cache +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + ## Install K6 RUN curl https://github.com/grafana/k6/releases/download/v0.48.0/k6-v0.48.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 RUN mv k6 /usr/local/bin/ @@ -18,4 +30,17 @@ RUN ./get_helm.sh RUN curl -LO https://dl.k8s.io/release/v1.29.0/bin/linux/amd64/kubectl RUN chmod +x kubectl -RUN mv kubectl /usr/local/bin/kubectl \ No newline at end of file +RUN mv kubectl /usr/local/bin/kubectl + +RUN mkdir $TEMP_DIR && \ + chmod 777 $TEMP_DIR && \ + mkdir $TEMP_DIR/s3cache && \ + chmod 777 $TEMP_DIR/s3cache + +COPY src/scripts/backup.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/backup.sh +COPY src/scripts/s3-mount.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/s3-mount.sh + +RUN ln -s /usr/local/bin/s3-mount.sh /usr/local/bin/s3-mount +RUN ln -s /usr/local/bin/backup.sh /usr/local/bin/backup diff --git a/src/scripts/backup.sh b/src/scripts/backup.sh new file mode 100644 index 0000000..59c9a5a --- /dev/null +++ b/src/scripts/backup.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +# S3 Backup & Restore + +# @author Jonas Kaninda +# @license MIT License +# @link https://github.com/jkaninda/toolkit +# +set -e + +TIME=$(date +%Y%m%d_%H%M%S) +arg0=$(basename "$0" .sh) +blnk=$(echo "$arg0" | sed 's/./ /g') +export OPERATION=backup +export S3_PATH=/mysql-bkup +export TIMEOUT=60 +export EXECUTION_MODE="default" +export SCHEDULE_PERIOD="0 1 * * *" +export FILE_COMPRESION=true +usage_info() +{ + echo "Usage: \\" + +} +version_info() +{ + echo "Version: $VERSION" + exit 0 +} +usage() +{ + exec 1>2 # Send standard output to standard error + usage_info + exit 0 +} + +error() +{ + echo "$arg0: $*" >&2 + exit 0 +} +info() { + { set +x; } 2> /dev/null + echo 'toolkit:' '[INFO] ' "$@" + #set -x +} +warning() { + { set +x; } 2> /dev/null + echo 'toolkit:' '[WARNING] ' "$@" +} +fatal() { + { set +x; } 2> /dev/null + echo 'toolkit:' '[ERROR] ' "$@" >&2 + exit 1 +} + +help() +{ + echo + echo " -o |--operation -- Set operation (default: backup)" + echo " |--path -- Set s3 path, without file name" + echo " -t |--timeout -- Set timeout (default: 120s)" + echo " -h |--help -- Print this help message and exit" + echo " -V |--version -- Print version information and exit" + exit 0 +} + +flags() +{ + while test $# -gt 0 + do + case "$1" in + (-o|--operation) + shift + [ $# = 0 ] && error "No operation specified - restore or backup" + export OPERATION="$1" + shift;; + (--enable-compresion) + shift + [ $# = 0 ] && error "No enable-compresion specified " + export FILE_COMPRESION="$1" + shift;; + (-m|--mode) + shift + [ $# = 0 ] && error "No execution mode specified" + export EXECUTION_MODE="$1" + shift;; + (-t|--timeout) + shift + [ $# = 0 ] && error "No timeout specified" + export TIMEOUT="$1" + shift;; + (-h|--help) + help;; + (-V|--version) + version_info;; + (--) + help;; + (*) usage;; + esac + done +} \ No newline at end of file diff --git a/src/scripts/s3-mount.sh b/src/scripts/s3-mount.sh new file mode 100644 index 0000000..358d43c --- /dev/null +++ b/src/scripts/s3-mount.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# S3 Backup & Restore + +# @author Jonas Kaninda +# @license MIT License +# @link https://github.com/jkaninda/toolkit +# +set -e + +usage_info() +{ + echo "Usage: \\" + +} +version_info() +{ + echo "Version: $VERSION" + exit 0 +} +usage() +{ + exec 1>2 # Send standard output to standard error + usage_info + exit 0 +} + +error() +{ + echo "$arg0: $*" >&2 + exit 0 +} +info() { + { set +x; } 2> /dev/null + echo 'toolkit:' '[INFO] ' "$@" + #set -x +} +warning() { + { set +x; } 2> /dev/null + echo 'toolkit:' '[WARNING] ' "$@" +} +fatal() { + { set +x; } 2> /dev/null + echo 'toolkit:' '[ERROR] ' "$@" >&2 + exit 1 +} + +help() +{ + echo + echo " -h |--help -- Print this help message and exit" + echo " -V |--version -- Print version information and exit" + exit 0 +} + +flags() +{ + while test $# -gt 0 + do + case "$1" in + (-t|--timeout) + shift + [ $# = 0 ] && error "No timeout specified" + export TIMEOUT="$1" + shift;; + (-h|--help) + help;; + (-V|--version) + version_info;; + (--) + help;; + (*) usage;; + esac + done +} +mount_s3() +{ +if [[ -z $ACCESS_KEY ]] || [[ -z $SECRET_KEY ]]; then +info "Please make sure all environment variables are set " +else + echo "$ACCESS_KEY:$SECRET_KEY" | tee /etc/passwd-s3fs + chmod 600 /etc/passwd-s3fs + info "Mounting Object storage in /s3mnt .... " + if [ -z "$(ls -A /s3mnt)" ]; then + s3fs $BUCKETNAME /s3mnt -o passwd_file=/etc/passwd-s3fs -o use_cache=/tmp/s3cache -o allow_other -o url=$S3_ENDPOINT -o use_path_request_style + if [ ! -d "/s3mnt" ]; then + mkdir -p /s3mnt + fi + else + info "Object storage already mounted in /s3mnt" + fi +fi +} +flags "$@" +mount_s3 \ No newline at end of file