Add s3fs

README.md

@@ -13,6 +13,7 @@ DevOps Portable toolkit
 - kustomize
 - jq
 - k6
+- s3fs
 
 ```yaml
 version: '3.7'
@@ -26,3 +27,22 @@ services:
       - ~/.kube:/root/.kube
       - ~/.ssh:/root/.ssh # If you use private CVS
 ```
+## Mount S3 
+
+> Command : s3-mount
+
+```yaml
+version: '3.7'
+services:
+  toolkit:
+    image: jkaninda/toolkit:latest
+    container_name: toolkit
+    command: ["/bin/bash", "sh"]
+    volumes:
+      - ./custome-volume:/custome-volume
+    environment:
+      - ACCESS_KEY=${ACCESS_KEY}
+      - SECRET_KEY=${SECRET_KEY}
+      - BUCKETNAME=${BUCKETNAME}
+      - S3_ENDPOINT=https://s3.us-west-2.amazonaws.com
+```

build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
- docker build -t jkaninda/toolkit:latest .
+ docker build -f src/Dockerfile -t jkaninda/toolkit:latest .
  
  docker compose up -d
 

go.mod

@@ -0,0 +1,3 @@
+module toolkit
+
+go 1.20

src/Dockerfile

@@ -1,9 +1,21 @@
-FROM ubuntu:23.10
+FROM ubuntu:22.04
+ENV BUCKETNAME=""
+ENV ACCESS_KEY=""
+ENV SECRET_KEY=""
+ENV S3_ENDPOINT=https://s3.amazonaws.com
+ARG DEBIAN_FRONTEND=noninteractive
+ENV VERSION="0.1"
+ARG TEMP_DIR=/temp
 RUN apt-get update -qq \
-    && apt-get install -qqy apt-transport-https ca-certificates curl gnupg2 software-properties-common jq git
+    && apt-get install -qqy apt-transport-https ca-certificates curl gnupg2 software-properties-common jq git build-essential libcurl4-openssl-dev libxml2-dev mime-support s3fs -y
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+
 RUN curl https://get.docker.com/ > dockerinstall && chmod 777 dockerinstall && ./dockerinstall
 
+
+# Clear cache
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
 ## Install K6
 RUN curl https://github.com/grafana/k6/releases/download/v0.48.0/k6-v0.48.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1
 RUN mv k6 /usr/local/bin/
@@ -19,3 +31,16 @@ RUN ./get_helm.sh
 RUN curl -LO https://dl.k8s.io/release/v1.29.0/bin/linux/amd64/kubectl
 RUN chmod +x kubectl
 RUN mv kubectl /usr/local/bin/kubectl
+
+RUN mkdir $TEMP_DIR && \
+ chmod 777 $TEMP_DIR && \
+ mkdir $TEMP_DIR/s3cache && \
+ chmod 777 $TEMP_DIR/s3cache
+
+COPY src/scripts/backup.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/backup.sh
+COPY src/scripts/s3-mount.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/s3-mount.sh
+
+RUN ln -s /usr/local/bin/s3-mount.sh /usr/local/bin/s3-mount
+RUN ln -s /usr/local/bin/backup.sh /usr/local/bin/backup

src/scripts/backup.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+# S3 Backup & Restore 
+
+# @author    Jonas Kaninda
+# @license   MIT License <https://opensource.org/licenses/MIT>
+# @link      https://github.com/jkaninda/toolkit
+#
+set -e
+
+TIME=$(date +%Y%m%d_%H%M%S)
+arg0=$(basename "$0" .sh)
+blnk=$(echo "$arg0" | sed 's/./ /g')
+export OPERATION=backup
+export S3_PATH=/mysql-bkup
+export TIMEOUT=60
+export EXECUTION_MODE="default"
+export SCHEDULE_PERIOD="0 1 * * *"
+export FILE_COMPRESION=true
+usage_info()
+{
+    echo "Usage: \\"
+
+}
+version_info()
+{
+   echo "Version: $VERSION"
+   exit 0
+}
+usage()
+{
+    exec 1>2   # Send standard output to standard error
+    usage_info
+    exit 0
+}
+
+error()
+{
+    echo "$arg0: $*" >&2
+    exit 0
+}
+info() {
+    { set +x; } 2> /dev/null
+    echo 'toolkit:' '[INFO] ' "$@"
+    #set -x
+}
+warning() {
+    { set +x; } 2> /dev/null
+    echo 'toolkit:' '[WARNING] ' "$@"
+}
+fatal() {
+    { set +x; } 2> /dev/null
+    echo 'toolkit:' '[ERROR] ' "$@" >&2
+    exit 1
+}
+
+help()
+{
+    echo
+    echo "  -o |--operation         -- Set operation (default: backup)"
+    echo "     |--path              -- Set s3 path, without file name"
+    echo "  -t |--timeout           -- Set timeout (default: 120s)"
+    echo "  -h |--help              -- Print this help message and exit"
+    echo "  -V |--version           -- Print version information and exit"
+    exit 0
+}
+
+flags()
+{
+    while test $# -gt 0
+    do
+        case "$1" in
+        (-o|--operation)
+            shift
+            [ $# = 0 ] && error "No operation specified - restore or backup"
+            export OPERATION="$1"
+            shift;;
+        (--enable-compresion)
+            shift
+            [ $# = 0 ] && error "No enable-compresion specified "
+            export FILE_COMPRESION="$1"
+            shift;;
+        (-m|--mode)
+            shift
+            [ $# = 0 ] && error "No execution mode specified"
+            export EXECUTION_MODE="$1"
+            shift;;
+        (-t|--timeout)
+            shift
+            [ $# = 0 ] && error "No timeout specified"
+            export TIMEOUT="$1"
+            shift;;   
+        (-h|--help)
+            help;;
+        (-V|--version)
+           version_info;;
+        (--)
+           help;;
+        (*) usage;;
+        esac
+    done
+}

src/scripts/s3-mount.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+# S3 Backup & Restore 
+
+# @author    Jonas Kaninda
+# @license   MIT License <https://opensource.org/licenses/MIT>
+# @link      https://github.com/jkaninda/toolkit
+#
+set -e
+
+usage_info()
+{
+    echo "Usage: \\"
+
+}
+version_info()
+{
+   echo "Version: $VERSION"
+   exit 0
+}
+usage()
+{
+    exec 1>2   # Send standard output to standard error
+    usage_info
+    exit 0
+}
+
+error()
+{
+    echo "$arg0: $*" >&2
+    exit 0
+}
+info() {
+    { set +x; } 2> /dev/null
+    echo 'toolkit:' '[INFO] ' "$@"
+    #set -x
+}
+warning() {
+    { set +x; } 2> /dev/null
+    echo 'toolkit:' '[WARNING] ' "$@"
+}
+fatal() {
+    { set +x; } 2> /dev/null
+    echo 'toolkit:' '[ERROR] ' "$@" >&2
+    exit 1
+}
+
+help()
+{
+    echo
+    echo "  -h |--help              -- Print this help message and exit"
+    echo "  -V |--version           -- Print version information and exit"
+    exit 0
+}
+
+flags()
+{
+    while test $# -gt 0
+    do
+        case "$1" in
+        (-t|--timeout)
+            shift
+            [ $# = 0 ] && error "No timeout specified"
+            export TIMEOUT="$1"
+            shift;;   
+        (-h|--help)
+            help;;
+        (-V|--version)
+           version_info;;
+        (--)
+           help;;
+        (*) usage;;
+        esac
+    done
+}
+mount_s3()
+{
+if [[ -z $ACCESS_KEY ]] ||  [[ -z $SECRET_KEY ]]; then
+info "Please make sure all environment variables are set "
+else
+    echo "$ACCESS_KEY:$SECRET_KEY" | tee /etc/passwd-s3fs
+    chmod 600 /etc/passwd-s3fs
+    info "Mounting Object storage in /s3mnt .... "
+    if [ -z "$(ls -A /s3mnt)" ]; then
+       s3fs $BUCKETNAME /s3mnt -o passwd_file=/etc/passwd-s3fs -o use_cache=/tmp/s3cache -o allow_other -o url=$S3_ENDPOINT -o use_path_request_style
+       if [ ! -d "/s3mnt" ]; then
+           mkdir -p /s3mnt
+        fi 
+    else
+     info "Object storage already mounted in /s3mnt"
+    fi
+fi
+}
+flags "$@"
+mount_s3