jkaninda/toolkit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add s3fs

Browse Source
This commit is contained in:
Jonas Kaninda
2024-01-10 03:39:09 +01:00
parent 905c64f395
commit 5c964c6e19
6 changed files with 249 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
README.md
View File

@@ -13,6 +13,7 @@ DevOps Portable toolkit
- kustomize
- jq
- k6
- s3fs
```yaml
version: '3.7'
@@ -25,4 +26,23 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
- ~/.kube:/root/.kube
- ~/.ssh:/root/.ssh # If you use private CVS
```
## Mount S3
> Command : s3-mount
```yaml
version: '3.7'
services:
toolkit:
image: jkaninda/toolkit:latest
container_name: toolkit
command: ["/bin/bash", "sh"]
volumes:
- ./custome-volume:/custome-volume
environment:
- ACCESS_KEY=${ACCESS_KEY}
- SECRET_KEY=${SECRET_KEY}
- BUCKETNAME=${BUCKETNAME}
- S3_ENDPOINT=https://s3.us-west-2.amazonaws.com
```

2
build.sh
View File

@@ -1,6 +1,6 @@
#!/bin/bash
docker build -t jkaninda/toolkit:latest .
docker build -f src/Dockerfile -t jkaninda/toolkit:latest .
docker compose up -d

3
go.mod Normal file
View File

@@ -0,0 +1,3 @@
module toolkit
go 1.20

31
Dockerfile → src/Dockerfile
View File

@@ -1,9 +1,21 @@
FROM ubuntu:23.10
FROM ubuntu:22.04
ENV BUCKETNAME=""
ENV ACCESS_KEY=""
ENV SECRET_KEY=""
ENV S3_ENDPOINT=https://s3.amazonaws.com
ARG DEBIAN_FRONTEND=noninteractive
ENV VERSION="0.1"
ARG TEMP_DIR=/temp
RUN apt-get update -qq \
&& apt-get install -qqy apt-transport-https ca-certificates curl gnupg2 software-properties-common jq git
&& apt-get install -qqy apt-transport-https ca-certificates curl gnupg2 software-properties-common jq git build-essential libcurl4-openssl-dev libxml2-dev mime-support s3fs -y
RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
RUN curl https://get.docker.com/ > dockerinstall && chmod 777 dockerinstall && ./dockerinstall
# Clear cache
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
## Install K6
RUN curl https://github.com/grafana/k6/releases/download/v0.48.0/k6-v0.48.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1
RUN mv k6 /usr/local/bin/
@@ -18,4 +30,17 @@ RUN ./get_helm.sh
RUN curl -LO https://dl.k8s.io/release/v1.29.0/bin/linux/amd64/kubectl
RUN chmod +x kubectl
RUN mv kubectl /usr/local/bin/kubectl
RUN mv kubectl /usr/local/bin/kubectl
RUN mkdir $TEMP_DIR && \
chmod 777 $TEMP_DIR && \
mkdir $TEMP_DIR/s3cache && \
chmod 777 $TEMP_DIR/s3cache
COPY src/scripts/backup.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/backup.sh
COPY src/scripts/s3-mount.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/s3-mount.sh
RUN ln -s /usr/local/bin/s3-mount.sh /usr/local/bin/s3-mount
RUN ln -s /usr/local/bin/backup.sh /usr/local/bin/backup

102
src/scripts/backup.sh Normal file
View File

@@ -0,0 +1,102 @@
#!/usr/bin/env bash
# S3 Backup & Restore
# @author Jonas Kaninda
# @license MIT License <https://opensource.org/licenses/MIT>
# @link https://github.com/jkaninda/toolkit
#
set -e
TIME=$(date +%Y%m%d_%H%M%S)
arg0=$(basename "$0" .sh)
blnk=$(echo "$arg0" | sed 's/./ /g')
export OPERATION=backup
export S3_PATH=/mysql-bkup
export TIMEOUT=60
export EXECUTION_MODE="default"
export SCHEDULE_PERIOD="0 1 * * *"
export FILE_COMPRESION=true
usage_info()
{
echo "Usage: \\"
}
version_info()
{
echo "Version: $VERSION"
exit 0
}
usage()
{
exec 1>2 # Send standard output to standard error
usage_info
exit 0
}
error()
{
echo "$arg0: $*" >&2
exit 0
}
info() {
{ set +x; } 2> /dev/null
echo 'toolkit:' '[INFO] ' "$@"
#set -x
}
warning() {
{ set +x; } 2> /dev/null
echo 'toolkit:' '[WARNING] ' "$@"
}
fatal() {
{ set +x; } 2> /dev/null
echo 'toolkit:' '[ERROR] ' "$@" >&2
exit 1
}
help()
{
echo
echo " -o |--operation -- Set operation (default: backup)"
echo " |--path -- Set s3 path, without file name"
echo " -t |--timeout -- Set timeout (default: 120s)"
echo " -h |--help -- Print this help message and exit"
echo " -V |--version -- Print version information and exit"
exit 0
}
flags()
{
while test $# -gt 0
do
case "$1" in
(-o|--operation)
shift
[ $# = 0 ] && error "No operation specified - restore or backup"
export OPERATION="$1"
shift;;
(--enable-compresion)
shift
[ $# = 0 ] && error "No enable-compresion specified "
export FILE_COMPRESION="$1"
shift;;
(-m|--mode)
shift
[ $# = 0 ] && error "No execution mode specified"
export EXECUTION_MODE="$1"
shift;;
(-t|--timeout)
shift
[ $# = 0 ] && error "No timeout specified"
export TIMEOUT="$1"
shift;;
(-h|--help)
help;;
(-V|--version)
version_info;;
(--)
help;;
(*) usage;;
esac
done
}

95
src/scripts/s3-mount.sh Normal file
View File

@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# S3 Backup & Restore
# @author Jonas Kaninda
# @license MIT License <https://opensource.org/licenses/MIT>
# @link https://github.com/jkaninda/toolkit
#
set -e
usage_info()
{
echo "Usage: \\"
}
version_info()
{
echo "Version: $VERSION"
exit 0
}
usage()
{
exec 1>2 # Send standard output to standard error
usage_info
exit 0
}
error()
{
echo "$arg0: $*" >&2
exit 0
}
info() {
{ set +x; } 2> /dev/null
echo 'toolkit:' '[INFO] ' "$@"
#set -x
}
warning() {
{ set +x; } 2> /dev/null
echo 'toolkit:' '[WARNING] ' "$@"
}
fatal() {
{ set +x; } 2> /dev/null
echo 'toolkit:' '[ERROR] ' "$@" >&2
exit 1
}
help()
{
echo
echo " -h |--help -- Print this help message and exit"
echo " -V |--version -- Print version information and exit"
exit 0
}
flags()
{
while test $# -gt 0
do
case "$1" in
(-t|--timeout)
shift
[ $# = 0 ] && error "No timeout specified"
export TIMEOUT="$1"
shift;;
(-h|--help)
help;;
(-V|--version)
version_info;;
(--)
help;;
(*) usage;;
esac
done
}
mount_s3()
{
if [[ -z $ACCESS_KEY ]] || [[ -z $SECRET_KEY ]]; then
info "Please make sure all environment variables are set "
else
echo "$ACCESS_KEY:$SECRET_KEY" | tee /etc/passwd-s3fs
chmod 600 /etc/passwd-s3fs
info "Mounting Object storage in /s3mnt .... "
if [ -z "$(ls -A /s3mnt)" ]; then
s3fs $BUCKETNAME /s3mnt -o passwd_file=/etc/passwd-s3fs -o use_cache=/tmp/s3cache -o allow_other -o url=$S3_ENDPOINT -o use_path_request_style
if [ ! -d "/s3mnt" ]; then
mkdir -p /s3mnt
fi
else
info "Object storage already mounted in /s3mnt"
fi
fi
}
flags "$@"
mount_s3