Commit 80c75072 authored by Combava Orange's avatar Combava Orange
Browse files

feat copy to 2 buckets, replace s3cmd by aws s3

parent c334f9cb
Pipeline #257710 passed with stages
in 12 minutes and 38 seconds
......@@ -8,27 +8,37 @@ RUN sed -i 's/^override_install_langs=.*$/override_install_langs=fr_FR/g' /etc/y
&& yum -y install epel-release \
&& yum -y install python3 python3-pip \
&& yum -y install nc \
&& yum -y install unzip \
&& useradd -g root javaapp
ENV LC_ALL fr_FR.utf8
ENV LANG fr_FR.utf8
ENV PATH $PATH:~/.local/bin
WORKDIR /tmp
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
&& unzip awscliv2.zip \
&& ./aws/install -i /usr/local/aws-cli -b /usr/local/bin \
&& rm awscliv2.zip rm -rf aws
WORKDIR /home/javaapp
USER javaapp
# Integration WebHook
RUN pip3 install --user s3cmd \
&& pip3 install --user flask flask_autoindex
FROM installer
COPY ./target/clea*-exec.jar ./clea-batch.jar
COPY ./src/main/scripts/* ./
COPY ./src/main/docker/* ./http/
COPY ./src/main/docker/aws ./.aws
USER root
RUN chmod +x *.sh
RUN mkdir /logs \
&& touch /logs/clea-batch.log \
&& touch /logs/clea-batch.error.log \
&& chmod a+rw /logs/*.log \
&& chmod +x *.sh
USER javaapp
ENTRYPOINT [ "./http/web.sh" ]
ENTRYPOINT [ "./clea-batch.sh" ]
[profile s3outscale]
region = eu-west-2
[profile s3scaleway]
region = fr-par
[s3outscale]
aws_access_key_id = AKIAIOSFODNN7EXAMPLE
aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
[s3scaleway]
aws_access_key_id = AKIAIOSFODNN7EXAMPLE
aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
......@@ -2,8 +2,11 @@
PROGNAM=$(basename $0)
die() { echo "[$PROGNAM] $*" 1>&2 ; exit 1; }
info() { echo "[$PROGNAM] $*"; }
# redirect logs to log file
CLEA_BATCH_LOG_FILE_PATH=${CLEA_BATCH_LOG_FILE_PATH:-/logs}
CLEA_BATCH_LOG_FILE_NAME=${CLEA_BATCH_LOG_FILE_NAME:-clea-batch}
test -f "${CLEA_BATCH_LOG_FILE_PATH}/${CLEA_BATCH_LOG_FILE_NAME}.log" || die "log file: ${CLEA_BATCH_LOG_FILE_PATH}/${CLEA_BATCH_LOG_FILE_NAME}.log does not exist"
test -f "${CLEA_BATCH_LOG_FILE_PATH}/${CLEA_BATCH_LOG_FILE_NAME}.error.log" || die "error log file: ${CLEA_BATCH_LOG_FILE_PATH}/${CLEA_BATCH_LOG_FILE_NAME}.error.log does not exist"
......@@ -13,6 +16,17 @@ exec 2>> "${CLEA_BATCH_LOG_FILE_PATH}/${CLEA_BATCH_LOG_FILE_NAME}.error.log"
WORKDIR=${CLEA_BATCH_CLUSTER_OUTPUT_PATH:-/tmp/v1}
BUCKET=${BUCKET:-}
BUCKET_OUTSCALE=${BUCKET_OUTSCALE:-$BUCKET}
BUCKET_SCALEWAY=${BUCKET_SCALEWAY:-$BUCKET}
PROFILE_OUTSCALE=${PROFILE_OUTSCALE:-s3outscale}
PROFILE_SCALEWAY=${PROFILE_SCALEWAY:-s3scaleway}
ENDPOINT_OUTSCALE=${ENDPOINT_OUTSCALE:-} # use https://oos.eu-west-2.outscale.com/ https://oos.cloudgouv-eu-west-1.outscale.com
ENDPOINT_SCALEWAY=${ENDPOINT_SCALEWAY:-} # https://s3.fr-par.scw.cloud
set -o pipefail # trace ERR through pipes
set -o errtrace # trace ERR through 'time command' and other functions
set -o nounset ## set -u : exit the script if you try to use an uninitialised variable
......@@ -20,7 +34,7 @@ set -o nounset ## set -u : exit the script if you try to use an uninitialised
set +e
[ -n "${BUCKET}" ] || die "Environment variable BUCKET required"
[ -n "${BUCKET_OUTSCALE}" ] || die "Environment variable BUCKET_OUTSCALE required"
if ! java -jar clea-batch.jar $@ ; then
die "Java batch fails"
......@@ -51,11 +65,29 @@ fi
# Copy clusterfiles to s3
# =======================
# All files except indexCluster.json
s3cmd sync --acl-public --exclude=clusterIndex.json $WORKDIR s3://${BUCKET}/ || die "S3cmd fails to copy cluster files to bucket"
info "Copying to OUTSCALE ...."
AWS_OPTS="--profile=$PROFILE_OUTSCALE --endpoint-url=$ENDPOINT_OUTSCALE"
BUCKET="$BUCKET_OUTSCALE"
# All files except indexCluster.json
aws $AWS_OPTS s3 sync --acl public-read --exclude=clusterIndex.json $WORKDIR s3://${BUCKET}/v1 || die "S3cmd fails to copy cluster files to bucket"
# only indexCluster.json at the root of "v1"
s3cmd put --acl-public $(find $WORKDIR -type f -name clusterIndex.json) s3://${BUCKET}/v1/ || die "S3cmd fails to copy clusterIndex file to bucket"
aws $AWS_OPTS s3 cp --acl public-read $(find $WORKDIR -type f -name clusterIndex.json) s3://${BUCKET}/v1/ || die "S3cmd fails to copy clusterIndex file to bucket"
# COPY TO SCALEWAY (optional)
# --------------------
if [ -n "$BUCKET_SCALEWAY" ] && [ -n "$PROFILE_SCALEWAY" ] && [ -n "$ENDPOINT_SCALEWAY" ] ; then
info "Copying to SCALEWAY ...."
AWS_OPTS="--profile=$PROFILE_SCALEWAY --endpoint-url=$ENDPOINT_SCALEWAY"
BUCKET=$BUCKET_SCALEWAY
# All files except indexCluster.json
aws $AWS_OPTS s3 sync --acl public-read --exclude=clusterIndex.json $WORKDIR s3://${BUCKET}/v1 || die "S3cmd fails to copy cluster files to bucket"
# only indexCluster.json at the root of "v1"
aws $AWS_OPTS s3 cp --acl public-read $(find $WORKDIR -type f -name clusterIndex.json) s3://${BUCKET}/v1/ || die "S3cmd fails to copy clusterIndex file to bucket"
fi
# purge batch temporary files
rm -rf $WORKDIR
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment