Commit b0547e7b authored by Figue Orange's avatar Figue Orange Committed by Combava Orange
Browse files

[clea-bath] compose with kafdrop

use clea images from clea/demo branch (replace vars USER by DB_USER by example)
remove cmak/kafka-manager: kafdrop simpler, allow to see messages
fix installation of extension uuid-ossp in clea.sql
parent 54a97d27
FROM centos:7 AS installer
# Install all the locales
RUN sed -i 's/^override_install_langs=.*$/override_install_langs=fr_FR/g' /etc/yum.conf \
&& yum -y update \
&& yum -y install java-11-openjdk \
&& yum -y install epel-release \
&& yum -y install python3 python3-pip \
&& pip3 install s3cmd \
&& useradd javaapp
WORKDIR /home/javaapp
FROM installer
COPY ./target/clea*-exec.jar ./clea-batch.jar
COPY ./src/main/scripts/* ./
RUN chmod +x *.sh
RUN mkdir /v1 && chown -R javaapp:root /v1
USER javaapp
ENTRYPOINT [ "./clea-batch.sh" ]
# ~/.s3cfg used by s3cmd
[default]
access_key = AKIAIOSFODNN7EXAMPLE
secret_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
host_base = http://minio:9000
host_bucket = %(bucket).minio.xip.io:9000
access_token =
add_encoding_exts =
add_headers =
bucket_location = FR
ca_certs_file =
cache_file =
check_ssl_certificate = True
check_ssl_hostname = True
cloudfront_host = cloudfront.amazonaws.com
connection_pooling = True
content_disposition =
content_type =
default_mime_type = binary/octet-stream
delay_updates = False
delete_after = False
delete_after_fetch = False
delete_removed = False
dry_run = False
enable_multipart = True
encoding = ANSI_X3.4-1968
encrypt = False
expiry_date =
expiry_days =
expiry_prefix =
follow_symlinks = False
force = False
get_continue = False
gpg_command = /usr/bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase =
guess_mime_type = True
human_readable_sizes = False
invalidate_default_index_on_cf = False
invalidate_default_index_root_on_cf = True
invalidate_on_cf = False
kms_key =
limit = -1
limitrate = 0
list_md5 = False
log_target_prefix =
long_listing = False
max_delete = -1
mime_type =
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
preserve_attrs = True
progress_meter = True
proxy_host =
proxy_port = 0
public_url_use_https = False
put_continue = False
recursive = False
recv_chunk = 65536
reduced_redundancy = False
requester_pays = False
restore_days = 1
restore_priority = Standard
send_chunk = 65536
server_side_encryption = False
signature_v2 = False
signurl_use_https = False
simpledb_host = sdb.amazonaws.com
skip_existing = False
socket_timeout = 300
stats = False
stop_on_error = False
storage_class =
throttle_max = 100
upload_id =
urlencoding_mode = normal
use_http_expect = False
use_https = False
use_mime_magic = True
verbosity = WARNING
website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
website_error =
website_index = index.html
#! /bin/bash
SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
SCRIPTNAME=`basename $0`
OPTIONS="-p $SCRIPTNAME -f $SCRIPTPATH/docker-compose.yml"
# IP of the host (bridge)
export EXTERNAL_IP=$(docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}')
if [ "$1" == "restart" ] ; then
shift
docker-compose $OPTIONS stop $*
docker-compose $OPTIONS rm -f $*
docker-compose $OPTIONS up -d $*
else
docker-compose $OPTIONS $*
fi
version: "3.8"
services:
nginx:
image: nginx:1.19
volumes:
- ./nginx/conf.d:/etc/nginx/conf.d:ro
ports:
- "80:80"
networks:
- public
- clea-network
kong:
image: kong:2.3
environment:
KONG_DATABASE: "off"
KONG_PROXY_ACCESS_LOG: /dev/stdout
KONG_ADMIN_ACCESS_LOG: /dev/stdout
KONG_PROXY_ERROR_LOG: /dev/stderr
KONG_ADMIN_ERROR_LOG: /dev/stderr
KONG_ADMIN_LISTEN: "0.0.0.0:8001"
KONG_DECLARATIVE_CONFIG: /home/kong/kong.yml
ports:
- "8000:8000"
- "8001:8001"
volumes:
- ./kong/kong.yml:/home/kong/kong.yml:ro
networks:
- clea-network
konga:
image: pantsel/konga:next
environment:
BASE_URL: /konga/
TOKEN_SECRET: km1GUr4RkcQD7DewhJPNXrCuZwcKmqjb
NODE_ENV: production
#ports:
# - "1337:1337"
networks:
- clea-network
postgres:
image: postgres:9
hostname: postgres
ports:
- "5432:5432"
environment:
#POSTGRES_USER: admin # replace default guest/guest
POSTGRES_PASSWORD: "admin123!"
volumes:
- ./postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
networks:
- clea-network
pgadmin:
image: dpage/pgadmin4:5.1
environment:
PGADMIN_DEFAULT_EMAIL: "admin@pgadmin.local"
PGADMIN_DEFAULT_PASSWORD: "admin123!"
networks:
- clea-network
minio:
image: minio/minio:edge
command: ["server", "/data"]
environment:
MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
MINIO_SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
#ports:
# - "9000:9000"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
networks:
- clea-network
# From https://github.com/confluentinc/cp-all-in-one/blob/6.1.1-post/cp-all-in-one/docker-compose.yml
zookeeper:
image: confluentinc/cp-zookeeper:6.1.1
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
networks:
- clea-network
kafka:
image: confluentinc/cp-server:6.1.1
hostname: broker
container_name: broker
depends_on:
- zookeeper
ports:
- "9092:9092"
- "9101:9101"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:9092 #,PLAINTEXT_HOST://broker:9092
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
#KAFKA_JMX_PORT: 9101
#KAFKA_JMX_HOSTNAME: localhost
#KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:9092
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
CONFLUENT_METRICS_ENABLE: 'true'
#CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
networks:
- clea-network
# cmak allow to manage kafka but is not so easy and don't allow to see messages'
# kafka-man:
# image: kafkamanager/kafka-manager
# command: ["cmak-3.0.0.4/bin/cmak", "-Dhttp.port=8080","-Dplay.http.context=/cmak"]
# environment:
# ZK_HOSTS: zookeeper
# KAFKA_MANAGER_PASSWORD: "admin123!"
# #ports:
# # - '9001:8080'
# networks:
# - clea-network
# kafdrop is a single page console already connected to kafka that allow to see messages
kafdrop:
image: "obsidiandynamics/kafdrop:latest"
environment:
JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
KAFKA_BROKERCONNECT: "kafka:9092"
SERVER_SERVLET_CONTEXTPATH: "/kafdrop"
depends_on:
- kafka
#ports:
# - "9000:9000"
networks:
- clea-network
clea-ws-rest:
image: clea-ws-rest:latest
container_name: clea-ws-rest
build: ../../clea-ws-rest/
restart: always
environment:
SPRING_PROFILES_ACTIVE: docker
KAFKA_URL: kafka:9092
depends_on:
- kafka
#ports:
# - "8080:8080"
networks:
- clea-network
clea-venue-consumer:
image: clea-venue-consumer:latest
container_name: clea-venue-consumer
build: ../../clea-venue-consumer/
environment:
SPRING_PROFILES_ACTIVE: docker
DB_URL: jdbc:postgresql://postgres:5432/clea
DB_USER: clea
DB_PASSWORD: aelc
JPA_DDL_AUTO: none
KAFKA_URL: kafka:9092
CLEA_CRYPTO_AUTHORITY_SECRET: "c991482ff078a3d83203dfcee763bd33366e0c2c89050e29b05334555964f736"
depends_on:
- postgres
- kafka
networks:
- clea-network
clea-batch:
image: clea-batch:latest
build: ../
#entrypoint: ["/bin/tail","-f","/dev/null"]
environment:
SPRING_PROFILES_ACTIVE: docker
DB_URL: jdbc:postgresql://postgres:5432/clea
DB_USER: clea
DB_PASSWORD: "aelc"
BUCKET: cleacluster-eu-west-3
CLEA_BATCH_CLUSTER_OUTPUT_PATH: /tmp/v1
volumes:
- ./batch/s3cfg:/home/javaapp/.s3cfg
depends_on:
- postgres
networks:
- clea-network
restart: "no"
networks:
public:
driver: bridge
name: public
clea-network:
#driver: bridge
name: clea-network
# ------------------------------------------------------------------------------
# This is an example file to get you started with using
# declarative configuration in Kong.
# ------------------------------------------------------------------------------
# Metadata fields start with an underscore (_)
# Fields that do not start with an underscore represent Kong entities and attributes
# _format_version is mandatory,
# it specifies the minimum version of Kong that supports the format
_format_version: "2.1"
# _transform is optional, defaulting to true.
# It specifies whether schema transformations should be applied when importing this file
# as a rule of thumb, leave this setting to true if you are importing credentials
# with plain passwords, which need to be encrypted/hashed before storing on the database.
# On the other hand, if you are reimporting a database with passwords already encrypted/hashed,
# set it to false.
_transform: true
# Each Kong entity (core entity or custom entity introduced by a plugin)
# can be listed in the top-level as an array of objects:
# services:
# - name: example-service
# url: http://example.com
# # Entities can store tags as metadata
# tags:
# - example
# # Entities that have a foreign-key relationship can be nested:
# routes:
# - name: example-route
# paths:
# - /
# plugins:
# - name: key-auth
# - name: another-service
# url: https://example.org
# routes:
# - name: another-route
# # Relationships can also be specified between top-level entities,
# # either by name or by id
# service: example-service
# hosts: ["hello.com"]
# consumers:
# - username: example-user
# # Custom entities from plugin can also be specified
services:
- name: report-service
url: http://clea-ws-rest:8080
path: /api
tags:
- clea
routes:
- name: api-route
paths:
- /api
strip_path: false
#default.conf
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
location /konga/ {
proxy_set_header Host $host;
proxy_pass http://konga:1337/;
# proxy_redirect off;
}
location /minio/ {
proxy_set_header Host $host;
proxy_pass http://minio:9000/minio/;
proxy_redirect off;
}
location /pgadmin4/ {
proxy_set_header X-Script-Name /pgadmin4;
proxy_set_header Host $host;
proxy_pass http://pgadmin:8080/;
proxy_redirect off;
}
#location /cmak {
# proxy_set_header Host $host;
# proxy_pass http://kafka-man:8080/cmak;
# # proxy_redirect off;
#}
location /kafdrop/ {
proxy_set_header Host $host;
proxy_pass http://kafdrop:9000;
proxy_redirect off;
}
location ~ (/swagger-ui/|/swagger-resources|/v3/api-docs) {
proxy_set_header Host $host;
proxy_pass http://clea-ws-rest:8080;
proxy_redirect off;
}
location /api {
proxy_set_header Host $host;
proxy_pass http://kong:8000;
#proxy_pass http://10.100.0.1:8088;
proxy_redirect off;
}
}
CREATE DATABASE CLEA;
CREATE ROLE cleaowner NOLOGIN;
CREATE ROLE clea LOGIN PASSWORD 'aelc';
-- ALTER DATABASE clea OWNER cleaowner;
GRANT ALL PRIVILEGES ON DATABASE clea TO clea WITH GRANT OPTION;
\c clea;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
# Working with the bucket:
install awscli-local that add the ability to specify the AWS api endpoint url.
```
$ pip3 install awscli-local
$ awslocal --version
aws-cli/2.1.14 Python/3.7.3 Linux/5.4.0-67-generic exe/x86_64.ubuntu.18 prompt/off
```
## Option 1
define a profile in ~/.aws/credentials :
```
[inria]
# minio secrets
aws_access_key_id = AKIAIOSFODNN7EXAMPLE
aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
```
and use awslocal with --profile=inria
```bashRiskConfigurationService
awslocal --endpoint-url=http://localhost:9000 --profile=inria s3 ls
```
or
```bash
$ export AWS_PROFILE=inria
awslocal --endpoint-url=http://localhost:9000 s3 ls
```
## Option 2 (don't use it because secret are more exposed)
```bash
export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
awslocal --endpoint-url=http://localhost:9000 s3 ls
```
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>clea-server</artifactId>
<groupId>fr.gouv.clea</groupId>
<version>0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>clea-batch</artifactId>
<properties>
<java.version>11</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-batch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-bootstrap</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-consul-config</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-vault-config-consul</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>