[clea-bath] compose with kafdrop

use clea images from clea/demo branch (replace vars USER by DB_USER by example)
remove cmak/kafka-manager: kafdrop simpler, allow to see messages
fix installation of extension uuid-ossp in clea.sql

clea-batch/Dockerfile

+FROM centos:7 AS installer
+
+# Install all the locales
+RUN sed -i 's/^override_install_langs=.*$/override_install_langs=fr_FR/g' /etc/yum.conf \
+    && yum -y update \
+    && yum -y install java-11-openjdk \
+    && yum -y install epel-release \
+    && yum -y install python3 python3-pip \
+    && pip3 install s3cmd \ 
+    && useradd javaapp
+
+WORKDIR /home/javaapp
+
+FROM installer
+COPY ./target/clea*-exec.jar ./clea-batch.jar
+COPY ./src/main/scripts/* ./
+RUN chmod +x *.sh
+RUN mkdir /v1 && chown -R javaapp:root /v1
+
+
+USER javaapp
+
+ENTRYPOINT [ "./clea-batch.sh" ]

clea-batch/docker-compose/batch/s3cfg

+# ~/.s3cfg used by s3cmd
+[default]
+access_key = AKIAIOSFODNN7EXAMPLE
+secret_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+host_base = http://minio:9000
+host_bucket = %(bucket).minio.xip.io:9000
+access_token = 
+add_encoding_exts = 
+add_headers = 
+bucket_location = FR
+ca_certs_file = 
+cache_file = 
+check_ssl_certificate = True
+check_ssl_hostname = True
+cloudfront_host = cloudfront.amazonaws.com
+connection_pooling = True
+content_disposition = 
+content_type = 
+default_mime_type = binary/octet-stream
+delay_updates = False
+delete_after = False
+delete_after_fetch = False
+delete_removed = False
+dry_run = False
+enable_multipart = True
+encoding = ANSI_X3.4-1968
+encrypt = False
+expiry_date = 
+expiry_days = 
+expiry_prefix = 
+follow_symlinks = False
+force = False
+get_continue = False
+gpg_command = /usr/bin/gpg
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = True
+human_readable_sizes = False
+invalidate_default_index_on_cf = False
+invalidate_default_index_root_on_cf = True
+invalidate_on_cf = False
+kms_key = 
+limit = -1
+limitrate = 0
+list_md5 = False
+log_target_prefix = 
+long_listing = False
+max_delete = -1
+mime_type = 
+multipart_chunk_size_mb = 15
+multipart_max_chunks = 10000
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+public_url_use_https = False
+put_continue = False
+recursive = False
+recv_chunk = 65536
+reduced_redundancy = False
+requester_pays = False
+restore_days = 1
+restore_priority = Standard
+send_chunk = 65536
+server_side_encryption = False
+signature_v2 = False
+signurl_use_https = False
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+socket_timeout = 300
+stats = False
+stop_on_error = False
+storage_class = 
+throttle_max = 100
+upload_id = 
+urlencoding_mode = normal
+use_http_expect = False
+use_https = False
+use_mime_magic = True
+verbosity = WARNING
+website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
+website_error = 
+website_index = index.html

clea-batch/docker-compose/clea

+#! /bin/bash
+
+SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+SCRIPTNAME=`basename $0`
+OPTIONS="-p $SCRIPTNAME -f $SCRIPTPATH/docker-compose.yml"
+
+# IP of the host (bridge)
+export EXTERNAL_IP=$(docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}')
+
+if [ "$1" == "restart" ] ; then
+  shift
+  docker-compose $OPTIONS stop $*
+  docker-compose $OPTIONS rm -f $*
+  docker-compose $OPTIONS up -d $*
+else 
+  docker-compose $OPTIONS $*
+fi
+
+

clea-batch/docker-compose/docker-compose.yml

+version: "3.8"
+services:
+
+  nginx:
+    image: nginx:1.19
+    volumes:
+      - ./nginx/conf.d:/etc/nginx/conf.d:ro
+    ports:
+      - "80:80"
+    networks:
+      - public
+      - clea-network
+
+  kong:
+    image: kong:2.3
+    environment:
+      KONG_DATABASE: "off"
+      KONG_PROXY_ACCESS_LOG: /dev/stdout
+      KONG_ADMIN_ACCESS_LOG: /dev/stdout
+      KONG_PROXY_ERROR_LOG: /dev/stderr 
+      KONG_ADMIN_ERROR_LOG: /dev/stderr 
+      KONG_ADMIN_LISTEN: "0.0.0.0:8001"
+      KONG_DECLARATIVE_CONFIG: /home/kong/kong.yml
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+    volumes:
+    - ./kong/kong.yml:/home/kong/kong.yml:ro
+    networks:
+      - clea-network
+  konga:
+    image: pantsel/konga:next
+    environment:
+      BASE_URL: /konga/
+      TOKEN_SECRET: km1GUr4RkcQD7DewhJPNXrCuZwcKmqjb
+      NODE_ENV: production
+    #ports:
+    #  - "1337:1337"
+    networks:
+      - clea-network
+
+  postgres:
+    image: postgres:9
+    hostname: postgres
+    ports:
+      - "5432:5432"
+    environment:
+      #POSTGRES_USER: admin # replace default guest/guest
+      POSTGRES_PASSWORD: "admin123!"
+    volumes:
+    - ./postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+    networks:
+      - clea-network
+
+  pgadmin:
+    image: dpage/pgadmin4:5.1
+    environment:
+      PGADMIN_DEFAULT_EMAIL: "admin@pgadmin.local"
+      PGADMIN_DEFAULT_PASSWORD: "admin123!"
+    networks:
+      - clea-network
+
+  minio:
+    image: minio/minio:edge
+    command: ["server", "/data"]
+    environment:
+       MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
+       MINIO_SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+    #ports:
+    #  - "9000:9000"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      - clea-network
+
+
+  # From https://github.com/confluentinc/cp-all-in-one/blob/6.1.1-post/cp-all-in-one/docker-compose.yml
+  zookeeper:
+    image: confluentinc/cp-zookeeper:6.1.1
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    networks:
+      - clea-network
+
+  kafka:
+    image: confluentinc/cp-server:6.1.1
+    hostname: broker
+    container_name: broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+      - "9101:9101"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:9092 #,PLAINTEXT_HOST://broker:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      #KAFKA_JMX_PORT: 9101
+      #KAFKA_JMX_HOSTNAME: localhost
+      #KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:9092
+      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
+      CONFLUENT_METRICS_ENABLE: 'true'
+      #CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
+    networks:
+      - clea-network
+
+  # cmak allow to manage kafka but is not so easy and don't allow to see messages'
+  # kafka-man:
+  #   image: kafkamanager/kafka-manager
+  #   command: ["cmak-3.0.0.4/bin/cmak", "-Dhttp.port=8080","-Dplay.http.context=/cmak"]
+  #   environment:
+  #     ZK_HOSTS: zookeeper
+  #     KAFKA_MANAGER_PASSWORD: "admin123!"
+  #   #ports:
+  #   #  - '9001:8080'
+  #   networks:
+  #     - clea-network
+
+  # kafdrop is a single page console already connected to kafka that allow to see messages 
+  kafdrop:
+    image: "obsidiandynamics/kafdrop:latest"
+    environment:
+      JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
+      KAFKA_BROKERCONNECT: "kafka:9092"
+      SERVER_SERVLET_CONTEXTPATH: "/kafdrop"
+    depends_on:
+      - kafka
+    #ports:
+    #  - "9000:9000"
+    networks:
+      - clea-network
+
+
+  clea-ws-rest:
+    image: clea-ws-rest:latest
+    container_name: clea-ws-rest
+    build: ../../clea-ws-rest/
+    restart: always
+    environment:
+      SPRING_PROFILES_ACTIVE: docker
+      KAFKA_URL: kafka:9092
+    depends_on:
+      - kafka
+    #ports:
+    #  - "8080:8080"
+    networks:
+      - clea-network
+
+
+  clea-venue-consumer:
+    image: clea-venue-consumer:latest
+    container_name: clea-venue-consumer
+    build: ../../clea-venue-consumer/
+    environment:
+      SPRING_PROFILES_ACTIVE: docker
+      DB_URL: jdbc:postgresql://postgres:5432/clea
+      DB_USER: clea
+      DB_PASSWORD: aelc
+      JPA_DDL_AUTO: none
+      KAFKA_URL: kafka:9092
+      CLEA_CRYPTO_AUTHORITY_SECRET: "c991482ff078a3d83203dfcee763bd33366e0c2c89050e29b05334555964f736"
+    depends_on:
+      - postgres
+      - kafka
+    networks:
+      - clea-network
+
+  clea-batch:
+    image: clea-batch:latest
+    build: ../
+    #entrypoint: ["/bin/tail","-f","/dev/null"]
+    environment:
+      SPRING_PROFILES_ACTIVE: docker
+      DB_URL: jdbc:postgresql://postgres:5432/clea
+      DB_USER: clea
+      DB_PASSWORD: "aelc"
+      BUCKET: cleacluster-eu-west-3
+      CLEA_BATCH_CLUSTER_OUTPUT_PATH: /tmp/v1
+    volumes:
+    - ./batch/s3cfg:/home/javaapp/.s3cfg
+    depends_on:
+      - postgres
+    networks:
+      - clea-network
+    restart: "no"
+
+networks:
+  public: 
+    driver: bridge
+    name: public
+  clea-network:
+    #driver: bridge
+    name: clea-network
+
+    

clea-batch/docker-compose/kong/kong.yml

+# ------------------------------------------------------------------------------
+# This is an example file to get you started with using
+# declarative configuration in Kong.
+# ------------------------------------------------------------------------------
+
+# Metadata fields start with an underscore (_)
+# Fields that do not start with an underscore represent Kong entities and attributes
+
+# _format_version is mandatory,
+# it specifies the minimum version of Kong that supports the format
+
+_format_version: "2.1"
+
+# _transform is optional, defaulting to true.
+# It specifies whether schema transformations should be applied when importing this file
+# as a rule of thumb, leave this setting to true if you are importing credentials
+# with plain passwords, which need to be encrypted/hashed before storing on the database.
+# On the other hand, if you are reimporting a database with passwords already encrypted/hashed,
+# set it to false.
+
+_transform: true
+
+# Each Kong entity (core entity or custom entity introduced by a plugin)
+# can be listed in the top-level as an array of objects:
+
+# services:
+# - name: example-service
+#   url: http://example.com
+#   # Entities can store tags as metadata
+#   tags:
+#   - example
+#   # Entities that have a foreign-key relationship can be nested:
+#   routes:
+#   - name: example-route
+#     paths:
+#     - /
+#   plugins:
+#   - name: key-auth
+# - name: another-service
+#   url: https://example.org
+
+# routes:
+# - name: another-route
+#   # Relationships can also be specified between top-level entities,
+#   # either by name or by id
+#   service: example-service
+#   hosts: ["hello.com"]
+
+# consumers:
+# - username: example-user
+#   # Custom entities from plugin can also be specified
+
+services:
+ - name: report-service
+   url: http://clea-ws-rest:8080
+   path: /api
+   tags:
+   - clea
+   routes:
+   - name: api-route
+     paths:
+     - /api
+     strip_path: false  

clea-batch/docker-compose/nginx/conf.d/default.conf

+#default.conf 
+server {
+    listen       80;
+    server_name  localhost;
+
+    #charset koi8-r;
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+    }
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
+    #
+    #location ~ \.php$ {
+    #    proxy_pass   http://127.0.0.1;
+    #}
+
+    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+    #
+    #location ~ \.php$ {
+    #    root           html;
+    #    fastcgi_pass   127.0.0.1:9000;
+    #    fastcgi_index  index.php;
+    #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
+    #    include        fastcgi_params;
+    #}
+
+    # deny access to .htaccess files, if Apache's document root
+    # concurs with nginx's one
+    #
+    #location ~ /\.ht {
+    #    deny  all;
+    #}
+
+    location /konga/ {
+        proxy_set_header Host $host;
+        proxy_pass http://konga:1337/;
+        # proxy_redirect off;
+    }
+
+    location /minio/ {
+        proxy_set_header Host $host;
+        proxy_pass http://minio:9000/minio/;
+        proxy_redirect off;
+    }
+
+    location /pgadmin4/ {
+        proxy_set_header X-Script-Name /pgadmin4;
+        proxy_set_header Host $host;
+        proxy_pass http://pgadmin:8080/;
+        proxy_redirect off;
+    }
+
+    #location /cmak {
+    #    proxy_set_header Host $host;
+    #    proxy_pass http://kafka-man:8080/cmak;
+    #    # proxy_redirect off;
+    #}
+    location /kafdrop/ {
+        proxy_set_header Host $host;
+        proxy_pass http://kafdrop:9000;
+        proxy_redirect off;
+    }
+
+    location ~ (/swagger-ui/|/swagger-resources|/v3/api-docs) {
+        proxy_set_header Host $host;
+        proxy_pass http://clea-ws-rest:8080;
+        proxy_redirect off;
+	}
+    location /api {
+        proxy_set_header Host $host;
+        proxy_pass http://kong:8000;
+        #proxy_pass http://10.100.0.1:8088;
+        proxy_redirect off;
+    }
+ 
+}

clea-batch/docker-compose/postgres/docker-entrypoint-initdb.d/clea.sql

+CREATE DATABASE CLEA;
+CREATE ROLE cleaowner NOLOGIN;
+
+CREATE ROLE clea LOGIN PASSWORD 'aelc';
+
+-- ALTER DATABASE clea OWNER cleaowner;
+GRANT ALL PRIVILEGES ON DATABASE clea TO clea WITH GRANT OPTION;
+
+\c clea;
+
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

clea-batch/docker-compose/readme.md

+
+# Working with the bucket:
+
+install awscli-local that add the ability to specify the AWS api endpoint url.
+
+```
+$ pip3 install awscli-local
+$ awslocal --version
+aws-cli/2.1.14 Python/3.7.3 Linux/5.4.0-67-generic exe/x86_64.ubuntu.18 prompt/off
+```
+
+## Option 1 
+
+define a profile in ~/.aws/credentials :
+
+```
+[inria]
+# minio secrets
+aws_access_key_id = AKIAIOSFODNN7EXAMPLE 
+aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+```
+and use awslocal with --profile=inria
+
+```bashRiskConfigurationService
+awslocal --endpoint-url=http://localhost:9000 --profile=inria s3 ls
+```
+
+or
+
+```bash
+$ export AWS_PROFILE=inria
+
+awslocal --endpoint-url=http://localhost:9000 s3 ls
+```
+
+
+## Option 2 (don't use it because secret are more exposed)
+
+```bash
+export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+awslocal --endpoint-url=http://localhost:9000 s3 ls
+```

clea-batch/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>clea-server</artifactId>
+        <groupId>fr.gouv.clea</groupId>
+        <version>0.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>clea-batch</artifactId>
+
+    <properties>
+        <java.version>11</java.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-batch</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.data</groupId>
+            <artifactId>spring-data-jdbc</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.flywaydb</groupId>
+            <artifactId>flyway-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-bootstrap</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-consul-config</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-vault-config-consul</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>