From 1a78776a5a5b3258dc8321f1d889f37c70c677e4 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:37:59 +0100
Subject: [PATCH 01/21] docker compose; rename vidjil metrics service
---
docker-compose.dev.yml | 2 +-
docker-compose.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 78c113e..5d648a3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,6 +1,6 @@
version: '3.3'
services:
- metrics_vidjil:
+ vidjil:
volumes:
- ./metrics/entrypoints:/entrypoints
- ./metrics/vidjil/tools/api_vidjil.py:/app/vidjil/tools/api_vidjil.py
diff --git a/docker-compose.yml b/docker-compose.yml
index 0c1c3d0..0e3d9ff 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -68,7 +68,7 @@ services:
# replicas: 1
# network_mode: "host"
- metrics_vidjil:
+ vidjil:
image: vidjil/metrics-app:latest
container_name: monitoring_vidjil_instance
env_file:
--
GitLab
From ea1164b0985b3be7c42f830f50e4cf8bc625723d Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:39:33 +0100
Subject: [PATCH 02/21] docker compose: use network and internal hostname
---
docker-compose.yml | 31 ++++++++++++++++++++++++-------
grafana/datasource.yml | 2 +-
prometheus/prometheus.yml | 6 +++---
3 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 0e3d9ff..d66772f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,8 @@ services:
prometheus:
image: prom/prometheus:latest
container_name: monitoring_prometheus
- user: "1000:1000"
+ ports:
+ - "9090:9090"
deploy:
replicas: 1
env_file:
@@ -18,7 +19,10 @@ services:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention.time=365d'
- network_mode: "host"
+ # network_mode: "host"
+ networks:
+ - metrics
+ hostname: monitoring_prometheus
# alertmanager:
# image: prom/alertmanager:v0.18.0
@@ -35,7 +39,8 @@ services:
grafana:
image: grafana/grafana:latest
container_name: monitoring_grafana
- user: "1000:1000"
+ ports:
+ - "3000:3000"
environment:
- GF_PATHS_CONFIG=/etc/grafana/grafana.ini
env_file:
@@ -50,7 +55,10 @@ services:
- ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
- ./grafana/grafana.ini:/etc/grafana/grafana.ini
- ./grafana/dashboards:/var/lib/grafana/dashboards
- network_mode: "host"
+ # network_mode: "host"
+ networks:
+ - metrics
+ hostname: monitoring_grafana
# blackbox:
# image: prom/blackbox-exporter:v0.14.0
@@ -70,7 +78,9 @@ services:
vidjil:
image: vidjil/metrics-app:latest
- container_name: monitoring_vidjil_instance
+ container_name: monitoring_vidjil
+ ports:
+ - "5000:5000"
env_file:
# cp .env-sample as .env file and modify it with user/password of metrics declared user on server
# Don't use an admin account. See XXX page to create metrics user if not already done
@@ -85,9 +95,16 @@ services:
- ./certs:/app/certs
- ./metrics/conf.py:/app/conf.py
command: bash entrypoints/flask-entrypoint.sh
- network_mode: "host"
+ # network_mode: "host"
+ networks:
+ - metrics
+ hostname: monitoring_vidjil
build:
./metrics/
+
# volumes:
# prometheus-data:
- # grafana-data:
\ No newline at end of file
+ # grafana-data:
+
+networks:
+ metrics:
\ No newline at end of file
diff --git a/grafana/datasource.yml b/grafana/datasource.yml
index a0ea46a..b67f29a 100644
--- a/grafana/datasource.yml
+++ b/grafana/datasource.yml
@@ -4,7 +4,7 @@ datasources:
- name: Prometheus
type: prometheus
access: proxy
- url: http://localhost:9090
+ url: http://monitoring_prometheus:9090
# - name: Prometheus-perso2
# type: prometheus-perso2
# access: proxy
diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml
index f61ddff..1c629d6 100644
--- a/prometheus/prometheus.yml
+++ b/prometheus/prometheus.yml
@@ -26,11 +26,11 @@ scrape_configs:
- job_name: 'measure server' # <-- j'ai mis ce titre de job mais le truc c'est de mettre un nom de job explicite
static_configs:
- - targets: ['127.0.0.1:9100', '192.168.1.77:9100']
- scrape_interval: 5s
+ - targets: ['monitoring_vidjil:9100', '192.168.1.77:9100']
+ scrape_interval: 30s
- job_name: 'vidjil'
static_configs:
- - targets: ['127.0.0.1:5000']
+ - targets: ['monitoring_vidjil:5000']
scrape_interval: 90s
--
GitLab
From d088604a4e8da6c4dc62a15d6c2ec0c2349dfbb3 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:39:48 +0100
Subject: [PATCH 03/21] update gitignore
---
.gitignore | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index 0621410..e484e18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,7 @@ vidjil/
__pycache__
.env
-volumes/
+volumes*/
volumes_dev/
certs/*.pem
--
GitLab
From 8c48081c2f0bf7b6deb40843c06d530ee7f5d0f6 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:40:48 +0100
Subject: [PATCH 04/21] main.py, fix error of missing quote in
unavailableServer function
---
metrics/main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/metrics/main.py b/metrics/main.py
index 600bf44..b9acdd5 100644
--- a/metrics/main.py
+++ b/metrics/main.py
@@ -44,7 +44,7 @@ def unavailableServer(url) -> str:
return f"\
# HELP vidjil_api_request_unavailable_server {'A server is unreachable'}\n\
# TYPE vidjil_api_request_unavailable_server {'gauge'}\n\
-vidjil_api_request_unavailable_server {{server={cleanUrlServer(url)}}}"
+vidjil_api_request_unavailable_server {{server='{cleanUrlServer(url)}'}}\n"
@app.route('/metrics')
--
GitLab
From 33d40d4859edb77d346cba37ff47d0bba6cd5b34 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:42:15 +0100
Subject: [PATCH 05/21] main.py; fix error of legacy file size that is
sometimes 0
Old data ofn vidjil server have no provider field filled, and sizes where not corectly computed. In this case, value is none and can't be interpreted by metrics.
---
metrics/main.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/metrics/main.py b/metrics/main.py
index b9acdd5..ffc7cb7 100644
--- a/metrics/main.py
+++ b/metrics/main.py
@@ -313,9 +313,11 @@ def convert_sequence_size_by_user(metrics_key, formater, data, url_server):
# HELP vidjil_api_request_sequence_size_by_user {formater['description']}\n\
# TYPE vidjil_api_request_sequence_size_by_user {formater['type']}\n"
for elt in data:
+ # Some old data have none value (no provider filled); needto replace None by 0
+ size = elt['size_file_sum'] if not elt['size_file_sum'] == None else -1
string += f"\
vidjil_api_request_sequence_size_by_user\
-{{user_id=\"{elt['user_id']}\", server=\"{url_server}\"}} {elt['size_file_sum']}\n"
+{{user_id=\"{elt['user_id']}\", server=\"{url_server}\"}} {size}\n"
return string
--
GitLab
From 22925492d3b5589810f31081433ba165eff16f4f Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:42:45 +0100
Subject: [PATCH 06/21] Update readme
---
readme.md | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
diff --git a/readme.md b/readme.md
index 73d3c81..a84c620 100644
--- a/readme.md
+++ b/readme.md
@@ -14,10 +14,8 @@ You can find information on third-party-tools at:
#### Grafana
-J'ai changé le login/pass par défaut, voici les credentials de grafana :
-
-- login : `unicorn`
-- password : `UnicornsExists!`
+Default password of grafana server is `admin/admin`.
+At first login, a new password will be asked.
#### Vidjil servers
@@ -35,9 +33,5 @@ Host network will be used. You can find services on **localhost** at these port
<!-- - Node exporter : `9100` -->
-## ouidou
-
-Chez ouidou, nous aimons jouer avec les nouvelles technologies et nous sommes toujours curieux de découvrir des projets sympa et de nouvelles personnes. On recrute, donc n'hésite pas à nous contacter [ici](mailto:contact@ouidou.fr) et de visiter notre site [ici](https://ouidou.fr).
-
<!-- Based on a projet of ouidou found on github -->
<!-- Ce dépôt est en lien avec mon article medium : https://medium.com/ouidou/un-monitoring-complet-en-quelques-minutes-avec-prometheus-33e849e6392e -->
\ No newline at end of file
--
GitLab
From cc9a22ad1913dc4bb0c1fc29da32536b41b408fa Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:47:01 +0100
Subject: [PATCH 07/21] main.py: clean log
---
metrics/main.py | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/metrics/main.py b/metrics/main.py
index ffc7cb7..30e62e1 100644
--- a/metrics/main.py
+++ b/metrics/main.py
@@ -18,8 +18,6 @@ def get_env_list():
@app.route('/metrics_test')
def metrics_test():
content = ""
- # user = os.getenv('METRICS_USER_EMAIL')
- # password = os.getenv('METRICS_USER_PASSWORD')
dev_mode = os.getenv('DEV_MODE')
print( "====== /metrics_test")
@@ -112,6 +110,9 @@ def formated_metrics(metrics, url_server):
'set_generics_by_group': {"type": "gauge", "description": "set_generics_by_group.", "function": convert_set_generics_by_group},
'set_runs_by_group': {"type": "gauge", "description": "set_runs_by_group.", "function": convert_set_runs_by_group},
'set_patients_by_group': {"type": "gauge", "description": "set_patients_by_group.", "function": convert_set_patients_by_group},
+
+ ### Not used
+ # 'request_times': {"type": "gauge", "description": "request_times.", "function": convert_request_times},
}
string = f"# Information : {metrics['message']}\n"
@@ -205,7 +206,6 @@ def convert_login_count(metrics_key, formater, data, url_server):
# HELP vidjil_api_request_login_count {formater['description']}\n\
# TYPE vidjil_api_request_login_count {formater['type']}\n"
for elt in data:
- #Â print(elt)
if elt['auth_user']['email'] == "metrics@vidjil.org":
continue
string += f"\
@@ -225,7 +225,6 @@ def convert_status_analysis(metrics_key, formater, data, url_server):
# HELP vidjil_api_request_status_analysis {formater['description']}\n\
# TYPE vidjil_api_request_status_analysis {formater['type']}\n"
for elt in data:
- print(elt)
string += f"\
vidjil_api_request_status_analysis\
{{status=\"{elt['scheduler_task']['status']}\", task_name=\"{elt['scheduler_task']['task_name']}\", server=\"{url_server}\"}} \
@@ -367,15 +366,16 @@ vidjil_api_request_set_generics_count\
return string
+# def convert_request_times(metrics_key, formater, data, url_server):
+# return f"=*=*= convert_request_times ==> {data}"
+# string = f"\
+# # HELP vidjil_api_request_request_times {formater['description']}\n\
+# # TYPE vidjil_api_request_request_times {formater['type']}\n"
+# return string
+
def convert_config_analysis_by_users_patients(metrics_key, formater, data, url_server):
- # "status_analysis": [{
- # 'config_id': 2,
- # 'config_name': 'multi+inc+xxx',
- # 'count': 3,
- # 'user_id': 1}, ...]
- # print( "=====\n===== convert_config_analysis_by_users_patients")
- # print(f"{data[:2]=}")
+ # return f"=*=*= convert_config_analysis_by_users_patients ==> {data[:2]}"
string = f"\
# HELP vidjil_api_request_config_analysis_by_users_patients {formater['description']}\n\
# TYPE vidjil_api_request_config_analysis_by_users_patients {formater['type']}\n"
--
GitLab
From 1d0c20cc0c45f293e5566bbddc3a06570867e5d8 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:47:46 +0100
Subject: [PATCH 08/21] docker compose; init grafana and prometheus with root
user for right reasons on volumes
---
docker-compose.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/docker-compose.yml b/docker-compose.yml
index d66772f..c57b6b0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,6 +5,7 @@ services:
prometheus:
image: prom/prometheus:latest
container_name: monitoring_prometheus
+ user: root
ports:
- "9090:9090"
deploy:
@@ -39,6 +40,7 @@ services:
grafana:
image: grafana/grafana:latest
container_name: monitoring_grafana
+ user: root
ports:
- "3000:3000"
environment:
--
GitLab
From fbdd6a980c43d430627b7b7024b11e5db1017b3b Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 10:58:52 +0100
Subject: [PATCH 09/21] docker compose; comment ports for prometheus and vidjil
metrics
All data will be serve with no security.
be removing port, data will be only accesible inside metrics docker network
---
docker-compose.yml | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index c57b6b0..3a1afa5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,8 +6,10 @@ services:
image: prom/prometheus:latest
container_name: monitoring_prometheus
user: root
- ports:
- - "9090:9090"
+ # Commented ports to be only avaialble fron metrics network and not exposed
+ # If needed for dev, you can expose this port but data are fully exposed
+ # ports:
+ # - "9090:9090"
deploy:
replicas: 1
env_file:
@@ -81,8 +83,10 @@ services:
vidjil:
image: vidjil/metrics-app:latest
container_name: monitoring_vidjil
- ports:
- - "5000:5000"
+ # Commented ports to be only avaialble fron metrics network and not exposed
+ # If needed for dev, you can expose this port but data are fully exposed
+ # ports:
+ # - "5000:5000"
env_file:
# cp .env-sample as .env file and modify it with user/password of metrics declared user on server
# Don't use an admin account. See XXX page to create metrics user if not already done
--
GitLab
From 4afbe53e7678e3503bae8f4c8d2dae519cb12533 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 11:13:56 +0100
Subject: [PATCH 10/21] Node exporter; create a dedicated docker compose file
As this service should be launched on target server, move it inside a dedicated compose file
---
docker-compose-node.yml | 12 ++++++++++++
docker-compose.yml | 6 ------
2 files changed, 12 insertions(+), 6 deletions(-)
create mode 100644 docker-compose-node.yml
diff --git a/docker-compose-node.yml b/docker-compose-node.yml
new file mode 100644
index 0000000..23b0c70
--- /dev/null
+++ b/docker-compose-node.yml
@@ -0,0 +1,12 @@
+version: '3.3'
+
+services:
+
+ ## This service allow to get cpu/memory/... information.
+ # It should be launch on target server and not in this project
+ node_exporter:
+ image: prom/node-exporter:v0.18.1
+ container_name: monitoring_node_exporter
+ hostname: monitoring_node_exporter
+ ports:
+ - "9100:9100"
diff --git a/docker-compose.yml b/docker-compose.yml
index 3a1afa5..ac239ba 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -73,12 +73,6 @@ services:
# - ./blackbox:/config
# network_mode: "host"
- # node_exporter:
- # image: prom/node-exporter:v0.18.1
- # container_name: monitoring_node_exporter
- # deploy:
- # replicas: 1
- # network_mode: "host"
vidjil:
image: vidjil/metrics-app:latest
--
GitLab
From c088bff828671e97e53ab88f1650316f4a93813d Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 11:14:20 +0100
Subject: [PATCH 11/21] main; return all metrics for the moment.
---
metrics/main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/metrics/main.py b/metrics/main.py
index 30e62e1..e9c9c9b 100644
--- a/metrics/main.py
+++ b/metrics/main.py
@@ -63,7 +63,7 @@ def metrics():
vidjil = Vidjil(url_server, ssl=certificat)
vidjil.login(user, password)
- metrics = vidjil.metrics()
+ metrics = vidjil.metrics(metrics_list="all")
# print(f"{metrics=}")
formated_response += formated_metrics(metrics,cleanUrlServer(url_server))
except Exception as e:
--
GitLab
From e0076aaaa8b5d8a51b3ebacfae444f63f6db2890 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 11:16:43 +0100
Subject: [PATCH 12/21] prometheus.yml; clean yaml conf file
---
prometheus/prometheus.yml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml
index 1c629d6..f5f4605 100644
--- a/prometheus/prometheus.yml
+++ b/prometheus/prometheus.yml
@@ -1,5 +1,5 @@
global:
- scrape_interval: 15s # Par défaut prometheus va scrapper les cibles toutes les 15 secondes
+ scrape_interval: 90s # Par défaut prometheus va scrapper les cibles toutes les 15 secondes
# Attach these labels to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
@@ -21,12 +21,12 @@ global:
# rule_files:
# - "/alertmanager/alert.rules/alerts.rules.yml"
-# la scrape configuration de prometheus, les hosts qu'il va contacter pour chercher les métrique
+
scrape_configs:
- - job_name: 'measure server' # <-- j'ai mis ce titre de job mais le truc c'est de mettre un nom de job explicite
+ - job_name: 'measure server' # Ask for node reporter (cpu, memory, ...) on each targets server at port 9100
static_configs:
- - targets: ['monitoring_vidjil:9100', '192.168.1.77:9100']
+ - targets: []
scrape_interval: 30s
- job_name: 'vidjil'
--
GitLab
From 41a95738b6c814ed4aa44556944d28be71c6c95a Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 11:17:12 +0100
Subject: [PATCH 13/21] clean
---
metrics/conf.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/metrics/conf.py b/metrics/conf.py
index ab099b7..522d669 100644
--- a/metrics/conf.py
+++ b/metrics/conf.py
@@ -10,9 +10,6 @@ vidjil_servers = [
"certificat": "./certs/cert_localhost.pem",
"user":"metrics@vidjil.org",
"password": "foobartest"
- # }, {
- # "url": "https://app.vidjil.org/vidjil/",
- # "certificat": "./certs/cert_app.vidjil.org.pem"
}
]
--
GitLab
From f991c552114194641ff5edc44fbce585313ce599 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 11:41:57 +0100
Subject: [PATCH 14/21] Readme; improve readme with lot of documentation
---
metrics/Makefile | 4 +++
metrics/{conf.py => conf.sample.py} | 0
readme.md | 55 +++++++++++++++++++++++++----
3 files changed, 52 insertions(+), 7 deletions(-)
create mode 100644 metrics/Makefile
rename metrics/{conf.py => conf.sample.py} (100%)
diff --git a/metrics/Makefile b/metrics/Makefile
new file mode 100644
index 0000000..4dc30ae
--- /dev/null
+++ b/metrics/Makefile
@@ -0,0 +1,4 @@
+init_submodule:
+ git submodule init
+ git submodule update --remote
+
diff --git a/metrics/conf.py b/metrics/conf.sample.py
similarity index 100%
rename from metrics/conf.py
rename to metrics/conf.sample.py
diff --git a/readme.md b/readme.md
index a84c620..2f18363 100644
--- a/readme.md
+++ b/readme.md
@@ -7,30 +7,71 @@ You can find information on third-party-tools at:
- [Grafana](https://github.com/grafana/grafana)
<!-- - [Alertmanager](https://github.com/prometheus/alertmanager) -->
<!-- - [Blackbox exporter](https://github.com/prometheus/blackbox_exporter) -->
-<!-- - [Node exporter](https://github.com/prometheus/node_exporter) -->
+- [Node exporter](https://github.com/prometheus/node_exporter)
-### Set parameters
+### Set project
+
+#### Vidjil submodules initialisation
+
+This project depend of vidjil component.
+A first step is needed to get vidjil reposirory fetch inside this project.
+
+You need to launch this command to set it up.
+
+```
+make init_submodules
+```
+
+If everything work fine, Vidjil submodule will be downloaded under `metrics/vidjil`.
+
#### Grafana
Default password of grafana server is `admin/admin`.
At first login, a new password will be asked.
+<!-- TODO; filled information to get correct dashboard at starting -->
+
+#### Prometheus
+
+Configuration should be setted in `prometheus/prometheus.yml` file.
+For vidjil metrics, everything should work out of the box.
+However, you need to declare target servers for node exporter in `measure server` job.
+
#### Vidjil servers
-TODO
-## lancer la stack
+Configuration can be set in a conf file `metrics/conf.py`. A sample if present under `metrics/conf.sample.py`.
+Please copy and rename this file as `conf.py` to be sure to not send your credential on repository.
+You can after that modifiy configuration to set urln, metrics user and password
+
+You can also use a SSL certificate for API as described in [vidjil documentation](https://www.vidjil.org/doc/api/).
+On target server, you need to set up a metrics user if not already done.
+See this [page](https://www.vidjil.org/doc/admin/#server-monitoring) for more information on how to do that.
+
+
+## Launch services
+
To launch this stack, you will only need to launch a docker-compose file and set some variable on server to follow :
-```console
-docker-compose up -d
+
+```
+docker-compose up -d
```
+Note that for the moment, no image is avaialbe for vidjil metrics.
+At first lauch, you should add `--build` at `docker compsoe up -d` to build metrics server image.
+
+By default, only grafana server will be serve on port 3000.
+As metrics and prometheus don't allow to manage user and credential,
+we use an internal network allowing a communication between service but not with external world.
+However, you can open network by uncomment `ports` declaration of these services.
+
Host network will be used. You can find services on **localhost** at these port
- Prometheus : `9090`
- Grafana : `3000`
+- Metrics vidjil: `5000`
+- Node exporter : `9100`
<!-- - Alertmanager : `9093` -->
-<!-- - Node exporter : `9100` -->
<!-- Based on a projet of ouidou found on github -->
--
GitLab
From 18ada5ff81d5eae3fd80a11c54d5a898bdafa3a2 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 11:46:11 +0100
Subject: [PATCH 15/21] Makefile, add build informations
---
metrics/Makefile | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/metrics/Makefile b/metrics/Makefile
index 4dc30ae..cd6143b 100644
--- a/metrics/Makefile
+++ b/metrics/Makefile
@@ -2,3 +2,13 @@ init_submodule:
git submodule init
git submodule update --remote
+build_metrics:
+ docker compose build vidjil
+
+init:
+ python3 -m pip install -U pip
+ python3 -m pip install --no-cache -r requirements.txt
+
+unit:
+ #python3 -m pytest --cov=./ --cov-report html:cov.html -v -s .
+ python3 -m unittest test_main.py
\ No newline at end of file
--
GitLab
From 4d1171e7fa97d5a3c5c2325eb4b24c5fb99a066e Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 11:54:20 +0100
Subject: [PATCH 16/21] gitlab-ci; add job to build and push metrics images
---
.gitlab-ci.yml | 25 +++++++++++++++++++++++++
docker-compose.yml | 2 +-
2 files changed, 26 insertions(+), 1 deletion(-)
create mode 100644 .gitlab-ci.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..b10d4c7
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,25 @@
+# Metrics
+
+stages :
+ - publish_release
+
+
+deploy_docker:
+ stage: publish_release
+ image: docker:latest
+ services:
+ - docker:dind
+ script:
+ - export CUR_DATE=`date +%Y-%m-%d`
+ - export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
+ - export TAG="vidjil/metrics:$CUR_DATE-$SHA"
+ - docker login -u "$CI_DOCKER_USER" -p "$CI_DOCKER_PASSWORD" $CI_DOCKER
+ - docker build --no-cache -t $TAG -f metrics/Dockerfile ./
+ - docker push $TAG
+ - docker tag $TAG "vidjil/metrics:latest"
+ - docker push "vidjil/metrics:latest"
+ - docker rmi $TAG "vidjil/metrics:latest"
+ when: manual
+ tags:
+ - ci.inria.fr
+ - small
diff --git a/docker-compose.yml b/docker-compose.yml
index ac239ba..5bccfcc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -75,7 +75,7 @@ services:
vidjil:
- image: vidjil/metrics-app:latest
+ image: vidjil/metrics:latest
container_name: monitoring_vidjil
# Commented ports to be only avaialble fron metrics network and not exposed
# If needed for dev, you can expose this port but data are fully exposed
--
GitLab
From 023f31665046fa323e303783ca0a7bd43e8fd0bc Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 12:00:42 +0100
Subject: [PATCH 17/21] Update readme to add information on node-reporter
---
readme.md | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/readme.md b/readme.md
index 2f18363..aad156a 100644
--- a/readme.md
+++ b/readme.md
@@ -49,6 +49,13 @@ You can also use a SSL certificate for API as described in [vidjil documentation
On target server, you need to set up a metrics user if not already done.
See this [page](https://www.vidjil.org/doc/admin/#server-monitoring) for more information on how to do that.
+#### Node reporter
+
+Node reporter isa dedicated service to follow hardware of a server (CPU and memory usage, disk space, ...).
+It should be launch on each target and declared in prometheus as specified below.
+
+You can use for that dedicated `docker-compose-node.yml` file.
+
## Launch services
@@ -59,7 +66,8 @@ docker-compose up -d
```
Note that for the moment, no image is avaialbe for vidjil metrics.
-At first lauch, you should add `--build` at `docker compsoe up -d` to build metrics server image.
+At first lauch, you should launch `make build_metrics` to build metrics docker images that will be used.
+Note that if you made some dev and want to build again this image, you can add `--build` at `docker compsoe up -d` to build metrics server image at launch.
By default, only grafana server will be serve on port 3000.
As metrics and prometheus don't allow to manage user and credential,
--
GitLab
From f65c4ae4aa96d03fcc47b84d57884edb899a9779 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 13:54:46 +0100
Subject: [PATCH 18/21] gitlab-ci; add submodules init step
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b10d4c7..417b054 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,6 +10,7 @@ deploy_docker:
services:
- docker:dind
script:
+ - make init_submodules
- export CUR_DATE=`date +%Y-%m-%d`
- export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
- export TAG="vidjil/metrics:$CUR_DATE-$SHA"
--
GitLab
From 2cef4f3f1fd14d5e4ff63c63bd7296d79cd9cc33 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 13:56:36 +0100
Subject: [PATCH 19/21] gitlab-ci, docker build, change runners
---
.gitlab-ci.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 417b054..dc5803f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,5 +22,4 @@ deploy_docker:
- docker rmi $TAG "vidjil/metrics:latest"
when: manual
tags:
- - ci.inria.fr
- - small
+ - cidocker
--
GitLab
From 7da72ccd5dceed81be41169fff8d6b99007594a9 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 13:57:19 +0100
Subject: [PATCH 20/21] gitlab-ci, don't use missing make command
---
.gitlab-ci.yml | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dc5803f..8915760 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,10 +7,15 @@ stages :
deploy_docker:
stage: publish_release
image: docker:latest
+ services:
+ - name: docker:dind
+ alias: docker
+ command: ["--tls=false"]
services:
- docker:dind
script:
- - make init_submodules
+ - git submodule init
+ - git submodule update --remote
- export CUR_DATE=`date +%Y-%m-%d`
- export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
- export TAG="vidjil/metrics:$CUR_DATE-$SHA"
--
GitLab
From 4716948fcc359b88db7133de5fb5279046fc9f03 Mon Sep 17 00:00:00 2001
From: Thonier Florian <florian.thonier@inria.fr>
Date: Fri, 10 Jan 2025 14:28:27 +0100
Subject: [PATCH 21/21] Try to fix docker puclication
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8915760..9f65577 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,11 +20,11 @@ deploy_docker:
- export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
- export TAG="vidjil/metrics:$CUR_DATE-$SHA"
- docker login -u "$CI_DOCKER_USER" -p "$CI_DOCKER_PASSWORD" $CI_DOCKER
- - docker build --no-cache -t $TAG -f metrics/Dockerfile ./
+ - docker build --no-cache -t $TAG -f metrics/Dockerfile ./metrics/
- docker push $TAG
- docker tag $TAG "vidjil/metrics:latest"
- docker push "vidjil/metrics:latest"
- docker rmi $TAG "vidjil/metrics:latest"
- when: manual
+ # when: manual
tags:
- cidocker
--
GitLab