From 9c88262ad93f224c2fd057373a491362f7176ba0 Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Mon, 4 Jun 2018 15:34:38 +0100
Subject: [PATCH] Support deploying Monasca Log Transformer
The Monasca Log Transformer takes raw, unstandardised logs from one
Kafka topic, standardises them with whatever rules the operator wants
to use, and then writes them to a standardised logs topic in Kafka. It
is currently implemented as a Logstash config file.
Since Kolla does a fairly good job of standardising logs, this service
does very little processing. However, when other sources of logs
are used, it may be useful to add rules to the Transformer, particularly
if it's not possible to standardise the logs at source.
Ingesting the logs from this service with Fluentd will be covered under
a different commit.
Change-Id: I31cbb7e9a40a848391f517a56a67e3fd5bc12529
Partially-Implements: blueprint monasca-roles
---
ansible/inventory/all-in-one | 3 +++
ansible/inventory/multinode | 3 +++
ansible/roles/monasca/defaults/main.yml | 15 +++++++++++
ansible/roles/monasca/handlers/main.yml | 20 +++++++++++++++
ansible/roles/monasca/tasks/config.yml | 25 ++++++++++++++++---
ansible/roles/monasca/tasks/deploy.yml | 6 +++--
.../log-transformer.conf.j2 | 25 +++++++++++++++++++
.../monasca-log-transformer.json.j2 | 18 +++++++++++++
...asca-log-transformer-80d985fc77603478.yaml | 5 ++++
9 files changed, 114 insertions(+), 6 deletions(-)
create mode 100644 ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
create mode 100644 ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
create mode 100644 releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 0eb6b6641..845b5c03a 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -446,6 +446,9 @@ monasca
[monasca-log-api:children]
monasca
+[monasca-log-transformer:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 1a223722c..1466025b7 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -455,6 +455,9 @@ monasca
[monasca-log-api:children]
monasca
+[monasca-log-transformer:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml
index b5fc75a69..4db2feb1c 100644
--- a/ansible/roles/monasca/defaults/main.yml
+++ b/ansible/roles/monasca/defaults/main.yml
@@ -18,6 +18,15 @@ monasca_services:
- "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla"
+ monasca-log-transformer:
+ container_name: monasca_log_transformer
+ group: monasca-log-transformer
+ enabled: true
+ image: "{{ monasca_logstash_image_full }}"
+ volumes:
+ - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "kolla_logs:/var/log/kolla"
####################
# Databases
@@ -34,10 +43,12 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
# Monasca
####################
monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_metrics_topic: "metrics"
monasca_raw_logs_topic: "logs"
+monasca_transformed_logs_topic: "transformed-logs"
####################
# Docker
@@ -53,6 +64,10 @@ monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{
monasca_log_api_tag: "{{ monasca_tag }}"
monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}"
+monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-logstash"
+monasca_logstash_tag: "{{ monasca_tag }}"
+monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}"
+
####################
# OpenStack
####################
diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml
index dffdbc7d5..6caadb6df 100644
--- a/ansible/roles/monasca/handlers/main.yml
+++ b/ansible/roles/monasca/handlers/main.yml
@@ -42,3 +42,23 @@
or monasca_log_api_confs.changed | bool
or monasca_log_api_wsgi_confs.changed | bool
or monasca_log_api_container.changed | bool
+
+- name: Restart monasca-log-transformer container
+ vars:
+ service_name: "monasca-log-transformer"
+ service: "{{ monasca_services[service_name] }}"
+ config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ monasca_log_transformer_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ volumes: "{{ service.volumes }}"
+ when:
+ - action != "config"
+ - inventory_hostname in groups[service.group]
+ - service.enabled | bool
+ - config_json.changed | bool
+ or monasca_log_transformer_confs.changed | bool
+ or monasca_log_transformer_container.changed | bool
diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml
index 164b97086..42c40f38d 100644
--- a/ansible/roles/monasca/tasks/config.yml
+++ b/ansible/roles/monasca/tasks/config.yml
@@ -25,8 +25,7 @@
- item.value.enabled | bool
with_dict: "{{ monasca_services }}"
notify:
- - Restart monasca-api container
- - Restart monasca-log-api container
+ - "Restart {{ item.key }} container"
- name: Copying over monasca-api config
vars:
@@ -100,6 +99,25 @@
notify:
- Restart monasca-log-api container
+- name: Copying over monasca-log-transformer config
+ vars:
+ service: "{{ monasca_services['monasca-log-transformer'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf"
+ mode: "0660"
+ become: true
+ register: monasca_log_transformer_confs
+ with_first_found:
+ - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf"
+ - "{{ node_custom_config }}/monasca/log-transformer.conf"
+ - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2"
+ when:
+ - inventory_hostname in groups[service['group']]
+ - service.enabled | bool
+ notify:
+ - Restart monasca-log-transformer container
+
- name: Check monasca containers
become: true
kolla_docker:
@@ -115,5 +133,4 @@
- item.value.enabled | bool
with_dict: "{{ monasca_services }}"
notify:
- - Restart monasca-api container
- - Restart monasca-log-api container
+ - "Restart {{ item.key }} container"
diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml
index decfa5ff9..b386d15e3 100644
--- a/ansible/roles/monasca/tasks/deploy.yml
+++ b/ansible/roles/monasca/tasks/deploy.yml
@@ -5,7 +5,8 @@
- include: config.yml
when: inventory_hostname in groups['monasca-api'] or
- inventory_hostname in groups['monasca-log-api']
+ inventory_hostname in groups['monasca-log-api'] or
+ inventory_hostname in groups['monasca-log-transformer']
- include: bootstrap.yml
when: inventory_hostname in groups['monasca-api']
@@ -15,4 +16,5 @@
- include: check.yml
when: inventory_hostname in groups['monasca-api'] or
- inventory_hostname in groups['monasca-log-api']
+ inventory_hostname in groups['monasca-log-api'] or
+ inventory_hostname in groups['monasca-log-transformer']
diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
new file mode 100644
index 000000000..c57162331
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
@@ -0,0 +1,25 @@
+# Provide input/output streams for transforming Monasca logs.
+# Filters should be provided in other configuration files.
+
+input {
+ kafka {
+ zk_connect => "{{ monasca_zookeeper_servers }}"
+ topic_id => "{{ monasca_raw_logs_topic }}"
+ group_id => "transformer-logstash-consumer"
+ }
+}
+
+filter {
+ # Update the timestamp of the event based on the time in the message.
+ date {
+ match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"]
+ remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
+ }
+}
+
+output {
+ kafka {
+ bootstrap_servers => "{{ monasca_kafka_servers }}"
+ topic_id => "{{ monasca_transformed_logs_topic }}"
+ }
+}
diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
new file mode 100644
index 000000000..94cb1cf37
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
@@ -0,0 +1,18 @@
+{
+ "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
+ "config_files": [
+ {
+ "source": "{{ container_config_directory }}/log-transformer.conf",
+ "dest": "/etc/logstash/conf.d/log-transformer.conf",
+ "owner": "logstash",
+ "perm": "0600"
+ }
+ ],
+ "permissions": [
+ {
+ "path": "/var/log/kolla/logstash",
+ "owner": "logstash:kolla",
+ "recurse": true
+ }
+ ]
+}
diff --git a/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml
new file mode 100644
index 000000000..3e1eb0963
--- /dev/null
+++ b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml
@@ -0,0 +1,5 @@
+---
+features:
+ - |
+ Add support for deploying the Monasca Log Transformer for
+ providing log standardisation in Monasca.
--
GitLab