Support deploying Monasca Log Persister

This is a Logstash component which reads processed logs from Kafka
and writes them to Elasticsearch (or some other backend supported by
Logstash).

Ingesting the logs from this service with Fluentd will be covered under
a different commit.

Change-Id: I2d722991ab2072c54c4715507b19a4c9279f921b
Partially-Implements: blueprint monasca-roles

ansible/inventory/all-in-one

@@ -449,6 +449,9 @@ monasca
 [monasca-log-transformer:children]
 monasca
 
+[monasca-log-persister:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/inventory/multinode

@@ -458,6 +458,9 @@ monasca
 [monasca-log-transformer:children]
 monasca
 
+[monasca-log-persister:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/roles/monasca/defaults/main.yml

@@ -27,6 +27,15 @@ monasca_services:
       - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
+  monasca-log-persister:
+    container_name: monasca_log_persister
+    group: monasca-log-persister
+    enabled: true
+    image: "{{ monasca_logstash_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
 
 ####################
 # Databases
@@ -45,6 +54,7 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
 monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
 
 monasca_metrics_topic: "metrics"
 monasca_raw_logs_topic: "logs"

ansible/roles/monasca/handlers/main.yml

@@ -62,3 +62,24 @@
     - config_json.changed | bool
       or monasca_log_transformer_confs.changed | bool
       or monasca_log_transformer_container.changed | bool
+
+- name: Restart monasca-log-persister container
+  vars:
+    service_name: "monasca-log-persister"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_log_persister_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_log_persister_confs.changed | bool
+      or monasca_log_persister_elasticsearch_template.changed | bool
+      or monasca_log_persister_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -118,6 +118,45 @@
   notify:
     - Restart monasca-log-transformer container
 
+- name: Copying over monasca-log-persister config
+  vars:
+    service: "{{ monasca_services['monasca-log-persister'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf"
+    mode: "0660"
+  become: true
+  register: monasca_log_persister_confs
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf"
+    - "{{ node_custom_config }}/monasca/log-persister.conf"
+    - "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-persister container
+
+- name: Copying over monasca-log-persister elasticsearch template
+  vars:
+    service: "{{ monasca_services['monasca-log-persister'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json"
+    mode: "0660"
+  become: true
+  register: monasca_log_persister_elasticsearch_template
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json"
+    - "{{ node_custom_config }}/monasca/elasticsearch-template.json"
+    - "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-persister container
+
+
 - name: Check monasca containers
   become: true
   kolla_docker:

ansible/roles/monasca/tasks/deploy.yml

@@ -6,7 +6,8 @@
 - include: config.yml
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
-        inventory_hostname in groups['monasca-log-transformer']
+        inventory_hostname in groups['monasca-log-transformer'] or
+        inventory_hostname in groups['monasca-log-persister']
 
 - include: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -17,4 +18,5 @@
 - include: check.yml
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
-        inventory_hostname in groups['monasca-log-transformer']
+        inventory_hostname in groups['monasca-log-transformer'] or
+        inventory_hostname in groups['monasca-log-persister']

ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json

+{
+  "aliases": {},
+  "mappings": {
+    "log": {
+      "_all": {
+        "enabled": true,
+        "omit_norms": true
+      },
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "mapping": {
+              "fielddata": {
+                "format": "disabled"
+              },
+              "index": "analyzed",
+              "omit_norms": true,
+              "type": "string"
+            },
+            "match": "message",
+            "match_mapping_type": "string"
+          }
+        },
+        {
+          "other_fields": {
+            "mapping": {
+              "index": "not_analyzed",
+              "type": "string"
+            },
+            "match": "*",
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "creation_time": {
+          "type": "date"
+        }
+      }
+    }
+  },
+  "order": 0,
+  "settings": {
+    "index": {
+      "refresh_interval": "5s"
+    }
+  },
+  "template": "monasca-*"
+}

ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2

+# Persist transformed logs to Elasticsearch
+
+input {
+    kafka {
+        zk_connect => "{{ monasca_zookeeper_servers }}"
+        topic_id => "{{ monasca_transformed_logs_topic }}"
+        group_id => "transformer-logstash-consumer"
+    }
+}
+
+output {
+    elasticsearch {
+        index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}"
+        hosts => [{{ monasca_elasticsearch_servers }}]
+        document_type => "log"
+        template_name => "monasca"
+        template => "/etc/logstash/elasticsearch-template.json"
+    }
+}

ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2

+{
+    "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/monasca/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/log-persister.conf",
+            "dest": "/etc/logstash/conf.d/log-persister.conf",
+            "owner": "logstash",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/elasticsearch-template.json",
+            "dest": "/etc/logstash/elasticsearch-template.json",
+            "owner": "logstash",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "logstash:kolla",
+            "recurse": true
+        }
+    ]
+}

releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml

+---
+features:
+  - |
+    Add support for deploying the Monasca Log Persister. The Log
+    Persister is responsible for reading logs from the Kafka processed
+    logs topic and writing them to Elasticsearch.