diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 0eb6b6641d52b69dfea1001e30722b4034901f3e..845b5c03adcd1645bcfeab270f9f38905cd74034 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -446,6 +446,9 @@ monasca [monasca-log-api:children] monasca +[monasca-log-transformer:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 1a223722c581bd884434e82821f88ae530efa41a..1466025b7569838c222a44581a0d14b13c874c95 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -455,6 +455,9 @@ monasca [monasca-log-api:children] monasca +[monasca-log-transformer:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index b5fc75a6956ff98c16a353d4da0b5690d77c0c75..4db2feb1c6ca89f60d703f855480cdf944175920 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -18,6 +18,15 @@ monasca_services: - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" + monasca-log-transformer: + container_name: monasca_log_transformer + group: monasca-log-transformer + enabled: true + image: "{{ monasca_logstash_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" #################### # Databases @@ -34,10 +43,12 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}" # Monasca #################### monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}" +monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_metrics_topic: "metrics" monasca_raw_logs_topic: "logs" +monasca_transformed_logs_topic: "transformed-logs" #################### # Docker @@ -53,6 +64,10 @@ monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ monasca_log_api_tag: "{{ monasca_tag }}" monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}" +monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-logstash" +monasca_logstash_tag: "{{ monasca_tag }}" +monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}" + #################### # OpenStack #################### diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index dffdbc7d5962ff92c59f781b7482ac84576a9c8e..6caadb6df07a32eb97eb3e4b2926649fbf331d61 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -42,3 +42,23 @@ or monasca_log_api_confs.changed | bool or monasca_log_api_wsgi_confs.changed | bool or monasca_log_api_container.changed | bool + +- name: Restart monasca-log-transformer container + vars: + service_name: "monasca-log-transformer" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_transformer_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_transformer_confs.changed | bool + or monasca_log_transformer_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 164b970868d0f439bea043c27d3985db8ee309c1..42c40f38dc2ffb695b6636ccdaf35c5b57c9f7b7 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -25,8 +25,7 @@ - item.value.enabled | bool with_dict: "{{ monasca_services }}" notify: - - Restart monasca-api container - - Restart monasca-log-api container + - "Restart {{ item.key }} container" - name: Copying over monasca-api config vars: @@ -100,6 +99,25 @@ notify: - Restart monasca-log-api container +- name: Copying over monasca-log-transformer config + vars: + service: "{{ monasca_services['monasca-log-transformer'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf" + mode: "0660" + become: true + register: monasca_log_transformer_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf" + - "{{ node_custom_config }}/monasca/log-transformer.conf" + - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-transformer container + - name: Check monasca containers become: true kolla_docker: @@ -115,5 +133,4 @@ - item.value.enabled | bool with_dict: "{{ monasca_services }}" notify: - - Restart monasca-api container - - Restart monasca-log-api container + - "Restart {{ item.key }} container" diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index decfa5ff9fcb3c86ad89bc1981072e07ede468e8..b386d15e3f4468ce6a8ba77a2ded5775531f18fe 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -5,7 +5,8 @@ - include: config.yml when: inventory_hostname in groups['monasca-api'] or - inventory_hostname in groups['monasca-log-api'] + inventory_hostname in groups['monasca-log-api'] or + inventory_hostname in groups['monasca-log-transformer'] - include: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -15,4 +16,5 @@ - include: check.yml when: inventory_hostname in groups['monasca-api'] or - inventory_hostname in groups['monasca-log-api'] + inventory_hostname in groups['monasca-log-api'] or + inventory_hostname in groups['monasca-log-transformer'] diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 new file mode 100644 index 0000000000000000000000000000000000000000..c571623317376d5382d4594deded312d3e8189f1 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 @@ -0,0 +1,25 @@ +# Provide input/output streams for transforming Monasca logs. +# Filters should be provided in other configuration files. + +input { + kafka { + zk_connect => "{{ monasca_zookeeper_servers }}" + topic_id => "{{ monasca_raw_logs_topic }}" + group_id => "transformer-logstash-consumer" + } +} + +filter { + # Update the timestamp of the event based on the time in the message. + date { + match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"] + remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] + } +} + +output { + kafka { + bootstrap_servers => "{{ monasca_kafka_servers }}" + topic_id => "{{ monasca_transformed_logs_topic }}" + } +} diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 new file mode 100644 index 0000000000000000000000000000000000000000..94cb1cf3751cb5d114de929e2c32d231a39a32e6 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-transformer.conf", + "dest": "/etc/logstash/conf.d/log-transformer.conf", + "owner": "logstash", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/logstash", + "owner": "logstash:kolla", + "recurse": true + } + ] +} diff --git a/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e1eb09639accdf746d06c46d47a30b49f9ebd1a --- /dev/null +++ b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for deploying the Monasca Log Transformer for + providing log standardisation in Monasca.