Merge "Update Monasca Log Transformer config"

ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2

@@ -5,7 +5,9 @@ input {
     kafka {
         zk_connect => "{{ monasca_zookeeper_servers }}"
         topic_id => "{{ monasca_raw_logs_topic }}"
-        group_id => "transformer-logstash-consumer"
+        group_id => "log_transformer"
+        consumer_id => "log_transformer_{{ ansible_hostname }}"
+        consumer_threads => "{{ monasca_log_pipeline_threads }}"
     }
 }
 
@@ -15,11 +17,23 @@ filter {
         match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"]
         remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
     }
+
+    # OpenStack log levels are uppercase, and syslog are lowercase.
+    # Furthermore, syslog has more log levels that OpenStack. To avoid
+    # mapping syslog log levels to OpenStack log levels, we standardise
+    # on the syslog style here.
+    if [log][dimensions][log_level] {
+        mutate {
+            lowercase => [ "[log][dimensions][log_level]" ]
+        }
+    }
 }
 
 output {
     kafka {
         bootstrap_servers => "{{ monasca_kafka_servers }}"
         topic_id => "{{ monasca_transformed_logs_topic }}"
+        client_id => "log_transformer_{{ ansible_hostname }}"
+        workers => {{ monasca_log_pipeline_threads|int }}
     }
 }

ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2

 {
-    "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
+    "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/log-transformer.conf",