From 30fa50f48df1a1845e882df813434895f725efd9 Mon Sep 17 00:00:00 2001
From: aynsix <asr@esokia-webagency.com>
Date: Fri, 24 Apr 2020 20:27:11 +0300
Subject: [PATCH] put worker log to ELK

---
 .gitignore                  |  2 ++
 config/filebeat.yml         |  9 +++++++++
 config/logstash.conf        | 36 ++++++++++++++++++++++++++++++++++++
 docker-compose.override.yml | 24 ++++++++++++++++++++++++
 4 files changed, 71 insertions(+)
 create mode 100644 config/filebeat.yml
 create mode 100644 config/logstash.conf

diff --git a/.gitignore b/.gitignore
index ead2712de2..3f96bc4dfd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,8 @@
 # Exclude configuration folder exception the configuration sample file
 /config
 !/config/configuration.sample.yml
+!/config/filebeat.yml
+!/config/logstash.conf
 
 # Exclude generated proxies from doctrine2
 #/resources/proxies
diff --git a/config/filebeat.yml b/config/filebeat.yml
new file mode 100644
index 0000000000..13f1adb53d
--- /dev/null
+++ b/config/filebeat.yml
@@ -0,0 +1,9 @@
+filebeat:
+  inputs:
+    - type: log
+      enabled: true
+      paths:
+      - /var/alchemy/Phraseanet/logs/task_*.log
+      - /var/alchemy/Phraseanet/logs/worker_service*.log
+output.logstash:
+  hosts: ["logstash:5044"]
\ No newline at end of file
diff --git a/config/logstash.conf b/config/logstash.conf
new file mode 100644
index 0000000000..48eece44aa
--- /dev/null
+++ b/config/logstash.conf
@@ -0,0 +1,36 @@
+input {
+  beats {
+    port => 5044
+  }
+}
+filter {
+  clone {
+    clones => ["to_stdout"]
+  }
+
+  if [type] == "to_stdout" {
+  #display only message field in the logstash stdout
+    prune {
+        whitelist_names => ["message"]
+    }
+    mutate {
+        add_field => { "[@metadata][type]" => "to_stdout" }
+    }
+  }
+  else {
+    mutate {
+        add_field => { "[@metadata][type]" => "to_elasticsearch" }
+    }
+  }
+
+}
+output {
+    if [@metadata][type][1] == "to_stdout" {
+        stdout {
+            codec => rubydebug
+        }
+    }
+    else {
+        elasticsearch { hosts => ["elasticsearch:9200"] }
+    }
+}
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
index e058d2886a..ef5b56c55f 100644
--- a/docker-compose.override.yml
+++ b/docker-compose.override.yml
@@ -91,6 +91,30 @@ services:
     image: kibana:4.6.6
     ports:
     - 5601:5601
+    links:
+    - elasticsearch
+    depends_on:
+    - elasticsearch
+
+  logstash:
+    image: logstash:7.6.2
+    links:
+    - elasticsearch
+    volumes:
+    - ${PHRASEANET_CONFIG_DIR}:/var/alchemy/Phraseanet/config:rw
+    command: logstash -f /var/alchemy/Phraseanet/config/logstash.conf
+    depends_on:
+    - elasticsearch
+    restart: on-failure
+
+  filebeat:
+    hostname: filebeat
+    image: "docker.elastic.co/beats/filebeat:7.6.2"
+    volumes:
+    - ${PHRASEANET_CONFIG_DIR}/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+    - ${PHRASEANET_LOGS_DIR}:/var/alchemy/Phraseanet/logs:ro
+    command: filebeat run -e --strict.perms=false
+    restart: on-failure
 
 networks:
   default: