Added Logstash logger

Dockerfile

-FROM azacrknowledgelens.azurecr.io/ai-forecasting/batch-multi-karios-read-component:v0.6
+FROM python:3.7-slim
 ADD . /opt
 WORKDIR /opt
+RUN pip install -r requirements.txt
 CMD python main.py
\ No newline at end of file

Dockerfile_bkp

-FROM python:3.7-slim
-ADD . /opt
-WORKDIR /opt
-RUN pip install -r requirements.txt
-CMD python main.py
\ No newline at end of file

conf/configuration.yml

@@ -2,3 +2,5 @@
 SERVICE_CONFIG:
   LOG_LEVEL: info
   LOG_HANDLER_NAME: ReadFromKairos
+  LOGSTASH_HOST: 192.168.1.47
+  LOGSTASH_PORT: 5000

requirements.txt

 requests
 pandas
 pyyaml
+python-logstash-async
\ No newline at end of file

scripts/common/config_parser.py

@@ -47,6 +47,10 @@ if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
 
 LOG_LEVEL = os.environ.get("LOG_LEVEL" ,_config.get('SERVICE_CONFIG', {}).get("LOG_LEVEL", "INFO")).upper()
 LOG_HANDLER_NAME = _config.get('SERVICE_CONFIG', {}).get("LOG_HANDLER_NAME", "ReadFromKairos")
+ENABLE_LOGSTASH_LOG = os.environ.get("ENABLE_LOGSTASH_LOG", 'False').lower()
+LOGSTASH_HOST = os.environ.get("LOGSTASH_HOST", _config.get('SERVICE_CONFIG', {}).get('LOGSTASH_HOST'))
+LOGSTASH_PORT = str(os.environ.get("LOGSTASH_PORT", _config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT')))
+
 
 config = {
     "shared_volume": os.environ.get("shared_volume"),

scripts/common/logsetup.py

 import os
 import logging
 from logging.handlers import RotatingFileHandler
+from logstash_async.handler import AsynchronousLogstashHandler
+
+from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH, LOGSTASH_HOST, LOGSTASH_PORT, \
+    ENABLE_LOGSTASH_LOG
 
-from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH
 
 DEFAULT_FORMAT = '%(asctime)s %(levelname)5s %(name)s %(message)s'
 DEBUG_FORMAT = '%(asctime)s %(levelname)5s %(name)s [%(threadName)5s:%(filename)5s:%(funcName)5s():%(lineno)s] %(' \
@@ -43,6 +46,8 @@ def get_logger(log_handler_name):
     handler.setFormatter(formatter)
     _logger.addHandler(log_handler)
     _logger.addHandler(handler)
+    if ENABLE_LOGSTASH_LOG == 'true' and LOGSTASH_PORT is not None and LOGSTASH_HOST is not None and LOGSTASH_PORT.isdigit():
+        _logger.addHandler(AsynchronousLogstashHandler(LOGSTASH_HOST, int(LOGSTASH_PORT), database_path=None))
     return _logger