feat(logging): Add structured JSON logging support for console (#1265)

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>

Author: tomchop

contrib/analytics/ivre_api/ivre_api.py

@@ -36,11 +36,7 @@
 import logging
 from pprint import pformat
 
-from ivre.db import db
-from ivre.utils import encode_b64
-
 from core.analytics import InlineAnalytics, OneShotAnalytics
-from core.errors import ObservableValidationError
 from core.observables import (
     AutonomousSystem,
     Certificate,
@@ -51,6 +47,10 @@
     Text,
     Url,
 )
+from ivre.db import db
+from ivre.utils import encode_b64
+
+from core.errors import ObservableValidationError
 
 LOG = logging.getLogger("Yeti-Ivre")
 

contrib/feeds/otx/otx.py

@@ -1,12 +1,12 @@
 from datetime import timedelta
 
 import dateutil.parser
+from core.feed import Feed
+from core.observables import Email, Hash, Hostname, Ip, Url
 from mongoengine import DictField
 from OTXv2 import OTXv2
 
 from core.config.config import yeti_config
-from core.feed import Feed
-from core.observables import Email, Hash, Hostname, Ip, Url
 
 OBSERVABLE_TYPES = {
     "IPv4": Ip,

contrib/feeds/proofpoint/proofpoint.py

@@ -7,15 +7,15 @@
 
 import hammock
 import requests
+from core.entities import TTP, Actor, Campaign
+from core.entities.malware import Malware
+from core.feed import Feed
+from core.observables import Email, File, Hash, Hostname, Ip, Observable, Tag, Text, Url
 from mongoengine.errors import DoesNotExist
 from requests import auth
 
 from core.config.config import yeti_config
-from core.entities import TTP, Actor, Campaign
-from core.entities.malware import Malware
 from core.errors import ObservableValidationError
-from core.feed import Feed
-from core.observables import Email, File, Hash, Hostname, Ip, Observable, Tag, Text, Url
 
 log = logging.getLogger("pp2yeti")
 

contrib/feeds/threattracking/threattracking.py

@@ -6,11 +6,11 @@
 import hammock
 import requests
 import simplejson as json
+from core.entities import Actor, Campaign, Malware
+from core.feed import Feed
 from mongoengine.errors import DoesNotExist
 
 from core.config.config import yeti_config
-from core.entities import Actor, Campaign, Malware
-from core.feed import Feed
 
 # # add configuration to yeti.conf
 # [threattracking]

core/logger.py

@@ -69,19 +69,30 @@ def __init__(self):
     def format(self, record):
         json_record = {}
         json_record["message"] = record.getMessage()
+        json_record["severity"] = record.levelname
+        json_record["timestamp"] = datetime.datetime.fromtimestamp(
+            record.created, datetime.timezone.utc
+        ).isoformat()
+        json_record["logger"] = record.name
         if "username" in record.__dict__:
             json_record["username"] = record.__dict__["username"]
         if "path" in record.__dict__:
             json_record["path"] = record.__dict__["path"]
         if "method" in record.__dict__:
             json_record["method"] = record.__dict__["method"]
         if "body" in record.__dict__ and record.__dict__["body"]:
-            if record.__dict__["content-type"] == "application/json":
-                json_record["body"] = json.loads(
-                    record.__dict__["body"].decode("utf-8")
-                )
-            else:
-                json_record["body"] = record.__dict__["body"].decode("utf-8")
+            try:
+                body_content = record.__dict__["body"]
+                if isinstance(body_content, bytes):
+                    body_content = body_content.decode("utf-8")
+
+                if record.__dict__.get("content-type") == "application/json":
+                    json_record["body"] = json.loads(body_content)
+                else:
+                    json_record["body"] = body_content
+            except Exception:
+                json_record["body"] = str(record.__dict__["body"])
+
         if "client" in record.__dict__:
             json_record["client"] = record.__dict__["client"]
         if "status_code" in record.__dict__:
@@ -149,7 +160,10 @@ def filter(self, record):
 
 console_handler = logging.StreamHandler()
 console_handler.addFilter(LogFilter())
-console_handler.setFormatter(logging.Formatter(log_format))
+if os.environ.get("YETI_K8S_RUNTIME", "false").lower() == "true":
+    console_handler.setFormatter(JsonFormatter())
+else:
+    console_handler.setFormatter(logging.Formatter(log_format))
 handlers.append(console_handler)
 
 audit_logfile = yeti_config.get("system", "audit_logfile")