Merge pull request #88 from geoadmin/develop

New Release v4.4.0 - #minor

Author: msom

.env.default

@@ -7,4 +7,11 @@ ALLOWED_DOMAINS=localhost,.*\.geo\.admin\.ch,.*\.bgdi\.ch
 STAGING=local
 
 # OTEL
-OTEL_SDK_DISABLED=true
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_BOTO=true
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-shortlink
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.pylintrc

@@ -36,10 +36,6 @@ load-plugins=pylint_flask
 # Pickle collected data for later comparisons.
 persistent=yes
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no

Makefile

@@ -28,6 +28,7 @@ YAPF_CMD := $(PIPENV_RUN) yapf
 ISORT_CMD := $(PIPENV_RUN) isort
 NOSE_CMD := $(PIPENV_RUN) nose2
 PYLINT_CMD := $(PIPENV_RUN) pylint
+EDOT_BOOTSTRAP_CMD := $(PIPENV_RUN) edot-bootstrap
 
 # AWS variables
 AWS_DEFAULT_REGION = eu-central-1
@@ -70,7 +71,6 @@ help:
 	@echo "- dockerpush         Build and push the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG))"
 	@echo "- dockerrun          Run the project using the gunicorn WSGI server inside a container. (Exposed_port: $(HTTP_PORT)"
 	@echo "- shutdown           Stop the aforementioned container"
-	@echo "- otelrequirements   Get a list of available otel instrumentation libraries to add to the pipfile of this project"
 	@echo -e " \033[1mCLEANING TARGETS\033[0m "
 	@echo "- clean              Clean genereated files"
 	@echo "- clean_venv         Clean python venv"
@@ -125,7 +125,7 @@ serve: clean_logs $(LOGS_DIR)
 
 .PHONY: gunicornserve
 gunicornserve: clean_logs $(LOGS_DIR)
-	OTEL_SDK_DISABLED=true ENV_FILE=.env.default LOGS_DIR=$(LOGS_DIR) ${PYTHON_CMD} wsgi.py
+	ENV_FILE=.env.default LOGS_DIR=$(LOGS_DIR) ${PYTHON_CMD} wsgi.py
 
 # Docker related functions.
 .PHONY: dockerlogin
@@ -179,9 +179,5 @@ clean: clean_venv clean_logs
 
 # Actual builds targets with dependencies
 
-.PHONY: otelrequirements
-otelrequirements:
-	edot-bootstrap --action=requirements
-
 $(LOGS_DIR):
 	mkdir -p -m=777 $(LOGS_DIR)

Pipfile

@@ -5,7 +5,7 @@ name = "pypi"
 
 [packages]
 boto3 = "~=1.40"
-logging-utilities = "~=5.1"
+logging-utilities = "~=5.3"
 Flask = "~=3.1"
 gevent = "~=25.8"
 gunicorn = "~=23.0"
@@ -18,18 +18,17 @@ opentelemetry-exporter-otlp = "~=1.37"
 opentelemetry-instrumentation = "*"
 
 # Specific OpenTelemetry instrumentation libraries used to instrument this app
-# Full list or a subset of libraries listed by 'make otelrequirements'
+# Full list or a subset of libraries listed by 'edot-bootstrap --action=requirements'
 opentelemetry-instrumentation-botocore = "*"
 opentelemetry-instrumentation-flask = "*"
-opentelemetry-instrumentation-requests = "*"
+opentelemetry-instrumentation-logging = "*"
 
 [dev-packages]
 yapf = "~=0.43"
 moto = "~=5.1"
 nose2 = "*"
 pylint = "*"
 pylint-flask = "*"
-elastic-opentelemetry = "*"
 
 [requires]
 python_version = "3.13"

Pipfile.lock

@@ -1,25 +1,56 @@
-import os
+from os import getenv
 
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \
     OTLPSpanExporter
+from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
-from app.helpers.utils import strtobool
 
+def strtobool(value) -> bool:
+    """Convert a string representation of truth to true (1) or false (0).
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
+    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
+    'val' is anything else.
+    """
+    value = value.lower()
+    if value in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    if value in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    raise ValueError(f"invalid truth value \'{value}\'")
 
-def setup_trace_provider(worker_pid):
-    trace.set_tracer_provider(TracerProvider(resource=Resource.create()))
 
-    # Since we created a new tracer, the default span processor is gone. We need to
-    # create a new one using the default OTEL env variables and ad it to the tracer.
-    span_processor = BatchSpanProcessor(
-        OTLPSpanExporter(
-            endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
-            headers=os.getenv('OTEL_EXPORTER_OTLP_HEADERS'),
-            insecure=strtobool(os.getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+def initialize() -> None:
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_BOTO", "false")):
+            BotocoreInstrumentor().instrument()
+        if strtobool(getenv("OTEL_ENABLE_LOGGING", "false")):
+            LoggingInstrumentor().instrument()
+
+
+def initialize_flask(app):
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_FLASK", "false")):
+            FlaskInstrumentor().instrument_app(app)
+
+
+def setup_trace_provider():
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        # Since we created a new tracer, the default span processor is gone. We need to
+        # create a new one using the default OTEL env variables and ad it to the tracer.
+        span_processor = BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
+                headers=getenv('OTEL_EXPORTER_OTLP_HEADERS'),
+                insecure=strtobool(getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+            )
         )
-    )
-    trace.get_tracer_provider().add_span_processor(span_processor)
+
+        provider = TracerProvider(resource=Resource.create())
+        provider.add_span_processor(span_processor)
+        trace.set_tracer_provider(provider)

README.md

@@ -15,6 +15,7 @@
 from flask import make_response
 from flask import request
 
+from app.helpers.otel import strtobool
 from app.settings import ALLOWED_DOMAINS_PATTERN
 from app.settings import SHORT_ID_ALPHABET
 from app.settings import SHORT_ID_SIZE
@@ -124,20 +125,6 @@ def get_url():
     return url
 
 
-def strtobool(value) -> bool:
-    """Convert a string representation of truth to true (1) or false (0).
-    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
-    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
-    'val' is anything else.
-    """
-    value = value.lower()
-    if value in ('y', 'yes', 't', 'true', 'on', '1'):
-        return True
-    if value in ('n', 'no', 'f', 'false', 'off', '0'):
-        return False
-    raise ValueError(f"invalid truth value \'{value}\'")
-
-
 def is_domain_allowed(url):
     """Check if the url contain a domain that is allowed
     """

app/helpers/otel.py

@@ -70,3 +70,16 @@ services:
       - AWS_ACCESS_KEY_ID=dummy123
       - AWS_SECRET_ACCESS_KEY=dummy123
       - AWS_DEFAULT_REGION=wonderland
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: --config otel-local-config.yaml
+    volumes:
+      - ./otel-local-config.yaml:/otel-local-config.yaml
+    ports:
+      - "4317:4317"
+
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"

app/helpers/utils.py

@@ -88,6 +88,9 @@ formatters:
         duration: "%(duration)s"
         payload: "%(response.json).128s"
       message: message
+      otelSpanID: otelSpanID
+      otelTraceID: otelTraceID
+      otelTraceSampled: otelTraceSampled
 
 handlers:
   console: