PB-1447: Improve tracing

.env.default

@@ -7,4 +7,11 @@ ALLOWED_DOMAINS=localhost,.*\.geo\.admin\.ch,.*\.bgdi\.ch
 STAGING=local
 
 # OTEL
-OTEL_SDK_DISABLED=true
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_BOTO=true
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-shortlink
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.pylintrc

@@ -36,10 +36,6 @@ load-plugins=pylint_flask
 # Pickle collected data for later comparisons.
 persistent=yes
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no

Makefile

@@ -71,7 +71,6 @@ help:
 	@echo "- dockerpush         Build and push the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG))"
 	@echo "- dockerrun          Run the project using the gunicorn WSGI server inside a container. (Exposed_port: $(HTTP_PORT)"
 	@echo "- shutdown           Stop the aforementioned container"
-	@echo "- otelrequirements   Get a list of available otel instrumentation libraries to add to the pipfile of this project"
 	@echo -e " \033[1mCLEANING TARGETS\033[0m "
 	@echo "- clean              Clean genereated files"
 	@echo "- clean_venv         Clean python venv"
@@ -126,7 +125,7 @@ serve: clean_logs $(LOGS_DIR)
 
 .PHONY: gunicornserve
 gunicornserve: clean_logs $(LOGS_DIR)
-	OTEL_SDK_DISABLED=true ENV_FILE=.env.default LOGS_DIR=$(LOGS_DIR) ${PYTHON_CMD} wsgi.py
+	ENV_FILE=.env.default LOGS_DIR=$(LOGS_DIR) ${PYTHON_CMD} wsgi.py
 
 # Docker related functions.
 .PHONY: dockerlogin
@@ -180,9 +179,5 @@ clean: clean_venv clean_logs
 
 # Actual builds targets with dependencies
 
-.PHONY: otelrequirements
-otelrequirements:
-	$(EDOT_BOOTSTRAP_CMD) --action=requirements
-
 $(LOGS_DIR):
 	mkdir -p -m=777 $(LOGS_DIR)

Pipfile

@@ -5,7 +5,7 @@ name = "pypi"
 
 [packages]
 boto3 = "~=1.40"
-logging-utilities = "~=5.1"
+logging-utilities = "~=5.3"
 Flask = "~=3.1"
 gevent = "~=25.8"
 gunicorn = "~=23.0"
@@ -18,18 +18,17 @@ opentelemetry-exporter-otlp = "~=1.37"
 opentelemetry-instrumentation = "*"
 
 # Specific OpenTelemetry instrumentation libraries used to instrument this app
-# Full list or a subset of libraries listed by 'make otelrequirements'
+# Full list or a subset of libraries listed by 'edot-bootstrap --action=requirements'
 opentelemetry-instrumentation-botocore = "*"
 opentelemetry-instrumentation-flask = "*"
-opentelemetry-instrumentation-requests = "*"
+opentelemetry-instrumentation-logging = "*"
 
 [dev-packages]
 yapf = "~=0.43"
 moto = "~=5.1"
 nose2 = "*"
 pylint = "*"
 pylint-flask = "*"
-elastic-opentelemetry = "*"
 
 [requires]
 python_version = "3.13"

README.md

@@ -139,7 +139,7 @@ Is the command you're looking for.
 
 A curl example for testing the generation of shortlinks on the local db is:
 
-    curl -X POST -H "Content-Type: application/json" -H "Origin: http://localhost:8000" -d '{"url":"http://localhost:8000"}' http://localhost:5000
+    curl -X POST -H "Content-Type: application/json" -H "Origin: https://map.geo.admin.ch" -d '{"url":"https://map.geo.admin.ch"}' http://localhost:5000
 
 ### Docker helpers
 
@@ -219,40 +219,18 @@ The service is configured by Environment Variable:
 ## OTEL
 
 [OpenTelemetry instrumentation](https://opentelemetry.io/docs/concepts/instrumentation/) can be done in many different ways, from fully automated zero-code instrumentation (otel-operator) to purely manual instrumentation.
-Since we are kubernetes, the ideal solution would be to use the [otel-operator zero-code instrumentation](https://www.elastic.co/docs/solutions/observability/get-started/opentelemetry/use-cases/kubernetes/instrumenting-applications).
-
-For reasons unclear (possibly related to how we do gevent monkey patching), zero-code auto-instrumentation does not work. Thus, we fall back to programmatic instrumentation as described in the [Python Opentelemetry Manual-Instrumentation Sample App](https://github.com/aws-observability/aws-otel-community/tree/master/sample-apps/python-manual-instrumentation-sample-app). We may revisit this once we figure out how to make auto-instrumentation work for this service.
-
-To still use as less code as we can, we use the so called `OTEL programmatical instrumentation` approach. Unfortunately there are different understandings,
-levels of integration and examples of this approach. We use the [method described here](https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/opentelemetry-instrumentation#programmatic-auto-instrumentation), since it provides the highest level of automatic instrumentation. I.e. we can use a initialize() method to automatically initialize all installed instrumentation libraries.
-
-Other examples like these:
-
-- [aws-otel-community](https://github.com/aws-observability/aws-otel-community/blob/master/sample-apps/python-manual-instrumentation-sample-app/app.py)
-- [OTEL examples](https://opentelemetry.io/docs/zero-code/python/example/#programmatically-instrumented-server)
-
-import the specific instrumentation libraries and initialize them with the instrument() method of each library.
-
-It can be expected that documentations will improve and consolidate over time, as well that zero-code instrumentaton can be used in the future.
-
-### Bootstrap
-
-As mentioned above, all available and desired instrumentation libraries need to be installed first, i.e. added to the pipfile.
-Well known libraries like flask, request and botocore could be added manually. To get a better overview and add broader instrumentation
-support, a otel bootstrap tool can be used to create a list of supported libraries for a given project.
-
-Usage:
-
-1. Make setup
-2. `make otelbootstrap` to get the list of libraries
-3. Add all or the desired ones to the Pipfile. Versions are set to "*" for the moment.
+We use the so called `OTEL programmatical instrumentation` approach where we import the specific instrumentation libraries and initialize them with the instrument() method of each library.
 
 ### Environment variables
 
 The following env variables can be used to configure OTEL
 
 | Env Variable                                              | Default                    | Description                                                                                                                                          |
 | --------------------------------------------------------- | -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
+| OTEL_SDK_DISABLED                                         | false                      | If set to "true", OTEL is disabled. See: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#general-sdk-configuration |
+| OTEL_ENABLE_BOTO                                          | false                      | If opentelemetry-instrumentation-botocore should be enabled or not.                                                                                  |
+| OTEL_ENABLE_FLASK                                         | false                      | If opentelemetry-instrumentation-django should be enabled or not.                                                                                    |
+| OTEL_ENABLE_LOGGING                                       | false                      | If opentelemetry-instrumentation-logging should be enabled or not.                                                                                   |
 | OTEL_EXPERIMENTAL_RESOURCE_DETECTORS                      |                            | OTEL resource detectors, adding resource attributes to the OTEL output. e.g. `os,process`                                                            |
 | OTEL_EXPORTER_OTLP_ENDPOINT                               | http://localhost:4317      | The OTEL Exporter endpoint, e.g. `opentelemetry-kube-stack-gateway-collector.opentelemetry-operator-system:4317`                                     |
 | OTEL_EXPORTER_OTLP_HEADERS                                |                            | A list of key=value headers added in outgoing data. https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/#header-configuration    |
@@ -261,4 +239,36 @@ The following env variables can be used to configure OTEL
 | OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE |                            | A comma separated list of request headers added in outgoing data. Regex supported. Use '.*' for all headers                                          |
 | OTEL_PYTHON_FLASK_EXCLUDED_URLS                           |                            | A comma separated list of url's to exclude, e.g. `checker`                                                                                           |
 | OTEL_RESOURCE_ATTRIBUTES                                  |                            | A comma separated list of custom OTEL resource attributes, Must contain at least the service-name `service.name=service-shortlink`                   |
-| OTEL_SDK_DISABLED                                         |                            | If set to "true", OTEL is disabled. See: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#general-sdk-configuration |
+| OTEL_TRACES_SAMPLER                                       | parentbased_always_on      | Sampler to be used, see https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.sampling.html#module-opentelemetry.sdk.trace.sampling.       |
+| OTEL_TRACES_SAMPLER_ARG                                   |                            | Optional additional arguments for sampler.                                                                                                           |
+| OTEL_TRACES_SAMPLER                                       | parentbased_always_on      | Sampler to be used, see https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.sampling.html#module-opentelemetry.sdk.trace.sampling.       |
+| OTEL_TRACES_SAMPLER_ARG                                   |                            | Optional additional arguments for sampler.                                                                                                           |
+
+### Adding a New Instrumentation
+
+1. Use `edot-bootstrap --action=requirements` to get a list of possible instrumentation libraries
+2. Add all or the desired ones to the Pipfile.
+3. Add the initalization to [otel.py](app/helpers/otel.py) together with a feature flag
+
+Note: `edot-bootstrap` should be already installed via `infra-ansible-bgdi-dev`. If not, install it with `pipx install elastic-opentelemetry`.
+
+### Log Correlation
+
+The OpenTelemetry logging integration automatically injects tracing context into log statements. The following keys are injected into log record objects:
+
+- otelSpanID
+- otelTraceID
+- otelTraceSampled
+
+Note that although otelServiceName is injected, it will be empty. This is because the logging integration tries to read the service name from the trace provider, but our trace provider instance does not contain this resource attribute.
+
+### Sampling
+
+The python SDK supports ratio based [head sampling](https://opentelemetry.io/docs/concepts/sampling/#head-sampling). To enable, set
+
+- OTEL_TRACES_SAMPLER=parentbased_traceidratio|traceidratio
+- and OTEL_TRACES_SAMPLER_ARG=[0.0,1.0]
+
+### Local Telemetry
+
+Local telemetry can be tested by using one of the serve commands that use gunicorn, either `make gunicornserve`  or `make dockerrun`, and visiting the Zipkin dashboard at [http://localhost:9411](http://localhost:9411).

app/helpers/otel.py

@@ -1,25 +1,56 @@
-import os
+from os import getenv
 
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \
     OTLPSpanExporter
+from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
-from app.helpers.utils import strtobool
 
+def strtobool(value) -> bool:
+    """Convert a string representation of truth to true (1) or false (0).
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
+    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
+    'val' is anything else.
+    """
+    value = value.lower()
+    if value in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    if value in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    raise ValueError(f"invalid truth value \'{value}\'")
 
-def setup_trace_provider(worker_pid):
-    trace.set_tracer_provider(TracerProvider(resource=Resource.create()))
 
-    # Since we created a new tracer, the default span processor is gone. We need to
-    # create a new one using the default OTEL env variables and ad it to the tracer.
-    span_processor = BatchSpanProcessor(
-        OTLPSpanExporter(
-            endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
-            headers=os.getenv('OTEL_EXPORTER_OTLP_HEADERS'),
-            insecure=strtobool(os.getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+def initialize() -> None:
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_BOTO", "false")):
+            BotocoreInstrumentor().instrument()
+        if strtobool(getenv("OTEL_ENABLE_LOGGING", "false")):
+            LoggingInstrumentor().instrument()
+
+
+def initialize_flask(app):
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_FLASK", "false")):
+            FlaskInstrumentor().instrument_app(app)
+
+
+def setup_trace_provider():
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        # Since we created a new tracer, the default span processor is gone. We need to
+        # create a new one using the default OTEL env variables and ad it to the tracer.
+        span_processor = BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
+                headers=getenv('OTEL_EXPORTER_OTLP_HEADERS'),
+                insecure=strtobool(getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+            )
         )
-    )
-    trace.get_tracer_provider().add_span_processor(span_processor)
+
+        provider = TracerProvider(resource=Resource.create())
+        provider.add_span_processor(span_processor)
+        trace.set_tracer_provider(provider)

app/helpers/utils.py

@@ -15,6 +15,7 @@
 from flask import make_response
 from flask import request
 
+from app.helpers.otel import strtobool
 from app.settings import ALLOWED_DOMAINS_PATTERN
 from app.settings import SHORT_ID_ALPHABET
 from app.settings import SHORT_ID_SIZE
@@ -124,20 +125,6 @@ def get_url():
     return url
 
 
-def strtobool(value) -> bool:
-    """Convert a string representation of truth to true (1) or false (0).
-    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
-    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
-    'val' is anything else.
-    """
-    value = value.lower()
-    if value in ('y', 'yes', 't', 'true', 'on', '1'):
-        return True
-    if value in ('n', 'no', 'f', 'false', 'off', '0'):
-        return False
-    raise ValueError(f"invalid truth value \'{value}\'")
-
-
 def is_domain_allowed(url):
     """Check if the url contain a domain that is allowed
     """

docker-compose.yml

@@ -70,3 +70,16 @@ services:
       - AWS_ACCESS_KEY_ID=dummy123
       - AWS_SECRET_ACCESS_KEY=dummy123
       - AWS_DEFAULT_REGION=wonderland
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: --config otel-local-config.yaml
+    volumes:
+      - ./otel-local-config.yaml:/otel-local-config.yaml
+    ports:
+      - "4317:4317"
+
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"

logging-cfg-local.yaml

@@ -88,6 +88,9 @@ formatters:
         duration: "%(duration)s"
         payload: "%(response.json).128s"
       message: message
+      otelSpanID: otelSpanID
+      otelTraceID: otelTraceID
+      otelTraceSampled: otelTraceSampled
 
 handlers:
   console:

otel-local-config.yaml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+  zipkin:
+    endpoint: http://zipkin:9411/api/v2/spans
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, zipkin]

wsgi.py

@@ -10,7 +10,7 @@
     isort:skip_file
 """
 
-# pylint: disable=wrong-import-position,wrong-import-order
+# pylint: disable=wrong-import-position,wrong-import-order,ungrouped-imports
 
 import gevent.monkey
 
@@ -21,8 +21,7 @@
 # The order has a impact on how the libraries are instrumented. If called after app import,
 # e.g. the flask instrumentation has no effect. See:
 # https://github.com/open-telemetry/opentelemetry.io/blob/main/content/en/docs/zero-code/python/troubleshooting.md#use-programmatic-auto-instrumentation
-
-from opentelemetry.instrumentation.auto_instrumentation import initialize
+from app.helpers.otel import initialize, initialize_flask, setup_trace_provider
 
 initialize()
 
@@ -32,18 +31,10 @@
 
 from app.app import app as application
 from app.helpers.utils import get_logging_cfg
-from app.helpers.utils import strtobool
-from app.helpers import otel
 from app.settings import GUNICORN_WORKER_TMP_DIR
 from app.settings import GUNICORN_KEEPALIVE
 
-
-def post_fork(server, worker):
-    server.log.info("Worker spawned (pid: %s)", worker.pid)
-
-    # Setup OTEL providers for this worker
-    if not strtobool(os.getenv("OTEL_SDK_DISABLED", "false")):
-        otel.setup_trace_provider(worker.pid)
+initialize_flask(application)
 
 
 class StandaloneApplication(BaseApplication):  # pylint: disable=abstract-method
@@ -66,6 +57,13 @@ def load(self):
         return self.application
 
 
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+    # Setup OTEL providers for this worker
+    setup_trace_provider()
+
+
 # We use the port 5000 as default, otherwise we set the HTTP_PORT env variable within the container.
 if __name__ == '__main__':