New Release v2.8.0 - #minor

.env.local

@@ -1 +1,10 @@
 ALLOWED_DOMAINS=.*localhost,.*\.admin\.ch,.*\.bgdi\.ch
+
+# OTEL
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-qrcode
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.env.test

@@ -1 +1,10 @@
-ALLOWED_DOMAINS=some_random_domain,.*\.geo\.admin\.ch,http://localhost
+ALLOWED_DOMAINS=some_random_domain,^.*\.geo\.admin\.ch$,http://localhost
+
+# OTEL
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-qrcode
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.pylintrc

@@ -36,10 +36,6 @@ load-plugins=pylint_flask
 # Pickle collected data for later comparisons.
 persistent=yes
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no
@@ -62,6 +58,7 @@ confidence=
 # --disable=W".
 disable=missing-docstring,
     missing-module-docstring,
+    unused-argument,
     useless-object-inheritance,
 
 

Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.12-slim-bullseye
-ENV HTTP_PORT 8080
+ENV HTTP_PORT=8080
 RUN groupadd -r geoadmin && useradd -r -s /bin/false -g geoadmin geoadmin
 
 

Makefile

@@ -67,6 +67,7 @@ help:
 	@echo -e " \033[1mSetup TARGETS\033[0m "
 	@echo "- setup              Create the python virtual environment with developper tools and activate it"
 	@echo "- ci                 Create the python virtual environment and install requirements based on the Pipfile.lock"
+	@echo "- otelrequirements   Get a list of available otel instrumentation libraries to add to the pipfile of this project"
 	@echo -e " \033[1mFORMATING, LINTING AND TESTING TOOLS TARGETS\033[0m "
 	@echo "- format             Format the python source code"
 	@echo "- lint               Lint the python source code"
@@ -99,6 +100,9 @@ ci: $(REQUIREMENTS)
 	# Create virtual env with all packages for development using the Pipfile.lock
 	pipenv sync --dev
 
+.PHONY: otelrequirements
+otelrequirements:
+	edot-bootstrap --action=requirements
 
 # linting target, calls upon yapf to make sure your code is easier to read and respects some conventions.
 

Pipfile

@@ -10,9 +10,15 @@ gunicorn = "~=23.0.0"
 Flask = "~=3.0.3"
 Pillow = "~=10.4.0"
 qrcode = "~=7.4"
-logging-utilities = "~=4.4.1"
+logging-utilities = "~=5.3.0"
 python-dotenv = "~=1.0.1"
 
+# OpenTelemetry packages
+opentelemetry-sdk = "*"
+opentelemetry-exporter-otlp = "*"
+opentelemetry-instrumentation-flask = "*"
+opentelemetry-instrumentation-logging = "*"
+
 [dev-packages]
 yapf = "*"
 nose2 = "*"

app/helpers/otel.py

@@ -0,0 +1,41 @@
+from os import getenv
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \
+    OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+from app.helpers.utils import strtobool
+
+
+def initialize() -> None:
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_LOGGING", "false")):
+            LoggingInstrumentor().instrument()
+
+
+def initialize_flask(app):
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_FLASK", "false")):
+            FlaskInstrumentor().instrument_app(app)
+
+
+def setup_trace_provider():
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        # Since we created a new tracer, the default span processor is gone. We need to
+        # create a new one using the default OTEL env variables and ad it to the tracer.
+        span_processor = BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
+                headers=getenv('OTEL_EXPORTER_OTLP_HEADERS'),
+                insecure=strtobool(getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+            )
+        )
+
+        provider = TracerProvider(resource=Resource.create())
+        provider.add_span_processor(span_processor)
+        trace.set_tracer_provider(provider)

app/helpers/url.py

@@ -32,7 +32,7 @@ def validate_url(url):
         logger.error('Invalid URL, could not determine the hostname, url=%s', url)
         abort(400, 'Invalid URL, could not determine the hostname')
 
-    if not re.match(ALLOWED_DOMAINS_PATTERN, result.hostname):
+    if not re.fullmatch(ALLOWED_DOMAINS_PATTERN, result.hostname):
         logger.error('URL domain not allowed: %s', result.hostname)
         abort(400, 'URL domain not allowed')
 

app/helpers/utils.py

@@ -30,3 +30,18 @@ def get_logging_cfg():
 def init_logging():
     config = get_logging_cfg()
     logging.config.dictConfig(config)
+
+
+def strtobool(val):
+    """Convert a string representation of truth to True or False.
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1';
+    False values are 'n', 'no', 'f', 'false', 'off', and '0'.
+    Raises ValueError if 'val' is anything else.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    if val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    raise ValueError(f"invalid truth value: {val}")

docker-compose-otel.yml

@@ -0,0 +1,14 @@
+version: "2"
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: --config otel-local-config.yaml
+    volumes:
+      - ./otel-local-config.yaml:/otel-local-config.yaml
+    ports:
+      - "4317:4317"
+
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"

otel-local-config.yaml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+  zipkin:
+    endpoint: http://zipkin:9411/api/v2/spans
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, zipkin]

tests/unit_tests/test_qrcode.py

@@ -114,7 +114,7 @@ def test_referer_check(self):
     def test_generate_url_domain_restriction(self):
         response = self.app.get(
             url_for('generate_get'),
-            query_string={'url': 'https://www.example.com/test'},
+            query_string={'url': 'https://map.geo.admin.ch.attacker.com/test/lore/ipsum'},
             headers=self.valid_origin_header
         )
         self.assertEqual(response.status_code, 400, msg="Domain restriction not applied")
@@ -134,17 +134,17 @@ def test_generate_url_domain_restriction(self):
 
     @params(
         None,
-        {'Origin': 'www.example'},
+        {'Origin': 'www.example.com'},
         {
-            'Origin': 'www.example', 'Sec-Fetch-Site': 'cross-site'
+            'Origin': 'www.example.com', 'Sec-Fetch-Site': 'cross-site'
         },
         {
-            'Origin': 'www.example', 'Sec-Fetch-Site': 'same-site'
+            'Origin': 'www.example.com', 'Sec-Fetch-Site': 'same-site'
         },
         {
-            'Origin': 'www.example', 'Sec-Fetch-Site': 'same-origin'
+            'Origin': 'www.example.com', 'Sec-Fetch-Site': 'same-origin'
         },
-        {'Referer': 'http://www.example'},
+        {'Referer': 'http://www.example.com'},
     )
     def test_generate_origin_not_allowed(self, headers):
         response = self.app.get(

wsgi.py

@@ -1,3 +1,30 @@
+"""
+    The gevent monkey import and patch suppress a warning, and a potential problem.
+    Gunicorn would call it anyway, but if it tries to call it after the ssl module
+    has been initialized in another module (like, in our code, by the botocore library),
+    then it could lead to inconsistencies in how the ssl module is used. Thus we patch
+    the ssl module through gevent.monkey.patch_all before any other import, especially
+    the app import, which would cause the boto module to be loaded, which would in turn
+    load the ssl module.
+"""
+
+# pylint: disable=wrong-import-position,wrong-import-order,ungrouped-imports
+import gevent.monkey
+
+gevent.monkey.patch_all()
+
+# Initialize OTEL.
+# Initialize should be called as early as possible, but at least before the app is imported
+# The order has a impact on how the libraries are instrumented. If called after app import,
+# e.g. the flask instrumentation has no effect. See:
+# https://github.com/open-telemetry/opentelemetry.io/blob/main/content/en/docs/zero-code/python/troubleshooting.md#use-programmatic-auto-instrumentation
+
+from app.helpers.otel import initialize
+from app.helpers.otel import initialize_flask
+from app.helpers.otel import setup_trace_provider
+
+initialize()
+
 import os
 
 from gunicorn.app.base import BaseApplication
@@ -6,6 +33,15 @@
 from app.helpers.utils import get_logging_cfg
 from app.settings import GUNICORN_KEEPALIVE
 
+initialize_flask(application)
+
+
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+    # Setup OTEL providers for this worker
+    setup_trace_provider()
+
 
 class StandaloneApplication(BaseApplication):  # pylint: disable=abstract-method
 
@@ -16,8 +52,9 @@ def __init__(self, app, options=None):  # pylint: disable=redefined-outer-name
 
     def load_config(self):
         config = {
-            key: value for key,
-            value in self.options.items() if key in self.cfg.settings and value is not None
+            key: value
+            for key, value in self.options.items()
+            if key in self.cfg.settings and value is not None
         }
         for key, value in config.items():
             self.cfg.set(key.lower(), value)
@@ -36,6 +73,7 @@ def load(self):
         'workers': 2,  # scaling horizontally is left to Kubernetes
         'timeout': 60,
         'keepalive': GUNICORN_KEEPALIVE,
-        'logconfig_dict': get_logging_cfg()
+        'logconfig_dict': get_logging_cfg(),
+        'post_fork': post_fork,
     }
     StandaloneApplication(application, options).run()