Add prometheus integration

Author: Grotax

README.md

@@ -0,0 +1,129 @@
+# ffshmon
+
+`ffshmon` monitors a WireGuard connection and attempts to recover it when the VPN check fails. It checks the configured FastD service, tests the WireGuard interface through Mullvad, regenerates the WireGuard configuration once after a failure, and alerts the NOC if recovery fails.
+
+It also provides a Prometheus endpoint with the latest up/down status.
+
+## Requirements
+
+- Linux with `systemd` and `systemctl`
+- Python 3.10 or newer
+- `curl`
+- A WireGuard interface named `exit` by default
+- A FastD service named `fastd@ffsh.service` by default
+- `/opt/wg-conf-gen/wg-conf-gen.py` for automatic configuration recovery
+- SMTP access to the configured mail host for failure alerts
+
+## Installation
+
+Create a virtual environment and install the Python dependencies:
+
+```bash
+python3 -m venv .venv
+.venv/bin/python -m pip install -r requirements.txt
+```
+
+## One-shot check
+
+Run the existing scheduled check with mail credentials and a log file:
+
+```bash
+.venv/bin/python wireguard.py check \
+  --user noc@example.org \
+  --password 'mail-password' \
+  --log /var/log/ffshmon.log
+```
+
+The command exits after one health cycle. If the FastD service is down, the connection probe is skipped and the status is considered down. If the probe fails, `ffshmon` regenerates the WireGuard configuration and retries once. A second failure stops FastD and WireGuard and sends an email alert.
+
+## Prometheus endpoint
+
+Start the long-running monitor with:
+
+```bash
+.venv/bin/python wireguard.py serve \
+  --user noc@example.org \
+  --password 'mail-password' \
+  --log /var/log/ffshmon.log
+```
+
+By default, the process:
+
+- Runs an immediate health check, then repeats every 60 seconds.
+- Listens on `127.0.0.1:8000`.
+- Exposes the latest completed result at `/metrics`.
+- Does not run a new health check when Prometheus scrapes the endpoint.
+
+Example request:
+
+```bash
+curl http://127.0.0.1:8000/metrics
+```
+
+The relevant metric is:
+
+```text
+wireguard_up{interface="exit"} 1.0
+```
+
+A value of `1` means the latest check succeeded. A value of `0` means the FastD service or WireGuard connectivity check is down.
+
+The listener and polling interval can be changed with `--host`, `--port`, and `--interval`:
+
+```bash
+.venv/bin/python wireguard.py serve \
+  --user noc@example.org \
+  --password 'mail-password' \
+  --log /var/log/ffshmon.log \
+  --host 127.0.0.1 \
+  --port 8000 \
+  --interval 60
+```
+
+## Prometheus configuration
+
+Add a scrape job for the host running `ffshmon`:
+
+```yaml
+scrape_configs:
+  - job_name: ffshmon
+    static_configs:
+      - targets: ["127.0.0.1:8000"]
+```
+
+If Prometheus runs on another host, bind `serve` to an appropriate reachable address and protect the endpoint with firewall rules or a reverse proxy. The endpoint has no built-in authentication.
+
+## Running as a service
+
+Run `serve` as a supervised systemd service so the endpoint remains available. A minimal unit could look like this:
+
+```ini
+[Unit]
+Description=WireGuard connectivity monitor
+After=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/ffshmon
+ExecStart=/opt/ffshmon/.venv/bin/python /opt/ffshmon/wireguard.py serve --user noc@example.org --password mail-password --log /var/log/ffshmon.log
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Avoid storing real credentials directly in a world-readable unit file. Use a protected environment file or another systemd credential mechanism in production.
+
+## Development
+
+Run the focused tests with:
+
+```bash
+.venv/bin/python -m unittest -v test_wireguard.py
+```
+
+Run Pylint with:
+
+```bash
+.venv/bin/python -m pylint wireguard.py test_wireguard.py
+```

requirements.txt

@@ -1,2 +1,3 @@
 requests
 click
+prometheus-client

test_wireguard.py

@@ -0,0 +1,41 @@
+"""Tests for WireGuard status checks and Prometheus metrics."""
+
+import socket
+import unittest
+from unittest.mock import patch
+from urllib.request import urlopen
+
+from prometheus_client import start_http_server
+
+import wireguard
+
+
+class WireguardTests(unittest.TestCase):
+    """Test the public monitoring behavior without system services."""
+
+    def test_interface_probe_uses_requested_interface(self):
+        """The connectivity probe should use the requested interface."""
+        curl_result = type("Result", (), {"stdout": '{"mullvad_exit_ip": true}'})()
+        with patch("wireguard.subprocess.run", return_value=curl_result) as run:
+            self.assertTrue(wireguard.test_interface("wg-test"))
+
+        command = run.call_args.args[0]
+        self.assertIn("wg-test", command)
+
+    def test_metrics_endpoint_exposes_cached_status(self):
+        """The HTTP endpoint should expose the latest cached gauge value."""
+        with socket.socket() as sock:
+            sock.bind(("127.0.0.1", 0))
+            port = sock.getsockname()[1]
+
+        wireguard.wireguard_up.labels(interface="exit").set(1)
+        start_http_server(port, addr="127.0.0.1")
+
+        with urlopen(f"http://127.0.0.1:{port}/metrics") as response:
+            metrics = response.read().decode("utf-8")
+
+        self.assertIn('wireguard_up{interface="exit"} 1.0', metrics)
+
+
+if __name__ == "__main__":
+    unittest.main()

wireguard.py

@@ -1,17 +1,30 @@
-import subprocess
+"""Monitor WireGuard connectivity and expose its status to Prometheus."""
+
 import json
 import logging
+import subprocess
+import time
 import click
+from prometheus_client import Gauge, start_http_server
 from config_manager import new_config
 from hard_stop import stop_fastd, stop_wg
 from inform_admin import send_mail
 
 
+WIREGUARD_INTERFACE = "exit"
+FASTD_SERVICE = "ffsh"
+wireguard_up = Gauge(
+    "wireguard_up", "Whether the WireGuard connection is up", ["interface"]
+)
+
+
 def is_service_running(service_name):
+    """Return whether the configured FastD service is running."""
     result = subprocess.run(
         ["systemctl", "show", "-p", "SubState", f"fastd@{service_name}.service"],
         capture_output=True,
         text=True,
+        check=False,
     )
     return result.stdout.strip() == "SubState=running"
 
@@ -23,7 +36,7 @@ def test_interface(interface_name):
         "--connect-timeout",
         "10",
         "--interface",
-        "exit",
+        interface_name,
         "https://am.i.mullvad.net/json",
     ]
     try:
@@ -34,20 +47,22 @@ def test_interface(interface_name):
         logging.error(e)
         return False
     try:
-        if data["mullvad_exit_ip"] is True:
-            logging.info("Everything ok.")
-            return True
-        else:
-            # something went wrong, Mullvad says we are not connected to Mullvad
-            logging.error("Mullvad says we are not connected to Mullvad")
-            return False
+        connected = data["mullvad_exit_ip"] is True
     except KeyError:
         # something went wrong the json did not contain mullvad_exit_ip
         logging.error("mullvad_exit_ip was not in the json")
         return False
 
+    if connected:
+        logging.info("Everything ok.")
+        return True
+
+    logging.error("Mullvad says we are not connected to Mullvad")
+    return False
+
 
 def verify(interface_name, fastd_name, mail_config):
+    """Check the connection and attempt recovery once if it is down."""
     result = test_interface(interface_name)
 
     if result is False:
@@ -61,56 +76,97 @@ def verify(interface_name, fastd_name, mail_config):
             stop_wg(interface_name)
             send_mail(
                 mail_config,
-                "VPN connection did not work, new VPN config did not help.\nFastd and wireguard stopped.",
+                "VPN connection did not work, new VPN config did not help.\n"
+                "Fastd and wireguard stopped.",
             )
+    return result
 
 
-# Cli group, could add more commands in the future
-@click.group()
-def cli():
-    pass
+def run_check(
+    mail_config, interface_name=WIREGUARD_INTERFACE, fastd_name=FASTD_SERVICE
+):
+    """Run one health cycle and return the resulting up/down state."""
+    if is_service_running(service_name=fastd_name):
+        return verify(
+            interface_name=interface_name,
+            fastd_name=fastd_name,
+            mail_config=mail_config,
+        )
 
+    logging.info("Fastd service is down, not checking connection")
+    return False
 
-@cli.command()
-@click.option("--user", help="Mail address", required=True)
-@click.option("--password", help="Password for Mail Address", required=True)
-@click.option("--log", help="Path to log file", required=True)
-def check(user, password, log):
-    """Check Status of wireguard interface"""
 
-    # Create log file if it does not exist
+def configure_logging(log):
+    """Create the log file if needed and configure application logging."""
     try:
-        with open(log, "x"):
-            # This part will only execute if the file is created successfully
+        with open(log, "x", encoding="utf-8"):
             pass
     except FileExistsError:
         pass
 
-    # Logging Config
-    # LogLevel DEBUG, INFO, WARNING, ERROR
-    log_level = logging.INFO
-    log_format = "%(asctime)s %(levelname)-8s %(message)s"
-    date_format = "%Y-%m-%d %H:%M:%S"
     logging.basicConfig(
-        format=log_format,
-        datefmt=date_format,
+        format="%(asctime)s %(levelname)-8s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
         filename=log,
         encoding="utf-8",
-        level=log_level,
+        level=logging.INFO,
     )
 
-    # Mail Config
-    config = {
+
+def build_mail_config(user, password):
+    """Build the mail settings used by the recovery alert."""
+    return {
         "target": "noc@freifunk-suedholstein.de",
         "host": "mail.freifunk-suedholstein.de",
         "port": "465",
         "user": user,
         "password": password,
     }
-    if is_service_running(service_name="ffsh"):
-        verify(interface_name="exit", fastd_name="ffsh", mail_config=config)
-    else:
-        logging.info("Fastd service is down, not checking connection")
+
+
+@click.group()
+def cli():
+    """WireGuard monitoring commands."""
+
+
+@cli.command()
+@click.option("--user", help="Mail address", required=True)
+@click.option("--password", help="Password for Mail Address", required=True)
+@click.option("--log", help="Path to log file", required=True)
+def check(user, password, log):
+    """Check the status of the WireGuard interface once."""
+    configure_logging(log)
+    run_check(build_mail_config(user, password))
+
+
+@cli.command()
+@click.option("--user", help="Mail address", required=True)
+@click.option("--password", help="Password for Mail Address", required=True)
+@click.option("--log", help="Path to log file", required=True)
+@click.option("--interval", type=float, default=60.0, show_default=True)
+@click.option("--host", default="127.0.0.1", show_default=True)
+@click.option("--port", type=int, default=8000, show_default=True)
+@click.pass_context
+def serve(context):
+    """Run checks and expose the latest WireGuard status as Prometheus metrics."""
+    user = context.params["user"]
+    password = context.params["password"]
+    log = context.params["log"]
+    interval = context.params["interval"]
+    host = context.params["host"]
+    port = context.params["port"]
+    configure_logging(log)
+    start_http_server(port, addr=host)
+    config = build_mail_config(user, password)
+    try:
+        while True:
+            wireguard_up.labels(interface=WIREGUARD_INTERFACE).set(
+                1 if run_check(config) else 0
+            )
+            time.sleep(interval)
+    except KeyboardInterrupt:
+        logging.info("Stopping WireGuard metrics server")
 
 
 if __name__ == "__main__":