Consensys · alexinthesky · Feb 14, 2026 · Feb 13, 2026
@@ -1,5 +1,8 @@
-# Environment variables for LLM Copilot App
-LLM_API_KEY=sk-your-llm-api-key-here
+# LLM API Key for grafana-llm-app
+# REQUIRED for E2E tests that involve AI chat functionality
+# Get your API key from https://console.anthropic.com/
+# Example: LLM_API_KEY=sk-ant-api03-xxxxx
+LLM_API_KEY=your-anthropic-api-key-here
 
 # Grafana Configuration
 GRAFANA_VERSION=12.3.1
@@ -12,3 +15,9 @@ ANONYMOUS_AUTH_ENABLED=false
 # Grant it Admin or Editor role to allow full access to Grafana resources
 GRAFANA_SERVICE_ACCOUNT_TOKEN=your-grafana-service-account-token-here
 
+# Prometheus Datasource Configuration (optional)
+# When set, a Prometheus datasource will be provisioned with basic auth
+# Example: PROM_URL=https://prometheus.example.com
+PROM_URL=
+O11Y_USER=
+O11Y_PWD=
@@ -123,7 +123,7 @@ jobs:
 
       - name: Start Grafana
         env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
         run: |
           docker compose up -d grafana
 
@@ -137,7 +137,7 @@ jobs:
 
       - name: Run E2E tests with coverage
         id: run-tests
-        run: npm run e2e:coverage -- --workers=1
+        run: npm run e2e:coverage
 
       - name: Generate coverage report
         if: always()

@@ -60,4 +60,5 @@ consensys-asko11y-app/
 consensys-asko11y-app.zip
 
 # Cursor plan file
-.cursor/plans/
+.cursor/plans/
+.vscode/
@@ -45,23 +45,31 @@ This project uses the following Claude Code plugins. They are configured in `.cl
 - **Fix ALL critical, major, and medium severity issues** before proceeding
 
 #### Step 2: Code Review
-- Run the `code-review` agent (via Task tool with `subagent_type: "pr-review-toolkit:code-reviewer"`) on your changes
+- Run the `code-review` skill (via Skill tool with `skill: "code-review"`) on your changes
 - **Fix ALL critical, major, and medium severity issues** reported by the reviewer
 - Only low/info severity issues may be left as-is with justification
 
 #### Step 3: Code Simplification
 - Run the `code-simplifier` agent (via Task tool with `subagent_type: "pr-review-toolkit:code-simplifier"`) on modified code
 - Apply simplifications that improve clarity without changing behavior
 
-#### Step 4: Tests & Lint
+#### Step 4: Remove AI Slop & Excessive Comments
+- Review all modified code for AI-generated noise: remove unnecessary comments, redundant docstrings, and obvious explanations
+- Delete comments that merely restate the code (e.g., `// increment counter` above `counter++`)
+- Remove filler phrases in comments like "Note:", "Important:", "This function...", "Helper to..."
+- Strip auto-generated JSDoc/GoDoc that adds no value beyond what types and names already convey
+- Do NOT add comments, docstrings, or type annotations to code you didn't change
+- Only keep comments where the **why** is non-obvious — never comment the **what**
+
+#### Step 5: Tests & Lint
 - Run `nvm use 22 && npm run test:ci` (frontend unit tests)
 - Run `go test ./pkg/...` (backend tests)
 - Run `nvm use 22 && npm run lint` (linting)
 - Run `nvm use 22 && npm run typecheck` (type checking)
 - **Fix any failures** - iterate until all pass
 
-#### Step 5: PR Review (before commit/PR)
-- Run the full `pr-review-toolkit:review-pr` skill for comprehensive analysis
+#### Step 6: PR Review (before commit/PR)
+- Run the full `pr-review-toolkit:review-pr` and the skill for comprehensive analysis
 - Fix critical/major/medium issues from: code-reviewer, silent-failure-hunter, type-design-analyzer
 
 **Issue Severity Policy:**
@@ -126,9 +134,18 @@ nvm use 22 && npm install
 # Start full development environment (Docker: Grafana + Redis + MCP servers)
 nvm use 22 && npm run server
 
+# Start multi-org development environment (uses docker-compose-full.yaml + full.yaml_)
+nvm use 22 && npm run server:full
+
 # Access: http://localhost:3000 (admin/admin)
 ```
 
+**Multi-org testing (`server:full`):**
+- Swaps `app.yaml` ↔ `full.yaml_` provisioning (with bash trap for cleanup)
+- Uses `docker-compose-full.yaml` with external `mcp-grafana` sidecar + Redis + mcpo
+- Provisions two orgs with separate MCP server configs
+- Create Org 2 manually in Grafana UI after startup
+
 ### Building
 ```bash
 # Full production build (frontend + backend for all platforms)
@@ -258,7 +275,8 @@ go test ./pkg/plugin -run TestFunctionName
 │                           ↕ HTTP                             │
 │  ┌────────────────────────────────────────────────────────┐ │
 │  │  Go Backend Plugin                                      │ │
-│  │  ├─ HTTP Routes (/api/mcp/*, /health)                  │ │
+│  │  ├─ HTTP Routes (/api/mcp/*, /api/agent/*, /health)    │ │
+│  │  ├─ Agentic Loop (LLM ↔ MCP tool-call cycle)          │ │
 │  │  ├─ RBAC Enforcement (Admin/Editor/Viewer)             │ │
 │  │  ├─ Session Sharing (in-memory or Redis)               │ │
 │  │  ├─ OAuth Flow Management (PKCE)                       │ │
@@ -303,18 +321,23 @@ src/
 ```
 pkg/
 ├── main.go             # Backend entry point
+├── agent/              # Agentic loop (LLM ↔ MCP tool-call cycle)
+│   ├── loop.go         # Core agent loop: LLM call → tool calls → repeat
+│   ├── llm_client.go   # HTTP client for grafana-llm-app OpenAI endpoint
+│   ├── tools.go        # MCP tool ↔ OpenAI function conversion + execution
+│   ├── context_window.go # Token-aware message truncation
+│   └── types.go        # SSE event types, request/response models
 ├── plugin/             # Plugin implementation
 │   ├── plugin.go       # Main plugin logic, HTTP routes, RBAC
-│   │                   # - Lines 140-191: isReadOnlyTool() (RBAC config)
-│   │                   # - Lines 192-233: filterToolsByRole() + canAccessTool()
-│   │                   # - Lines 600-850: Session sharing API endpoints
 │   ├── shares.go       # In-memory share store
 │   ├── shares_redis.go # Redis-backed share store
 │   ├── ratelimit.go    # Rate limiting (50 shares/hour/user)
 │   └── config.go       # Plugin configuration
+├── rbac/               # Role-based access control
+│   └── rbac.go         # Annotation-based RBAC using MCP ToolAnnotations
 └── mcp/                # MCP client & proxy
     ├── client.go       # MCP client implementation
-    │                   # - Lines 49-75: Multi-tenant header injection
+    │                   # - customRoundTripper: Multi-tenant header injection
     ├── proxy.go        # MCP proxy (aggregates servers)
     ├── health.go       # Health monitoring
     ├── oauth_*.go      # OAuth PKCE flow implementation
@@ -360,19 +383,48 @@ pkg/
 ### RBAC System
 
 **Role Hierarchy:**
-- Admin/Editor: Full access (56 tools: read + write)
-- Viewer: Read-only access (45 tools: get*, list*, query*, search*, find*, generate*)
+- Admin/Editor: Full access to all tools
+- Viewer: Read-only access (tools with `readOnlyHint: true` annotation)
+
+**Annotation-Based Enforcement:**
+RBAC uses MCP protocol `ToolAnnotations` (specifically `ReadOnlyHint`) advertised by MCP servers. No hardcoded tool lists — the server is the source of truth. Tools without annotations are treated as not read-only (denied to Viewers).
 
 **Enforcement Points:**
-1. Tool listing (filtered by role)
-2. Tool execution (permission check before execution)
+1. Tool listing — `rbac.FilterToolsByRole()` filters by annotations
+2. Tool execution — `rbac.CanAccessTool()` double-checks via `proxy.FindToolByName()` annotation lookup
+3. Agent loop — execution-time RBAC check in `executeTool()` before calling MCP server
 
 **Implementation:**
-- `pkg/plugin/plugin.go:140-191`: `isReadOnlyTool()` - defines read-only tools
-- `pkg/plugin/plugin.go:192-233`: `filterToolsByRole()` + `canAccessTool()`
+- `pkg/rbac/rbac.go`: `IsReadOnlyTool()`, `FilterToolsByRole()`, `CanAccessTool()` — all annotation-based
+- `pkg/mcp/types.go`: `ToolAnnotations` struct with `ReadOnlyHint`, `DestructiveHint`, etc.
+- `pkg/mcp/proxy.go`: `FindToolByName()` for execution-time annotation lookup
 - Double-check pattern (list AND execute)
 
-### OAuth Integration (New in Current Branch)
+### Agentic Backend Loop
+
+The AI conversation loop runs server-side in Go (not in the browser). The frontend sends the full message history and receives SSE events back.
+
+**Flow:**
+1. Frontend POSTs to `/api/agent/run` with messages + systemPrompt
+2. Backend streams SSE events: `content`, `tool_call_start`, `tool_call_result`, `done`, `error`
+3. Agent loop: LLM call → parse tool calls → execute via MCP proxy → feed results back → repeat
+4. Max 25 iterations per request (configurable via `AgentMaxIterations`)
+
+**Key Components:**
+- `pkg/agent/loop.go` — Core loop orchestration
+- `pkg/agent/llm_client.go` — Calls `grafana-llm-app` OpenAI-compatible endpoint using SA token
+- `pkg/agent/tools.go` — Converts MCP tools to OpenAI function format, executes tool calls via MCP proxy
+- `pkg/agent/context_window.go` — Token-aware message truncation to stay within model limits
+- `pkg/plugin/plugin.go:handleAgentRun()` — HTTP handler, SSE streaming, request validation
+
+**Authentication for LLM calls:**
+- Uses Grafana service account (SA) token from `backend.GrafanaConfigFromContext()`
+- SA token is passed to `grafana-llm-app` via `Authorization: Bearer` header
+- `X-Grafana-Org-Id` header forwarded for org context
+
+### OAuth Integration
+
+The plugin supports OAuth 2.0 authentication flows for MCP servers that require OAuth:
 
 The plugin supports OAuth 2.0 authentication flows for MCP servers that require OAuth:
 
@@ -399,6 +451,7 @@ Ask O11y supports three MCP modes for flexible tool integration:
 - Provides 56+ native Grafana observability tools
 - Automatically configured when grafana-llm-app is installed
 - Enabled via `useBuiltInMCP: true` in plugin settings
+- **LIMITATION: Only works for Org 1** (see Multi-Org Constraints below)
 
 **2. External Only**: Use user-configured external MCP servers
 - Supports OpenAPI, SSE, Standard MCP, and Streamable HTTP protocols
@@ -421,6 +474,26 @@ Ask O11y supports three MCP modes for flexible tool integration:
 - Error isolation: If one source fails, the other continues to work
 - RBAC: Filtering applied by each underlying client independently
 
+### Multi-Org Constraints (CRITICAL)
+
+**`externalServiceAccounts` creates a service account scoped to Org 1 only.** This is a known Grafana limitation:
+- [grafana/grafana#91844](https://github.com/grafana/grafana/issues/91844): SA token from `backend.GrafanaConfigFromContext()` is always Org 1
+- [grafana-llm-app#829](https://github.com/grafana/grafana-llm-app/issues/829): Built-in MCP not compatible with multi-org
+
+**Impact on this plugin:**
+- `useBuiltInMCP: true` injects grafana-llm-app's MCP endpoint using the SA token → **only works for Org 1**
+- The LLM client (`pkg/agent/llm_client.go`) uses the SA token for `grafana-llm-app` calls. It intentionally **does NOT send `X-Grafana-Org-Id`** when using SA token auth — sending a non-Org-1 header with the Org-1-scoped SA token causes a 401 from grafana-llm-app. This means **all orgs share Org 1's LLM configuration** (API key, model settings). Org isolation is enforced at the MCP tool-call layer, not the LLM layer.
+- Grafana 12 strips `Cookie` headers from backend plugin requests, so user session cookies cannot be forwarded to grafana-llm-app as an alternative auth mechanism
+
+**Multi-org workaround for MCP tools (used in `docker-compose-full.yaml`):**
+- Run `mcp-grafana` as an external sidecar container with basic auth (`GRAFANA_USERNAME`/`GRAFANA_PASSWORD` = admin/admin)
+- Basic auth credentials have cross-org access (unlike SA tokens)
+- The MCP proxy's `customRoundTripper` in `pkg/mcp/client.go` forwards `X-Grafana-Org-Id` and `X-Scope-OrgID` headers
+- The frontend sends `X-Grafana-Org-Id` on the `/api/agent/run` request, which flows through to MCP tool calls
+- Configure via external MCP server entries in provisioning (NOT `useBuiltInMCP`)
+
+**NEVER set `useBuiltInMCP: true` in multi-org provisioning files (`full.yaml_`).** Always use external `mcp-grafana` sidecar for multi-org deployments.
+
 ## Critical Implementation Details
 
 ### Theme Integration (CRITICAL)
@@ -445,17 +518,17 @@ Ask O11y supports three MCP modes for flexible tool integration:
 
 ### Adding a New MCP Tool
 
-1. **Backend RBAC Configuration:**
-   - If read-only: Add tool name to `isReadOnlyTool()` in `pkg/plugin/plugin.go:140-191`
-   - If write operation: No changes needed (auto-restricted to Admin/Editor)
-   - Viewers automatically get: `get*`, `list*`, `query*`, `search*`, `find*`, `generate*`
+1. **RBAC — No plugin changes needed:**
+   - RBAC is driven by MCP `ToolAnnotations` from the server
+   - The MCP server must set `readOnlyHint: true` on read-only tools
+   - Tools without annotations are restricted to Admin/Editor only
 
 2. **Frontend Tool Implementation:**
    - Follow existing patterns in `src/tools/` directory
    - Include schema validation, error handling, TypeScript types
 
 3. **Testing:**
-   - Verify RBAC: Viewer cannot access write operations
+   - Verify RBAC: Viewer cannot access tools without `readOnlyHint: true`
    - Test with different org contexts
    - Validate error handling
 
@@ -539,6 +612,8 @@ One-click RCA from alert notifications. URL params trigger auto-send of investig
 
 ## Code Style & Conventions
 
+**Guiding Principle:** Always make the simplest change possible. Code readability matters most — we're happy to make bigger structural changes to achieve it. Don't worry about backwards compatibility or migration paths; just write the clearest code.
+
 **Formatting:**
 - Linter: `@grafana/eslint-config` with Prettier
 - Indentation: 2 spaces, semicolons required, single quotes preferred
@@ -601,9 +676,10 @@ All routes under: `/api/plugins/consensys-asko11y-app/resources/`
 
 ```go
 // In pkg/plugin/plugin.go
-func (p *App) registerRoutes(mux *http.ServeMux) {
+func (p *Plugin) registerRoutes(mux *http.ServeMux) {
     mux.HandleFunc("/api/mcp/tools", p.handleMCPTools)
     mux.HandleFunc("/api/mcp/call-tool", p.handleMCPCallTool)
+    mux.HandleFunc("/api/agent/run", p.handleAgentRun)  // SSE streaming agentic loop
     // etc.
 }
 ```
@@ -651,9 +727,11 @@ docker compose exec grafana curl http://mcp-grafana:8000/mcp
 
 ## Configuration Files
 
-- `provisioning/plugins/apps.yaml` - Plugin & MCP server config
+- `provisioning/plugins/app.yaml` - Plugin & MCP server config (single-org, default)
+- `provisioning/plugins/full.yaml_` - Multi-org provisioning (trailing `_` prevents Grafana from loading; `server:full` swaps it in)
 - `.env` - Environment variables (not committed, see `.env.example`)
-- `docker-compose.yaml` - Local development environment
+- `docker-compose.yaml` - Local development environment (single-org)
+- `docker-compose-full.yaml` - Multi-org development environment (external mcp-grafana + Redis + mcpo)
 - `webpack.config.ts` - Frontend build config
 - `.config/` - DO NOT EDIT (scaffolded by `@grafana/create-plugin`)
 

@@ -9,6 +9,8 @@ services:
       GF_AUTH_BASIC_ENABLED: true
       GF_AUTH_ANONYMOUS_ENABLED: false
       GF_SECURITY_ALLOW_EMBEDDING: true
+      GF_FEATURE_TOGGLES_ENABLE: 'externalServiceAccounts'
+      GF_AUTH_MANAGED_SERVICE_ACCOUNTS_ENABLED: 'true'
       LLM_API_KEY: ${LLM_API_KEY:-}
       GF_PLUGIN_ASKO11Y_REDIS: ${GF_PLUGIN_ASKO11Y_REDIS:-redis://redis:6379/0}
     depends_on:
@@ -55,4 +57,3 @@ services:
 
 volumes:
   redis-data:
-
@@ -11,3 +11,6 @@ services:
       GF_FEATURE_TOGGLES_ENABLE: 'externalServiceAccounts'
       GF_AUTH_MANAGED_SERVICE_ACCOUNTS_ENABLED: 'true'
       LLM_API_KEY: ${LLM_API_KEY:-}
+      PROM_URL: ${PROM_URL:-}
+      O11Y_USER: ${O11Y_USER:-}
+      O11Y_PWD: ${O11Y_PWD:-}
@@ -22,6 +22,7 @@
     "coverage:merge": "mkdir -p coverage-combined/.nyc_output && cp coverage-unit/coverage-final.json coverage-combined/.nyc_output/unit.json 2>/dev/null; cp coverage-e2e/.nyc_output/*.json coverage-combined/.nyc_output/ 2>/dev/null; nyc report --temp-dir coverage-combined/.nyc_output --report-dir coverage-combined",
     "coverage": "npm run coverage:clean && npm run coverage:unit && npm run build:frontend:coverage && npm run e2e:coverage && npm run coverage:merge",
     "server": "docker compose up --build",
+    "server:full": "bash -c 'cleanup() { mv provisioning/plugins/full.yaml provisioning/plugins/full.yaml_ 2>/dev/null; mv provisioning/plugins/app.yaml_ provisioning/plugins/app.yaml 2>/dev/null; }; trap cleanup EXIT; mv provisioning/plugins/app.yaml provisioning/plugins/app.yaml_ 2>/dev/null; mv provisioning/plugins/full.yaml_ provisioning/plugins/full.yaml 2>/dev/null; docker compose -f docker-compose-full.yaml up --build'",
     "sign": "npx --yes @grafana/sign-plugin@latest",
     "backend:mod": "go mod tidy",
     "validate": "npm run validate:build && npm run validate:archive && npm run validate:run",