fix

Author: Radovan Fuchs

tests/e2e-prow/rhoai/pipeline-konflux.sh

@@ -236,7 +236,9 @@ fi
 # So behave/e2e-ops can kill this listener before rebinding 8080 (restart-lightspeed hooks).
 # Debug hook/port churn: export E2E_OPS_VERBOSE=1 before running pipeline.sh
 export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}"
+export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}"
 rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE"
+rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE"
 
 oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE
 
@@ -275,6 +277,7 @@ PF_JWKS_PID=$!
 log "Starting port-forward for llama-stack (MCP / llama_stack_client hooks)..."
 oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE &
 PF_LLAMA_PID=$!
+echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE"
 
 # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop)
 log "Waiting for port-forward to lightspeed-stack to be ready..."
@@ -324,6 +327,7 @@ for i in $(seq 1 36); do
     log "Llama port-forward died, restarting (attempt $i)..."
     oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE &
     PF_LLAMA_PID=$!
+    echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE"
   fi
   sleep 5
 done
@@ -364,6 +368,13 @@ if [[ -n "${E2E_LSC_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_LSC_PORT_FORWARD_PID_F
   fi
   rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE"
 fi
+if [[ -n "${E2E_LLAMA_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" ]]; then
+  read -r _ll_pf <"$E2E_LLAMA_PORT_FORWARD_PID_FILE" 2>/dev/null || true
+  if [[ "${_ll_pf:-}" =~ ^[0-9]+$ ]]; then
+    kill -9 "$_ll_pf" 2>/dev/null || true
+  fi
+  rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE"
+fi
 kill $PF_LCS_PID 2>/dev/null || true
 kill $PF_JWKS_PID 2>/dev/null || true
 kill $PF_LLAMA_PID 2>/dev/null || true

tests/e2e-prow/rhoai/scripts/e2e-ops.sh

@@ -8,11 +8,17 @@
 # - In-cluster pod logs (Uvicorn up, Llama OK) do not reflect localhost bind races; "address already in use"
 #   is the CI runner, not the application.
 # - E2E_LSC_PORT_FORWARD_PID_FILE coordinates the handoff.
+# - pipeline-konflux.sh (and hooks) forward llama-stack-service-svc to localhost:8321 for
+#   Behave steps that call Llama Stack directly (MCP toolgroups, shields). When the llama
+#   pod is recreated, that forward must be restarted or you get "PodSandbox ... not found" /
+#   APIConnectionError on subsequent scenarios.
+# - E2E_LLAMA_PORT_FORWARD_PID_FILE coordinates killing/restarting the 8321 forward.
 #
 # Commands:
 #   restart-lightspeed              - Restart lightspeed-stack pod and port-forward
-#   restart-llama-stack             - Restart/restore llama-stack pod
+#   restart-llama-stack             - Restart/restore llama-stack pod and localhost:8321 forward
 #   restart-port-forward            - Re-establish port-forward for lightspeed
+#   restart-llama-port-forward      - Re-establish port-forward for Llama Stack (8321)
 #   wait-for-pod <name> [attempts]  - Wait for a pod to be ready
 #   update-configmap <name> <file>  - Update ConfigMap from file
 #   get-configmap-content <name>    - Get ConfigMap content (outputs to stdout)
@@ -25,6 +31,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 MANIFEST_DIR="$SCRIPT_DIR/../manifests/lightspeed"
 # Written by pipeline.sh when it starts LCS port-forward; e2e-ops kills this PID before rebinding 8080.
 E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}"
+E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}"
 
 # ============================================================================
 # Helper functions
@@ -123,6 +130,24 @@ kill_stale_lightspeed_forward() {
     free_local_tcp_port "$port"
 }
 
+# Kill anything likely to hold the Llama Stack local forward (localhost:8321).
+kill_stale_llama_forward() {
+    local port="${1:-8321}"
+    local saved_pf
+    if [[ -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" ]]; then
+        read -r saved_pf <"$E2E_LLAMA_PORT_FORWARD_PID_FILE" 2>/dev/null || true
+        if [[ "$saved_pf" =~ ^[0-9]+$ ]]; then
+            kill -9 "$saved_pf" 2>/dev/null || true
+        fi
+    fi
+    pkill -9 -f "port-forward.*llama-stack-service-svc.*${port}:${port}" 2>/dev/null || true
+    pkill -9 -f "oc port-forward svc/llama-stack-service-svc ${port}:${port}" 2>/dev/null || true
+    pkill -9 -f "port-forward pod/llama-stack-service.*${port}:${port}" 2>/dev/null || true
+    free_local_tcp_port "$port"
+    sleep 1
+    free_local_tcp_port "$port"
+}
+
 # After oc port-forward dies in <2s, show recent oc stderr from the log file.
 e2e_ops_emit_port_forward_immediate_failure_diag() {
     echo "[e2e-ops] /tmp/port-forward.log (tail 25):"
@@ -266,6 +291,11 @@ cmd_restart_llama_stack() {
         oc label pod llama-stack-service pod=llama-stack-service -n "$NAMESPACE" --overwrite
     fi
 
+    if ! cmd_restart_llama_port_forward; then
+        echo "ERROR: Llama pod is up but localhost:${LOCAL_LLAMA_PORT:-8321} port-forward failed"
+        exit 1
+    fi
+
     echo "===== Llama-stack restore complete ====="
 }
 
@@ -334,6 +364,84 @@ cmd_restart_port_forward() {
     return 1
 }
 
+verify_llama_local_forward() {
+    local max_attempts="${1:-15}"
+    local http_code=""
+    local attempt
+
+    for ((attempt=1; attempt<=max_attempts; attempt++)); do
+        http_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://127.0.0.1:8321/v1/health" 2>/dev/null) || http_code="000"
+        if [[ "$http_code" == "200" ]]; then
+            return 0
+        fi
+        if [[ $attempt -lt $max_attempts ]]; then
+            sleep 2
+        fi
+    done
+    echo "Llama Stack localhost:8321 connectivity check failed (HTTP: ${http_code:-unknown})"
+    return 1
+}
+
+cmd_restart_llama_port_forward() {
+    local local_port="${LOCAL_LLAMA_PORT:-8321}"
+    local remote_port="${REMOTE_LLAMA_PORT:-8321}"
+    local max_attempts=6
+    local pf_pid
+    local pf_resource
+    local llama_pf_log="/tmp/port-forward-llama.log"
+
+    echo "Re-establishing Llama Stack port-forward on $local_port:$remote_port..."
+
+    for ((attempt=1; attempt<=max_attempts; attempt++)); do
+        kill_stale_llama_forward "$local_port"
+        sleep 3
+
+        if [[ $attempt -le 2 ]]; then
+            pf_resource="svc/llama-stack-service-svc"
+        else
+            pf_resource="pod/llama-stack-service"
+        fi
+        echo "Llama port-forward attempt $attempt/$max_attempts -> $pf_resource"
+
+        : >"$llama_pf_log"
+        nohup oc port-forward "$pf_resource" "$local_port:$remote_port" -n "$NAMESPACE" \
+            </dev/null >"$llama_pf_log" 2>&1 &
+        pf_pid=$!
+        disown "$pf_pid" 2>/dev/null || true
+        sleep 3
+
+        if ! kill -0 "$pf_pid" 2>/dev/null; then
+            echo "Llama port-forward process exited immediately:"
+            if [[ -s "$llama_pf_log" ]]; then
+                tail -25 "$llama_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true
+            fi
+            kill_stale_llama_forward "$local_port"
+            sleep 2
+            continue
+        fi
+        sleep 4
+
+        if verify_llama_local_forward 12; then
+            echo "$pf_pid" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE"
+            echo "[e2e-ops] Llama through port-forward: GET http://127.0.0.1:$local_port/v1/health -> OK"
+            echo "✓ Llama Stack port-forward established (PID: $pf_pid, $pf_resource)"
+            return 0
+        fi
+
+        if [[ $attempt -lt $max_attempts ]]; then
+            echo "Llama forward attempt $attempt failed, retrying..."
+            kill -9 "$pf_pid" 2>/dev/null || true
+            sleep 2
+        fi
+    done
+
+    echo "Failed to establish Llama Stack port-forward on :$local_port"
+    if [[ -s "$llama_pf_log" ]]; then
+        tail -30 "$llama_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true
+    fi
+    return 1
+}
+
 cmd_wait_for_pod() {
     local pod_name="${1:?Pod name required}"
     local max_attempts="${2:-24}"
@@ -395,6 +503,9 @@ case "$COMMAND" in
     restart-llama-stack)
         cmd_restart_llama_stack
         ;;
+    restart-llama-port-forward)
+        cmd_restart_llama_port_forward
+        ;;
     restart-port-forward)
         cmd_restart_port_forward
         ;;
@@ -416,6 +527,7 @@ case "$COMMAND" in
         echo "Commands:"
         echo "  restart-lightspeed              - Restart lightspeed-stack pod and port-forward"
         echo "  restart-llama-stack             - Restart/restore llama-stack pod"
+        echo "  restart-llama-port-forward      - Re-establish port-forward for Llama (8321)"
         echo "  restart-port-forward            - Re-establish port-forward for lightspeed"
         echo "  wait-for-pod <name> [attempts]  - Wait for a pod to be ready"
         echo "  update-configmap <name> <file>  - Update ConfigMap from file"

tests/e2e/test_list.txt

@@ -1,18 +1,5 @@
+features/faiss.feature
 features/smoketests.feature
 features/authorized_noop.feature
-features/authorized_noop_token.feature
-features/authorized_rh_identity.feature
-features/rbac.feature
-features/conversations.feature
-features/conversation_cache_v2.feature
-features/feedback.feature
-features/health.feature
-features/info.feature
-features/responses.feature
-features/query.feature
-features/rlsapi_v1.feature
-features/rlsapi_v1_errors.feature
-features/streaming_query.feature
-features/rest_api.feature
 features/mcp.feature
 features/models.feature