pause localqueues with namespace annotations

this change proposes to add support for a Namespace annotation to use
to drive the value of the LocalQueue's `spec.stopPolicy` field.
With this we can allow/disallow admission of Workloads at tenant
namespace level.

The release of this change needs to go through the 3-phases
rollout strategy described at
https://github.com/redhat-appstudio/infra-deployments/tree/main/components/policies\#deletion-of-generated-resources-not-acceptable

Signed-off-by: Francesco Ilario <[email protected]>

rh-pre-commit.version: 2.3.2
rh-pre-commit.check-secrets: ENABLED

Author: filariow

components/policies/development/kueue/queue-config/.chainsaw-test/chainsaw-test.yaml

@@ -6,7 +6,8 @@ metadata:
 spec:
   description: |
     Tests that a LocalQueue is created in a namespace labeled with
-    `konflux-ci.dev/type=tenant`.
+    `konflux-ci.dev/type=tenant` and no `kueue.konflux-ci.dev/stop-policy`
+    annotation.
   concurrent: false
   namespace: kueue-queue-new
   bindings:
@@ -51,6 +52,241 @@ spec:
 # yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
+metadata:
+  name: kueue-bootstrap-queue-new-tenant-labeled-namespace-unpaused
+spec:
+  description: |
+    Tests that a LocalQueue is created in a namespace labeled with
+    `konflux-ci.dev/type=tenant` and the `kueue.konflux-ci.dev/stop-policy`
+    annotation set to a value different from `hold`.
+  concurrent: false
+  namespace: kueue-queue-new
+  bindings:
+  - name: suffix
+    value: labeled
+  steps:
+  - name: given-localqueue-crd-exists
+    description: |
+      Install the Kueue LocalQueue CRD so the API server accepts LocalQueue objects in the test cluster.
+    try:
+    - apply:
+        file: ./resources/localqueue-crd.yaml
+  - name: given-kyverno-has-permission-on-resources
+    description: |
+      Apply Kyverno RBAC so the policy engine can generate and reconcile LocalQueue resources.
+    try:
+    - apply:
+        file: ../kyverno-rbac.yaml
+  - name: given-cluster-policy-is-ready
+    description: |
+      Apply the queue bootstrap ClusterPolicy and assert Kyverno reports the policy as ready before exercising generation.
+    try:
+    - apply:
+        file: ../cluster-policy.yaml
+    - assert:
+        file: chainsaw-assert-clusterpolicy.yaml
+  - name: when-tenant-labeled-namespace-is-created
+    description: |
+      Create a namespace labeled konflux-ci.dev/type=tenant so the policy should generate a LocalQueue.
+    try:
+    - apply:
+        file: resources/namespace-tenant-unpaused.yaml
+        template: true
+  - name: then-localqueue-is-created
+    description: |
+      Assert the expected pipelines-queue LocalQueue exists and matches the fixture (name and spec).
+    try:
+    - assert:
+        file: resources/expected-localqueue.yaml
+        template: true
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: kueue-bootstrap-queue-new-tenant-labeled-paused-namespace-resumed
+spec:
+  description: |
+    Tests that a LocalQueue with `stopPolicy` set to `Hold` is created
+    in a namespace labeled with `konflux-ci.dev/type=tenant` with the
+    `kueue.konflux-ci.dev/stop-policy` annotation set to `hold`. The
+    `stopPolicy` is then set to `None` when the annotation is removed
+    from the namespace.
+  concurrent: false
+  namespace: kueue-queue-new
+  bindings:
+  - name: suffix
+    value: labeled
+  steps:
+  - name: given-localqueue-crd-exists
+    description: |
+      Install the Kueue LocalQueue CRD so the API server accepts LocalQueue objects in the test cluster.
+    try:
+    - apply:
+        file: ./resources/localqueue-crd.yaml
+  - name: given-kyverno-has-permission-on-resources
+    description: |
+      Apply Kyverno RBAC so the policy engine can generate and reconcile LocalQueue resources.
+    try:
+    - apply:
+        file: ../kyverno-rbac.yaml
+  - name: given-cluster-policy-is-ready
+    description: |
+      Apply the queue bootstrap ClusterPolicy and assert Kyverno reports the policy as ready before exercising generation.
+    try:
+    - apply:
+        file: ../cluster-policy.yaml
+    - assert:
+        file: chainsaw-assert-clusterpolicy.yaml
+  - name: when-tenant-labeled-namespace-is-created-paused
+    description: |
+      update a namespace labeled konflux-ci.dev/type=tenant so the policy should generate a LocalQueue.
+    try:
+    - apply:
+        file: resources/namespace-tenant-paused.yaml
+        template: true
+  - name: then-localqueue-is-created-paused
+    description: |
+      Assert the expected pipelines-queue LocalQueue exists and matches the fixture (name and spec).
+    try:
+    - assert:
+        file: resources/expected-localqueue-paused.yaml
+        template: true
+  - name: when-tenant-labeled-namespace-is-updated-to-unpause
+    description: |
+      Create a namespace labeled konflux-ci.dev/type=tenant so the policy should generate a LocalQueue.
+    try:
+    - apply:
+        file: resources/namespace-tenant.yaml
+        template: true
+  - name: then-localqueue-is-unpaused
+    description: |
+      Assert the expected pipelines-queue LocalQueue exists and matches the fixture (name and spec).
+    try:
+    - assert:
+        file: resources/expected-localqueue.yaml
+        template: true
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: kueue-bootstrap-queue-new-tenant-labeled-namespace-update-paused
+spec:
+  description: |
+    Tests that a LocalQueue with `stopPolicy` set to `None` is created
+    in a namespace labeled with `konflux-ci.dev/type=tenant` with no
+    `kueue.konflux-ci.dev/stop-policy` annotation set. The `stopPolicy`
+    is then set to `Hold` when the annotation is added to the namespace.
+  concurrent: false
+  namespace: kueue-queue-new
+  bindings:
+  - name: suffix
+    value: labeled
+  steps:
+  - name: given-localqueue-crd-exists
+    description: |
+      Install the Kueue LocalQueue CRD so the API server accepts LocalQueue objects in the test cluster.
+    try:
+    - apply:
+        file: ./resources/localqueue-crd.yaml
+  - name: given-kyverno-has-permission-on-resources
+    description: |
+      Apply Kyverno RBAC so the policy engine can generate and reconcile LocalQueue resources.
+    try:
+    - apply:
+        file: ../kyverno-rbac.yaml
+  - name: given-cluster-policy-is-ready
+    description: |
+      Apply the queue bootstrap ClusterPolicy and assert Kyverno reports the policy as ready before exercising generation.
+    try:
+    - apply:
+        file: ../cluster-policy.yaml
+    - assert:
+        file: chainsaw-assert-clusterpolicy.yaml
+  - name: when-tenant-labeled-namespace-is-created
+    description: |
+      Create a namespace labeled konflux-ci.dev/type=tenant so the policy should generate a LocalQueue.
+    try:
+    - apply:
+        file: resources/namespace-tenant.yaml
+        template: true
+  - name: then-localqueue-is-created
+    description: |
+      Assert the expected pipelines-queue LocalQueue exists and matches the fixture (name and spec).
+    try:
+    - assert:
+        file: resources/expected-localqueue.yaml
+        template: true
+  - name: when-tenant-labeled-namespace-is-updated
+    description: |
+      update a namespace labeled konflux-ci.dev/type=tenant so the policy should generate a LocalQueue.
+    try:
+    - apply:
+        file: resources/namespace-tenant-paused.yaml
+        template: true
+  - name: then-localqueue-is-updated
+    description: |
+      Assert the expected pipelines-queue LocalQueue exists and matches the fixture (name and spec).
+    try:
+    - assert:
+        file: resources/expected-localqueue-paused.yaml
+        template: true
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: kueue-bootstrap-queue-new-tenant-labeled-namespace-paused
+spec:
+  description: |
+    Tests that a LocalQueue is created in a namespace labeled with
+    `konflux-ci.dev/type=tenant` and with annotation to pause the
+    LocalQueue.
+  concurrent: false
+  namespace: kueue-queue-new
+  bindings:
+  - name: suffix
+    value: labeled
+  steps:
+  - name: given-localqueue-crd-exists
+    description: |
+      Install the Kueue LocalQueue CRD so the API server accepts LocalQueue objects in the test cluster.
+    try:
+    - apply:
+        file: ./resources/localqueue-crd.yaml
+  - name: given-kyverno-has-permission-on-resources
+    description: |
+      Apply Kyverno RBAC so the policy engine can generate and reconcile LocalQueue resources.
+    try:
+    - apply:
+        file: ../kyverno-rbac.yaml
+  - name: given-cluster-policy-is-ready
+    description: |
+      Apply the queue bootstrap ClusterPolicy and assert Kyverno reports the policy as ready before exercising generation.
+    try:
+    - apply:
+        file: ../cluster-policy.yaml
+    - assert:
+        file: chainsaw-assert-clusterpolicy.yaml
+  - name: when-tenant-labeled-namespace-is-created
+    description: |
+      Create a namespace labeled konflux-ci.dev/type=tenant so the policy should generate a LocalQueue.
+    try:
+    - apply:
+        file: resources/namespace-tenant-paused.yaml
+        template: true
+  - name: then-localqueue-is-created
+    description: |
+      Assert the expected pipelines-queue LocalQueue exists and matches the fixture (name and spec).
+    try:
+    - assert:
+        file: resources/expected-localqueue-paused.yaml
+        template: true
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
 metadata:
   name: kueue-bootstrap-queue-unlabeled-namespace-negative
 spec:

components/policies/development/kueue/queue-config/.chainsaw-test/resources/expected-localqueue-kanary.yaml

@@ -5,3 +5,4 @@ metadata:
   namespace: appstudio-kanary-exporter
 spec:
   clusterQueue: cluster-pipeline-queue
+  stopPolicy: None

components/policies/development/kueue/queue-config/.chainsaw-test/resources/expected-localqueue-mintmaker.yaml

@@ -5,3 +5,4 @@ metadata:
   namespace: mintmaker
 spec:
   clusterQueue: cluster-pipeline-queue
+  stopPolicy: None

components/policies/development/kueue/queue-config/.chainsaw-test/resources/expected-localqueue-paused.yaml

@@ -0,0 +1,8 @@
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: LocalQueue
+metadata:
+  name: pipelines-queue
+  namespace: (join('-', [$namespace, $suffix]))
+spec:
+  clusterQueue: cluster-pipeline-queue
+  stopPolicy: Hold

components/policies/development/kueue/queue-config/.chainsaw-test/resources/expected-localqueue.yaml

@@ -5,3 +5,4 @@ metadata:
   namespace: (join('-', [$namespace, $suffix]))
 spec:
   clusterQueue: cluster-pipeline-queue
+  stopPolicy: None

components/policies/development/kueue/queue-config/.chainsaw-test/resources/namespace-tenant-paused.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: (join('-', [$namespace, $suffix]))
+  annotations:
+    kueue.konflux-ci.dev/stop-policy: hold
+  labels:
+    konflux-ci.dev/type: tenant

components/policies/development/kueue/queue-config/.chainsaw-test/resources/namespace-tenant-unpaused.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: (join('-', [$namespace, $suffix]))
+  annotations:
+    kueue.konflux-ci.dev/stop-policy: none
+  labels:
+    konflux-ci.dev/type: tenant

components/policies/development/kueue/queue-config/cluster-policy.yaml

@@ -21,6 +21,11 @@ spec:
           names:
           - mintmaker
           - appstudio-kanary-exporter
+    context:
+    - name: stopPolicy
+      variable:
+        jmesPath: >- 
+          (request.object.metadata.annotations."kueue.konflux-ci.dev/stop-policy" || '') == 'hold' && 'Hold' || 'None'
     generate:
       generateExisting: true
       orphanDownstreamOnPolicyDelete: true
@@ -32,3 +37,4 @@ spec:
       data:
         spec:
           clusterQueue: cluster-pipeline-queue
+          stopPolicy: "{{stopPolicy}}"