feat: automate extensions local testing

.github/workflows/bake_targets.yml

@@ -138,33 +138,17 @@ jobs:
       - name: Set up environment
         run: |
           task e2e:setup-env
-          task e2e:export-kubeconfig
 
       - name: Generate Chainsaw testing values
-        uses: dagger/dagger-for-github@d913e70051faf3b907d4dd96ef1161083c88c644 # v8.2.0
-        env:
-          # renovate: datasource=github-tags depName=dagger/dagger versioning=semver
-          DAGGER_VERSION: 0.19.8
-        with:
-          version: ${{ env.DAGGER_VERSION }}
-          verb: call
-          module: ./dagger/maintenance/
-          args: generate-testing-values --target ${{ inputs.extension_name }} --extension-image ${{ matrix.image }} export --path=${{ inputs.extension_name }}/values.yaml
-
-      - name: Install Chainsaw
-        uses: kyverno/action-install-chainsaw@06560d18422209e9c1e08e931d477d04bf2674c1 # v0.2.14
+        run: |
+          task e2e:generate-values EXTENSION_IMAGE="${{ matrix.image }}" TARGET="${{ inputs.extension_name }}"
 
-      - name: Run Kyverno/Chainsaw
-        env:
-          EXT_NAME: ${{ inputs.extension_name }}
+      - name: Run e2e tests
         run: |
-          # Common smoke tests
-          chainsaw test ./test --values "$EXT_NAME/values.yaml"
+          # Get Kind cluster internal kubeconfig
+          task e2e:export-kubeconfig KUBECONFIG_PATH=./kubeconfig INTERNAL=true
 
-          # Specific smoke tests
-          if [ -d "$EXT_NAME/test" ]; then
-            chainsaw test "$EXT_NAME/test" --values "$EXT_NAME/values.yaml"
-          fi
+          task e2e:test TARGET="${{ inputs.extension_name }}" KUBECONFIG_PATH="./kubeconfig"
 
   copytoproduction:
     name: Copy images to production

.gitignore

@@ -28,3 +28,6 @@
 
 # Go workspace file
 go.work
+
+# Chainsaw values files
+**/values.yaml

BUILD.md

@@ -98,14 +98,54 @@ Docker network and installs CloudNativePG by default.
 task e2e:setup-env
 ```
 
-The container registry will be exposed locally at `localhost:5000`.
+The local container registry will be exposed locally at `localhost:5000`.
 
 The Kubeconfig to connect to the Kind cluster can be retrieved with:
 
 ```bash
 task e2e:export-kubeconfig KUBECONFIG_PATH=<path-to-export-kubeconfig>
 ```
 
+### Build & Push the images to the registry
+
+Any public registry can be used to test the extension images, but using the local registry
+is the recommended approach for local testing.
+The `task bake` command can be used to build and push the images, which by default
+targets the local registry, but can be configured differently by setting the `registry` env variable.
+
+```bash
+task bake TARGET=<extension> PUSH=true
+```
+
+### Generate testing values for Chainsaw
+
+Local testing is performed using [Chainsaw](https://github.com/kyverno/chainsaw), which requires a set
+of specific values to be generated for the targeted extension image.
+The `e2e:generate-values` task generates these values and export them in the extension directory:
+
+```bash
+task e2e:generate-values EXTENSION_IMAGE="<my-local-image>" TARGET="<extension>"
+```
+
+### Execute the end-to-end tests
+
+The first step to run the end-to-end tests is getting the internal Kubeconfig for the Kind cluster so it
+can be provided to the test command:
+
+```bash
+task e2e:export-kubeconfig KUBECONFIG_PATH=./kubeconfig INTERNAL=true
+```
+This command will export the Kubeconfig file to `./kubeconfig` file.
+
+The end-to-end tests can then be executed using the `e2e:test` task:
+
+```bash
+task e2e:test TARGET="<extension>" KUBECONFIG_PATH="./kubeconfig"
+```
+The framework executes by default a set of generic tests defined in the `test` folder, but
+more specific tests can be defined in the extension directory under the `test` folder which will be
+included automatically.
+
 ### Tear down the local test environment
 
 To clean up all the resources created by the `e2e:setup-env` task, run:

Taskfile.yml

@@ -128,6 +128,22 @@ tasks:
           TARGET: "{{.ITEM}}"
         task: update-os-libs
 
+  generate-values:
+    desc: Generate Chainsaw testing values for the specified target
+    deps:
+      - prereqs
+    prefix: 'generate-values-{{.TARGET}}'
+    vars:
+        EXTENSION_IMAGE: '{{ .EXTENSION_IMAGE| default "" }}'
+    cmds:
+      - echo -e "{{.BLUE}}Generating values for target {{.TARGET}}...{{.NC}}"
+      - >
+        dagger call -sm ./dagger/maintenance/ generate-testing-values
+        --target {{ .TARGET }} --extension-image="{{ .EXTENSION_IMAGE }}" export --path {{.TARGET}}/values.yaml
+    requires:
+      vars:
+        - name: TARGET
+
   e2e:create-docker-network:
     desc: Create Docker network to connect all the services, such as the Registry, Kind nodes, Chainsaw, etc.
     run: once
@@ -177,7 +193,7 @@ tasks:
     internal: true
     run: once
     vars:
-      REGISTRY_DIR: /etc/containerd/certs.d/localhost:{{ .REGISTRY_HOST_PORT }}
+      REGISTRY_DIR: /etc/containerd/certs.d/{{ .REGISTRY_NAME }}:{{ .REGISTRY_HOST_PORT }}
       DOCKER_SOCKET:
         sh: docker context inspect -f {{`'{{json .Endpoints.docker.Host}}'`}} $(docker context show)
     env:
@@ -254,6 +270,7 @@ tasks:
       - e2e:setup-kind
     vars:
       KUBECONFIG_PATH: '{{.KUBECONFIG_PATH| default "~/.kube/config"}}'
+      INTERNAL: '{{.INTERNAL| default "false"}}'
       DOCKER_SOCKET:
         sh: docker context inspect -f {{`'{{json .Endpoints.docker.Host}}'`}} $(docker context show)
     env:
@@ -262,7 +279,7 @@ tasks:
       - |
         CONFIG=$(dagger call -m github.com/aweris/daggerverse/kind@{{ .DAGGER_KIND_SHA }} \
           --socket {{ .DOCKER_SOCKET }} container \
-          with-exec --args "kind","get","kubeconfig","--name","{{ .KIND_CLUSTER_NAME }}" stdout)
+          with-exec --args "kind","get","kubeconfig","--name","{{ .KIND_CLUSTER_NAME }}","--internal={{ .INTERNAL }}" stdout)
         mkdir -p $(dirname {{ .KUBECONFIG_PATH }})
         echo "${CONFIG}" > {{ .KUBECONFIG_PATH }}
 
@@ -276,6 +293,53 @@ tasks:
     cmds:
       - echo -e "{{.GREEN}}--- E2E environment setup complete ---{{.NC}}"
 
+  e2e:generate-values:
+    desc: Generate Chainsaw testing values for the specified target in e2e environment
+    deps:
+      - e2e:start-dagger-engine
+      - e2e:start-container-registry
+    prefix: 'e2e:generate-values-{{ .TARGET }}'
+    silent: true
+    vars:
+      EXTENSION_IMAGE: '{{ .EXTENSION_IMAGE| default "" }}'
+    cmds:
+      - task: generate-values
+        vars:
+          TARGET: '{{ .TARGET }}'
+          EXTENSION_IMAGE:
+            # We need to replace localhost with the registry container name as it is how
+            # the registry is reachable from within the Docker network.
+            sh: sed -E 's/^localhost/{{ .REGISTRY_NAME }}/;t' <<< "{{ .EXTENSION_IMAGE }}"
+
+  e2e:test:
+    desc: Test target extension using Chainsaw
+    deps:
+      - e2e:start-dagger-engine
+    prefix: 'e2e:test-{{ .TARGET }}'
+    silent: true
+    vars:
+      KUBECONFIG_PATH: '{{ .KUBECONFIG_PATH | default "~/.kube/config" }}'
+    env:
+      _EXPERIMENTAL_DAGGER_RUNNER_HOST: container://{{ .DAGGER_ENGINE_NAME }}
+    cmds:
+      - echo -e "{{ .BLUE }}Testing {{ .TARGET }}...{{ .NC }}"
+      - dagger call -m ./dagger/maintenance/ test --source . --target {{ .TARGET }} --kubeconfig {{ .KUBECONFIG_PATH }}
+    requires:
+      vars:
+        - name: TARGET
+
+  e2e:test:all:
+    desc: Test all the available targets using Chainsaw
+    vars:
+      TARGETS:
+        sh: dagger call -sm ./dagger/maintenance/ get-targets | tr -d '[]"' | tr ',' '\n'
+    cmds:
+      - for:
+          var: TARGETS
+        vars:
+          TARGET: "{{ .ITEM }}"
+        task: e2e:test
+
   e2e:cleanup:
     desc: Cleanup E2E resources
     deps:

dagger/maintenance/image.go

@@ -19,7 +19,8 @@ const (
 
 // getImageAnnotations returns the OCI annotations given an image ref.
 func getImageAnnotations(imageRef string) (map[string]string, error) {
-	ref, err := name.ParseReference(imageRef)
+	// Setting Insecure option to allow fetching images from local registries with no TLS
+	ref, err := name.ParseReference(imageRef, name.Insecure)
 	if err != nil {
 		return nil, err
 	}

dagger/maintenance/main.go

@@ -10,6 +10,7 @@ import (
 	"maps"
 	"path"
 	"slices"
+	"time"
 
 	"go.yaml.in/yaml/v3"
 
@@ -189,3 +190,96 @@ func (m *Maintenance) GenerateTestingValues(
 
 	return result.File("values.yaml"), nil
 }
+
+// Tests the specified target using Chainsaw
+func (m *Maintenance) Test(
+	ctx context.Context,
+	// The source directory containing the extension folders. Defaults to the current directory
+	// +ignore=["dagger", ".github"]
+	// +defaultPath="/"
+	source *dagger.Directory,
+	// Kubeconfig to connect to the target K8s
+	// +required
+	kubeconfig *dagger.File,
+	// The target extension to test
+	// +default="all"
+	target string,
+	// Container image to use to run chainsaw
+	// renovate: datasource=docker depName=kyverno/chainsaw packageName=ghcr.io/kyverno/chainsaw versioning=docker
+	// +default="ghcr.io/kyverno/chainsaw:v0.2.14@sha256:c703e4d4ce7b89c5121fe957ab89b6e2d33f91fd15f8274a9f79ca1b2ba8ecef"
+	chainsawImage string,
+) error {
+	extDir := source
+	if target != "all" {
+		extDir = source.Filter(dagger.DirectoryFilterOpts{
+			Include: []string{path.Join(target, "**"), "test"},
+		})
+		hasMetadataFile, err := extDir.Exists(ctx, path.Join(target, metadataFile))
+		if err != nil {
+			return err
+		}
+		if !hasMetadataFile {
+			return fmt.Errorf("not a valid target, metadata.hcl file is missing. Target: %s", target)
+		}
+	}
+
+	targetExtensions, err := extensionsDirectories(ctx, extDir)
+	if err != nil {
+		return err
+	}
+
+	const valuesFile = "values.yaml"
+
+	for _, targetExtension := range targetExtensions {
+		extName, err := targetExtension.Name(ctx)
+		if err != nil {
+			return err
+		}
+
+		hasValues, err := targetExtension.Exists(ctx, valuesFile)
+		if err != nil {
+			return err
+		}
+		if !hasValues {
+			return fmt.Errorf("cannot execute tests for extension %q, values.yaml file is missing", target)
+		}
+
+		ctr := dag.Container().From(chainsawImage).
+			WithWorkdir("e2e").
+			WithEnvVariable("CACHEBUSTER", time.Now().String()).
+			WithDirectory("test", extDir.Directory("test")).
+			WithDirectory(extName, targetExtension).
+			WithFile("/etc/kubeconfig/config", kubeconfig).
+			WithEnvVariable("KUBECONFIG", "/etc/kubeconfig/config")
+
+		_, err = ctr.WithExec(
+			[]string{"test", "./test", "--values", path.Join(extName, valuesFile)},
+			dagger.ContainerWithExecOpts{
+				UseEntrypoint: true,
+			}).
+			Sync(ctx)
+
+		if err != nil {
+			return err
+		}
+
+		hasIndividualTests, err := targetExtension.Exists(ctx, "test")
+		if err != nil {
+			return err
+		}
+		if !hasIndividualTests {
+			continue
+		}
+		_, err = ctr.WithExec(
+			[]string{"test", path.Join(extName, "test"), "--values", path.Join(extName, valuesFile)},
+			dagger.ContainerWithExecOpts{
+				UseEntrypoint: true,
+			}).
+			Sync(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

renovate.json

@@ -46,6 +46,16 @@
         "# renovate: datasource=(?<datasource>[a-z-.]+?) depName=(?<depName>[^\\s]+?)(?: (?:lookupName|packageName)=(?<packageName>[^\\s]+?))?(?: versioning=(?<versioning>[^\\s]+?))?(?: extractVersion=(?<extractVersion>[^\\s]+?))?(?: currentValue=(?<currentValue>[^\\s]+?))?\\s+[A-Za-z0-9_]+?_SHA\\s*:\\s*[\"']?(?<currentDigest>[a-f0-9]+?)[\"']?\\s",
         "# renovate: datasource=(?<datasource>[a-z-.]+?) depName=(?<depName>[^\\s]+?)(?: (?:lookupName|packageName)=(?<packageName>[^\\s]+?))?(?: versioning=(?<versioning>[^\\s]+?))?(?: extractVersion=(?<extractVersion>[^\\s]+?))?\\s+[^:]+\\s*:\\s*[\"']?(?<currentValue>[^@]+?)(@(?<currentDigest>sha256:[0-9a-f]+))?[\"']?\\s"
       ]
+    },
+    {
+      "description": "updates the dagger dependencies",
+      "customType": "regex",
+      "managerFilePatterns": [
+        "dagger/**/*.go"
+      ],
+      "matchStrings": [
+        "\\/\\/\\s+renovate: datasource=(?<datasource>[a-z-.]+?) depName=(?<depName>[^\\s]+?)(?: (?:packageName)=(?<packageName>[^\\s]+?))?(?: versioning=(?<versioning>[^\\s]+?))\\s+\\/\\/\\s+\\+default=[\\\"']?[^:]+?:(?<currentValue>[^@]+?)(@(?<currentDigest>sha256:[0-9a-f]+))?[\"']?\\s"
+      ]
     }
   ],
   "packageRules": [