python buildpack update

Author: gauron99

docs/building-functions/on_cluster_build.md

@@ -14,8 +14,8 @@ kubectl apply -f https://storage.googleapis.com/tekton-releases/pipeline/previou
 
 ## Enabling a namespace to run Function related Tekton Pipelines
 
-Set up RBAC permissions for the `default` Service Account to deploy Functions: (This is not needed on OpenShift). 
-Depending on the to be used deployers, different permissions are required: 
+Set up RBAC permissions for the `default` Service Account to deploy Functions: (This is not needed on OpenShift).
+Depending on the to be used deployers, different permissions are required:
 
 ### Option A: Permissions for all deployers (knative, raw and Keda)
 
@@ -33,11 +33,11 @@ kubectl create rolebinding func-deployer-binding \
   --role=func-deployer \
   --serviceaccount=$NAMESPACE:default \
   --namespace=$NAMESPACE
-  
+
 kubectl create clusterrolebinding $NAMESPACE:knative-eventing-namespaced-admin \
   --clusterrole=knative-eventing-namespaced-admin \
   --serviceaccount=$NAMESPACE:default
-  
+
 kubectl create clusterrolebinding $NAMESPACE:knative-serving-namespaced-admin \
   --clusterrole=knative-serving-namespaced-admin \
   --serviceaccount=$NAMESPACE:default

e2e/e2e_matrix_test.go

@@ -217,10 +217,6 @@ func matrixExceptionsShared(t *testing.T, initArgs []string, funcRuntime, builde
 
 	// Python Special Treatment
 	// --------------------------
-	// Skip Pack builder (not supported)
-	if funcRuntime == "python" && builder == "pack" {
-		t.Skip("The pack builder does not currently support Python Functions")
-	}
 
 	// Echo Implementation
 	// Replace the simple "OK" implementation with an echo.

e2e/e2e_userdeps_test.go

@@ -0,0 +1,71 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// TestPython_UserDeps_Run verifies that user code and its local dependencies
+// survive the scaffolding process during a local pack build. The test template
+// includes a local mylib package inside function/ that func.py imports;
+// if scaffolding misplaces user files, the import fails at runtime.
+func TestPython_UserDeps_Run(t *testing.T) {
+	name := "func-e2e-python-userdeps-run"
+	_ = fromCleanEnv(t, name)
+	t.Cleanup(func() { cleanImages(t, name) })
+
+	// Init with testdata Python HTTP template (includes function/mylib/)
+	initArgs := []string{"init", "-l", "python", "-t", "http",
+		"--repository", "file://" + filepath.Join(Testdata, "templates-userdeps")}
+	if err := newCmd(t, initArgs...).Run(); err != nil {
+		t.Fatalf("init failed: %v", err)
+	}
+
+	// Run with pack builder
+	cmd := newCmd(t, "run", "--builder", "pack", "--json")
+	address := parseRunJSON(t, cmd)
+
+	if !waitFor(t, address,
+		withWaitTimeout(6*time.Minute),
+		withContentMatch("hello from mylib")) {
+		t.Fatal("function did not return mylib greeting — user code not preserved")
+	}
+
+	if err := cmd.Process.Signal(os.Interrupt); err != nil {
+		fmt.Fprintf(os.Stderr, "error interrupting: %v", err)
+	}
+}
+
+// TestPython_UserDeps_Remote verifies that user code and its local
+// dependencies survive a remote (Tekton) build.
+func TestPython_UserDeps_Remote(t *testing.T) {
+	name := "func-e2e-python-userdeps-remote"
+	_ = fromCleanEnv(t, name)
+	t.Cleanup(func() { cleanImages(t, name) })
+	t.Cleanup(func() { clean(t, name, Namespace) })
+
+	// Init with testdata Python HTTP template (includes function/mylib/)
+	initArgs := []string{"init", "-l", "python", "-t", "http",
+		"--repository", "file://" + filepath.Join(Testdata, "templates-userdeps")}
+	if err := newCmd(t, initArgs...).Run(); err != nil {
+		t.Fatalf("init failed: %v", err)
+	}
+
+	// Deploy remotely via Tekton
+	if err := newCmd(t, "deploy", "--builder", "pack", "--remote",
+		fmt.Sprintf("--registry=%s", ClusterRegistry)).Run(); err != nil {
+		t.Fatal(err)
+	}
+
+	if !waitFor(t, ksvcUrl(name),
+		withWaitTimeout(5*time.Minute),
+		withContentMatch("hello from mylib")) {
+		t.Fatal("function did not return mylib greeting — user code not preserved in remote build")
+	}
+}

e2e/testdata/templates-userdeps/python/http/function/__init__.py

@@ -0,0 +1,20 @@
+import logging
+from mylib import greet
+
+
+def new():
+    return Function()
+
+
+class Function:
+    async def handle(self, scope, receive, send):
+        logging.info("Request Received")
+        await send({
+            'type': 'http.response.start',
+            'status': 200,
+            'headers': [[b'content-type', b'text/plain']],
+        })
+        await send({
+            'type': 'http.response.body',
+            'body': greet().encode(),
+        })

e2e/testdata/templates-userdeps/python/http/function/func.py

@@ -0,0 +1,2 @@
+def greet():
+    return "hello from mylib"

e2e/testdata/templates-userdeps/python/http/function/mylib/__init__.py

@@ -0,0 +1,9 @@
+[project]
+name = "function"
+version = "0.1.0"
+requires-python = ">=3.9"
+license = "MIT"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

e2e/testdata/templates-userdeps/python/http/pyproject.toml

@@ -210,6 +210,30 @@ func (b *Builder) Build(ctx context.Context, f fn.Function, platforms []fn.Platf
 	// only trust our known builders
 	opts.TrustBuilder = TrustBuilder
 
+	// Python scaffolding via inline pre-buildpack.
+	// Injects a script that rearranges user code into fn/ and copies
+	// scaffolding from .func/build/ to the workspace root.
+	if f.Runtime == "python" {
+		opts.ProjectDescriptor.Build.Pre = types.GroupAddition{
+			Buildpacks: []types.Buildpack{
+				{
+					ID: "knative.dev/func/python-scaffolding",
+					Script: types.Script{
+						API:    "0.10",
+						Inline: pythonScaffoldScript(),
+						Shell:  "/bin/bash",
+					},
+				},
+			},
+		}
+		// Tell the procfile buildpack what command to run.
+		// The Procfile written by the pre-buildpack is only
+		// available during the build phase, but the procfile
+		// buildpack needs to detect it earlier. Setting this
+		// env var makes it detect from environment instead.
+		opts.Env["BP_PROCFILE_DEFAULT_PROCESS"] = "python -m service.main"
+	}
+
 	var impl = b.impl
 	// Instantiate the pack build client implementation
 	// (and update build opts as necessary)
@@ -230,19 +254,6 @@ func (b *Builder) Build(ctx context.Context, f fn.Function, platforms []fn.Platf
 			return fmt.Errorf("podman 4.3 is not supported, use podman 4.2 or 4.4")
 		}
 
-		if f.Runtime == "python" {
-			if fi, _ := os.Lstat(filepath.Join(f.Root, "Procfile")); fi == nil {
-				// HACK (of a hack): need to get the right invocation signature
-				// the standard scaffolding does this in toSignature() func.
-				// we know we have python here.
-				invoke := f.Invoke
-				if invoke == "" {
-					invoke = "http"
-				}
-				cli = pyScaffoldInjector{cli, invoke}
-			}
-		}
-
 		// Client with a logger which is enabled if in Verbose mode and a dockerClient that supports SSH docker daemon connection.
 		if impl, err = pack.NewClient(pack.WithLogger(b.logger), pack.WithDockerClient(cli)); err != nil {
 			return fmt.Errorf("cannot create pack client: %w", err)

pkg/buildpacks/builder.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	fn "knative.dev/func/pkg/functions"
 	"knative.dev/func/pkg/scaffolding"
@@ -21,14 +22,14 @@ func NewScaffolder(verbose bool) *Scaffolder {
 	return &Scaffolder{verbose: verbose}
 }
 
-// Scaffold the function so that it can be built via buildpacks builder.
-// 'path' is an optional override. Assign "" (empty string) most of the time
+// Scaffold writes scaffolding for the function's runtime to the target path
+// using embedded templates. Pass "" for path to use the default (.func/build/).
+// Runtime-specific processing is applied after scaffolding is written.
 func (s Scaffolder) Scaffold(ctx context.Context, f fn.Function, path string) error {
-	// Because of Python injector we currently dont scaffold python.
-	// Add it here once the injector is removed
-	if f.Runtime != "go" {
+	// TODO: can be written as switch statement when we add more runtimes for readability
+	if f.Runtime != "go" && f.Runtime != "python" {
 		if s.verbose {
-			fmt.Println("Scaffolding skipped. Currently available for runtime go")
+			fmt.Println("Scaffolding skipped. Currently available for runtimes go, python")
 		}
 		return nil
 	}
@@ -38,13 +39,90 @@ func (s Scaffolder) Scaffold(ctx context.Context, f fn.Function, path string) er
 		appRoot = filepath.Join(f.Root, defaultPath)
 	}
 	if s.verbose {
-		fmt.Printf("Writing buildpacks scaffolding at path '%v'\n", appRoot)
+		fmt.Printf("Writing %s buildpacks scaffolding at path '%v'\n", f.Runtime, appRoot)
 	}
 
 	embeddedRepo, err := fn.NewRepository("", "")
 	if err != nil {
 		return fmt.Errorf("unable to load embedded scaffolding: %w", err)
 	}
-	_ = os.RemoveAll(appRoot)
-	return scaffolding.Write(appRoot, f.Root, f.Runtime, f.Invoke, embeddedRepo.FS())
+	if err = os.RemoveAll(appRoot); err != nil {
+		return fmt.Errorf("cannot clean scaffolding directory: %w", err)
+	}
+	if err = scaffolding.Write(appRoot, f.Root, f.Runtime, f.Invoke, embeddedRepo.FS()); err != nil {
+		return err
+	}
+
+	if f.Runtime != "python" {
+		return nil
+	}
+	// Python specific: patch pyproject.toml for Poetry and write Procfile.
+	//
+	// Add procfile for python+pack scaffolding. This will be used for buildpacks
+	// build/run phases to tell pack what to run. Note that this is not for
+	// detection as pre-buildpack runs only at build phase.
+	// Variable BP_PROCFILE_DEFAULT_PROCESS (see builder.go) is used for detection
+	// for local deployment.
+	if err = patchPyprojectForPack(filepath.Join(appRoot, "pyproject.toml")); err != nil {
+		return err
+	}
+	return os.WriteFile(
+		filepath.Join(appRoot, "Procfile"),
+		[]byte(PythonScaffoldingProcfile()),
+		os.FileMode(0644),
+	)
+}
+
+// patchPyprojectForPack applies pack-specific modifications to the template
+// pyproject.toml:
+//   - Replaces {root:uri}/f with ./f — Poetry doesn't understand hatchling's
+//     {root:uri} context variable. The inline buildpack creates a symlink
+//     f -> fn/ so ./f resolves correctly.
+//   - Appends [tool.poetry.dependencies] — Poetry needs this section for
+//     dependency solving
+func patchPyprojectForPack(pyprojectPath string) error {
+	data, err := os.ReadFile(pyprojectPath)
+	if err != nil {
+		return fmt.Errorf("cannot read pyproject.toml for patching: %w", err)
+	}
+	content := strings.Replace(string(data), "{root:uri}/f", "./f", 1)
+	content += "[tool.poetry.dependencies]\npython = \">=3.9,<4.0\"\n"
+	if err = os.WriteFile(pyprojectPath, []byte(content), 0644); err != nil {
+		return fmt.Errorf("cannot write patched pyproject.toml: %w", err)
+	}
+	return nil
+}
+
+// PythonScaffoldingProcfile returns the Procfile content that tells the
+// buildpack how to start the service.
+func PythonScaffoldingProcfile() string {
+	return "web: python -m service.main\n"
+}
+
+// pythonScaffoldScript returns a bash script for use as an inline buildpack.
+// The script rearranges user code into fn/ and moves pre-written scaffolding
+// from .func/build/ (populated by Scaffold) to the workspace root.
+func pythonScaffoldScript() string {
+	return `#!/bin/bash
+set -eo pipefail
+
+# Move user code into fn/ subdirectory, preserving infrastructure entries
+shopt -s dotglob
+mkdir -p fn
+for item in *; do
+  case "$item" in
+    fn|.func|.git|.gitignore|func.yaml) continue ;;
+  esac
+  mv "$item" fn/
+done
+shopt -u dotglob
+
+# Move scaffolding from .func/build/ to root
+mv .func/build/* .
+rm -rf .func
+
+# Create symlink so ./f in pyproject.toml resolves to fn/
+# -n: treat existing symlink as file, not follow it to directory
+ln -snf fn f
+`
 }