new scaffolding (#3237)

Author: gauron99

cmd/build.go

@@ -226,6 +226,11 @@ For more options, run 'func build --help'`, err)
 	if err != nil {
 		return
 	}
+
+	if err = client.Scaffold(cmd.Context(), f, ""); err != nil {
+		return
+	}
+
 	if f, err = client.Build(cmd.Context(), f, buildOptions...); err != nil {
 		return
 	}
@@ -479,6 +484,7 @@ func (c buildConfig) clientOptions() ([]fn.Option, error) {
 	switch c.Builder {
 	case builders.Host:
 		o = append(o,
+			fn.WithScaffolder(oci.NewScaffolder(c.Verbose)),
 			fn.WithBuilder(oci.NewBuilder(builders.Host, c.Verbose)),
 			fn.WithPusher(oci.NewPusher(c.RegistryInsecure, false, c.Verbose,
 				oci.WithTransport(newTransport(c.RegistryInsecure)),
@@ -487,6 +493,7 @@ func (c buildConfig) clientOptions() ([]fn.Option, error) {
 		)
 	case builders.Pack:
 		o = append(o,
+			fn.WithScaffolder(buildpacks.NewScaffolder(c.Verbose)),
 			fn.WithBuilder(buildpacks.NewBuilder(
 				buildpacks.WithName(builders.Pack),
 				buildpacks.WithTimestamp(c.WithTimestamp),
@@ -497,6 +504,7 @@ func (c buildConfig) clientOptions() ([]fn.Option, error) {
 				docker.WithVerbose(c.Verbose))))
 	case builders.S2I:
 		o = append(o,
+			fn.WithScaffolder(s2i.NewScaffolder(c.Verbose)),
 			fn.WithBuilder(s2i.NewBuilder(
 				s2i.WithName(builders.S2I),
 				s2i.WithVerbose(c.Verbose))),

cmd/client.go

@@ -66,6 +66,7 @@ func NewClient(cfg ClientConfig, options ...fn.Option) (*fn.Client, func()) {
 			fn.WithVerbose(cfg.Verbose),
 			fn.WithTransport(t),
 			fn.WithRepositoriesPath(config.RepositoriesPath()),
+			fn.WithScaffolder(buildpacks.NewScaffolder(cfg.Verbose)),
 			fn.WithBuilder(buildpacks.NewBuilder(buildpacks.WithVerbose(cfg.Verbose))),
 			fn.WithRemovers(knative.NewRemover(cfg.Verbose), k8s.NewRemover(cfg.Verbose)),
 			fn.WithDescribers(knative.NewDescriber(cfg.Verbose), k8s.NewDescriber(cfg.Verbose)),

cmd/deploy.go

@@ -458,12 +458,7 @@ For more options, run 'func deploy --help'`, err)
 		if buildOptions, err = cfg.buildOptions(); err != nil {
 			return
 		}
-
-		var (
-			digested   bool
-			justBuilt  bool
-			justPushed bool
-		)
+		var digested bool
 
 		// Validate the image and check whether its digested or not
 		if cfg.Image != "" {
@@ -483,20 +478,30 @@ For more options, run 'func deploy --help'`, err)
 		if digested {
 			f.Deploy.Image = cfg.Image
 		} else {
-			// NOT digested, build & push the Function unless specified otherwise
-			if f, justBuilt, err = build(cmd, cfg.Build, f, client, buildOptions); err != nil {
+			// NOT digested: scaffold, build & push as per config
+			var (
+				shouldBuild bool
+				justPushed  bool
+			)
+			if shouldBuild, err = build(cmd, cfg.Build, f); err != nil {
 				return
 			}
+			if shouldBuild {
+				if err = client.Scaffold(cmd.Context(), f, ""); err != nil {
+					return
+				}
+				if f, err = client.Build(cmd.Context(), f, buildOptions...); err != nil {
+					return
+				}
+			}
 			if cfg.Push {
 				if f, justPushed, err = client.Push(cmd.Context(), f); err != nil {
 					return
 				}
 			}
-			// TODO: gauron99 - temporary fix for undigested image direct deploy
-			// (w/out build) This might be more complex to do than leaving like this
-			// image digests are created via the registry on push.
-			if (justBuilt || justPushed) && f.Build.Image != "" {
-				// f.Build.Image is set in Push for now, just set it as a deployed image
+			// image was just built and pushed to registry => potentially new image
+			if (shouldBuild || justPushed) && f.Build.Image != "" {
+				// f.Build.Image is set when pushed to registry, just set it as a deployed image
 				f.Deploy.Image = f.Build.Image
 			}
 		}
@@ -523,33 +528,23 @@ For more options, run 'func deploy --help'`, err)
 	return f.Stamp()
 }
 
-// build when flag == 'auto' and the function is out-of-date, or when the
-// flag value is explicitly truthy such as 'true' or '1'.  Error if flag
-// is neither 'auto' nor parseable as a boolean.  Return CLI-specific error
-// message verbeage suitable for both Deploy and Run commands which feature an
-// optional build step. Boolean return value signifies if the image has gone
-// through a build process.
-func build(cmd *cobra.Command, flag string, f fn.Function, client *fn.Client, buildOptions []fn.BuildOption) (fn.Function, bool, error) {
-	var err error
+// build determines if the function should be built based on given flag
+func build(cmd *cobra.Command, flag string, f fn.Function) (bool, error) {
 	if flag == "auto" {
 		if f.Built() {
 			fmt.Fprintln(cmd.OutOrStdout(), "function up-to-date. Force rebuild with --build")
-			return f, false, nil
+			return false, nil
 		} else {
-			if f, err = client.Build(cmd.Context(), f, buildOptions...); err != nil {
-				return f, false, err
-			}
+			return true, nil
 		}
 	} else if build, _ := strconv.ParseBool(flag); build {
-		if f, err = client.Build(cmd.Context(), f, buildOptions...); err != nil {
-			return f, false, err
-		}
-	} else if _, err = strconv.ParseBool(flag); err != nil {
-		return f, false, fmt.Errorf("invalid value for the build flag (%q), valid value is either 'auto' or a boolean", flag)
+		return true, nil
+	} else if _, err := strconv.ParseBool(flag); err != nil {
+		return false, fmt.Errorf("invalid value for the build flag (%q), valid value is either 'auto' or a boolean", flag)
 	} else if !build {
-		return f, false, nil
+		return false, nil
 	}
-	return f, true, nil
+	return false, nil
 }
 
 func NewRegistryValidator(path string) survey.Validator {

cmd/func-util/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/openshift/source-to-image/pkg/cmd/cli"
 	"k8s.io/klog/v2"
 
+	"knative.dev/func/pkg/buildpacks"
 	fn "knative.dev/func/pkg/functions"
 	"knative.dev/func/pkg/k8s"
 	"knative.dev/func/pkg/knative"
@@ -76,12 +77,12 @@ func socat(ctx context.Context) error {
 }
 
 func scaffold(ctx context.Context) error {
-
-	if len(os.Args) != 2 {
-		return fmt.Errorf("expected exactly one positional argument (function project path)")
+	if len(os.Args) != 3 {
+		return fmt.Errorf("expected exactly 2 positional arguments (function project path & builder)")
 	}
 
 	path := os.Args[1]
+	builder := os.Args[2]
 
 	f, err := fn.NewFunction(path)
 	if err != nil {
@@ -107,38 +108,17 @@ func scaffold(ctx context.Context) error {
 		return fmt.Errorf("cannot write middleware version as a file: %w", err)
 	}
 
-	if f.Runtime != "go" && f.Runtime != "python" {
-		// Scaffolding is for now supported/needed only for Go/Python.
-		return nil
-	}
-
-	appRoot := filepath.Join(f.Root, ".s2i", "builds", "last")
-	_ = os.RemoveAll(appRoot)
-
-	err = scaffolding.Write(appRoot, f.Root, f.Runtime, f.Invoke, embeddedRepo.FS())
-	if err != nil {
-		return fmt.Errorf("cannot write the scaffolding: %w", err)
-	}
-
-	if err := os.MkdirAll(filepath.Join(f.Root, ".s2i", "bin"), 0755); err != nil {
-		return fmt.Errorf("unable to create .s2i bin dir. %w", err)
-	}
-
-	var asm string
-	switch f.Runtime {
-	case "go":
-		asm = s2i.GoAssembler
-	case "python":
-		asm = s2i.PythonAssembler
+	var scaffolder fn.Scaffolder
+	switch builder {
+	case "pack":
+		scaffolder = buildpacks.NewScaffolder(true)
+	case "s2i":
+		scaffolder = s2i.NewScaffolder(true)
 	default:
-		panic("unreachable")
+		return fmt.Errorf("unknown builder in func-util image '%v'", builder)
 	}
 
-	if err := os.WriteFile(filepath.Join(f.Root, ".s2i", "bin", "assemble"), []byte(asm), 0755); err != nil {
-		return fmt.Errorf("unable to write go assembler. %w", err)
-	}
-
-	return nil
+	return fn.New(fn.WithScaffolder(scaffolder)).Scaffold(ctx, f, "")
 }
 
 func s2iCmd(ctx context.Context) error {
@@ -151,10 +131,6 @@ func s2iCmd(ctx context.Context) error {
 func deploy(ctx context.Context) error {
 	const imageDigestFileName = "image-digest"
 	var err error
-	deployer := knative.NewDeployer(
-		knative.WithDeployerVerbose(true),
-		knative.WithDeployerDecorator(deployDecorator{}))
-
 	var root string
 	if len(os.Args) > 1 {
 		root = os.Args[1]
@@ -182,7 +158,14 @@ func deploy(ctx context.Context) error {
 		f.Deploy.Image = f.Image
 	}
 
-	res, err := deployer.Deploy(ctx, f)
+	client := fn.New(
+		fn.WithDeployer(
+			knative.NewDeployer(
+				knative.WithDeployerDecorator(deployDecorator{}),
+				knative.WithDeployerVerbose(true)),
+		),
+	)
+	res, err := client.Deploy(ctx, f, fn.WithDeploySkipBuildCheck(true))
 	if err != nil {
 		return fmt.Errorf("cannot deploy the function: %w", err)
 	}

cmd/run.go

@@ -227,9 +227,19 @@ Or if you have an existing function:
 
 		// actual build step
 		if !digested {
-			if f, _, err = build(cmd, cfg.Build, f, client, buildOptions); err != nil {
+			shouldBuild, err := build(cmd, cfg.Build, f)
+			if err != nil {
 				return err
 			}
+			if shouldBuild {
+				if err := client.Scaffold(cmd.Context(), f, ""); err != nil {
+					return err
+				}
+				f, err = client.Build(cmd.Context(), f, buildOptions...)
+				if err != nil {
+					return err
+				}
+			}
 		}
 	} else { // if !container
 		// don't run digested image without a container

pkg/builders/builders_int_test.go

@@ -139,13 +139,15 @@ password=nbusr123
 
 	testCases := []struct {
 		Name         string
+		Scaffolder   fn.Scaffolder
 		Builder      fn.Builder
 		BuilderImage func(ctx context.Context, t *testing.T, certDir string) string
 		Envs         []fn.Env
 		Mounts       []fn.MountSpec
 	}{
 		{
 			Name:         "s2i",
+			Scaffolder:   s2i.NewScaffolder(true),
 			Builder:      s2i.NewBuilder(s2i.WithVerbose(true)),
 			BuilderImage: buildPatchedS2IBuilder,
 			Mounts: []fn.MountSpec{
@@ -156,6 +158,7 @@ password=nbusr123
 		},
 		{
 			Name:         "pack",
+			Scaffolder:   buildpacks.NewScaffolder(true),
 			Builder:      buildpacks.NewBuilder(buildpacks.WithVerbose(true)),
 			BuilderImage: buildPatchedBuildpackBuilder,
 			Envs: []fn.Env{
@@ -184,6 +187,10 @@ password=nbusr123
 			f.Build.Mounts = append(f.Build.Mounts, tt.Mounts...)
 			f.Build.BuildEnvs = append(f.Build.BuildEnvs, tt.Envs...)
 
+			err = tt.Scaffolder.Scaffold(ctx, f, "")
+			if err != nil {
+				t.Fatal(err)
+			}
 			err = tt.Builder.Build(ctx, f, nil)
 			if err != nil {
 				t.Fatal(err)
@@ -282,7 +289,7 @@ USER 1001:0
 // Builds a tiny paketo builder that trusts to our self-signed certificate (see createCertificate).
 func buildPatchedBuildpackBuilder(ctx context.Context, t *testing.T, certDir string) string {
 	tag := "localhost:50000/builder-jammy-tin:test"
-	dockerfile := `FROM ghcr.io/knative/builder-jammy-tiny:latest
+	dockerfile := `FROM ghcr.io/knative/builder-jammy-tiny:v2
 COPY 85c05568.0 /etc/ssl/certs/
 `
 	return buildPatchedBuilder(ctx, t, tag, dockerfile, certDir)

pkg/buildpacks/builder.go

@@ -28,8 +28,8 @@ import (
 // DefaultName when no WithName option is provided to NewBuilder
 const DefaultName = builders.Pack
 
-var DefaultBaseBuilder = "ghcr.io/knative/builder-jammy-base:latest"
-var DefaultTinyBuilder = "ghcr.io/knative/builder-jammy-tiny:latest"
+var DefaultBaseBuilder = "ghcr.io/knative/builder-jammy-base:v2"
+var DefaultTinyBuilder = "ghcr.io/knative/builder-jammy-tiny:v2"
 
 var (
 	DefaultBuilderImages = map[string]string{
@@ -186,6 +186,11 @@ func (b *Builder) Build(ctx context.Context, f fn.Function, platforms []fn.Platf
 		opts.Env["BPE_DEFAULT_LISTEN_ADDRESS"] = "[::]:8080"
 	}
 
+	// set scaffolding path to buildpacks builder
+	if f.Runtime == "go" {
+		opts.Env["BP_GO_WORKDIR"] = ".func/build"
+	}
+
 	var bindings = make([]string, 0, len(f.Build.Mounts))
 	for _, m := range f.Build.Mounts {
 		bindings = append(bindings, fmt.Sprintf("%s:%s", m.Source, m.Destination))

pkg/buildpacks/scaffolder.go

@@ -0,0 +1,50 @@
+package buildpacks
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	fn "knative.dev/func/pkg/functions"
+	"knative.dev/func/pkg/scaffolding"
+)
+
+const defaultPath = ".func/build"
+
+// Scaffolder for buildpacks builder
+type Scaffolder struct {
+	verbose bool
+}
+
+func NewScaffolder(verbose bool) *Scaffolder {
+	return &Scaffolder{verbose: verbose}
+}
+
+// Scaffold the function so that it can be built via buildpacks builder.
+// 'path' is an optional override. Assign "" (empty string) most of the time
+func (s Scaffolder) Scaffold(ctx context.Context, f fn.Function, path string) error {
+	// Because of Python injector we currently dont scaffold python.
+	// Add it here once the injector is removed
+	if f.Runtime != "go" {
+		if s.verbose {
+			fmt.Println("Scaffolding skipped. Currently available for runtime go")
+		}
+		return nil
+	}
+
+	appRoot := path
+	if appRoot == "" {
+		appRoot = filepath.Join(f.Root, defaultPath)
+	}
+	if s.verbose {
+		fmt.Printf("Writing buildpacks scaffolding at path '%v'\n", appRoot)
+	}
+
+	embeddedRepo, err := fn.NewRepository("", "")
+	if err != nil {
+		return fmt.Errorf("unable to load embedded scaffolding: %w", err)
+	}
+	_ = os.RemoveAll(appRoot)
+	return scaffolding.Write(appRoot, f.Root, f.Runtime, f.Invoke, embeddedRepo.FS())
+}