Add option for zstd compression to rules_oci (#98)

Add zstd compression

Author: egor-karavaev-datadog

.bazelrc

@@ -6,3 +6,7 @@ test:ci --keep_going
 # Only show failing test targets to avoid scrolling past a long list of
 # successful tests in order to see error logs.
 test:ci --test_summary=terse
+
+build:linux --sandbox_add_mount_pair=/tmp
+build:macos --sandbox_add_mount_pair=/var/tmp
+build:windows --sandbox_add_mount_pair=C:\Temp

MODULE.bazel

@@ -24,6 +24,7 @@ use_repo(
     "com_github_blakesmith_ar",
     "com_github_containerd_containerd",
     "com_github_containerd_log",
+    "com_github_datadog_zstd",
     "com_github_docker_docker_credential_helpers",
     "com_github_mitchellh_go_homedir",
     "com_github_opencontainers_go_digest",
@@ -56,3 +57,13 @@ oci_pull(
 register_toolchains(
     "@com_github_datadog_rules_oci//:oci_local_toolchain",
 )
+
+bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0")
+
+toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains")
+use_repo(toolchains, "zig_sdk")
+
+register_toolchains(
+    "@zig_sdk//libc_aware/toolchain:linux_amd64_gnu.2.28",
+    "@zig_sdk//libc_aware/toolchain:linux_arm64_gnu.2.28",
+)

MODULE.bazel.lock

@@ -173,7 +173,8 @@ The config file named after the rule, os, and arch
 <pre>
 load("@rules_oci//oci:defs.bzl", "oci_image_layer")
 
-oci_image_layer(<a href="#oci_image_layer-name">name</a>, <a href="#oci_image_layer-directory">directory</a>, <a href="#oci_image_layer-files">files</a>, <a href="#oci_image_layer-file_map">file_map</a>, <a href="#oci_image_layer-mode_map">mode_map</a>, <a href="#oci_image_layer-owner_map">owner_map</a>, <a href="#oci_image_layer-symlinks">symlinks</a>, <a href="#oci_image_layer-kwargs">kwargs</a>)
+oci_image_layer(<a href="#oci_image_layer-name">name</a>, <a href="#oci_image_layer-directory">directory</a>, <a href="#oci_image_layer-files">files</a>, <a href="#oci_image_layer-file_map">file_map</a>, <a href="#oci_image_layer-mode_map">mode_map</a>, <a href="#oci_image_layer-owner_map">owner_map</a>, <a href="#oci_image_layer-symlinks">symlinks</a>, <a href="#oci_image_layer-compression_method">compression_method</a>,
+                <a href="#oci_image_layer-kwargs">kwargs</a>)
 </pre>
 
 Creates a tarball and an OCI descriptor for it
@@ -190,6 +191,7 @@ Creates a tarball and an OCI descriptor for it
 | <a id="oci_image_layer-mode_map"></a>mode_map |  Dictionary of file location in tarball -> mode int (e.g. 0o755)   |  `None` |
 | <a id="oci_image_layer-owner_map"></a>owner_map |  Dictionary of file location in tarball -> owner:group string (e.g. '501:501')   |  `None` |
 | <a id="oci_image_layer-symlinks"></a>symlinks |  Dictionary of symlink -> target entries to place in the tarball   |  `None` |
+| <a id="oci_image_layer-compression_method"></a>compression_method |  A string, currently supports "gzip" and "zstd", defaults to "gzip"   |  `"gzip"` |
 | <a id="oci_image_layer-kwargs"></a>kwargs |  Additional arguments to pass to the rule, e.g. tags or visibility   |  none |
 
 

docs/docs.md

@@ -5,6 +5,7 @@ go 1.22.9
 toolchain go1.24.2
 
 require (
+	github.com/DataDog/zstd v1.5.7
 	github.com/bazelbuild/bazel-gazelle v0.43.0
 	github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
 	github.com/containerd/containerd v1.7.20

go.mod

@@ -1,5 +1,7 @@
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
+github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.12.3 h1:LS9NXqXhMoqNCplK1ApmVSfB4UnVLRDWRapB6EIlxE0=

go.sum

@@ -31,6 +31,7 @@ go_library(
         "@com_github_containerd_containerd//images:go_default_library",
         "@com_github_containerd_containerd//platforms:go_default_library",
         "@com_github_containerd_log//:go_default_library",
+        "@com_github_datadog_zstd//:go_default_library",
         "@com_github_opencontainers_go_digest//:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",

go/cmd/ocitool/BUILD.bazel

@@ -17,6 +17,7 @@ import (
 	"github.com/DataDog/rules_oci/go/internal/tarutil"
 	"github.com/DataDog/rules_oci/go/pkg/layer"
 	"github.com/DataDog/rules_oci/go/pkg/ociutil"
+	"github.com/DataDog/zstd"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/urfave/cli/v2"
@@ -32,14 +33,36 @@ func CreateLayerCmd(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to create file at %s: %w", config.OutputLayer, err)
 	}
+	defer out.Close()
 
 	digester := digest.SHA256.Digester()
 	wc := ociutil.NewWriterCounter(io.MultiWriter(out, digester.Hash()))
-	gw := gzip.NewWriter(wc)
-	gw.Name = path.Base(out.Name())
-	defer gw.Close()
 
-	tw := tar.NewWriter(gw)
+	var compressWriter io.Writer
+	var compressCloser io.Closer
+	var mediaType string
+	switch config.CompressionMethod {
+	case "gzip":
+		gzipWriter := gzip.NewWriter(wc)
+		gzipWriter.Name = path.Base(out.Name())
+		compressWriter = gzipWriter
+		compressCloser = gzipWriter
+		mediaType = ocispec.MediaTypeImageLayerGzip
+	case "zstd":
+		zstdWriter := zstd.NewWriter(wc)
+		compressWriter = zstdWriter
+		compressCloser = zstdWriter
+		mediaType = ocispec.MediaTypeImageLayerZstd
+	default:
+		return fmt.Errorf("uknown compress method %s", config.CompressionMethod)
+	}
+	defer func() {
+		if compressCloser != nil {
+			compressCloser.Close()
+		}
+	}()
+
+	tw := tar.NewWriter(compressWriter)
 	defer tw.Close()
 
 	slices.Sort(config.Files)
@@ -136,14 +159,14 @@ func CreateLayerCmd(c *cli.Context) error {
 
 	}
 
-	// Need to flush before we count bytes and digest, might as well close since
-	// it's not needed anymore.
+	// Need to flush before we count bytes and digest
 	tw.Close()
-	gw.Close()
+	compressCloser.Close()
+	compressCloser = nil
 
 	desc := ocispec.Descriptor{
 		Digest:    digester.Digest(),
-		MediaType: ocispec.MediaTypeImageLayerGzip,
+		MediaType: mediaType,
 		Size:      int64(wc.Count()),
 	}
 
@@ -164,15 +187,16 @@ func CreateLayerCmd(c *cli.Context) error {
 }
 
 type createLayerConfig struct {
-	BazelLabel     string            `json:"bazel-label" toml:"bazel-label" yaml:"bazel-label"`
-	Descriptor     string            `json:"outd" toml:"outd" yaml:"outd"`
-	Directory      string            `json:"dir" toml:"dir" yaml:"dir"`
-	FileMapping    map[string]string `json:"file-map" toml:"file-map" yaml:"file-map"`
-	Files          []string          `json:"file" toml:"file" yaml:"file"`
-	ModeMapping    map[string]int64  `json:"mode-map" toml:"mode-map" yaml:"mode-map"`
-	OutputLayer    string            `json:"out" toml:"out" yaml:"out"`
-	OwnerMapping   map[string]string `json:"owner-map" toml:"owner-map" yaml:"owner-map"`
-	SymlinkMapping map[string]string `json:"symlink" toml:"symlink" yaml:"symlink"`
+	BazelLabel        string            `json:"bazel-label" toml:"bazel-label" yaml:"bazel-label"`
+	Descriptor        string            `json:"outd" toml:"outd" yaml:"outd"`
+	Directory         string            `json:"dir" toml:"dir" yaml:"dir"`
+	FileMapping       map[string]string `json:"file-map" toml:"file-map" yaml:"file-map"`
+	Files             []string          `json:"file" toml:"file" yaml:"file"`
+	ModeMapping       map[string]int64  `json:"mode-map" toml:"mode-map" yaml:"mode-map"`
+	OutputLayer       string            `json:"out" toml:"out" yaml:"out"`
+	OwnerMapping      map[string]string `json:"owner-map" toml:"owner-map" yaml:"owner-map"`
+	SymlinkMapping    map[string]string `json:"symlink" toml:"symlink" yaml:"symlink"`
+	CompressionMethod string            `json:"compression-method" toml:"compression-method" yaml:"compression-method"`
 }
 
 func newCreateLayerConfig(c *cli.Context) (*createLayerConfig, error) {
@@ -184,16 +208,23 @@ func newCreateLayerConfig(c *cli.Context) (*createLayerConfig, error) {
 		}
 		modeMapping[path] = mode
 	}
+
+	compressionMethod := c.String("compression-method")
+	if compressionMethod == "" {
+		compressionMethod = "gzip"
+	}
+
 	return &createLayerConfig{
-		BazelLabel:     c.String("bazel-label"),
-		Descriptor:     c.String("outd"),
-		Directory:      c.String("dir"),
-		FileMapping:    c.Generic("file-map").(*flagutil.KeyValueFlag).Map,
-		Files:          c.StringSlice("file"),
-		ModeMapping:    modeMapping,
-		OutputLayer:    c.String("out"),
-		OwnerMapping:   c.Generic("owner-map").(*flagutil.KeyValueFlag).Map,
-		SymlinkMapping: c.Generic("symlink").(*flagutil.KeyValueFlag).Map,
+		BazelLabel:        c.String("bazel-label"),
+		Descriptor:        c.String("outd"),
+		Directory:         c.String("dir"),
+		FileMapping:       c.Generic("file-map").(*flagutil.KeyValueFlag).Map,
+		Files:             c.StringSlice("file"),
+		ModeMapping:       modeMapping,
+		OutputLayer:       c.String("out"),
+		OwnerMapping:      c.Generic("owner-map").(*flagutil.KeyValueFlag).Map,
+		SymlinkMapping:    c.Generic("symlink").(*flagutil.KeyValueFlag).Map,
+		CompressionMethod: compressionMethod,
 	}, nil
 }
 

go/cmd/ocitool/createlayer_cmd.go

@@ -78,6 +78,9 @@ var app = &cli.App{
 					Name:  "mode-map",
 					Value: &flagutil.KeyValueFlag{},
 				},
+				&cli.StringFlag{
+					Name: "compression-method",
+				},
 			},
 		},
 		{

go/cmd/ocitool/main.go

@@ -35,6 +35,7 @@ go_library(
         "@com_github_containerd_containerd//reference/docker:go_default_library",
         "@com_github_containerd_containerd//remotes:go_default_library",
         "@com_github_containerd_containerd//remotes/docker:go_default_library",
+        "@com_github_datadog_zstd//:go_default_library",
         "@com_github_opencontainers_go_digest//:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
         "@com_github_sethvargo_go_retry//:go_default_library",