Skip to content

Commit 92b531c

Browse files
committed
wip: exporter: new gateway exporter
Signed-off-by: Justin Chadwell <me@jedevc.com>
1 parent 721d5dc commit 92b531c

42 files changed

Lines changed: 4171 additions & 807 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

api/services/control/control.pb.go

Lines changed: 200 additions & 127 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

api/services/control/control.proto

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,10 +244,20 @@ message BuildResultInfo {
244244
map<int64, Descriptor> Results = 3;
245245
}
246246

247+
enum ExporterTarget {
248+
UNKNOWN = 0;
249+
NONE = 1;
250+
FILE = 2;
251+
DIRECTORY = 3;
252+
STORE = 4;
253+
}
254+
247255
// Exporter describes the output exporter
248256
message Exporter {
249257
// Type identifies the exporter
250258
string Type = 1;
251259
// Attrs specifies exporter configuration
252260
map<string, string> Attrs = 2;
261+
// Target indicates the target type of the exporter
262+
ExporterTarget Target = 3;
253263
}

api/services/control/control_vtproto.pb.go

Lines changed: 31 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

api/services/control/converters.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package moby_buildkit_v1
2+
3+
import (
4+
"github.com/moby/buildkit/exporter/containerimage/exptypes"
5+
)
6+
7+
func ExporterTargetFromPB(target ExporterTarget) exptypes.ExporterTarget {
8+
switch target {
9+
case ExporterTarget_UNKNOWN:
10+
return exptypes.ExporterTargetUnknown
11+
case ExporterTarget_NONE:
12+
return exptypes.ExporterTargetNone
13+
case ExporterTarget_FILE:
14+
return exptypes.ExporterTargetFile
15+
case ExporterTarget_DIRECTORY:
16+
return exptypes.ExporterTargetDirectory
17+
case ExporterTarget_STORE:
18+
return exptypes.ExporterTargetStore
19+
default:
20+
return exptypes.ExporterTargetUnknown
21+
}
22+
}
23+
24+
func ExporterTargetToPB(target exptypes.ExporterTarget) ExporterTarget {
25+
switch target {
26+
case exptypes.ExporterTargetUnknown:
27+
return ExporterTarget_UNKNOWN
28+
case exptypes.ExporterTargetNone:
29+
return ExporterTarget_NONE
30+
case exptypes.ExporterTargetFile:
31+
return ExporterTarget_FILE
32+
case exptypes.ExporterTargetDirectory:
33+
return ExporterTarget_DIRECTORY
34+
case exptypes.ExporterTargetStore:
35+
return ExporterTarget_STORE
36+
default:
37+
return ExporterTarget_UNKNOWN
38+
}
39+
}

client/build.go

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"maps"
66

77
"github.com/moby/buildkit/client/buildid"
8+
"github.com/moby/buildkit/exporter/containerimage/exptypes"
89
gateway "github.com/moby/buildkit/frontend/gateway/client"
910
"github.com/moby/buildkit/frontend/gateway/grpcclient"
1011
gatewayapi "github.com/moby/buildkit/frontend/gateway/pb"
@@ -14,7 +15,7 @@ import (
1415
"google.golang.org/grpc"
1516
)
1617

17-
func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
18+
func (c *Client) BuildExport(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, exportFunc gateway.ExportFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
1819
defer func() {
1920
if statusChan != nil {
2021
close(statusChan)
@@ -56,15 +57,25 @@ func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildF
5657
caps := g.BuildOpts().Caps
5758
gwClient.caps = &caps
5859

59-
if err := g.Run(ctx, buildFunc); err != nil {
60+
result, err := g.Build(ctx, buildFunc)
61+
if err != nil {
6062
return errors.Wrap(err, "failed to run Build function")
6163
}
64+
if exportFunc != nil {
65+
if err := exportFunc(ctx, g, c.conn, exptypes.ExporterTargetUnknown, result); err != nil {
66+
return errors.Wrap(err, "failed to run Export function")
67+
}
68+
}
6269
return nil
6370
}
6471

6572
return c.solve(ctx, nil, cb, opt, statusChan)
6673
}
6774

75+
func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
76+
return c.BuildExport(ctx, opt, product, buildFunc, nil, statusChan)
77+
}
78+
6879
func (c *Client) gatewayClientForBuild(buildid string) *gatewayClientForBuild {
6980
g := gatewayapi.NewLLBBridgeClient(c.conn)
7081
return &gatewayClientForBuild{
@@ -138,6 +149,11 @@ func (g *gatewayClientForBuild) Evaluate(ctx context.Context, in *gatewayapi.Eva
138149
return g.gateway.Evaluate(ctx, in, opts...)
139150
}
140151

152+
func (g *gatewayClientForBuild) GetRemote(ctx context.Context, in *gatewayapi.GetRemoteRequest, opts ...grpc.CallOption) (*gatewayapi.GetRemoteResponse, error) {
153+
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
154+
return g.gateway.GetRemote(ctx, in, opts...)
155+
}
156+
141157
func (g *gatewayClientForBuild) Ping(ctx context.Context, in *gatewayapi.PingRequest, opts ...grpc.CallOption) (*gatewayapi.PongResponse, error) {
142158
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
143159
return g.gateway.Ping(ctx, in, opts...)
@@ -148,6 +164,11 @@ func (g *gatewayClientForBuild) Return(ctx context.Context, in *gatewayapi.Retur
148164
return g.gateway.Return(ctx, in, opts...)
149165
}
150166

167+
func (g *gatewayClientForBuild) GetReturn(ctx context.Context, in *gatewayapi.GetReturnRequest, opts ...grpc.CallOption) (*gatewayapi.GetReturnResponse, error) {
168+
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
169+
return g.gateway.GetReturn(ctx, in, opts...)
170+
}
171+
151172
func (g *gatewayClientForBuild) Inputs(ctx context.Context, in *gatewayapi.InputsRequest, opts ...grpc.CallOption) (*gatewayapi.InputsResponse, error) {
152173
if g.caps != nil {
153174
if err := g.caps.Supports(gatewayapi.CapFrontendInputs); err != nil {

client/exporters.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package client
22

33
const (
4-
ExporterImage = "image"
5-
ExporterLocal = "local"
6-
ExporterTar = "tar"
7-
ExporterOCI = "oci"
8-
ExporterDocker = "docker"
4+
ExporterImage = "image"
5+
ExporterLocal = "local"
6+
ExporterTar = "tar"
7+
ExporterOCI = "oci"
8+
ExporterDocker = "docker"
9+
ExporterGateway = "gateway"
910
)

client/solve.go

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,9 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
168168
if supportFile && supportStore {
169169
return nil, errors.Errorf("both file and store output is not supported by %s exporter", ex.Type)
170170
}
171+
case ExporterGateway:
172+
supportFile = ex.Output != nil
173+
supportDir = ex.OutputDir != ""
171174
}
172175
if !supportFile && ex.Output != nil {
173176
return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
@@ -274,9 +277,19 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
274277
exportDeprecated = exp.Type
275278
exportAttrDeprecated = exp.Attrs
276279
}
280+
target := controlapi.ExporterTarget_NONE
281+
switch {
282+
case exp.Output != nil:
283+
target = controlapi.ExporterTarget_FILE
284+
case exp.OutputDir != "":
285+
target = controlapi.ExporterTarget_DIRECTORY
286+
case exp.OutputStore != nil:
287+
target = controlapi.ExporterTarget_STORE
288+
}
277289
exports = append(exports, &controlapi.Exporter{
278-
Type: exp.Type,
279-
Attrs: exp.Attrs,
290+
Type: exp.Type,
291+
Attrs: exp.Attrs,
292+
Target: target,
280293
})
281294
}
282295

cmd/buildctl/build/output.go

Lines changed: 62 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,10 @@ func parseOutputCSV(s string) (client.ExportEntry, error) {
4242
if v, ok := ex.Attrs["output"]; ok {
4343
return ex, errors.Errorf("output=%s not supported for --output, you meant dest=%s?", v, v)
4444
}
45-
ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs["dest"], ex.Attrs)
45+
ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs)
4646
if err != nil {
4747
return ex, errors.Wrap(err, "invalid output option: output")
4848
}
49-
if ex.Output != nil || ex.OutputDir != "" {
50-
delete(ex.Attrs, "dest")
51-
}
5249
return ex, nil
5350
}
5451

@@ -66,55 +63,83 @@ func ParseOutput(exports []string) ([]client.ExportEntry, error) {
6663
}
6764

6865
// resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path).
69-
func resolveExporterDest(exporter, dest string, attrs map[string]string) (filesync.FileOutputFunc, string, error) {
70-
wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
71-
return func(m map[string]string) (io.WriteCloser, error) {
72-
return wc, nil
73-
}
74-
}
75-
76-
var supportFile bool
77-
var supportDir bool
66+
func resolveExporterDest(exporter string, attrs map[string]string) (destFile filesync.FileOutputFunc, destDir string, _ error) {
67+
var destFilePath string
7868
switch exporter {
7969
case client.ExporterLocal:
80-
supportDir = true
70+
var ok bool
71+
destDir, ok = attrs["dest"]
72+
if !ok {
73+
return nil, "", errors.Errorf("%s exporter requires dest=<path>", client.ExporterLocal)
74+
}
75+
delete(attrs, "dest")
8176
case client.ExporterTar:
82-
supportFile = true
77+
var ok bool
78+
destFilePath, ok = attrs["dest"]
79+
if !ok {
80+
return nil, "", errors.Errorf("%s exporter requires dest=<file>", client.ExporterTar)
81+
}
82+
delete(attrs, "dest")
8383
case client.ExporterOCI, client.ExporterDocker:
8484
tar, err := strconv.ParseBool(attrs["tar"])
8585
if err != nil {
8686
tar = true
8787
}
88-
supportFile = tar
89-
supportDir = !tar
88+
var ok bool
89+
if tar {
90+
destFilePath, ok = attrs["dest"]
91+
} else {
92+
destDir, ok = attrs["dest"]
93+
}
94+
if !ok {
95+
return nil, "", errors.Errorf("%s exporter requires dest=<file|path>", exporter)
96+
}
97+
delete(attrs, "dest")
98+
case client.ExporterGateway:
99+
destFilePath = attrs["file"]
100+
destDir = attrs["dir"]
101+
if destFilePath == "" && destDir == "" {
102+
return nil, "", errors.Errorf("%s exporter requires file=<file> or dir=<path>", exporter)
103+
}
104+
if destFilePath != "" && destDir != "" {
105+
return nil, "", errors.Errorf("%s exporter requires only one of file=<file> or dir=<path>", exporter)
106+
}
107+
delete(attrs, "file")
108+
delete(attrs, "dir")
109+
default:
110+
dest, ok := attrs["dest"]
111+
if ok {
112+
return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter)
113+
}
90114
}
91115

92-
if supportDir {
93-
if dest == "" {
94-
return nil, "", errors.Errorf("output directory is required for %s exporter", exporter)
116+
if destFilePath != "" {
117+
wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
118+
return func(m map[string]string) (io.WriteCloser, error) {
119+
return wc, nil
120+
}
95121
}
96-
return nil, dest, nil
97-
} else if supportFile {
98-
if dest != "" && dest != "-" {
99-
fi, err := os.Stat(dest)
122+
if destFilePath != "" && destFilePath != "-" {
123+
fi, err := os.Stat(destFilePath)
100124
if err != nil && !errors.Is(err, os.ErrNotExist) {
101-
return nil, "", errors.Wrapf(err, "invalid destination file: %s", dest)
125+
return nil, "", errors.Wrapf(err, "invalid destination file: %s", destFilePath)
102126
}
103127
if err == nil && fi.IsDir() {
104128
return nil, "", errors.Errorf("destination file is a directory")
105129
}
106-
w, err := os.Create(dest)
107-
return wrapWriter(w), "", err
108-
}
109-
// if no output file is specified, use stdout
110-
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
111-
return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter)
130+
w, err := os.Create(destFilePath)
131+
if err != nil {
132+
return nil, "", err
133+
}
134+
destFile = wrapWriter(w)
135+
} else {
136+
// if no output file is specified, use stdout
137+
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
138+
return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter)
139+
}
140+
destFile = wrapWriter(os.Stdout)
112141
}
113-
return wrapWriter(os.Stdout), "", nil
114142
}
115-
// e.g. client.ExporterImage
116-
if dest != "" {
117-
return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter)
118-
}
119-
return nil, "", nil
143+
144+
return destFile, destDir, nil
120145
}

control/control.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
418418
return nil, err
419419
}
420420
bklog.G(ctx).Debugf("resolve exporter %s with %v", ex.Type, ex.Attrs)
421-
expi, err := exp.Resolve(ctx, i, ex.Attrs)
421+
expi, err := exp.Resolve(ctx, i, req.FrontendAttrs, ex.Attrs, controlapi.ExporterTargetFromPB(ex.Target))
422422
if err != nil {
423423
return nil, err
424424
}

0 commit comments

Comments
 (0)