gateway: create interface for reading from container filesystem

This creates an interface that can be used to read the filesystem of a
new container created through the gateway API. These filesystem reading
methods are tied to a specific container that has been created, but
aren't tied to the container itself.

Due to being run inside of buildkit, these containers have access to the
same mounts that a container request would have. This is useful for
features like the file explorer in `buildx dap` because it can access
container filesystem state from stages that error along with ones that
have completed successfully.

Signed-off-by: Jonathan A. Sternberg <[email protected]>

Author: jsternberg

client/build.go

@@ -178,6 +178,36 @@ func (g *gatewayClientForBuild) ReleaseContainer(ctx context.Context, in *gatewa
 	return g.gateway.ReleaseContainer(ctx, in, opts...)
 }
 
+func (g *gatewayClientForBuild) ReadFileContainer(ctx context.Context, in *gatewayapi.ReadFileRequest, opts ...grpc.CallOption) (*gatewayapi.ReadFileResponse, error) {
+	if g.caps != nil {
+		if err := g.caps.Supports(gatewayapi.CapGatewayExecFilesystem); err != nil {
+			return nil, err
+		}
+	}
+	ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
+	return g.gateway.ReadFileContainer(ctx, in, opts...)
+}
+
+func (g *gatewayClientForBuild) ReadDirContainer(ctx context.Context, in *gatewayapi.ReadDirRequest, opts ...grpc.CallOption) (*gatewayapi.ReadDirResponse, error) {
+	if g.caps != nil {
+		if err := g.caps.Supports(gatewayapi.CapGatewayExecFilesystem); err != nil {
+			return nil, err
+		}
+	}
+	ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
+	return g.gateway.ReadDirContainer(ctx, in, opts...)
+}
+
+func (g *gatewayClientForBuild) StatFileContainer(ctx context.Context, in *gatewayapi.StatFileRequest, opts ...grpc.CallOption) (*gatewayapi.StatFileResponse, error) {
+	if g.caps != nil {
+		if err := g.caps.Supports(gatewayapi.CapGatewayExecFilesystem); err != nil {
+			return nil, err
+		}
+	}
+	ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
+	return g.gateway.StatFileContainer(ctx, in, opts...)
+}
+
 func (g *gatewayClientForBuild) ExecProcess(ctx context.Context, opts ...grpc.CallOption) (gatewayapi.LLBBridge_ExecProcessClient, error) {
 	if g.caps != nil {
 		if err := g.caps.Supports(gatewayapi.CapGatewayExec); err != nil {

control/gateway/gateway.go

@@ -188,6 +188,30 @@ func (gwf *GatewayForwarder) ReleaseContainer(ctx context.Context, req *gwapi.Re
 	return fwd.ReleaseContainer(ctx, req)
 }
 
+func (gwf *GatewayForwarder) ReadFileContainer(ctx context.Context, req *gwapi.ReadFileRequest) (*gwapi.ReadFileResponse, error) {
+	fwd, err := gwf.lookupForwarder(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "forwarding ReadFileContainer")
+	}
+	return fwd.ReadFileContainer(ctx, req)
+}
+
+func (gwf *GatewayForwarder) ReadDirContainer(ctx context.Context, req *gwapi.ReadDirRequest) (*gwapi.ReadDirResponse, error) {
+	fwd, err := gwf.lookupForwarder(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "forwarding ReadDirContainer")
+	}
+	return fwd.ReadDirContainer(ctx, req)
+}
+
+func (gwf *GatewayForwarder) StatFileContainer(ctx context.Context, req *gwapi.StatFileRequest) (*gwapi.StatFileResponse, error) {
+	fwd, err := gwf.lookupForwarder(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "forwarding StatFileContainer")
+	}
+	return fwd.StatFileContainer(ctx, req)
+}
+
 func (gwf *GatewayForwarder) ExecProcess(srv gwapi.LLBBridge_ExecProcessServer) error {
 	fwd, err := gwf.lookupForwarder(srv.Context())
 	if err != nil {

frontend/gateway/client/client.go

@@ -66,6 +66,7 @@ type Mount struct {
 type Container interface {
 	Start(context.Context, StartRequest) (ContainerProcess, error)
 	Release(context.Context) error
+	MountReference
 }
 
 // StartRequest encapsulates the arguments to define a process within a
@@ -101,6 +102,10 @@ type ContainerProcess interface {
 type Reference interface {
 	ToState() (llb.State, error)
 	Evaluate(ctx context.Context) error
+	MountReference
+}
+
+type MountReference interface {
 	ReadFile(ctx context.Context, req ReadRequest) ([]byte, error)
 	StatFile(ctx context.Context, req StatRequest) (*fstypes.Stat, error)
 	ReadDir(ctx context.Context, req ReadDirRequest) ([]*fstypes.Stat, error)

frontend/gateway/container/container.go

@@ -4,6 +4,8 @@ import (
 	"cmp"
 	"context"
 	"fmt"
+	"io/fs"
+	"os"
 	"path/filepath"
 	"runtime"
 	"slices"
@@ -25,6 +27,7 @@ import (
 	"github.com/moby/buildkit/util/stack"
 	"github.com/moby/buildkit/worker"
 	"github.com/pkg/errors"
+	fstypes "github.com/tonistiigi/fsutil/types"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -281,22 +284,23 @@ func PrepareMounts(ctx context.Context, mm *mounts.MountManager, cm cache.Manage
 }
 
 type gatewayContainer struct {
-	id         string
-	netMode    opspb.NetMode
-	hostname   string
-	extraHosts []executor.HostIP
-	platform   *opspb.Platform
-	rootFS     executor.Mount
-	mounts     []executor.Mount
-	executor   executor.Executor
-	sm         *session.Manager
-	group      session.Group
-	started    bool
-	errGroup   *errgroup.Group
-	mu         sync.Mutex
-	cleanup    []func() error
-	ctx        context.Context
-	cancel     func(error)
+	id          string
+	netMode     opspb.NetMode
+	hostname    string
+	extraHosts  []executor.HostIP
+	platform    *opspb.Platform
+	rootFS      executor.Mount
+	mounts      []executor.Mount
+	executor    executor.Executor
+	sm          *session.Manager
+	group       session.Group
+	started     bool
+	errGroup    *errgroup.Group
+	mu          sync.Mutex
+	cleanup     []func() error
+	ctx         context.Context
+	cancel      func(error)
+	localMounts map[executor.Mount]fs.FS
 }
 
 func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartRequest) (client.ContainerProcess, error) {
@@ -419,6 +423,124 @@ func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
 	return stack.Enable(err2)
 }
 
+func (gwCtr *gatewayContainer) ReadFile(ctx context.Context, req client.ReadRequest) ([]byte, error) {
+	fsys, path, err := gwCtr.mount(ctx, req.Filename)
+	if err != nil {
+		return nil, err
+	}
+	return fs.ReadFile(fsys, path)
+}
+
+func (gwCtr *gatewayContainer) ReadDir(ctx context.Context, req client.ReadDirRequest) ([]*fstypes.Stat, error) {
+	fsys, path, err := gwCtr.mount(ctx, req.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	entries, err := fs.ReadDir(fsys, path)
+	if err != nil {
+		return nil, err
+	}
+
+	files := make([]*fstypes.Stat, len(entries))
+	for i, e := range entries {
+		fullpath := filepath.Join(path, e.Name())
+		fi, err := e.Info()
+		if err != nil {
+			return nil, err
+		}
+
+		files[i], err = mkstat(fsys, fullpath, e.Name(), fi)
+		if err != nil {
+			return nil, errors.Wrap(err, "mkstat")
+		}
+	}
+	return files, nil
+}
+
+func (gwCtr *gatewayContainer) StatFile(ctx context.Context, req client.StatRequest) (*fstypes.Stat, error) {
+	fsys, path, err := gwCtr.mount(ctx, req.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := fs.Stat(fsys, path)
+	if err != nil {
+		return nil, err
+	}
+	return mkstat(fsys, req.Path, filepath.Base(req.Path), fi)
+}
+
+func (gwCtr *gatewayContainer) mount(ctx context.Context, fullpath string) (fs.FS, string, error) {
+	mount, path := gwCtr.findMount(ctx, fullpath)
+
+	gwCtr.mu.Lock()
+	defer gwCtr.mu.Unlock()
+
+	// Check if this mount has already been mounted.
+	if f, ok := gwCtr.localMounts[mount]; ok {
+		return f, path, nil
+	}
+
+	ref, err := mount.Src.Mount(ctx, true)
+	if err != nil {
+		return nil, "", err
+	}
+
+	mounter := snapshot.LocalMounter(ref)
+	dir, err := mounter.Mount()
+	if err != nil {
+		return nil, "", err
+	}
+
+	// Register cleanup.
+	gwCtr.cleanup = append(gwCtr.cleanup, func() error {
+		return mounter.Unmount()
+	})
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return nil, "", err
+	}
+
+	gwCtr.cleanup = append(gwCtr.cleanup, func() error {
+		return root.Close()
+	})
+
+	if gwCtr.localMounts == nil {
+		gwCtr.localMounts = make(map[executor.Mount]fs.FS)
+	}
+
+	f := root.FS()
+	gwCtr.localMounts[mount] = f
+	return f, path, nil
+}
+
+func (gwCtr *gatewayContainer) findMount(ctx context.Context, fullpath string) (m executor.Mount, path string) {
+	m = gwCtr.rootFS
+	path, _ = filepath.Rel("/", fullpath)
+	if len(gwCtr.mounts) == 0 {
+		return m, path
+	}
+
+	for _, mount := range gwCtr.mounts {
+		if strings.HasPrefix(fullpath, mount.Dest) {
+			remainder, err := filepath.Rel(mount.Dest, fullpath)
+			if err != nil {
+				bklog.G(ctx).Warnf("skipping mount at %q because it could not be converted into a relative path from %q", mount.Dest, fullpath)
+				continue
+			}
+
+			if len(remainder) < len(path) {
+				// Prefix matches and the remaining path is shorter so the prefix
+				// must be longer. This match works better.
+				m, path = mount, remainder
+			}
+		}
+	}
+	return m, path
+}
+
 type gatewayContainerProcess struct {
 	errGroup *errgroup.Group
 	groupCtx context.Context
@@ -511,3 +633,44 @@ type mountable struct {
 func (m *mountable) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) {
 	return m.m.Mount(ctx, readonly, m.g)
 }
+
+// constructs a Stat object. path is where the path can be found right
+// now, relpath is the desired path to be recorded in the stat (so
+// relative to whatever base dir is relevant). fi is the os.Stat
+// info. inodemap is used to calculate hardlinks over a series of
+// mkstat calls and maps inode to the canonical (aka "first") path for
+// a set of hardlinks to that inode.
+func mkstat(fsys fs.FS, path, relpath string, fi os.FileInfo) (*fstypes.Stat, error) {
+	relpath = filepath.ToSlash(relpath)
+
+	stat := &fstypes.Stat{
+		Path:    filepath.FromSlash(relpath),
+		Mode:    uint32(fi.Mode()),
+		ModTime: fi.ModTime().UnixNano(),
+	}
+
+	if !fi.IsDir() {
+		stat.Size = fi.Size()
+		if fi.Mode()&os.ModeSymlink != 0 {
+			link, err := fs.ReadLink(fsys, path)
+			if err != nil {
+				return nil, errors.WithStack(err)
+			}
+			stat.Linkname = link
+		}
+	}
+
+	if runtime.GOOS == "windows" {
+		permPart := stat.Mode & uint32(os.ModePerm)
+		noPermPart := stat.Mode &^ uint32(os.ModePerm)
+		// Add the x bit: make everything +x from windows
+		permPart |= 0111
+		permPart &= 0755
+		stat.Mode = noPermPart | permPart
+	}
+
+	// Clear the socket bit since archive/tar.FileInfoHeader does not handle it
+	stat.Mode &^= uint32(os.ModeSocket)
+
+	return stat, nil
+}

frontend/gateway/gateway.go

@@ -1181,6 +1181,84 @@ func (lbf *llbBridgeForwarder) NewContainer(ctx context.Context, in *pb.NewConta
 	return &pb.NewContainerResponse{}, nil
 }
 
+func (lbf *llbBridgeForwarder) ReadFileContainer(ctx context.Context, in *pb.ReadFileRequest) (*pb.ReadFileResponse, error) {
+	bklog.G(ctx).Debugf("|<--- ReadFileContainer %s", in.Ref)
+	lbf.ctrsMu.Lock()
+	ctr, ok := lbf.ctrs[in.Ref]
+	lbf.ctrsMu.Unlock()
+	if !ok {
+		return nil, errors.Errorf("container details for %s not found", in.Ref)
+	}
+
+	var fileRange *gwclient.FileRange
+	if in.Range != nil {
+		fileRange = &gwclient.FileRange{
+			Length: int(in.Range.Length),
+			Offset: int(in.Range.Offset),
+		}
+	}
+	req := gwclient.ReadRequest{
+		Filename: in.FilePath,
+		Range:    fileRange,
+	}
+
+	data, err := ctr.ReadFile(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pb.ReadFileResponse{
+		Data: data,
+	}, nil
+}
+
+func (lbf *llbBridgeForwarder) ReadDirContainer(ctx context.Context, in *pb.ReadDirRequest) (*pb.ReadDirResponse, error) {
+	bklog.G(ctx).Debugf("|<--- ReadDirContainer %s", in.Ref)
+	lbf.ctrsMu.Lock()
+	ctr, ok := lbf.ctrs[in.Ref]
+	lbf.ctrsMu.Unlock()
+	if !ok {
+		return nil, errors.Errorf("container details for %s not found", in.Ref)
+	}
+
+	req := gwclient.ReadDirRequest{
+		Path:           in.DirPath,
+		IncludePattern: in.IncludePattern,
+	}
+
+	files, err := ctr.ReadDir(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pb.ReadDirResponse{
+		Entries: files,
+	}, nil
+}
+
+func (lbf *llbBridgeForwarder) StatFileContainer(ctx context.Context, in *pb.StatFileRequest) (*pb.StatFileResponse, error) {
+	bklog.G(ctx).Debugf("|<--- StatFileContainer %s", in.Ref)
+	lbf.ctrsMu.Lock()
+	ctr, ok := lbf.ctrs[in.Ref]
+	lbf.ctrsMu.Unlock()
+	if !ok {
+		return nil, errors.Errorf("container details for %s not found", in.Ref)
+	}
+
+	req := gwclient.StatRequest{
+		Path: in.Path,
+	}
+
+	stat, err := ctr.StatFile(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pb.StatFileResponse{
+		Stat: stat,
+	}, nil
+}
+
 func (lbf *llbBridgeForwarder) ReleaseContainer(ctx context.Context, in *pb.ReleaseContainerRequest) (*pb.ReleaseContainerResponse, error) {
 	bklog.G(ctx).Debugf("|<--- ReleaseContainer %s", in.ContainerID)
 	lbf.ctrsMu.Lock()