Skip to content

Commit d64bbab

Browse files
committed
patchpkg: start moving glibc-patch.bash to Go
Start porting the glibc patching script (which has expanded to patching other libraries too) over to Go. To make this change more incremental, this commit only ports copying the original package's files to the patched package (basically the `cp` command). Devbox still runs the embedded bash script until the rest of it is ported. The general patching steps are: - When generating the glibc-patch.nix flake, Devbox gets the absolute path to itself and adds it as a flake input. This makes devbox available in the flake's build. - Instead of running `glibc-patch.bash` as the flake's builder, run `devbox patch <pkg>`.
1 parent 8d48e38 commit d64bbab

File tree

7 files changed

+229
-35
lines changed

7 files changed

+229
-35
lines changed

internal/boxcli/patch.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package boxcli
2+
3+
import (
4+
"github.com/spf13/cobra"
5+
"go.jetpack.io/devbox/internal/patchpkg"
6+
)
7+
8+
func patchCmd() *cobra.Command {
9+
return &cobra.Command{
10+
Use: "patch <store-path>",
11+
Short: "Apply Devbox patches to a package to fix common linker errors",
12+
Args: cobra.ExactArgs(1),
13+
Hidden: true,
14+
RunE: func(cmd *cobra.Command, args []string) error {
15+
builder, err := patchpkg.NewDerivationBuilder()
16+
if err != nil {
17+
return err
18+
}
19+
return builder.Build(cmd.Context(), args[0])
20+
},
21+
}
22+
}

internal/boxcli/root.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ func RootCmd() *cobra.Command {
7070
command.AddCommand(integrateCmd())
7171
command.AddCommand(listCmd())
7272
command.AddCommand(logCmd())
73+
command.AddCommand(patchCmd())
7374
command.AddCommand(removeCmd())
7475
command.AddCommand(runCmd(runFlagDefaults{}))
7576
command.AddCommand(searchCmd())

internal/patchpkg/builder.go

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// patchpkg patches packages to fix common linker errors.
2+
package patchpkg
3+
4+
import (
5+
"bytes"
6+
"context"
7+
_ "embed"
8+
"fmt"
9+
"io"
10+
"io/fs"
11+
"iter"
12+
"log/slog"
13+
"os"
14+
"os/exec"
15+
"path/filepath"
16+
)
17+
18+
//go:embed glibc-patch.bash
19+
var glibcPatchScript []byte
20+
21+
// DerivationBuilder patches an existing package.
22+
type DerivationBuilder struct {
23+
// Out is the output directory that will contain the built derivation.
24+
// If empty it defaults to $out, which is typically set by Nix.
25+
Out string
26+
}
27+
28+
// NewDerivationBuilder initializes a new DerivationBuilder from the current
29+
// Nix build environment.
30+
func NewDerivationBuilder() (*DerivationBuilder, error) {
31+
d := &DerivationBuilder{}
32+
if err := d.init(); err != nil {
33+
return nil, err
34+
}
35+
return d, nil
36+
}
37+
38+
func (d *DerivationBuilder) init() error {
39+
if d.Out == "" {
40+
d.Out = os.Getenv("out")
41+
if d.Out == "" {
42+
return fmt.Errorf("patchpkg: $out is empty (is this being run from a nix build?)")
43+
}
44+
}
45+
return nil
46+
}
47+
48+
// Build applies patches to a package store path and puts the result in the
49+
// d.Out directory.
50+
func (d *DerivationBuilder) Build(ctx context.Context, pkgStorePath string) error {
51+
slog.DebugContext(ctx, "starting build of patched package", "pkg", pkgStorePath, "out", d.Out)
52+
53+
var err error
54+
pkgFS := os.DirFS(pkgStorePath)
55+
for path, entry := range allFiles(pkgFS, ".") {
56+
switch {
57+
case entry.IsDir():
58+
err = d.copyDir(path)
59+
case isSymlink(entry.Type()):
60+
err = d.copySymlink(pkgStorePath, path)
61+
default:
62+
err = d.copyFile(pkgFS, path)
63+
}
64+
65+
if err != nil {
66+
return err
67+
}
68+
}
69+
70+
bash := filepath.Join(os.Getenv("bash"), "bin/bash")
71+
cmd := exec.CommandContext(ctx, bash, "-s")
72+
cmd.Stdin = bytes.NewReader(glibcPatchScript)
73+
cmd.Stdout = os.Stdout
74+
cmd.Stderr = os.Stderr
75+
return cmd.Run()
76+
}
77+
78+
func (d *DerivationBuilder) copyDir(path string) error {
79+
osPath, err := filepath.Localize(path)
80+
if err != nil {
81+
return err
82+
}
83+
return os.Mkdir(filepath.Join(d.Out, osPath), 0o777)
84+
}
85+
86+
func (d *DerivationBuilder) copyFile(pkgFS fs.FS, path string) error {
87+
src, err := pkgFS.Open(path)
88+
if err != nil {
89+
return err
90+
}
91+
defer src.Close()
92+
93+
stat, err := src.Stat()
94+
if err != nil {
95+
return err
96+
}
97+
98+
// We only need to copy the executable permissions of a file.
99+
// Nix ends up making everything in the store read-only after
100+
// the build is done.
101+
perm := fs.FileMode(0o666)
102+
if isExecutable(stat.Mode()) {
103+
perm = fs.FileMode(0o777)
104+
}
105+
106+
osPath, err := filepath.Localize(path)
107+
if err != nil {
108+
return err
109+
}
110+
dstPath := filepath.Join(d.Out, osPath)
111+
112+
dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, perm)
113+
if err != nil {
114+
return err
115+
}
116+
defer dst.Close()
117+
118+
_, err = io.Copy(dst, src)
119+
if err != nil {
120+
return err
121+
}
122+
return dst.Close()
123+
}
124+
125+
func (d *DerivationBuilder) copySymlink(pkgStorePath, path string) error {
126+
// The fs package doesn't support symlinks, so we need to convert the
127+
// path back to an OS path to see what it points to.
128+
osPath, err := filepath.Localize(path)
129+
if err != nil {
130+
return err
131+
}
132+
target, err := os.Readlink(filepath.Join(pkgStorePath, osPath))
133+
if err != nil {
134+
return err
135+
}
136+
// TODO(gcurtis): translate absolute symlink targets to relative paths.
137+
return os.Symlink(target, filepath.Join(d.Out, osPath))
138+
}
139+
140+
// RegularFiles iterates over all files in fsys starting at root. It silently
141+
// ignores errors.
142+
func allFiles(fsys fs.FS, root string) iter.Seq2[string, fs.DirEntry] {
143+
return func(yield func(string, fs.DirEntry) bool) {
144+
_ = fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error {
145+
if err == nil {
146+
if !yield(path, d) {
147+
return filepath.SkipAll
148+
}
149+
}
150+
return nil
151+
})
152+
}
153+
}
154+
155+
func isExecutable(mode fs.FileMode) bool { return mode&0o111 != 0 }
156+
func isSymlink(mode fs.FileMode) bool { return mode&fs.ModeSymlink != 0 }

internal/shellgen/tmpl/glibc-patch.bash renamed to internal/patchpkg/glibc-patch.bash

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,6 @@ hash -p "$gnused/bin/sed" sed
2323
hash -p "$patchelf/bin/patchelf" patchelf
2424
hash -p "$ripgrep/bin/rg" rg
2525

26-
# Copy the contents of the original package so we can patch them.
27-
cp -R "$pkg" "$out"
28-
29-
# Because we copied an existing store path, our new $out directory might be
30-
# read-only. This might've caused issues with some versions of Nix, so make it
31-
# writable again just to be safe.
32-
chmod u+rwx "$out"
33-
3426
# Find the new linker that we'll patch into all of the package's executables as
3527
# the interpreter.
3628
interp="$(find "$glibc/lib" -type f -maxdepth 1 -executable -name 'ld-linux-*.so*' | head -n1)"

internal/shellgen/flake_plan.go

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package shellgen
33
import (
44
"context"
55
"fmt"
6+
"os"
67
"path/filepath"
78
"runtime/trace"
89
"strings"
@@ -68,6 +69,10 @@ func (f *flakePlan) needsGlibcPatch() bool {
6869
}
6970

7071
type glibcPatchFlake struct {
72+
// DevboxExecutable is the absolute path to the Devbox binary to use as
73+
// the flake's builder. It must not be the wrapper script.
74+
DevboxExecutable string
75+
7176
// NixpkgsGlibcFlakeRef is a flake reference to the nixpkgs flake
7277
// containing the new glibc package.
7378
NixpkgsGlibcFlakeRef string
@@ -85,7 +90,21 @@ type glibcPatchFlake struct {
8590
}
8691

8792
func newGlibcPatchFlake(nixpkgsGlibcRev string, packages []*devpkg.Package) (glibcPatchFlake, error) {
88-
flake := glibcPatchFlake{NixpkgsGlibcFlakeRef: "flake:nixpkgs/" + nixpkgsGlibcRev}
93+
// Get the path to the actual devbox binary (not the /usr/bin/devbox
94+
// wrapper) so the flake build can use it.
95+
exe, err := os.Executable()
96+
if err != nil {
97+
return glibcPatchFlake{}, err
98+
}
99+
exe, err = filepath.EvalSymlinks(exe)
100+
if err != nil {
101+
return glibcPatchFlake{}, err
102+
}
103+
104+
flake := glibcPatchFlake{
105+
DevboxExecutable: exe,
106+
NixpkgsGlibcFlakeRef: "flake:nixpkgs/" + nixpkgsGlibcRev,
107+
}
89108
for _, pkg := range packages {
90109
if !pkg.PatchGlibc() {
91110
continue
@@ -143,9 +162,5 @@ func (g *glibcPatchFlake) fetchClosureExpr(pkg *devpkg.Package) (string, error)
143162
}
144163

145164
func (g *glibcPatchFlake) writeTo(dir string) error {
146-
err := writeFromTemplate(dir, g, "glibc-patch.nix", "flake.nix")
147-
if err != nil {
148-
return err
149-
}
150-
return writeGlibcPatchScript(filepath.Join(dir, "glibc-patch.bash"))
165+
return writeFromTemplate(dir, g, "glibc-patch.nix", "flake.nix")
151166
}

internal/shellgen/generate.go

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"bytes"
99
"context"
1010
"embed"
11-
"io/fs"
1211
"os"
1312
"path/filepath"
1413
"runtime/trace"
@@ -102,20 +101,6 @@ func writeFromTemplate(path string, plan any, tmplName, generatedName string) er
102101
return nil
103102
}
104103

105-
// writeGlibcPatchScript writes the embedded glibc patching script to disk so
106-
// that a generated flake can use it.
107-
func writeGlibcPatchScript(path string) error {
108-
script, err := fs.ReadFile(tmplFS, "tmpl/glibc-patch.bash")
109-
if err != nil {
110-
return redact.Errorf("read embedded glibc-patch.bash: %v", redact.Safe(err))
111-
}
112-
err = overwriteFileIfChanged(path, script, 0o755)
113-
if err != nil {
114-
return redact.Errorf("write glibc-patch.bash to file: %v", err)
115-
}
116-
return nil
117-
}
118-
119104
// overwriteFileIfChanged checks that the contents of f == data, and overwrites
120105
// f if they differ. It also ensures that f's permissions are set to perm.
121106
func overwriteFileIfChanged(path string, data []byte, perm os.FileMode) error {

internal/shellgen/tmpl/glibc-patch.nix.tmpl

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,19 @@
22
description = "Patches packages to use a newer version of glibc";
33

44
inputs = {
5+
local-devbox = {
6+
url = "path://{{ .DevboxExecutable }}";
7+
flake = false;
8+
};
9+
510
nixpkgs-glibc.url = "{{ .NixpkgsGlibcFlakeRef }}";
611

712
{{- range $name, $flakeref := .Inputs }}
813
{{ $name }}.url = "{{ $flakeref }}";
914
{{- end }}
1015
};
1116

12-
outputs = args@{ self, nixpkgs-glibc {{- range $name, $_ := .Inputs -}}, {{ $name }} {{- end }} }:
17+
outputs = args@{ self, local-devbox, nixpkgs-glibc {{- range $name, $_ := .Inputs -}}, {{ $name }} {{- end }} }:
1318
{{ with .Outputs -}}
1419
let
1520
# Initialize each nixpkgs input into a new attribute set with the
@@ -30,17 +35,35 @@
3035
else null) args;
3136

3237
patchGlibc = pkg: derivation rec {
33-
name = "devbox-patched-glibc";
34-
system = pkg.system;
35-
3638
# The package we're patching.
3739
inherit pkg;
3840

41+
# Keep the name the same as the package we're patching so that the
42+
# length of the store path doesn't change. Otherwise patching binaries
43+
# becomes trickier.
44+
name = pkg.name;
45+
system = pkg.system;
46+
3947
# Programs needed by glibc-patch.bash.
4048
inherit (nixpkgs-glibc.legacyPackages."${system}") bash coreutils file findutils glibc gnused patchelf ripgrep;
4149

42-
builder = "${bash}/bin/bash";
43-
args = [ ./glibc-patch.bash ];
50+
# Create a package that puts the local devbox binary in the conventional
51+
# bin subdirectory. This also ensures that the executable is named
52+
# "devbox" and not "<hash>-source" (which is how Nix names the flake
53+
# input). Invoking it as anything other than "devbox" will break
54+
# testscripts which look at os.Args[0] to decide to run the real
55+
# entrypoint or the test entrypoint.
56+
devbox = derivation {
57+
name = "devbox";
58+
system = pkg.system;
59+
builder = "${bash}/bin/bash";
60+
61+
# exit 0 to work around https://github.com/NixOS/nix/issues/2176
62+
args = [ "-c" "${coreutils}/bin/mkdir -p $out/bin && ${coreutils}/bin/cp ${local-devbox} $out/bin/devbox && exit 0" ];
63+
};
64+
65+
builder = "${devbox}/bin/devbox";
66+
args = [ "patch" pkg ];
4467
};
4568
in
4669
{

0 commit comments

Comments
 (0)