fixes a bug where file paths with spaces in them would cause upload e…

.github/workflows/install.yaml

@@ -22,10 +22,18 @@ jobs:
           - { os: darwin, arch: arm64, runner: macos-latest }
 
     steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
       - name: Install Default Version (Latest Release)
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          BRANCH=${{ github.ref_name }}
-          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/calypr/git-drs/$BRANCH/install.sh)"
-          source $HOME/.bashrc
-          which git-drs
+          ./install.sh
+          # GITHUB_PATH is updated by the script, but we might need to refresh for the current step 
+          # but usually it's better to just use identifying path or rely on next steps
+          # In this single-step test, we check if it works by calling it directly if not in PATH yet
+          export PATH="$PATH:$HOME/.local/bin"
           git-drs --help
+

README.md

@@ -75,6 +75,7 @@ For detailed setup and usage information:
 
 - **[Getting Started](docs/getting-started.md)** - Repository setup and basic workflows
 - **[Commands Reference](docs/commands.md)** - Complete command documentation
+- **[Remote Remove Use Cases](docs/remote-remove-use-cases.md)** - Practical scenarios for remote cleanup
 - **[Installation Guide](docs/installation.md)** - Platform-specific installation
 - **[Troubleshooting](docs/troubleshooting.md)** - Common issues and solutions
 - **[S3 Integration](docs/adding-s3-files.md)** - Adding files via S3 URLs
@@ -98,6 +99,7 @@ For detailed setup and usage information:
 | `git drs remote add`   | Add a DRS remote server               |
 | `git drs remote list`  | List configured remotes               |
 | `git drs remote set`   | Set default remote                    |
+| `git drs remote remove`| Remove a configured remote            |
 | `git drs add-url`      | Add files via S3 URLs                 |
 | `git lfs track`        | Track file patterns with LFS          |
 | `git lfs ls-files`     | List tracked files                    |

client/indexd/add_url.go

@@ -151,6 +151,11 @@ func (inc *GitDrsIdxdClient) upsertIndexdRecord(ctx context.Context, url string,
 	if err != nil {
 		return nil, err
 	}
+	if len(drsObj.AccessMethods) > 0 {
+		drsObj.AccessMethods[0].AccessURL = drs.AccessURL{URL: url}
+	} else {
+		drsObj.AccessMethods = append(drsObj.AccessMethods, drs.AccessMethod{AccessURL: drs.AccessURL{URL: url}})
+	}
 
 	// Add authz explicitly since BuildDrsObj might not set it exactly as needed for all cases
 	// Actually BuildDrsObj does set authz.

client/indexd/client.go

@@ -130,7 +130,7 @@ func (cl *GitDrsIdxdClient) getDownloadURLFromRecords(ctx context.Context, oid s
 	}
 
 	// Find a record that matches the client's project ID
-	matchingRecord, err := drsmap.FindMatchingRecord(records, cl.Config.ProjectId)
+	matchingRecord, err := drsmap.FindMatchingRecord(records, cl.Config.ProjectId, "")
 	if err != nil {
 		cl.Logger.Debug(fmt.Sprintf("error finding matching record for project %s: %s", cl.Config.ProjectId, err))
 		return nil, fmt.Errorf("error finding matching record for project %s: %v", cl.Config.ProjectId, err)

client/indexd/register.go

@@ -24,7 +24,7 @@ func (cl *GitDrsIdxdClient) RegisterFile(ctx context.Context, oid string, path s
 		return nil, fmt.Errorf("error getting drs object for oid %s: %v", oid, err)
 	}
 
-	cl.Logger.InfoContext(ctx, fmt.Sprintf("registering record for oid %s in indexd (did: %s)", oid, drsObject.Id))
+	cl.Logger.DebugContext(ctx, fmt.Sprintf("registering record for oid %s in indexd (did: %s)", oid, drsObject.Id))
 	_, err = cl.RegisterRecord(ctx, drsObject)
 	if err != nil {
 		// handle "already exists" error ie upsert behavior
@@ -46,10 +46,10 @@ func (cl *GitDrsIdxdClient) RegisterFile(ctx context.Context, oid string, path s
 			return nil, fmt.Errorf("error saving oid %s indexd record: %v", oid, err)
 		}
 	}
-	cl.Logger.InfoContext(ctx, fmt.Sprintf("indexd record registration complete for oid %s", oid))
+	cl.Logger.DebugContext(ctx, fmt.Sprintf("indexd record registration complete for oid %s", oid))
 
 	// Now attempt to upload the file if not already available
-	cl.Logger.InfoContext(ctx, fmt.Sprintf("checking if oid %s is already downloadable", oid))
+	cl.Logger.DebugContext(ctx, fmt.Sprintf("checking if oid %s is already downloadable", oid))
 	downloadable, err := cl.isFileDownloadable(ctx, drsObject)
 	if err != nil {
 		return nil, fmt.Errorf("error checking if file is downloadable: oid %s %v", oid, err)
@@ -58,7 +58,7 @@ func (cl *GitDrsIdxdClient) RegisterFile(ctx context.Context, oid string, path s
 		cl.Logger.DebugContext(ctx, fmt.Sprintf("file %s is already available for download, skipping upload", oid))
 		return drsObject, nil
 	}
-	cl.Logger.InfoContext(ctx, fmt.Sprintf("file %s is not downloadable, proceeding to upload", oid))
+	cl.Logger.InfoContext(ctx, fmt.Sprintf("file %s is not downloadable, proceeding to upload", path))
 
 	// Proceed to upload the file
 	// Reuse the Gen3 interface

client/local/local_client_test.go

@@ -0,0 +1,153 @@
+package local
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/calypr/data-client/common"
+	drs "github.com/calypr/data-client/drs"
+)
+
+type mockBackend struct {
+	uploadURLCalls         int
+	uploadCalls            int
+	initMultipartCalls     int
+	partURLCalls           int
+	uploadPartCalls        int
+	completeMultipartCalls int
+}
+
+func (m *mockBackend) Name() string { return "mock" }
+func (m *mockBackend) Logger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+func (m *mockBackend) GetFileDetails(ctx context.Context, guid string) (*drs.DRSObject, error) {
+	return &drs.DRSObject{Id: guid, Size: 1, Name: "x"}, nil
+}
+func (m *mockBackend) GetObjectByHash(ctx context.Context, checksumType, checksum string) ([]drs.DRSObject, error) {
+	return nil, nil
+}
+func (m *mockBackend) BatchGetObjectsByHash(ctx context.Context, hashes []string) (map[string][]drs.DRSObject, error) {
+	return map[string][]drs.DRSObject{}, nil
+}
+func (m *mockBackend) GetDownloadURL(ctx context.Context, guid string, accessID string) (string, error) {
+	return "https://example.invalid/download", nil
+}
+func (m *mockBackend) Download(ctx context.Context, fdr *common.FileDownloadResponseObject) (*http.Response, error) {
+	return nil, nil
+}
+func (m *mockBackend) Register(ctx context.Context, obj *drs.DRSObject) (*drs.DRSObject, error) {
+	return obj, nil
+}
+func (m *mockBackend) BatchRegister(ctx context.Context, objs []*drs.DRSObject) ([]*drs.DRSObject, error) {
+	return objs, nil
+}
+func (m *mockBackend) GetUploadURL(ctx context.Context, guid string, filename string, metadata common.FileMetadata, bucket string) (string, error) {
+	m.uploadURLCalls++
+	return "https://example.invalid/upload", nil
+}
+func (m *mockBackend) InitMultipartUpload(ctx context.Context, guid string, filename string, bucket string) (*common.MultipartUploadInit, error) {
+	m.initMultipartCalls++
+	return &common.MultipartUploadInit{GUID: guid, UploadID: "up-1"}, nil
+}
+func (m *mockBackend) GetMultipartUploadURL(ctx context.Context, key string, uploadID string, partNumber int32, bucket string) (string, error) {
+	m.partURLCalls++
+	return "https://example.invalid/part", nil
+}
+func (m *mockBackend) CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []common.MultipartUploadPart, bucket string) error {
+	m.completeMultipartCalls++
+	return nil
+}
+func (m *mockBackend) Upload(ctx context.Context, url string, body io.Reader, size int64) error {
+	m.uploadCalls++
+	return nil
+}
+func (m *mockBackend) UploadPart(ctx context.Context, url string, body io.Reader, size int64) (string, error) {
+	m.uploadPartCalls++
+	return "etag-1", nil
+}
+
+func TestRegisterFileUsesSingleUploadPath(t *testing.T) {
+	tmp := t.TempDir()
+	path := tmp + "/small.bin"
+	if err := os.WriteFile(path, []byte("small"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	mb := &mockBackend{}
+	lc := &LocalClient{
+		Remote: LocalRemote{
+			BaseURL:      "http://localhost:8080",
+			Bucket:       "bucket-a",
+			ProjectID:    "p",
+			Organization: "o",
+		},
+		Logger:  slog.New(slog.NewTextHandler(io.Discard, nil)),
+		Backend: mb,
+		Config:  &LocalConfig{MultiPartThreshold: 1024 * 1024},
+	}
+
+	oid := "1111111111111111111111111111111111111111111111111111111111111111"
+	if _, err := lc.RegisterFile(context.Background(), oid, path); err != nil {
+		t.Fatalf("RegisterFile failed: %v", err)
+	}
+
+	if mb.uploadURLCalls != 1 || mb.uploadCalls != 1 {
+		t.Fatalf("expected single upload path, got uploadURLCalls=%d uploadCalls=%d", mb.uploadURLCalls, mb.uploadCalls)
+	}
+	if mb.initMultipartCalls != 0 || mb.uploadPartCalls != 0 || mb.completeMultipartCalls != 0 {
+		t.Fatalf("did not expect multipart path, got init=%d part=%d complete=%d", mb.initMultipartCalls, mb.uploadPartCalls, mb.completeMultipartCalls)
+	}
+}
+
+func TestRegisterFileUsesMultipartUploadPath(t *testing.T) {
+	tmp := t.TempDir()
+	path := tmp + "/large.bin"
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Use sparse file >100MB so OptimalChunkSize yields multipart chunks < file size.
+	if err := f.Truncate(120 * common.MB); err != nil {
+		_ = f.Close()
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	mb := &mockBackend{}
+	lc := &LocalClient{
+		Remote: LocalRemote{
+			BaseURL:      "http://localhost:8080",
+			Bucket:       "bucket-a",
+			ProjectID:    "p",
+			Organization: "o",
+		},
+		Logger:  slog.New(slog.NewTextHandler(io.Discard, nil)),
+		Backend: mb,
+		Config: &LocalConfig{
+			MultiPartThreshold: 1,
+			UploadConcurrency:  4,
+		},
+	}
+
+	oid := "2222222222222222222222222222222222222222222222222222222222222222"
+	if _, err := lc.RegisterFile(context.Background(), oid, path); err != nil {
+		t.Fatalf("RegisterFile failed: %v", err)
+	}
+
+	if mb.initMultipartCalls == 0 || mb.partURLCalls == 0 || mb.uploadPartCalls == 0 || mb.completeMultipartCalls == 0 {
+		t.Fatalf("expected multipart path, got init=%d partURL=%d uploadPart=%d complete=%d", mb.initMultipartCalls, mb.partURLCalls, mb.uploadPartCalls, mb.completeMultipartCalls)
+	}
+	if mb.uploadPartCalls <= 1 {
+		t.Fatalf("expected multiple multipart part uploads, got %d", mb.uploadPartCalls)
+	}
+	if mb.uploadURLCalls != 0 || mb.uploadCalls != 0 {
+		t.Fatalf("did not expect single upload path, got uploadURLCalls=%d uploadCalls=%d", mb.uploadURLCalls, mb.uploadCalls)
+	}
+}

cloud/inspect.go

@@ -93,7 +93,7 @@ func InspectS3ForLFS(ctx context.Context, in S3ObjectParameters) (*S3Object, err
 	metaSHA := extractSHA256FromMetadata(head.Metadata)
 
 	// Optional: validate provided SHA256 against metadata if both exist.
-	expected := normalizeSHA256(in.SHA256)
+	expected := NormalizeSHA256(in.SHA256)
 	if expected != "" && metaSHA != "" && !strings.EqualFold(expected, metaSHA) {
 		return nil, fmt.Errorf("sha256 mismatch: expected=%s head.meta=%s", expected, metaSHA)
 	}
@@ -211,7 +211,7 @@ func newS3Client(ctx context.Context, in S3ObjectParameters) (*s3.Client, error)
 
 var sha256HexRe = regexp.MustCompile(`(?i)^[0-9a-f]{64}$`)
 
-func normalizeSHA256(s string) string {
+func NormalizeSHA256(s string) string {
 	s = strings.TrimSpace(s)
 	s = strings.TrimPrefix(strings.ToLower(s), "sha256:")
 	s = strings.TrimSpace(s)
@@ -243,7 +243,7 @@ func extractSHA256FromMetadata(md map[string]string) string {
 
 	for _, k := range candidates {
 		if v, ok := md[k]; ok {
-			n := normalizeSHA256(v)
+			n := NormalizeSHA256(v)
 			if n != "" {
 				return n
 			}
@@ -252,7 +252,7 @@ func extractSHA256FromMetadata(md map[string]string) string {
 
 	// Sometimes people stash "sha256:<hex>"
 	for _, v := range md {
-		if n := normalizeSHA256(v); n != "" {
+		if n := NormalizeSHA256(v); n != "" {
 			return n
 		}
 	}

cloud/inspect_test.go

@@ -1,8 +1,6 @@
 package cloud
 
 import (
-	"os"
-	"os/exec"
 	"strings"
 	"testing"
 )
@@ -49,15 +47,15 @@ func TestParseS3URL_HTTPSVirtualHosted(t *testing.T) {
 func TestNormalizeSHA256(t *testing.T) {
 	hex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
 
-	if got := normalizeSHA256(hex); got != hex {
+	if got := NormalizeSHA256(hex); got != hex {
 		t.Fatalf("expected %q, got %q", hex, got)
 	}
 
-	if got := normalizeSHA256("sha256:" + strings.ToUpper(hex)); got != hex {
+	if got := NormalizeSHA256("sha256:" + strings.ToUpper(hex)); got != hex {
 		t.Fatalf("expected %q, got %q", hex, got)
 	}
 
-	if got := normalizeSHA256("not-a-sha"); got != "" {
+	if got := NormalizeSHA256("not-a-sha"); got != "" {
 		t.Fatalf("expected empty for invalid, got %q", got)
 	}
 }
@@ -97,27 +95,3 @@ func TestExtractSHA256FromMetadata_SearchValues(t *testing.T) {
 		t.Fatalf("expected %q, got %q", hex, got)
 	}
 }
-
-// --- test helpers ---
-
-func mustRun(t *testing.T, dir string, name string, args ...string) {
-	t.Helper()
-	cmd := exec.Command(name, args...)
-	cmd.Dir = dir
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("command failed: %s %v\nerr=%v\nout=%s", name, args, err, string(out))
-	}
-}
-
-func mustChdir(t *testing.T, dir string) string {
-	t.Helper()
-	old, err := os.Getwd()
-	if err != nil {
-		t.Fatalf("Getwd: %v", err)
-	}
-	if err := os.Chdir(dir); err != nil {
-		t.Fatalf("Chdir(%s): %v", dir, err)
-	}
-	return old
-}

cmd/addurl/main.go

@@ -97,10 +97,14 @@ func parseAddURLInput(cmd *cobra.Command, args []string) (addURLInput, error) {
 		return addURLInput{}, err
 	}
 
-	sha256Param, err := cmd.Flags().GetString("sha256")
+	sha256ParamRaw, err := cmd.Flags().GetString("sha256")
 	if err != nil {
 		return addURLInput{}, fmt.Errorf("read flag sha256: %w", err)
 	}
+	sha256Param := cloud.NormalizeSHA256(sha256ParamRaw)
+	if sha256ParamRaw != "" && sha256Param == "" {
+		return addURLInput{}, fmt.Errorf("invalid sha256: %s", sha256ParamRaw)
+	}
 
 	awsKey, err := cmd.Flags().GetString(cloud.AWS_KEY_FLAG_NAME)
 	if err != nil {