Use bazel's downloader for Go modules when possible

Author: dzbarsky

cmd/fetch_repo/main.go

@@ -27,7 +27,9 @@ package main
 import (
 	"flag"
 	"log"
+	"os"
 
+	"golang.org/x/mod/sumdb/dirhash"
 	"golang.org/x/tools/go/vcs"
 )
 
@@ -70,7 +72,19 @@ func main() {
 	}
 
 	if *no_fetch {
-		// Nothing to do
+		if *sum != "" {
+			repoSum, err := dirhash.HashDir(*dest, *importpath+"@"+*version, dirhash.Hash1)
+			if err != nil {
+				log.Fatalf("failed computing sum: %v", err)
+			}
+
+			if repoSum != *sum {
+				if goModCache := os.Getenv("GOMODCACHE"); goModCache != "" {
+					log.Fatalf("resulting module with sum %s; expected sum %s, Please try clearing your module cache directory %q", repoSum, *sum, goModCache)
+				}
+				log.Fatalf("resulting module with sum %s; expected sum %s", repoSum, *sum)
+			}
+		}
 	} else if *path != "" {
 		if *importpath != "" {
 			log.Fatal("-importpath must not be set")

internal/go_repository.bzl

@@ -117,10 +117,56 @@ Authorization: Bearer RANDOM-TOKEN
 # We can't disable timeouts on Bazel, but we can set them to large values.
 _GO_REPOSITORY_TIMEOUT = 86400
 
+def _escape_go_module_path(module_path):
+    """Escapes a Go module path according to the Go proxy protocol."""
+    result = ""
+    for c in module_path.elems():
+        if "A" <= c and c <= "Z":
+            result += "!" + c.lower()
+        else:
+            result += c
+    return result
+
+def _get_goproxy_url_for_module(env, module_path):
+    """Return proxy URL to be used for module_path, or None if no proxy will be used."""
+    goproxy = env.get("GOPROXY", "")
+    goprivate = env.get("GOPRIVATE", "")
+    gonoproxy = env.get("GONOPROXY", "")
+
+    if goproxy == "off":
+        return None
+
+    def matches_pattern(pattern, module_path):
+        pat = pattern.strip()
+        if not pat:
+            return False
+        if pat.endswith("/..."):
+            return module_path.startswith(pat[:-4])
+        if "*" in pat:
+            return pat.replace("*", "") in module_path
+        return module_path.startswith(pat)
+
+    for pat in goprivate.split(",") + gonoproxy.split(","):
+        if matches_pattern(pat, module_path):
+            return None
+
+    if not goproxy:
+        return "https://proxy.golang.org"
+
+    idx = goproxy.find(",")
+    if idx != -1:
+        goproxy = goproxy[:idx]
+
+    if goproxy == "direct":
+        return None
+
+    return goproxy
+
 def _go_repository_impl(ctx):
     # TODO(#549): vcs repositories are not cached and still need to be fetched.
     # Download the repository or module.
     fetch_repo_args = None
+    sum_args = []
     gazelle_path = None
 
     is_module_extension_repo = bool(ctx.attr.internal_only_do_not_use_apparent_name)
@@ -137,6 +183,7 @@ def _go_repository_impl(ctx):
         gazelle_path = ctx.path(Label(_gazelle))
         watch(ctx, gazelle_path)
 
+    can_use_direct_download = False
     reproducible = False
     if ctx.attr.local_path:
         if hasattr(ctx, "watch_tree"):
@@ -204,14 +251,15 @@ def _go_repository_impl(ctx):
             else:
                 fail("if version is specified, sum must also be")
         reproducible = True
+        can_use_direct_download = True
 
         fetch_path = ctx.attr.replace if ctx.attr.replace else ctx.attr.importpath
+        sum_args = ["-sum=" + ctx.attr.sum]
         fetch_repo_args = [
             "-dest=" + str(ctx.path("")),
             "-importpath=" + fetch_path,
             "-version=" + ctx.attr.version,
-            "-sum=" + ctx.attr.sum,
-        ]
+        ] + sum_args
     else:
         fail("one of urls, commit, tag, or version must be specified")
 
@@ -286,6 +334,28 @@ def _go_repository_impl(ctx):
     # Override external GO111MODULE, because it is needed by module mode, no-op in repository mode
     fetch_repo_env["GO111MODULE"] = "on"
 
+    if can_use_direct_download:
+        module = ctx.attr.replace if ctx.attr.replace else ctx.attr.importpath
+        proxy_url = _get_goproxy_url_for_module(env, module)
+
+        if proxy_url:
+            result = ctx.download_and_extract(
+                url = "%s/%s/@v/%s.zip" % (
+                    proxy_url,
+                    _escape_go_module_path(module),
+                    ctx.attr.version,
+                ),
+                sha256 = ctx.attr.sha256,
+                canonical_id = ctx.attr.canonical_id,
+                stripPrefix = module + "@" + ctx.attr.version,
+                type = ctx.attr.type,
+                auth = use_netrc(read_user_netrc(ctx), ctx.attr.urls, ctx.attr.auth_patterns),
+                allow_fail = True,
+            )
+            if result.success:
+                # All we need to do now is verify the sum!
+                fetch_repo_args = ["-no-fetch"] + fetch_repo_args
+
     result = env_execute(
         ctx,
         [fetch_repo] + fetch_repo_args,

internal/go_repository_test.go

@@ -180,7 +180,7 @@ go_repository(
 }
 
 func TestModcacheRW(t *testing.T) {
-	if err := bazel_testing.RunBazel("query", "@errors_go_mod//:go_default_library"); err != nil {
+	if err := bazel_testing.RunBazel("query", "--repo_env=GOPROXY=direct", "@errors_go_mod//:go_default_library"); err != nil {
 		t.Fatal(err)
 	}
 	out, err := bazel_testing.BazelOutput("info", "output_base")