[AutoBuild] Fix default value of lazy_artifacts (#1191)

* [AutoBuild] Fix default value of `lazy_artifacts`

* [AutoBuild] Simplify writing of files in `.pkg` directory

* [AutoBuild] Add tests for platform augmentation

Author: giordano

Project.toml

@@ -28,6 +28,7 @@ RegistryTools = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 

src/AutoBuild.jl

@@ -1,7 +1,7 @@
 export build_tarballs, autobuild, print_artifacts_toml, build, get_meta_json
 import GitHub: gh_get_json, DEFAULT_API
 import SHA: sha256, sha1
-using Pkg.TOML, Dates, UUIDs
+using TOML, Dates, UUIDs
 using RegistryTools
 import LibGit2
 import PkgLicenses
@@ -104,7 +104,7 @@ const BUILD_HELP = (
                             instead of actually building.  Note that this can
                             (and often does) output multiple JSON objects for
                             multiple platforms, multi-stage builds, etc...
-        
+
         --skip-audit        Skips auditing of the output products.
 
         --help              Print out this message.
@@ -543,8 +543,8 @@ function register_jll(name, build_version, dependencies, julia_compat;
                       code_dir=joinpath(Pkg.devdir(), "$(name)_jll"),
                       gh_auth=Wizard.github_auth(;allow_anonymous=false),
                       gh_username=gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"],
-                      lazy_artifacts::Bool=false,
-                      augment_platform_block="",
+                      augment_platform_block::String="",
+                      lazy_artifacts::Bool=!isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
                       kwargs...)
     if !isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.6"
         error("Augmentation blocks cannot be used with Julia v1.5-.\nChange `julia_compat` to require at least Julia v1.6")
@@ -622,9 +622,10 @@ function get_meta_json(
                    products::Vector{<:Product},
                    dependencies::Vector{<:AbstractDependency};
                    julia_compat::String = DEFAULT_JULIA_VERSION_SPEC,
-                   lazy_artifacts::Bool = false,
                    init_block::String = "",
-                   augment_platform_block::String = "")
+                   augment_platform_block::String = "",
+                   lazy_artifacts::Bool=!isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
+    )
 
     dict = Dict(
         "name" => src_name,
@@ -654,7 +655,7 @@ function compose_debug_prompt(workspace)
             end
         end
     end
-    
+
     if length(log_files) > 0
         log_files_str = join(log_files, "\n  - ")
 
@@ -1064,7 +1065,6 @@ function rebuild_jll_package(obj::Dict;
                              build_version = nothing,
                              gh_org::String = "JuliaBinaryWrappers",
                              verbose::Bool = false,
-                             lazy_artifacts::Bool = false,
                              from_scratch::Bool = true)
     if build_version === nothing
         build_version = BinaryBuilder.get_next_wrapper_version(obj["name"], obj["version"])
@@ -1079,6 +1079,9 @@ function rebuild_jll_package(obj::Dict;
         error("If download_dir is specified, you must specify upload_prefix as well!")
     end
 
+    julia_compat = get(obj, "julia_compat", DEFAULT_JULIA_VERSION_SPEC)
+    augment_platform_block = get(obj, "augment_platform_block", "")
+    lazy_artifacts = get(obj, "lazy_artifacts", !isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7")
     return rebuild_jll_package(
         obj["name"],
         build_version,
@@ -1088,12 +1091,12 @@ function rebuild_jll_package(obj::Dict;
         obj["dependencies"],
         download_dir,
         upload_prefix;
-        verbose=verbose,
-        lazy_artifacts = lazy_artifacts,
-        julia_compat = get(obj, "julia_compat", DEFAULT_JULIA_VERSION_SPEC),
+        verbose,
+        lazy_artifacts,
+        julia_compat,
         init_block = get(obj, "init_block", ""),
-        augment_platform_block = get(obj, "augment_platform_block", ""),
-        from_scratch = from_scratch,
+        augment_platform_block,
+        from_scratch,
     )
 end
 
@@ -1196,8 +1199,8 @@ function build_jll_package(src_name::String,
                            bin_path::String;
                            verbose::Bool = false,
                            julia_compat::String = DEFAULT_JULIA_VERSION_SPEC,
-                           init_block = "",
-                           augment_platform_block = "",
+                           init_block::String = "",
+                           augment_platform_block::String = "",
                            # If we support versions older than Julia v1.7 the artifact
                            # should be lazy when we augment the platform.
                            lazy_artifacts::Bool = !isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
@@ -1327,33 +1330,28 @@ function build_jll_package(src_name::String,
     if !isempty(augment_platform_block)
         pkg_dir = joinpath(code_dir, ".pkg")
         !ispath(pkg_dir) && mkdir(pkg_dir)
-        open(joinpath(pkg_dir, "platform_augmentation.jl"), "w") do io
-            println(io, """
-            $(augment_platform_block)
-            """)
-        end
+        write(joinpath(pkg_dir, "platform_augmentation.jl"), augment_platform_block)
 
-        open(joinpath(pkg_dir, "select_artifacts.jl"), "w") do io
-            println(io, """
-            push!(Base.LOAD_PATH, dirname(@__DIR__))
+        write(joinpath(pkg_dir, "select_artifacts.jl"),
+              """
+              push!(Base.LOAD_PATH, dirname(@__DIR__))
 
-            using TOML, Artifacts, Base.BinaryPlatforms
-            include("./platform_augmentation.jl")
-            artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml")
+              using TOML, Artifacts, Base.BinaryPlatforms
+              include("./platform_augmentation.jl")
+              artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml")
 
-            # Get "target triplet" from ARGS, if given (defaulting to the host triplet otherwise)
-            target_triplet = get(ARGS, 1, Base.BinaryPlatforms.host_triplet())
+              # Get "target triplet" from ARGS, if given (defaulting to the host triplet otherwise)
+              target_triplet = get(ARGS, 1, Base.BinaryPlatforms.host_triplet())
 
-            # Augment this platform object with any special tags we require
-            platform = augment_platform!(HostPlatform(parse(Platform, target_triplet)))
+              # Augment this platform object with any special tags we require
+              platform = augment_platform!(HostPlatform(parse(Platform, target_triplet)))
 
-            # Select all downloadable artifacts that match that platform
-            artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy=true)
+              # Select all downloadable artifacts that match that platform
+              artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy=true)
 
-            #Output the result to `stdout` as a TOML dictionary
-            TOML.print(stdout, artifacts)
-            """)
-        end
+              #Output the result to `stdout` as a TOML dictionary
+              TOML.print(stdout, artifacts)
+              """)
     end
 
     # Generate target-demuxing main source file.
@@ -1419,9 +1417,9 @@ function build_jll_package(src_name::String,
         if is_yggdrasil()
             println(io, """
                         The originating [`build_tarballs.jl`](https://github.com/JuliaPackaging/Yggdrasil/blob/$(yggdrasil_head())/$(ENV["PROJECT"])/build_tarballs.jl) script can be found on [`Yggdrasil`](https://github.com/JuliaPackaging/Yggdrasil/), the community build tree.")
-                        
+
                         ## Bug Reports
-                        
+
                         If you have any issue, please report it to the Yggdrasil [bug tracker](https://github.com/JuliaPackaging/Yggdrasil/issues).
                         """)
         end
@@ -1505,7 +1503,7 @@ function build_jll_package(src_name::String,
     jllwrappers_compat = isempty(augment_platform_block) ? "1.2.0" : "1.4.0"
     project = build_project_dict(src_name, build_version, dependencies, julia_compat; lazy_artifacts, jllwrappers_compat, augment_platform_block)
     open(joinpath(code_dir, "Project.toml"), "w") do io
-        Pkg.TOML.print(io, project)
+        TOML.print(io, project)
     end
 
     # Add a `.gitignore`
@@ -1579,8 +1577,8 @@ end
 function build_project_dict(name, version, dependencies::Array{Dependency},
                             julia_compat::String=DEFAULT_JULIA_VERSION_SPEC;
                             jllwrappers_compat::String=DEFAULT_JLLWRAPPERS_VERSION_SPEC,
-                            lazy_artifacts::Bool=false,
-                            augment_platform_block="",
+                            augment_platform_block::String="",
+                            lazy_artifacts::Bool=!isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7",
                             kwargs...)
     Pkg.Types.semver_spec(julia_compat) # verify julia_compat is valid
     project = Dict(

test/jll.jl

@@ -1,3 +1,4 @@
+using TOML
 using JSON
 using UUIDs
 using GitHub
@@ -48,6 +49,17 @@ end
         platforms = [platform, freebsd]
         # We depend on Zlib_jll only on the host platform, but not on FreeBSD
         dependencies = [Dependency("Zlib_jll"; platforms=[platform])]
+        # Augment platform
+        augment_platform_block = """
+        using Base.BinaryPlatforms
+        function augment_platform!(platform::Platform)
+            platform["test"] = "enabled"
+            return platform
+        end
+        """
+        # Julia compat.  Include Julia v1.6 to exercise the code path which forces lazy
+        # artifacts when augmenting the platform
+        julia_compat = "1.6"
         # The buffer where we'll write the JSON meta data
         buff = IOBuffer()
 
@@ -65,6 +77,8 @@ end
                 # The products we expect to be build
                 libfoo_products,
                 dependencies;
+                julia_compat,
+                augment_platform_block,
             )
             # Generate the JSON file
             println(buff, JSON.json(dict))
@@ -143,13 +157,21 @@ end
                     @test occursin("MESON_TARGET_TOOLCHAIN", json_obj["script"])
                 end
 
-                BinaryBuilder.rebuild_jll_package(json_obj; download_dir=download_dir, upload_prefix=upload_prefix, verbose=false, lazy_artifacts=json_obj["lazy_artifacts"], from_scratch=from_scratch)
+                BinaryBuilder.rebuild_jll_package(json_obj; download_dir, upload_prefix, verbose=false, from_scratch)
             end
 
             env_dir = joinpath(build_path, "foo")
             mkpath(env_dir)
             Pkg.activate(env_dir)
             Pkg.develop(PackageSpec(path=code_dir))
+            @test isfile(joinpath(code_dir, ".pkg", "select_artifacts.jl"))
+            @test read(joinpath(code_dir, ".pkg", "platform_augmentation.jl"), String) == augment_platform_block
+            # Make sure the artifacts are lazy because we are augmenting the platform and
+            # Julia compat includes versions before v1.7.
+            artifacts_toml = TOML.parsefile(joinpath(code_dir, "Artifacts.toml"))
+            for artifact in artifacts_toml["libfoo"]
+                @test artifact["lazy"]
+            end
             # Make sure we use Zlib_jll only in the wrapper for the host
             # platform and not the FreeBSD one.
             platform_wrapper = joinpath(code_dir, "src", "wrappers", triplet(platform) * ".jl")