WIP: Add a loading mechanism for compat-dependent API sets

This is a first WIP implementation for providing the underlying
mechanism for a #54905. I haven't fully designed out what I want the
final thing to look like, but I've thought about it enough that I think
we need some hands-on examples for further design exploration, which is
what this PR is designed to support.

The core idea here is that if you specify in your package
```
[compact]
julia = "0.14"
MyDependency = "0.3,0.4"
```

Then this mechanism will ask `MyDependency` to provide you a view of its
API that is compatible with its 0.3 version, even if 0.4 is installed.
The objective of this mechanism is that downstreams of widely used packages
(including `Base` and standard libraries) can be upgraded more gradually
by allowing some portion of the packages in project to use the new 0.4
API set, while packages that have not yet upgraded can continue to use
the 0.3 API set. Currently, whenever a widely used package changes an
API there's a big frenzied rush to update downstreams, which just isn't
sustainable as the ecosystem grows.

In other languages, problems like these are solved by allowing multiple
versions of the same package to be loaded. However, in julia, this is
not particularly feasible by default because packages may have generic
functions that need to be extended and unified and which break if there
are multiple copies of such resources.

That said, for simple packages, this mechanism could be used to emulate
the multiple-version-loading strategy on an opt-in basis.

The way that this works is that `loading` gains an additional layer of
indirection that is provided the `compat` specification of the loading
package. Packages can opt into this by providing a
```
function _get_versioned_api(compat)

end
```
function. Where the default is equivalent to
`_get_versioned_api(compat) = @__MODULE__`.

The loader makes no further assumptions on either the structure of
`compat` or how the package processes it. This is intentional to
allow evolution without impacting the core loading logic (of course
Pkg also looks at the structure of compat, as would the
`_get_versioned_api` implementation in Base, so there are some
constraints).

That said, the envisioned usage of this is that we create a standard
(in the sense of being widely used, not in the sense of it being
a stdlib) package to help package authors define the APIs of their
packages more precisely and with version information. This package
would then create a set of modules under the hood that describe
these APIs to the sytem and would set up `_get_versioned_api`
appropriately (i.e. the mechanism is not intended to be used
directly in most cases).

To actually make this useful, we will likely need some additional
features in the binding system, in particular:
1. #59859 and a few variants
   thereon to make the API modueles look more transparent.

2. A mechanism to intercept extension of generic function overloads
   in order to be able to provide compatibility (Design/PR for this
   forthcoming).

Note that this PR itself is WIP, it is not yet fully complete and
there is also a Pkg.jl side of this required to put compat info
into the manifest.

Author: Keno

base/loading.jl

@@ -256,8 +256,8 @@ struct LoadingCache
     env_project_file::Dict{String, Union{Bool, String}}
     project_file_manifest_path::Dict{String, Union{Nothing, String}}
     require_parsed::Set{String}
-    identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, String}}}
-    identified::Dict{String, Union{Nothing, Tuple{PkgId, String}}}
+    identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{ApiId, String}}}
+    identified::Dict{String, Union{Nothing, Tuple{ApiId, String}}}
     located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{String, String}, Nothing}}
 end
 const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) # n.b.: all access to and through this are protected by require_lock
@@ -351,36 +351,36 @@ function identify_package_env(where::Union{PkgId, Nothing}, name::String)
     cache_key = where === nothing ? name : (where, name)
     if cache !== nothing
         env_cache = where === nothing ? cache.identified : cache.identified_where
-        pkg_env = get(env_cache, cache_key, missing)
-        pkg_env === missing || return pkg_env
+        api_env = get(env_cache, cache_key, missing)
+        api_env === missing || return api_env
     end
 
     # Main part: Search through all environments in the load path to see if we have
     # a matching entry.
-    pkg_env = nothing
+    api_env = nothing
     for env in load_path()
-        pkgid = environment_deps_get(env, where, name)
+        apiid = environment_deps_get(env, where, name)
         # If we didn't find `where` at all, keep looking through the environment stack
-        pkgid === nothing && continue
-        if pkgid.uuid !== nothing || where === nothing
-            pkg_env = pkgid, env
+        apiid === nothing && continue
+        if apiid.pkg.uuid !== nothing || where === nothing
+            api_env = apiid, env
         end
         # If we don't have pkgid.uuid, still break here - this is a sentinel that indicates
         # that we've found `where` but it did not have the required dependency. We terminate the search.
         break
     end
-    if pkg_env === nothing && where !== nothing && is_stdlib(where)
+    if api_env === nothing && where !== nothing && is_stdlib(where)
         # if not found it could be that manifests are from a different julia version/commit
         # where stdlib dependencies have changed, so look up deps based on the stdlib Project.toml
         # as a fallback
-        pkg_env = identify_stdlib_project_dep(where, name)
+        api_env = identify_stdlib_project_dep(where, name)
     end
 
     # Cache the result
     if cache !== nothing
-        env_cache[cache_key] = pkg_env
+        env_cache[cache_key] = api_env
     end
-    return pkg_env
+    return api_env
 end
 identify_package_env(name::String) = identify_package_env(nothing, name)
 
@@ -711,9 +711,9 @@ end
 function package_get_here(project_file, name::String)::Union{Nothing,ApiId}
     # if `where` matches the project, use [deps] section as manifest, and stop searching
     pkg_uuid = explicit_project_deps_get(project_file, name)
-    pkg_uuid === nothing && return PkgId(name)
-    return ApiId(PkgId(pkg_uuid, name),
-        explicit_project_compat_get(project_file, name))
+    pkg_compat = explicit_project_compat_get(project_file, name)
+    pkg_uuid === nothing && return ApiId(PkgId(name), pkg_compat)
+    return ApiId(PkgId(pkg_uuid, name), pkg_compat)
 end
 
 function package_get(project_file, where::Union{Nothing, PkgId}, name::String)::Union{Nothing,ApiId}
@@ -752,7 +752,7 @@ function package_extension_get(project_file, where::PkgId, name::String)::Union{
     return nothing
 end
 
-function environment_deps_get(env::String, where::Union{Nothing,PkgId}, name::String)::Union{Nothing,PkgId}
+function environment_deps_get(env::String, where::Union{Nothing,PkgId}, name::String)::Union{Nothing,ApiId}
     @assert where === nothing || where.uuid !== nothing
     project_file = env_project_file(env)
     implicit_manifest = !(project_file isa String)
@@ -762,7 +762,7 @@ function environment_deps_get(env::String, where::Union{Nothing,PkgId}, name::St
             # Toplevel load with a directory (implicit manifest) - all we look for is the
             # existence of the package name in the directory.
             pkg = implicit_manifest_pkgid(env, name)
-            return pkg
+            return ApiId(pkg, nothing)
         end
         project_file = implicit_manifest_project(env, where)
         project_file === nothing && return nothing
@@ -783,7 +783,7 @@ function environment_deps_get(env::String, where::Union{Nothing,PkgId}, name::St
     #       uses the same code path. Otherwise this is the active project.
     pkg = package_get(project_file, where, name)
     if pkg !== nothing
-        if where === nothing && pkg.uuid === nothing
+    if where === nothing && pkg.pkg.uuid === nothing
             # This is a top-level load - even though we didn't find the dependency
             # here, we still want to keep looking through the top-level environment stack.
             return nothing
@@ -804,7 +804,7 @@ function environment_deps_get(env::String, where::Union{Nothing,PkgId}, name::St
         # With an implicit manifest, getting here means that our (implicit) environment
         # *has* the package `where`. If we don't find it, it just means that `where` doesn't
         # have `name` as a dependency - c.f. the analogous case in `explicit_manifest_deps_get`.
-        return PkgId(name)
+        return ApiId(PkgId(name), nothing)
     end
 
     # All other cases, dependencies come from the (top-level) manifest
@@ -1032,18 +1032,19 @@ function dep_stanza_get(stanza::Dict{String, Any}, name)::Union{ApiId, Nothing}
         if dep == name
             if uuid_or_obj isa String
                 # uuid specified, but no API compat
-                return ApiId(PkgId(name, UUID(uuid_or_obj)), nothing)
+                return ApiId(PkgId(UUID(uuid_or_obj), name), nothing)
             else
                 api_compat = get(uuid_or_obj, "compat", nothing)::Union{Nothing, String}
                 uuid = get(uuid_or_obj, "uuid", nothing)::Union{Nothing, String}
-                uuid === nothing && @goto done
+                uuid === nothing && return nothing
+                return ApiId(PkgId(UUID(uuid), name), api_compat)
             end
         end
     end
     return nothing
 end
 
-function dep_stanza_get(stanza::Vector{String}, name::String)::Union{Nothing, PkgId}
+function dep_stanza_get(stanza::Vector{String}, name::String)::Union{Nothing, ApiId}
     name in stanza && return ApiId(PkgId(name), nothing)
     return nothing
 end
@@ -1063,7 +1064,7 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St
             # deps is either a list of names (deps = ["DepA", "DepB"]) or
             # a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."}
             deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
-            local dep::Union{Nothing, PkgId}
+            local dep::Union{Nothing, ApiId}
             if UUID(uuid) === where.uuid
                 dep = dep_stanza_get(deps, name)
 
@@ -1100,7 +1101,7 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St
             end
 
             @label have_dep
-            dep.uuid !== nothing && return dep
+            dep.pkg.uuid !== nothing && return dep
 
             # We have the dep, but it did not specify a UUID. In this case,
             # it must be that the name is unique in the manifest - so lookup
@@ -1112,7 +1113,7 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St
             entry = first(name_deps::Vector{Any})::Dict{String, Any}
             uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
             uuid === nothing && return PkgId(name)
-            return PkgId(UUID(uuid), name)
+            return ApiId(PkgId(UUID(uuid), name), nothing)
         end
     end
 
@@ -1486,6 +1487,7 @@ function run_module_init(mod::Module, i::Int=1)
     end
 end
 
+run_package_callbacks(modkey::ApiId) = run_package_callbacks(modkey.pkg)
 function run_package_callbacks(modkey::PkgId)
     run_extension_callbacks(modkey)
     assert_havelock(require_lock)
@@ -1520,6 +1522,7 @@ const EXT_PRIMED = Dict{PkgId,Vector{PkgId}}() # Extension -> Parent + Triggers
 const EXT_DORMITORY = Dict{PkgId,Vector{ExtensionId}}() # Trigger -> Extensions that can be triggered by it
 const EXT_DORMITORY_FAILED = ExtensionId[]
 
+insert_extension_triggers(api::ApiId) = insert_extension_triggers(api.pkg)
 function insert_extension_triggers(pkg::PkgId)
     pkg.uuid === nothing && return
     path_env_loc = locate_package_env(pkg)
@@ -2278,6 +2281,12 @@ function canstart_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
     return cond
 end
 
+function start_loading(modkey::ApiId, build_id::UInt128, stalecheck::Bool)
+    m = start_loading(modkey.pkg, build_id, true)
+    m === nothing && return nothing
+    return api_for_loaded_module(modkey, m)
+end
+
 function start_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
     # handle recursive and concurrent calls to require
     while true
@@ -2293,6 +2302,7 @@ function start_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
     end
 end
 
+end_loading(apikey::ApiId, @nospecialize loaded) = end_loading(apikey.pkg, loaded)
 function end_loading(modkey::PkgId, @nospecialize loaded)
     assert_havelock(require_lock)
     loading = pop!(package_locks, modkey)
@@ -2513,7 +2523,7 @@ function __require(into::Module, mod::Symbol)
         end
         uuidkey, env = uuidkey_env
         if _track_dependencies[]
-            path = binpack(uuidkey)
+            path = binpack(uuidkey.pkg)
             push!(_require_dependencies, (into, path, UInt64(0), UInt32(0), 0.0))
         end
         return _require_prelocked(uuidkey, env)
@@ -2580,8 +2590,8 @@ function require(uuidkey::PkgId)
     end
     return invoke_in_world(world, __require, uuidkey)
 end
-__require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey)
-function _require_prelocked(uuidkey::PkgId, env=nothing)
+__require(uuidkey::Union{PkgId, ApiId}) = @lock require_lock _require_prelocked(uuidkey)
+function _require_prelocked(uuidkey::Union{PkgId, ApiId}, env=nothing)
     assert_havelock(require_lock)
     m = start_loading(uuidkey, UInt128(0), true)
     if m === nothing
@@ -2840,16 +2850,18 @@ function __require_prelocked(pkg::PkgId, env)
     return loaded
 end
 
+__require_prelocked(apikey::ApiId, env) =
+    api_for_loaded_module(apikey, __require_prelocked(apikey.pkg, env))
+
 function lookup_api(loaded::Module, api::ApiId)
     if !isdefined(loaded, :_get_versioned_api)
         return loaded
     end
-    return loaded._get_versioned_api(api.api)
+    return loaded._get_versioned_api(api.compat)::Module
 end
 
-function __require_prelocked(apikey::ApiId, env)
-    loaded = __require_prelocked(apikey.pkg, env)
-    if apikey.api === nothing
+function api_for_loaded_module(apikey::ApiId, loaded::Module)
+    if apikey.compat === nothing
         return loaded
     end
     return invokelatest(lookup_api, loaded, apikey)

base/pkgid.jl

@@ -57,4 +57,4 @@ function hash(api::ApiId, h::UInt)
 end
 
 show(io::IO,  ::MIME"text/plain", api::ApiId) =
-    print(io, api.pkg, api.compat === nothing ? "" : " at version " * api.compat)
+    print(io, api.pkg, api.compat === nothing ? "" : " at version compat " * api.compat)