diff --git a/README.md b/README.md index f92284745..ef0abe76a 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,12 @@ Note: Hotspot is not supported on Ubuntu 20.04 for s390x arch. - 1809 - ltsc2016 -# Official and Unofficial Images -AdoptOpenJDK Docker Images are available as both Official Images (Maintained by Docker) and Unofficial Images (Maintained by AdoptOpenJDK). Please choose based on your requirements. +# musl libc based Alpine Images + +Starting from Java 16, hotspot builds are available natively built on musl libc instead of the regular glibc as part of the AdoptOpenJDK project. Currently these are available only for the x86_64 architecture. Accordingly we now have both regular and slim Docker Images for alpine musl based hotspot on x86_64. + +# Official and Non-official Images +AdoptOpenJDK Docker Images are available as both Official Images (Maintained by Docker) and Non-official Images (Maintained by AdoptOpenJDK). Please choose based on your requirements. * [Official Images](https://hub.docker.com/_/adoptopenjdk) are maintained by Docker and updated on every release from AdoptOpenJDK as well as when the underlying OSes are updated. Supported OSes and their versions and type of images are as below. - Linux - Ubuntu (20.04): Release diff --git a/build_latest.sh b/build_latest.sh index c30498cdc..ca683f39c 100755 --- a/build_latest.sh +++ b/build_latest.sh @@ -53,24 +53,26 @@ set_runtype "$4" # Get the image build time stored in the "build_time" array for the current arch # Build time is stored as the time since 1-1-1970 function get_image_build_time() { + local osfamily=$2 + if ! declare -p "$1" &>/dev/null; then return; fi # shellcheck disable=SC2154,SC1083 - btime=$(btarray=$1[${current_arch}]; eval btarch=\${"$btarray"}; echo "${btarch}"); + btime=$(btarray=$1[${osfamily}_${current_arch}]; eval btarch=\${"$btarray"}; echo "${btarch}"); echo "${btime}" } # Check if the adopt image is available, if not need to build it. function check_adopt_image_available() { - local tag=$1 + local image_name=$1 - echo "INFO: Checking when the adopt docker image ${tag} was built ..." - if ! docker pull "${tag}" &>/dev/null; then + echo "INFO: Checking when the adopt docker image ${image_name} was built ..." + if ! docker pull "${image_name}" &>/dev/null; then # Adopt image not available currently, build needed - echo "INFO: AdoptOpenJDK docker image for ${tag} does not exist. Docker build needed" + echo "INFO: AdoptOpenJDK docker image for ${image_name} does not exist. Docker build needed" build_needed=1 return; fi @@ -79,7 +81,7 @@ function check_adopt_image_available() { # Check if we have a newer base OS docker image # If a new base OS docker image was built in the last 24 hrs, then we need to rebuild the adopt docker image function check_new_os_image_available() { - local tag=$2 + local image_name=$2 # Get the date when the base image was created. Eg if the base OS is ubuntu, this # translates as the exact date/time when the Ubuntu image was created on DockerHub @@ -104,7 +106,7 @@ EOF )" # Get the shasums of the current Adopt Image layers - docker inspect "${tag}" > adopt_image.info + docker inspect "${image_name}" > adopt_image.info adopt_sha_arr="$(python3 <> "${push_cmdfile}" + auto_space_line=" " + printf -v expanded_tags "%s ${repo}:%s " "-t" "${tag}" # concatenate to single string : -t repo:tag -t repo:tag2 + expanded_tags=${expanded_tags%?} # remove trailing space + + echo "docker push ${image_name}" >> "${push_cmdfile}" echo "#####################################################" echo "INFO: docker build --no-cache ${expanded_tags} -f ${dockerfile} ." echo "#####################################################" @@ -264,11 +263,8 @@ function build_image() { echo "#####################################################" echo " Scanning with snyk for vulnerabilities " echo "#####################################################" - for i in "${!tags[@]}" - do - echo "...scanning ${repo}:${tags[$i]}" - snyk test --docker "${repo}:${tags[$i]}" --file="${dockerfile}" - done + echo "...scanning ${image_name}" + snyk test --docker "${image_name}" --file="${dockerfile}" fi echo "| ${image_name:0:80}${auto_space_line:0:$((76 - ${#image_name}))} | success |" >> ${summary_table_file} echo "+------------------------------------------------------------------------------+----------+" >> ${summary_table_file} @@ -295,11 +291,8 @@ function build_image() { echo "#####################################################" echo " Scanning with snyk for vulnerabilities " echo "#####################################################" - for i in "${!tags[@]}" - do - echo "...scanning ${repo}:${tags[$i]}" - snyk test --docker "${repo}:${tags[$i]}" --file="${dockerfile}" - done + echo "...scanning ${image_name}" + snyk test --docker "${image_name}" --file="${dockerfile}" fi echo "| ${image_name:0:80}${auto_space_line:0:$((76 - ${#image_name}))} | success |" >> ${summary_table_file} echo "+------------------------------------------------------------------------------+----------+" >> ${summary_table_file} @@ -309,15 +302,23 @@ function build_image() { # Build the docker image for a given VM, OS, BUILD and BUILD_TYPE combination. function build_dockerfile { - vm=$1; pkg=$2; os=$3; build=$4; btype=$5; - + local vm=$1; + local pkg=$2; + local build=$3; + local btype=$4; + local osfamily=$5; + local os=$6; + + local tag="" + echo "INFO: current_arch: ${current_arch}, osfamily: ${osfamily}, os: ${os}" if [ -z "${current_arch}" ]; then jverinfo="${shasums}[version]" else - jverinfo="${shasums}[version-${current_arch}]" + jverinfo="${shasums}[version-${osfamily}_${current_arch}]" fi # shellcheck disable=SC1083,SC2086 eval jrel=\${$jverinfo} + echo "INFO: release: ${jrel}" # Docker image tags cannot have "+" in them, replace it with "_" instead. # shellcheck disable=SC2154 rel=${jrel//+/_} @@ -354,7 +355,7 @@ function build_dockerfile { fi echo "INFO: Building ${trepo} ${tag} from $file ..." pushd "${dir}" >/dev/null || return - build_image "${trepo}" "${build}" "${btype}" "${tag}" + build_image "${trepo}" "${build}" "${btype}" "${osfamily}" "${tag}" popd >/dev/null || return } @@ -386,6 +387,7 @@ do # Type = Full or Slim btypes=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "Type:") dir=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "Directory:") + osfamily=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "OS_Family:") for build in ${builds} do @@ -411,12 +413,12 @@ do for btype in ${btypes} do file="${dir}/Dockerfile.${vm}.${build}.${btype}" - generate_dockerfile "${file}" "${package}" "${build}" "${btype}" "${os}" + generate_dockerfile "${file}" "${package}" "${build}" "${btype}" "${osfamily}" "${os}" if [ ! -f "${file}" ]; then continue; fi # Build the docker images for valid Dockerfiles - build_dockerfile "${vm}" "${package}" "${os}" "${build}" "${btype}" + build_dockerfile "${vm}" "${package}" "${build}" "${btype}" "${osfamily}" "${os}" done done done diff --git a/common_functions.sh b/common_functions.sh index 5d1607eee..cac465998 100755 --- a/common_functions.sh +++ b/common_functions.sh @@ -32,8 +32,11 @@ test_buckets_file="config/test_buckets.list" # shellcheck disable=SC2034 # used externally all_jvms="hotspot openj9" -# All supported arches -all_arches="aarch64 armv7l ppc64le s390x x86_64 windows-amd windows-nano" +# Supported arches for each of the os_families +os_families="linux alpine-linux windows" +linux_arches="aarch64 armv7l ppc64le s390x x86_64" +alpine_linux_arches="x86_64" +windows_arches="windows-amd windows-nano" # All supported packages # shellcheck disable=SC2034 # used externally @@ -59,7 +62,7 @@ export latest_version="16" export supported_builds="releases nightly" function check_version() { - version=$1 + local version=$1 case ${version} in 8|9|10|11|12|13|14|15|16) ;; @@ -101,22 +104,22 @@ function set_arch_os() { case ${machine} in armv7l|linux/arm/v7) current_arch="armv7l" - oses="ubuntu debian debianslim centos leap tumbleweed" + oses="centos debian debianslim leap tumbleweed ubuntu" os_family="linux" ;; aarch64) current_arch="aarch64" - oses="ubuntu debian debianslim ubi ubi-minimal centos leap tumbleweed" + oses="centos debian debianslim leap tumbleweed ubi ubi-minimal ubuntu" os_family="linux" ;; ppc64el|ppc64le) current_arch="ppc64le" - oses="ubuntu debian debianslim ubi ubi-minimal centos leap tumbleweed" + oses="centos debian debianslim leap tumbleweed ubi ubi-minimal ubuntu" os_family="linux" ;; s390x) current_arch="s390x" - oses="ubuntu debian debianslim ubi ubi-minimal clefos tumbleweed" + oses="clefos debian debianslim tumbleweed ubi ubi-minimal ubuntu" os_family="linux" ;; amd64|x86_64) @@ -142,9 +145,10 @@ function set_arch_os() { # shellcheck disable=SC2034 # used externally current_arch="x86_64" # shellcheck disable=SC2034 # used externally - oses="ubuntu alpine debian debianslim ubi ubi-minimal centos leap tumbleweed" + oses="alpine centos debian debianslim leap tumbleweed ubi ubi-minimal ubuntu" + # alpine-linux (musl libc) based builds are only available on x86_64 currently # shellcheck disable=SC2034 # used externally - os_family="linux" + os_family="alpine-linux linux" ;; esac ;; @@ -155,6 +159,22 @@ function set_arch_os() { esac } +# get shasums for a given architecture and os_family +# This is based on the hotspot_shasums_latest.sh/openj9_shasums_latest.sh +# arch = aarch64, armv7l, ppc64le, s390x, x86_64 +# os_family = alpine-linux, linux, windows +function get_shasum() { + local arch=$2; + local os_family=$3; + + if ! declare -p "$1" >/dev/null 2>/dev/null; then + return; + fi + # shellcheck disable=SC2154,SC1083 + local shasum=$(sarray=$1[${os_family}_${arch}]; eval esum=\${"$sarray"}; echo "${esum}"); + echo "${shasum}" +} + # Get the supported architectures for a given VM (Hotspot, OpenJ9). # This is based on the hotspot_shasums_latest.sh/openj9_shasums_latest.sh function get_arches() { @@ -162,11 +182,12 @@ function get_arches() { # corresponding build combination does not exist. # Eg. jdk_openj9_10_releases_sums does not exist as we do not have any # release builds for version 10 (Only nightly builds). - if ! declare -p "$1" 2>/dev/null; then + if ! declare -p "$1" >/dev/null 2>/dev/null; then return; fi - archsums="$(declare -p "$1")"; + local archsums="$(declare -p "$1")"; eval "declare -A sums=""${archsums#*=}"; + local arch="" for arch in "${!sums[@]}"; do if [[ "${arch}" == version* ]] ; then @@ -174,9 +195,10 @@ function get_arches() { fi # Arch is supported only if the shasum is not empty ! # shellcheck disable=SC2154,SC1083 - shasum=$(sarray=$1[${arch}]; eval esum=\${"$sarray"}; echo "${esum}"); + local shasum=$(sarray=$1[${arch}]; eval esum=\${"$sarray"}; echo "${esum}"); if [ -n "${shasum}" ]; then - echo "${arch} " + local arch_val=$(echo ${arch} | sed 's/alpine-linux_//; s/linux_//; s/windows_//') + echo "${arch_val} " fi done } @@ -193,8 +215,8 @@ function vm_supported_onarch() { test_arch=$(uname -m) fi - suparches=$(get_arches "${sums}") - sup=$(echo "${suparches}" | grep "${test_arch}") + local suparches=$(get_arches "${sums}") + local sup=$(echo "${suparches}" | grep "${test_arch}") echo "${sup}" } @@ -297,9 +319,9 @@ function build_tags() { rel="${rel//jdk/jre}" fi # Get the list of supported arches for this vm / ver /os combo - arches=$(parse_vm_entry "${vm}" "${ver}" "${pkg}" "${os}" "Architectures:") + local arches=$(parse_vm_entry "${vm}" "${ver}" "${pkg}" "${os}" "Architectures:") # Replace the proper version string in the tags - rtags=$(echo "${rawtags}" | sed "s/{{ JDK_${build}_VER }}/${rel}/gI; s/{{ OS }}/${os}/gI;"); + local rtags=$(echo "${rawtags}" | sed "s/{{ JDK_${build}_VER }}/${rel}/gI; s/{{ OS }}/${os}/gI;"); echo "${rtags}" | sed "s/{{ *ARCH *}}/{{ARCH}}/" | # Separate the arch and the generic alias tags awk '{ a=0; n=0; @@ -316,10 +338,13 @@ function build_tags() { }' > ${tmpfile} # shellcheck disable=SC2034 # used externally - tag_aliases=$( < "${tmpfile}" grep "^tag_aliases:" | sed "s/tag_aliases: //") - raw_arch_tags=$( < "${tmpfile}" grep "^arch_tags:" | sed "s/arch_tags: //") + local tag_aliases=$( < "${tmpfile}" grep "^tag_aliases:" | sed "s/tag_aliases: //") + local raw_arch_tags=$( < "${tmpfile}" grep "^arch_tags:" | sed "s/arch_tags: //") + # arch_tags is a global variable arch_tags="" # Iterate through the arch tags and expand to add the supported arches. + local tag="" + local arch="" for tag in ${raw_arch_tags} do for arch in ${arches} @@ -407,6 +432,7 @@ function get_v2_installer_url() { # url_impl = hotspot / openj9 # url_arch = aarch64 / ppc64le / s390x / x64 # url_pkg = jdk / jre +# url_os_family = linux / windows / alpine-linux # https://api.adoptopenjdk.net/v3/assets/feature_releases/11/ga?page=0&page_size=1&release_type=ga&sort_order=DESC&vendor=adoptopenjdk&jvm_impl=openj9&heap_size=normal&architecture=x64&os=linux&image_type=jdk function get_v3_url() { local request_type=$1 @@ -414,6 +440,7 @@ function get_v3_url() { local url_impl=$3 local url_pkg=$4 local url_arch=$5 + local url_os_family=$6 local url_heapsize="normal" if [ "${release_type}" == "releases" ]; then @@ -427,9 +454,14 @@ function get_v3_url() { windows_pat="windows.*" if [ -n "${url_arch}" ]; then if [[ "${url_arch}" =~ ${windows_pat} ]]; then - specifiers="${specifiers}&os=windows&architecture=x64" + specifiers="${specifiers}&architecture=x64&os=windows" else - specifiers="${specifiers}&os=linux&architecture=${url_arch}" + specifiers="${specifiers}&architecture=${url_arch}" + if [ -n "${url_os_family}" ]; then + specifiers="${specifiers}&os=${url_os_family}" + else + specifiers="${specifiers}&os=linux" + fi fi else specifiers="${specifiers}&os=linux" @@ -469,6 +501,7 @@ function get_v3_installer_url() { function get_nightly_short_version() { local arch_build=$1 local arch_full_version=$2 + if [ "${arch_build}" = "nightly" ]; then # Remove date and time at the end of full_version for nightly builds. # Handle both the old and new date-time formats used by the Adopt build system. @@ -491,25 +524,30 @@ function get_sums_for_build_arch() { local pkg=$3 local build=$4 local arch=$5 + local os_family=$6 + if [ -z ${os_family} ]; then + os_family=linux + fi case ${arch} in armv7l) - LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" arm); + LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" arm "${os_family}"); ;; aarch64) - LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" aarch64); + LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" aarch64 "${os_family}"); ;; ppc64le) - LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" ppc64le); + LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" ppc64le "${os_family}"); ;; s390x) - LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" s390x); + LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" s390x "${os_family}"); ;; x86_64) - LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" x64); + LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" x64 "${os_family}"); ;; windows-amd|windows-nano) - LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" windows); + os_family=windows + LATEST_URL=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}" windows "${os_family}"); ;; *) echo "Unsupported arch: ${arch}" @@ -566,10 +604,10 @@ function get_sums_for_build_arch() { arch_last_build_time="$(date --date "${arch_last_build_date}" +%s)" # Only print the entry if the shasum is not empty if [ -n "${shasum}" ]; then - printf "\t[version-%s]=\"%s\"\n" "${arch}" "${arch_build_version}" >> "${ofile_sums}" - printf "\t[version-%s]=\"%s\"\n" "${arch}" "${arch_build_version}" >> "${ofile_build_time}" - printf "\t[%s]=\"%s\"\n" "${arch}" "${shasum}" >> "${ofile_sums}" - printf "\t[%s]=\"%s\"\n" "${arch}" "${arch_last_build_time}" >> "${ofile_build_time}" + printf "\t[version-%s]=\"%s\"\n" "${os_family}_${arch}" "${arch_build_version}" >> "${ofile_sums}" + printf "\t[version-%s]=\"%s\"\n" "${os_family}_${arch}" "${arch_build_version}" >> "${ofile_build_time}" + printf "\t[%s]=\"%s\"\n" "${os_family}_${arch}" "${shasum}" >> "${ofile_sums}" + printf "\t[%s]=\"%s\"\n" "${os_family}_${arch}" "${arch_last_build_time}" >> "${ofile_build_time}" fi fi break; @@ -584,7 +622,6 @@ function get_sums_for_build() { local vm=$2 local pkg=$3 local build=$4 - local arch=$5 info_url=$(get_v3_url feature_releases "${build}" "${vm}" "${pkg}"); # Repeated requests from a script triggers a error threshold on adoptopenjdk.net @@ -603,14 +640,19 @@ function get_sums_for_build() { # Capture the full version according to adoptopenjdk printf "\t[version]=\"%s\"\n" "${full_version}" >> "${ofile_sums}" printf "\t[version]=\"%s\"\n" "${full_version}" >> "${ofile_build_time}" - if [ -n "${arch}" ]; then - get_sums_for_build_arch "${ver}" "${vm}" "${pkg}" "${build}" "${arch}" - else - for arch in ${all_arches} + # Need to get shasums for each of the OS Families + # families = alpine-linux, linux and windows + for os_fam in ${os_families} + do + # bash doesnt support '-' in the name of a variable + # So make alpine-linux as alpine_linux + local fam="${os_fam//-/_}" + local arches="${fam}_arches" + for arch in ${!arches} do - get_sums_for_build_arch "${ver}" "${vm}" "${pkg}" "${build}" "${arch}" + get_sums_for_build_arch "${ver}" "${vm}" "${pkg}" "${build}" "${arch}" "${os_fam}" done - fi + done printf ")\n" >> "${ofile_sums}" printf ")\n" >> "${ofile_build_time}" @@ -626,7 +668,6 @@ function get_shasums() { local vm=$2 local pkg=$3 local build=$4 - local arch=$5 local ofile_sums="${vm}_shasums_latest.sh" local ofile_build_time="${vm}_build_time_latest.sh" @@ -634,21 +675,21 @@ function get_shasums() { if [ -f "${ofile_sums}" ]; then # shellcheck disable=SC1090 source ./"${vm}"_shasums_latest.sh - sums="${pkg}_${vm}_${ver}_${build}_sums" + local sums="${pkg}_${vm}_${ver}_${build}_sums" # File exists, which means shasums for the VM exists. # Now check for the specific Ver/VM/Pg/Build combo - suparches=$(get_arches "${sums}") + local suparches=$(get_arches "${sums}") if [ -n "${suparches}" ]; then return; fi fi if [ -n "${build}" ]; then - get_sums_for_build "${ver}" "${vm}" "${pkg}" "${build}" "${arch}" + get_sums_for_build "${ver}" "${vm}" "${pkg}" "${build}" else for build in ${supported_builds} do - get_sums_for_build "${ver}" "${vm}" "${pkg}" "${build}" "${arch}" + get_sums_for_build "${ver}" "${vm}" "${pkg}" "${build}" done fi chmod +x "${ofile_sums}" "${ofile_build_time}" diff --git a/dockerfile_functions.sh b/dockerfile_functions.sh index 38c62db7d..906f7089a 100755 --- a/dockerfile_functions.sh +++ b/dockerfile_functions.sh @@ -125,13 +125,13 @@ print_tumbleweed_ver() { # Print the supported Windows OS print_windows_ver() { - os=$4 + local os=$4 case $os in - *20h2) os_version="20H2" ;; *ltsc2019) os_version="ltsc2019" ;; *1909) os_version="1909" ;; *ltsc2016) os_version="ltsc2016" ;; *1809) os_version="1809" ;; + *20h2) os_version="20H2" ;; esac servertype=$(echo "$file" | cut -f4 -d"/") @@ -140,6 +140,7 @@ print_windows_ver() { cat >> "$1" <<-EOI FROM mcr.microsoft.com/windows/servercore:${os_version} as installer + EOI else cat >> "$1" <<-EOI @@ -160,7 +161,7 @@ print_alpine_ver() { # Print the locale and language print_lang_locale() { - os=$2 + local os=$2 if [ "$os" != "windows" ]; then cat >> "$1" <<-EOI ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' @@ -189,7 +190,7 @@ print_debianslim_pkg() { } print_windows_pkg() { - cat >> "$1" <<'EOI' + cat >> "$1" <<'EOI' # $ProgressPreference: https://github.com/PowerShell/PowerShell/issues/2138#issuecomment-251261324 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] EOI @@ -198,6 +199,17 @@ EOI # Select the alpine OS packages. # Install GNU glibc as this OpenJDK build is compiled against glibc and not musl. print_alpine_pkg() { + local osfamily=$2 + + if [ "${osfamily}" == "alpine-linux" ]; then + print_alpine_musl_pkg "$1" "$2" + else + print_alpine_glibc_pkg "$1" "$2" + fi + +} + +print_alpine_glibc_pkg() { cat >> "$1" <<'EOI' RUN apk add --no-cache tzdata --virtual .build-deps curl binutils zstd \ && GLIBC_VER="2.31-r0" \ @@ -234,6 +246,14 @@ RUN apk add --no-cache tzdata --virtual .build-deps curl binutils zstd \ EOI } +# Select the alpine OS musl based packages. +print_alpine_musl_pkg() { + cat >> "$1" <<'EOI' +RUN apk add --no-cache tzdata musl-locales musl-locales-lang java-cacerts \ + && rm -rf /var/cache/apk/* +EOI +} + # Select the ubi OS packages. print_ubi_pkg() { cat >> "$1" <<'EOI' @@ -279,13 +299,12 @@ print_tumbleweed_pkg() { # Print the Java version that is being installed here print_env() { + local osfamily=$2 + local os=$3 + # shellcheck disable=SC2154 shasums="${package}"_"${vm}"_"${version}"_"${build}"_sums - if [ -z "${arch}" ]; then - jverinfo="${shasums}[version]" - else - jverinfo="${shasums}[version-${arch}]" - fi + jverinfo="${shasums}[version]" # shellcheck disable=SC1083,SC2086 # TODO not sure about intention here eval jver=\${$jverinfo} jver="${jver}" # to satifsy shellcheck SC2154 @@ -312,45 +331,54 @@ EOI # OS independent portion (Works for both Alpine and Ubuntu) print_java_install_pre() { - pkg=$2 - bld=$3 - btype=$4 - reldir="openjdk${version}"; + local pkg=$2 + local bld=$3 + local btype=$4 + local osfamily=$5 + local os=$6 + local reldir="openjdk${version}"; + if [ "${vm}" != "hotspot" ]; then reldir="${reldir}-${vm}"; fi - supported_arches=$(get_arches "${shasums}" | sort) + # First get the arches for which the builds are available as per shasums file + local sup_arches_for_build=$(get_arches "${shasums}" | sort | uniq) + # Next, check the arches that are supported for the underlying OS + local sup_arches_for_os=$(parse_vm_entry "${vm}" "${version}" "${pkg}" "${os}" "Architectures:") + # Now the actual arches are the intersection of the above two + local merge_arches="${sup_arches_for_build} ${sup_arches_for_os}" + local supported_arches=$(echo ${merge_arches} | tr ' ' '\n' | sort | uniq -d) for sarch in ${supported_arches} do if [ "${sarch}" == "aarch64" ]; then - JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" aarch64); + JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" aarch64 "${osfamily}"); cat >> "$1" <<-EOI aarch64|arm64) \\ - ESUM='$(sarray="${shasums}[aarch64]"; eval esum=\${$sarray}; echo "${esum}")'; \\ + ESUM='$(get_shasum "${shasums}" aarch64 "${osfamily}")'; \\ BINARY_URL='$(get_v3_binary_url "${JAVA_URL}")'; \\ ;; \\ EOI elif [ "${sarch}" == "armv7l" ]; then - JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" arm); + JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" arm "${osfamily}"); cat >> "$1" <<-EOI armhf|armv7l) \\ - ESUM='$(sarray="${shasums}[armv7l]"; eval esum=\${$sarray}; echo "${esum}")'; \\ + ESUM='$(get_shasum "${shasums}" armv7l "${osfamily}")'; \\ BINARY_URL='$(get_v3_binary_url "${JAVA_URL}")'; \\ ;; \\ EOI elif [ "${sarch}" == "ppc64le" ]; then - JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" ppc64le); + JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" ppc64le "${osfamily}"); cat >> "$1" <<-EOI ppc64el|ppc64le) \\ - ESUM='$(sarray="${shasums}[ppc64le]"; eval esum=\${$sarray}; echo "${esum}")'; \\ + ESUM='$(get_shasum "${shasums}" ppc64le "${osfamily}")'; \\ BINARY_URL='$(get_v3_binary_url "${JAVA_URL}")'; \\ ;; \\ EOI elif [ "${sarch}" == "s390x" ]; then - JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" s390x); + JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" s390x "${osfamily}"); cat >> "$1" <<-EOI s390x) \\ - ESUM='$(sarray="${shasums}[s390x]"; eval esum=\${$sarray}; echo "${esum}")'; \\ + ESUM='$(get_shasum "${shasums}" s390x "${osfamily}")'; \\ BINARY_URL='$(get_v3_binary_url "${JAVA_URL}")'; \\ EOI # Ubuntu 20.04 has a newer version of libffi (libffi7) @@ -370,10 +398,10 @@ EOI ;; \\ EOI elif [ "${sarch}" == "x86_64" ]; then - JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" x64); + JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" x64 "${osfamily}"); cat >> "$1" <<-EOI amd64|x86_64) \\ - ESUM='$(sarray="${shasums}[x86_64]"; eval esum=\${$sarray}; echo "${esum}")'; \\ + ESUM='$(get_shasum "${shasums}" x86_64 "${osfamily}")'; \\ BINARY_URL='$(get_v3_binary_url "${JAVA_URL}")'; \\ ;; \\ EOI @@ -446,15 +474,18 @@ EOI # Print the main RUN command that installs Java on ubuntu. print_ubuntu_java_install() { - pkg=$2 - bld=$3 - btype=$4 + local pkg=$2 + local bld=$3 + local btype=$4 + local osfamily=$5 + local os=$6 + cat >> "$1" <<-EOI RUN set -eux; \\ ARCH="\$(dpkg --print-architecture)"; \\ case "\${ARCH}" in \\ EOI - print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" + print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" "${osfamily}" "${os}" if [ "${btype}" == "slim" ]; then print_ubuntu_slim_package "$1" fi @@ -462,17 +493,18 @@ EOI } print_debian_java_install() { - print_ubuntu_java_install "$1" "$2" "$3" "$4" + print_ubuntu_java_install "$1" "$2" "$3" "$4" "$5" "$6" } print_debianslim_java_install() { - print_ubuntu_java_install "$1" "$2" "$3" "$4" + print_ubuntu_java_install "$1" "$2" "$3" "$4" "$5" "$6" } print_windows_java_install_post() { - servertype="$2" - version="$3" - os=$4 + local servertype="$2" + local version="$3" + local os=$4 + case $os in *20h2) os_version="20H2" ;; *ltsc2019) os_version="ltsc2019" ;; @@ -507,16 +539,16 @@ EOI # Print the main RUN command that installs Java on ubuntu. print_windows_java_install() { - pkg=$2 - bld=$3 - btype=$4 - os=$5 + local pkg=$2 + local bld=$3 + local btype=$4 + local os=$5 - servertype=$(echo -n "${file}" | cut -f4 -d"/" | cut -f1 -d"-" | head -qn1) - version=$(echo -n "${file}" | cut -f1 -d "/" | head -qn1) + local servertype=$(echo -n "${file}" | cut -f4 -d"/" | cut -f1 -d"-" | head -qn1) + local version=$(echo -n "${file}" | cut -f1 -d "/" | head -qn1) if [ "${servertype}" == "windowsservercore" ]; then - JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" windows-amd); - ESUM=$(sarray="${shasums}[windows-amd]"; eval esum=\${$sarray}; echo "${esum}"); + JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" windows-amd windows); + ESUM=$(get_shasum "${shasums}" windows-amd "${osfamily}"); BINARY_URL=$(get_v3_installer_url "${JAVA_URL}"); cat >> "$1" <<-EOI @@ -536,9 +568,9 @@ RUN Write-Host ('Downloading ${BINARY_URL} ...'); \\ Remove-Item -Path C:\temp -Recurse | Out-Null; \\ EOI else - JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" windows-nano); - # shellcheck disable=SC1083 - ESUM=$(sarray="${shasums}[windows-nano]"; eval esum=\${"$sarray"}; echo "${esum}"); + JAVA_URL=$(get_v3_url feature_releases "${bld}" "${vm}" "${pkg}" windows-nano windows); + # shellcheck disable=SC1083 + ESUM=$(get_shasum "${shasums}" windows-nano "${osfamily}"); BINARY_URL=$(get_v3_binary_url "${JAVA_URL}"); cat >> "$1" <<-EOI @@ -566,16 +598,19 @@ EOI # Print the main RUN command that installs Java on alpine. print_alpine_java_install() { - pkg=$2 - bld=$3 - btype=$4 + local pkg=$2 + local bld=$3 + local btype=$4 + local osfamily=$5 + local os=$6 + cat >> "$1" <<-EOI RUN set -eux; \\ apk add --no-cache --virtual .fetch-deps curl; \\ ARCH="\$(apk --print-arch)"; \\ case "\${ARCH}" in \\ EOI - print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" + print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" "${osfamily}" "${os}" if [ "${btype}" == "slim" ]; then print_alpine_slim_package "$1" fi @@ -588,67 +623,74 @@ EOI # Print the main RUN command that installs Java on ubi print_ubi_java_install() { - pkg=$2 - bld=$3 - btype=$4 + local pkg=$2 + local bld=$3 + local btype=$4 + local osfamily=$5 + local os=$6 + cat >> "$1" <<-EOI RUN set -eux; \\ ARCH="\$(uname -m)"; \\ case "\${ARCH}" in \\ EOI - print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" + print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" "${osfamily}" "${os}" print_java_install_post "$1" } # Print the main RUN command that installs Java on ubi-minimal print_ubi-minimal_java_install() { - print_ubi_java_install "$1" "$2" "$3" "$4" + print_ubi_java_install "$1" "$2" "$3" "$4" "$5" "$6" } # Print the main RUN command that installs Java on CentOS print_centos_java_install() { - pkg=$2 - bld=$3 - btype=$4 + local pkg=$2 + local bld=$3 + local btype=$4 + local osfamily=$5 + local os=$6 + cat >> "$1" <<-EOI RUN set -eux; \\ ARCH="\$(uname -m)"; \\ case "\${ARCH}" in \\ EOI - print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" + print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" "${osfamily}" "${os}" print_java_install_post "$1" } # Print the main RUN command that installs Java on ClefOS print_clefos_java_install() { - print_centos_java_install "$1" "$2" "$3" "$4" + print_centos_java_install "$1" "$2" "$3" "$4" "$5" "$6" } # Print the main RUN command that installs Java on Leap print_leap_java_install() { - pkg=$2 - bld=$3 - btype=$4 + local pkg=$2 + local bld=$3 + local btype=$4 cat >> "$1" <<-EOI RUN set -eux; \\ ARCH="\$(uname -m)"; \\ case "\${ARCH}" in \\ EOI - print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" + print_java_install_pre "${file}" "${pkg}" "${bld}" "${btype}" "${osfamily}" "${os}" print_java_install_post "$1" } # Print the main RUN command that installs Java on Tumbleweed print_tumbleweed_java_install() { - print_leap_java_install "$1" "$2" "$3" "$4" + print_leap_java_install "$1" "$2" "$3" "$4" "$5" "$6" } # Print the JAVA_HOME and PATH. # Currently Java is installed at a fixed path "/opt/java/openjdk" print_java_env() { # e.g 11 or 8 - version=$(echo "$file" | cut -f1 -d"/") - os=$4 + local version=$(echo "$file" | cut -f1 -d"/") + local os=$4 + if [ "$os" != "windows" ]; then cat >> "$1" <<-EOI @@ -729,7 +771,11 @@ print_cmd() { } print_scc_gen() { - if [[ "${vm}" == "openj9" && "${os_family}" != "windows" ]]; then + local vm=$2; + local osfamily=$3; + local os=$4; + + if [[ "${vm}" == "openj9" && "${osfamily}" != "windows" ]]; then cat >> "$1" <<'EOI' # Create OpenJ9 SharedClassCache (SCC) for bootclasses to improve the java startup. @@ -740,7 +786,7 @@ print_scc_gen() { RUN set -eux; \ EOI - if [[ "${os_family}" == "alpine" ]]; then + if [[ "${os}" == "alpine" ]]; then cat >> "$1" <<'EOI' apk add --no-cache --virtual .scc-deps curl; \ EOI @@ -787,7 +833,7 @@ EOI fi; \ \ EOI - if [[ "${os_family}" == "alpine" ]]; then + if [[ "${os}" == "alpine" ]]; then cat >> "$1" <<'EOI' apk del --purge .scc-deps; \ rm -rf /var/cache/apk/*; \ @@ -802,18 +848,12 @@ EOI # Generate the dockerfile for a given build, build_type and OS generate_dockerfile() { - file=$1 - pkg=$2 - bld=$3 - btype=$4 - case $5 in - windows*|nanoserver*) - os_family=windows - os=$5 ;; - *) - os_family=$5 - os=$5 ;; - esac + local file=$1 + local pkg=$2 + local bld=$3 + local btype=$4 + local osfamily=$5 + local os=$6 jhome="/opt/java/openjdk" @@ -821,18 +861,27 @@ generate_dockerfile() { echo echo -n "Writing ${file} ... " print_legal "${file}"; - print_"${os_family}"_ver "${file}" "${bld}" "${btype}" "${os}"; - print_lang_locale "${file}" "${os_family}"; - print_"${os_family}"_pkg "${file}"; - print_env "${file}" "${bld}" "${btype}"; - copy_slim_script "${file}"; - print_"${os_family}"_java_install "${file}" "${pkg}" "${bld}" "${btype}" "${os}"; - print_java_env "${file}" "${bld}" "${btype}" "${os_family}"; - print_java_options "${file}" "${bld}" "${btype}"; - print_scc_gen "${file}"; - print_cmd "${file}"; + if [ "${osfamily}" == "windows" ]; then + print_"${osfamily}"_ver "${file}" "${bld}" "${btype}" "${os}"; + print_lang_locale "${file}" "${osfamily}"; + print_"${osfamily}"_pkg "${file}" "${osfamily}"; + print_env "${file}" "${osfamily}" "${os}"; + print_"${osfamily}"_java_install "${file}" "${pkg}" "${bld}" "${btype}" "${osfamily}" "${os}"; + print_java_env "${file}" "${bld}" "${btype}" "${osfamily}"; + print_java_options "${file}" "${bld}" "${btype}"; + print_cmd "${file}"; + else + print_"${os}"_ver "${file}" "${bld}" "${btype}" "${os}"; + print_lang_locale "${file}" "${osfamily}"; + print_"${os}"_pkg "${file}" "${osfamily}"; + print_env "${file}" "${osfamily}" "${os}"; + copy_slim_script "${file}"; + print_"${os}"_java_install "${file}" "${pkg}" "${bld}" "${btype}" "${osfamily}" "${os}"; + print_java_env "${file}" "${bld}" "${btype}" "${osfamily}"; + print_java_options "${file}" "${bld}" "${btype}"; + print_scc_gen "${file}" "${vm}" "${osfamily}" "${os}"; + print_cmd "${file}"; + fi echo "done" echo - # Reset os back to the value of disto - os_family="$os" } diff --git a/generate_manifest_script.sh b/generate_manifest_script.sh index a450ee8ce..0d2ad9257 100755 --- a/generate_manifest_script.sh +++ b/generate_manifest_script.sh @@ -58,13 +58,13 @@ function print_annotate_cmd() { *) ;; esac - echo "\"${manifest_tool}\" manifest annotate ${main_tag} ${arch_tag} --os ${os_family} --arch ${march}" >> "${man_file}" + echo "\"${manifest_tool}\" manifest annotate ${main_tag} ${arch_tag} --os linux --arch ${march}" >> "${man_file}" } # Space separated list of tags function print_manifest_cmd() { - trepo=$1; shift; - img_list=$* + local trepo=$1; shift; + local img_list=$* # Global variable tag_aliases has the alias list for talias in ${tag_aliases} @@ -82,8 +82,8 @@ function print_manifest_cmd() { # Check each of the images in the global variable arch_tags exist and # Create the tag list from the arch_tags list. function print_tags() { - repo="$1" - img_list="" + local repo="$1" + local img_list="" # Check if all the individual docker images exist for each expected arch for arch_tag in ${arch_tags} do @@ -104,8 +104,7 @@ function print_tags() { check_manifest_tool # Set the OSes that we will be generating manifests for -oses="alpine centos clefos debian debianslim ubi ubi-minimal ubuntu" -os_family="linux" +oses="alpine centos clefos debian debianslim leap tumbleweed ubi ubi-minimal ubuntu" # Which JVMs are available for the current version ./generate_latest_sums.sh "${version}" @@ -140,13 +139,15 @@ do builds=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "Build:") # Type = Full or Slim btypes=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "Type:") + osfamily=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "OS_Family:") + for build in ${builds} do shasums="${package}"_"${vm}"_"${version}"_"${build}"_sums if [ -z "${arch}" ]; then jverinfo="${shasums}[version]" else - jverinfo="${shasums}[version-${arch}]" + jverinfo="${shasums}[version-${osfamily}_${arch}]" fi # shellcheck disable=SC1083,SC2086 eval jrel=\${$jverinfo} diff --git a/update_multiarch.sh b/update_multiarch.sh index a518a3972..b8a73325f 100755 --- a/update_multiarch.sh +++ b/update_multiarch.sh @@ -42,6 +42,7 @@ do # Build Type = Full or Slim btypes=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "Type:") dir=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "Directory:") + osfamily=$(parse_vm_entry "${vm}" "${version}" "${package}" "${os}" "OS_Family:") for build in ${builds} do @@ -64,7 +65,7 @@ do for btype in ${btypes} do file="${dir}/Dockerfile.${vm}.${build}.${btype}" - generate_dockerfile "${file}" "${package}" "${build}" "${btype}" "${os}" + generate_dockerfile "${file}" "${package}" "${build}" "${btype}" "${osfamily}" "${os}" # Copy the script to generate slim builds. if [ "${btype}" = "slim" ]; then if [ "${os}" == "windows" ]; then