Build 5.6.1 on Ubuntu 24.04

src/ue4docker/build.py

@@ -97,9 +97,20 @@ def build():
         )
 
     # Create an auto-deleting temporary directory to hold our build context
-    with tempfile.TemporaryDirectory() as tempDir:
+    with tempfile.TemporaryDirectory(prefix="ue4docker", delete=not config.debug) as tempDir:
         contextOrig = join(os.path.dirname(os.path.abspath(__file__)), "dockerfiles")
 
+        template_context = {
+            'namespace': GlobalConfiguration.getTagNamespace(),
+            'tag': config.release + config.suffix,
+            'prereqs_tag': config.prereqsTag
+        }
+
+        template_context.update(config.opts)
+
+        if config.debug:
+            logger.info(f"Temp build context dir: {tempDir}", False)
+
         # Create the builder instance to build the Docker images
         builder = ImageBuilder(
             join(tempDir, "dockerfiles"),
@@ -108,7 +119,7 @@ def build():
             config.rebuild,
             config.dryRun,
             config.layoutDir,
-            config.opts,
+            template_context,
             config.combine,
         )
 
@@ -527,7 +538,6 @@ def build():
                     mainTags,
                     commonArgs
                     + config.platformArgs
-                    + ue4BuildArgs
                     + infrastructureFlags,
                 )
                 builtImages.append("ue4-full")

src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile

@@ -15,7 +15,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},compute
 
 # Add the "display" driver capability for NVIDIA GPUs
 # (This allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket)
-ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},display
+ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},display,graphics
 
 # Enable NVENC support for use by Unreal Engine plugins that depend on it (e.g. Pixel Streaming)
 # (Note that adding `video` seems to implicitly enable `compute` as well, but we include separate directives here to clearly indicate the purpose of both)
@@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 		git \
 		git-lfs \
 		gpg-agent \
+		ispc \
 		python3 \
 		python3-dev \
 		python3-pip \
@@ -45,7 +46,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Install the X11 runtime libraries required by CEF so we can cook Unreal Engine projects that use the WebBrowserWidget plugin
 # (Starting in Unreal Engine 5.0, we need these installed before creating an Installed Build to prevent cooking failures related to loading the Quixel Bridge plugin)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-			libasound2 \
+			libasound2t64 \
 			libatk1.0-0 \
 			libatk-bridge2.0-0 \
 			libcairo2 \
@@ -74,24 +75,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 			xkb-data && \
 	rm -rf /var/lib/apt/lists/*
 
-{% if enable_dso_patch %}
-# Install the glibc DSO patch to improve Editor startup times
-RUN add-apt-repository -y ppa:slonopotamus/glibc-dso && \
-	apt-get update && \
-	apt upgrade -y libc6 && \
-	rm -rf /var/lib/apt/lists/*
-
-# Enable the glibc DSO patch
-ENV GLIBC_TUNABLES=glibc.rtld.dynamic_sort=2
-{% endif %}
-
 # Disable the default "lecture" message the first time a user runs a command using sudo
 RUN echo 'Defaults lecture="never"' >> /etc/sudoers
 
 # Unreal refuses to run as the root user, so create a non-root user with no password and allow them to run commands using sudo
-RUN useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \
+RUN usermod --move-home --home /home/ue4 --login ue4 ubuntu && \
+	usermod -a -G audio,video,sudo,plugdev,dialout ue4 && \
 	passwd -d ue4 && \
-	usermod -a -G audio,video,sudo ue4
+	groupmod --new-name ue4 ubuntu
+
 USER ue4
 
 {% if enable_ushell %}

src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile

@@ -1,12 +1,12 @@
 ARG UE4CLI_VERSION="ue4cli>=0.0.45"
-ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27"
+ARG CONAN_UE4CLI_VERSION="git+https://github.com/Greenroom-Robotics/conan-ue4cli@feature/fix-libcxx"
 ARG CONAN_VERSION="conan>=1.59.0,<2"
 {% if combine %}
 FROM source as conan
 {% else %}
-ARG NAMESPACE
-ARG TAG
-ARG PREREQS_TAG
+ARG NAMESPACE="{{ namespace }}" 
+ARG TAG="{{ tag }}"
+ARG PREREQS_TAG="{{ prereqs_tag }}"
 FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS conan
 {% endif %}
 ARG UE4CLI_VERSION
@@ -15,13 +15,21 @@ ARG CONAN_VERSION
 
 # Install ue4cli and conan-ue4cli
 USER root
-RUN pip3 install --upgrade pip setuptools wheel
-RUN pip3 install "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION"
+RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED
+RUN pip3 install --ignore-installed  "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION"
 USER ue4
 
+# Copy in UBT from minimal image, if present (required for `ue4 conan generate` to work on UE5)
+ARG UBT_DIR="/home/ue4/UnrealEngine/Engine/Binaries/DotNET"
+{% if combine %}
+COPY --from=minimal --chown=ue4:ue4 $UBT_DIR/UnrealBuildTool $UBT_DIR/UnrealBuildTool
+{% else %}
+COPY --from={{ namespace }}/ue4-minimal:{{ tag }}-{{ prereqs_tag }} --chown=ue4:ue4 $UBT_DIR/UnrealBuildTool $UBT_DIR/UnrealBuildTool
+{% endif %}
+
 # Extract the third-party library details from UBT
 RUN ue4 setroot /home/ue4/UnrealEngine
-RUN ue4 conan generate
+# RUN ue4 conan generate
 
 # Copy the generated Conan packages into a new image with our Installed Build
 {% if combine %}
@@ -35,21 +43,22 @@ ARG CONAN_VERSION
 
 # Install CMake, ue4cli, conan-ue4cli, and ue4-ci-helpers
 USER root
+RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED
+
 RUN apt-get update && apt-get install -y --no-install-recommends cmake
-RUN pip3 install --upgrade pip setuptools wheel
-RUN pip3 install "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" ue4-ci-helpers
+RUN pip3 install --ignore-installed "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" ue4-ci-helpers
 USER ue4
 
 # Explicitly set the configuration directory for ue4cli
 # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable)
 ENV UE4CLI_CONFIG_DIR=/home/ue4/.config/ue4cli
 
 # Copy the Conan configuration settings and package cache from the previous build stage
-COPY --from=conan --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan
+# COPY --from=conan --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan
 
 # Install conan-ue4cli (just generate the profile, since we've already copied the generated packages)
 RUN ue4 setroot /home/ue4/UnrealEngine
-RUN ue4 conan generate --profile-only
+# RUN ue4 conan generate --profile-only
 
 # Enable PulseAudio support
 USER root

src/ue4docker/dockerfiles/ue4-minimal/linux/BuildConfiguration.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8" ?>
+
+<Configuration xmlns="https://www.unrealengine.com/BuildConfiguration">
+<BuildConfiguration>
+ <bAllowUBALocalExecutor>false</bAllowUBALocalExecutor>
+</BuildConfiguration>
+</Configuration>

src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile

@@ -19,6 +19,9 @@ RUN if [ -f ./Engine/Build/BatchFiles/BuildUBT.sh ]; then \
 		./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -SkipBuild -buildubt; \
 	fi
 
+# Copy the BuildConfiguration.xml file to disable UnrealBuildAccelerator
+COPY --chown=ue4:ue4 ["BuildConfiguration.xml", "/home/ue4/.config/Unreal Engine/UnrealBuildTool/BuildConfiguration.xml"]
+
 # Create an Installed Build of the Engine
 WORKDIR /home/ue4/UnrealEngine
 RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \
@@ -78,8 +81,11 @@ COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DebugSymbo
 COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/TemplatesAndSamples /home/ue4/UnrealEngine
 {% endif %}
 
+
 # Copy Install.ini from the builder image, so it can be used by tools that read the list of engine installations (e.g. ushell)
 COPY --from=builder --chown=ue4:ue4 /home/ue4/.config/Epic/UnrealEngine/Install.ini /home/ue4/.config/Epic/UnrealEngine/Install.ini
+# Copy the BuildConfiguration.xml file to disable UnrealBuildAccelerator
+COPY --chown=ue4:ue4 ["BuildConfiguration.xml", "/home/ue4/.config/Unreal Engine/UnrealBuildTool/BuildConfiguration.xml"]
 WORKDIR /home/ue4/UnrealEngine
 
 {% if not disable_labels %}
@@ -109,9 +115,17 @@ RUN EDITOR_TARGET=$(python3 /tmp/print-editor-target.py /home/ue4/UnrealEngine)
 
 # Enable Vulkan support for NVIDIA GPUs
 USER root
-RUN apt-get update && apt-get install -y --no-install-recommends libvulkan1 && \
-	rm -rf /var/lib/apt/lists/* && \
-	VULKAN_API_VERSION=`dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9|\.]+'` && \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	libvulkan1 \
+	libglvnd0 \
+	libgl1 \
+	libglx0 \
+	libegl1  \
+	libgles2  \
+	libxcb1-dev \
+	vulkan-tools \
+	&& rm -rf /var/lib/apt/lists/* && \
+	VULKAN_API_VERSION=`dpkg -s libvulkan1 | grep -oP 'Version: [0-9]+\.[0-9]+\.[0-9]+' | grep -oP '[0-9]+\.[0-9]+\.[0-9]+'` && \
 	mkdir -p /etc/vulkan/icd.d/ && \
 	echo \
 	"{\

src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile

@@ -91,7 +91,6 @@ RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT
 {% endif %}
 
 # Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update`
-{% if credential_mode == "secrets" %}
 
 # Ensure Setup.sh uses the same cache path when building either UE4 or UE5
 ENV UE_GITDEPS=/home/ue4/gitdeps

src/ue4docker/infrastructure/BuildConfiguration.py

@@ -14,10 +14,12 @@
 
 # The base images for Linux containers
 LINUX_BASE_IMAGES = {
-    "opengl": "nvidia/opengl:1.0-glvnd-devel-{ubuntu}",
-    "cuda": "nvidia/cuda:{cuda}-devel-{ubuntu}",
+    "opengl": "{org}/opengl:1.0-glvnd-devel-{ubuntu}",
+    "cuda": "{org}/cuda:{cuda}-devel-{ubuntu}",
 }
 
+DEFAULT_BASE_IMG_ORG = "nvidia"
+
 # The default ubuntu base to use
 DEFAULT_LINUX_VERSION = "ubuntu22.04"
 
@@ -122,12 +124,12 @@ def addArguments(parser):
         parser.add_argument(
             "release",
             nargs="?",  # aka "required = False", but that doesn't work in positionals
-            help='UE4 release to build, in semver format (e.g. 4.27.0) or "custom" for a custom repo and branch (deprecated, use --ue-version instead)',
+            help='Unreal Engine release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch (deprecated, use --ue-version instead)',
         )
         parser.add_argument(
             "--ue-version",
             default=None,
-            help='UE4 release to build, in semver format (e.g. 4.27.0) or "custom" for a custom repo and branch',
+            help='Unreal Engine release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch',
         )
         parser.add_argument(
             "--linux",
@@ -144,6 +146,11 @@ def addArguments(parser):
             action="store_true",
             help="Print `docker build` commands instead of running them",
         )
+        parser.add_argument(
+            "--debug",
+            action="store_true",
+            help="Use buildx debug features",
+        )
         parser.add_argument(
             "--no-minimal",
             action="store_true",
@@ -227,6 +234,11 @@ def addArguments(parser):
             default=None,
             help="Set the isolation mode to use for Windows containers (process or hyperv)",
         )
+        parser.add_argument(
+            "-baseorg",
+            default=DEFAULT_BASE_IMG_ORG,
+            help=f"The Docker organisation to pull the base images from (defaults to '{DEFAULT_BASE_IMG_ORG}')",
+        )
         parser.add_argument(
             "-basetag",
             default=None if platform.system() == "Windows" else DEFAULT_LINUX_VERSION,
@@ -238,8 +250,15 @@ def addArguments(parser):
         )
         parser.add_argument(
             "-m",
+            "--memory",
             default=None,
-            help="Override the default memory limit under Windows (also overrides --random-memory)",
+            help="Override the default memory limit (also overrides --random-memory)",
+        )
+        parser.add_argument(
+            "--cpus",
+            type=int,
+            default=None,
+            help="Override the number of CPUs available during build",
         )
         parser.add_argument(
             "-ue4cli",
@@ -446,7 +465,7 @@ def __init__(self, parser, argv, logger):
 
                 # Use the default repository and the release tag for the specified version
                 self.repository = DEFAULT_GIT_REPO
-                self.branch = "{}-release".format(self.release)
+                self.branch = self.args.ue_version if self.args.ue_version else "{}-release".format(self.release)
                 self.custom = False
 
                 # If the user specified a .0 release of the Unreal Engine and did not specify a changelist override then
@@ -470,6 +489,7 @@ def __init__(self, parser, argv, logger):
             if platform.system() == "Windows" and self.args.linux == False
             else "linux"
         )
+        self.debug = self.args.debug
         self.dryRun = self.args.dry_run
         self.rebuild = self.args.rebuild
         self.suffix = self.args.suffix
@@ -582,6 +602,16 @@ def __init__(self, parser, argv, logger):
                     False,
                 )
 
+        # If the user has explicitly specified a memory limit then use it, otherwise auto-detect
+        self.memLimit = None
+        if self.args.memory is not None:
+            try:
+                self.memLimit = humanfriendly.parse_size(self.args.memory) / (
+                    1000 * 1000 * 1000
+                )
+            except:
+                raise RuntimeError('invalid memory limit "{}"'.format(self.args.memory))
+
         # If we're building Windows containers, generate our Windows-specific configuration settings
         if self.containerPlatform == "windows":
             self._generateWindowsConfig()
@@ -590,6 +620,14 @@ def __init__(self, parser, argv, logger):
         if self.containerPlatform == "linux":
             self._generateLinuxConfig()
 
+        # Set the memory limit Docker flag
+        if self.memLimit is not None:
+            self.platformArgs.extend(["--memory", "{:.2f}GB".format(self.memLimit)])
+
+        # Set the number of CPUs to use for building
+        if self.args.cpus is not None:
+            self.platformArgs.extend(["--cpuset-cpus", "0-{:d}".format(self.args.cpus - 1)])
+
         # If the user-specified suffix passed validation, prefix it with a dash
         self.suffix = "-{}".format(self.suffix) if self.suffix != "" else ""
 
@@ -695,17 +733,12 @@ def _generateWindowsConfig(self):
                     )
                 )
 
-        # Set the memory limit Docker flag
-        if self.memLimit is not None:
-            self.platformArgs.extend(["-m", "{:.2f}GB".format(self.memLimit)])
-
     def _generateLinuxConfig(self):
         # Verify that any user-specified tag suffix does not collide with our base tags
         if self.suffix.startswith("opengl") or self.suffix.startswith("cuda"):
             raise RuntimeError('tag suffix cannot begin with "opengl" or "cuda".')
 
         # Determine if we are building CUDA-enabled container images
-        self.cuda = None
         if self.args.cuda is not None:
             # Verify that the specified CUDA version is valid
             self.cuda = self.args.cuda if self.args.cuda != "" else DEFAULT_CUDA_VERSION
@@ -717,7 +750,7 @@ def _generateLinuxConfig(self):
             self.prereqsTag = "opengl-{ubuntu}"
 
         self.baseImage = self.baseImage.format(
-            cuda=self.args.cuda, ubuntu=self.args.basetag
+            org=self.args.baseorg, cuda=self.args.cuda, ubuntu=self.args.basetag
         )
         self.prereqsTag = self.prereqsTag.format(
             cuda=self.args.cuda, ubuntu=self.args.basetag