diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ae983625..b3e3ae97c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,6 +12,8 @@ jobs: run: bash ci.sh deps - name: Build LLVM run: bash ci.sh llvm + - name: Build Rust + run: bash ci.sh rust - name: Build binutils run: bash ci.sh binutils - name: Build kernel diff --git a/README.md b/README.md index 28727c17b..fc98d7f76 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ These scripts have been tested in a Docker image of the following distributions lld \ make \ ninja-build \ + pkg-config \ python3-dev \ texinfo \ u-boot-tools \ @@ -145,6 +146,19 @@ bfd plugin: LLVM gold plugin has failed to create LTO module: Unknown attribute Having a standalone copy of binutils (ideally in the same folder at the LLVM toolchain so that only one `PATH` modification is needed) works around this without any adverse side effects. Another workaround is bind mounting the new `LLVMgold.so` to `/usr/lib/LLVMgold.so`. +## build-rust.py + +By default, `./build-rust.py` will clone Rust and build it using an LLVM previously built by `./build-llvm.py`, e.g.: + +```sh +./build-llvm.py +./build-rust.py +``` + +This script does not apply any Rust-specific patches to LLVM. + +Run `./build-rust.py -h` for more options and information. + ## Contributing This repository openly welcomes pull requests! There are a few presubmit checks that run to make sure the code stays consistently formatted and free of bugs. diff --git a/build-llvm.py b/build-llvm.py index f2f94b727..c1d3907f3 100755 --- a/build-llvm.py +++ b/build-llvm.py @@ -193,7 +193,7 @@ By default, the script will clone the llvm-project into the tc-build repo. If you have another LLVM checkout that you would like to work out of, pass it to this parameter. This can either be an absolute or relative path. Implies '--no-update'. When this - option is supplied, '--ref' and '--use-good-revison' do nothing, as the script does + option is supplied, '--ref' and '--use-good-revision' do nothing, as the script does not manipulate a repository it does not own. '''), diff --git a/build-rust.py b/build-rust.py new file mode 100755 index 000000000..871d9b0e8 --- /dev/null +++ b/build-rust.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# pylint: disable=invalid-name + +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path +import textwrap +import time + +import tc_build.utils + +from tc_build.rust import RustBuilder, RustSourceManager + +# This is a known good revision of Rust for building the kernel +GOOD_REVISION = '69b3959afec9b5468d5de15133b199553f6e55d2' + +parser = ArgumentParser(formatter_class=RawTextHelpFormatter) +clone_options = parser.add_mutually_exclusive_group() + +parser.add_argument('--debug', + help=textwrap.dedent('''\ + Build a debug compiler and standard library. This enables debug assertions, + debug logging, overflow checks and debug info. The debug assertions and overflow + checks can help catch issues when compiling. + + '''), + action='store_true') +parser.add_argument('-b', + '--build-folder', + help=textwrap.dedent('''\ + By default, the script will create a "build/rust" folder in the same folder as this + script and build each requested stage within that containing folder. To change the + location of the containing build folder, pass it to this parameter. This can be either + an absolute or relative path. If it is provided, then a custom LLVM install folder + needs to be provided as well to prevent mistakes. + + '''), + type=str) +parser.add_argument('-i', + '--install-folder', + help=textwrap.dedent('''\ + By default, the script will leave the toolchain in its build folder. To install it + outside the build folder for persistent use, pass the installation location that you + desire to this parameter. This can be either an absolute or relative path. + + '''), + type=str) +parser.add_argument('-l', + '--llvm-install-folder', + help=textwrap.dedent('''\ + By default, the script will try to use a built LLVM by './build-llvm.py'. To use + another LLVM installation (perhaps from './build-llvm.py --install-folder'), pass + it to this parameter. + + '''), + type=str) +parser.add_argument('-R', + '--rust-folder', + help=textwrap.dedent('''\ + By default, the script will clone the Rust project into the tc-build repo. If you have + another Rust checkout that you would like to work out of, pass it to this parameter. + This can either be an absolute or relative path. Implies '--no-update'. When this + option is supplied, '--ref' and '--use-good-revision' do nothing, as the script does + not manipulate a repository it does not own. + + '''), + type=str) +parser.add_argument('-n', + '--no-update', + help=textwrap.dedent('''\ + By default, the script always updates the Rust repo before building. This prevents + that, which can be helpful during something like bisecting or manually managing the + repo to pin it to a particular revision. + + '''), + action='store_true') +parser.add_argument('-r', + '--ref', + help=textwrap.dedent('''\ + By default, the script builds the main branch (tip of tree) of Rust. If you would + like to build an older branch, use this parameter. This may be helpful in tracking + down an older bug to properly bisect. This value is just passed along to 'git checkout' + so it can be a branch name, tag name, or hash. This will have no effect if + '--rust-folder' is provided, as the script does not manipulate a repository that it + does not own. + + '''), + default='master', + type=str) +parser.add_argument('--show-build-commands', + help=textwrap.dedent('''\ + By default, the script only shows the output of the comands it is running. When this option + is enabled, the invocations of the build tools will be shown to help with reproducing + issues outside of the script. + + '''), + action='store_true') +clone_options.add_argument('--use-good-revision', + help=textwrap.dedent('''\ + By default, the script updates Rust to the latest tip of tree revision, which may at times be + broken or not work right. With this option, it will checkout a known good revision of Rust + that builds and works properly. If you use this option often, please remember to update the + script as the known good revision will change. This option may work best with a matching good + revision used to build LLVM by './build-llvm.py'. + + '''), + action='store_const', + const=GOOD_REVISION, + dest='ref') +parser.add_argument('--vendor-string', + help=textwrap.dedent('''\ + Add this value to the Rust version string (like "rustc ... (ClangBuiltLinux)"). Useful when + reverting or applying patches on top of upstream Rust to differentiate a toolchain built + with this script from upstream Rust or to distinguish a toolchain built with this script + from the system's Rust. Defaults to ClangBuiltLinux, can be set to an empty string to + override this and have no vendor in the version string. + + '''), + type=str, + default='ClangBuiltLinux') +args = parser.parse_args() + +# Start tracking time that the script takes +script_start = time.time() + +# Folder validation +tc_build_folder = Path(__file__).resolve().parent +src_folder = Path(tc_build_folder, 'src') + +if args.build_folder: + build_folder = Path(args.build_folder).resolve() + + if not args.llvm_install_folder: + raise RuntimeError( + 'Build folder customized, but no custom LLVM install folder provided -- this is likely a mistake. Provide both if you want to build in a custom folder?' + ) +else: + build_folder = Path(tc_build_folder, 'build/rust') + +if args.llvm_install_folder: + llvm_install_folder = Path(args.llvm_install_folder).resolve() +else: + llvm_install_folder = Path(tc_build_folder, 'build/llvm/final') + +# Validate and configure Rust source +if args.rust_folder: + if not (rust_folder := Path(args.rust_folder).resolve()).exists(): + raise RuntimeError(f"Provided Rust folder ('{args.rust_folder}') does not exist?") +else: + rust_folder = Path(src_folder, 'rust') +rust_source = RustSourceManager(rust_folder) +rust_source.download(args.ref) +if not (args.rust_folder or args.no_update): + rust_source.update(args.ref) + +# Build Rust +tc_build.utils.print_header('Building Rust') + +# Final build +final = RustBuilder() +final.folders.source = rust_folder +final.folders.build = Path(build_folder, 'final') +final.folders.install = Path(args.install_folder).resolve() if args.install_folder else None +final.llvm_install_folder = llvm_install_folder +final.debug = args.debug +final.vendor_string = args.vendor_string +final.show_commands = args.show_build_commands + +final.configure() +final.build() +final.show_install_info() + +print(f"Script duration: {tc_build.utils.get_duration(script_start)}") diff --git a/ci.sh b/ci.sh index e3daf8cac..1596dea60 100755 --- a/ci.sh +++ b/ci.sh @@ -9,7 +9,7 @@ set -eu function parse_parameters() { while (($#)); do case $1 in - all | binutils | deps | kernel | llvm) action=$1 ;; + all | binutils | deps | kernel | llvm | rust) action=$1 ;; *) exit 33 ;; esac shift @@ -19,6 +19,7 @@ function parse_parameters() { function do_all() { do_deps do_llvm + do_rust do_binutils do_kernel } @@ -54,6 +55,7 @@ function do_deps() { lld \ make \ ninja-build \ + pkg-config \ python3 \ texinfo \ xz-utils \ @@ -111,5 +113,13 @@ function do_llvm() { "${extra_args[@]}" } +function do_rust() { + "$base"/build-rust.py \ + --debug \ + --llvm-install-folder "$install" \ + --install-folder "$install" \ + --show-build-commands +} + parse_parameters "$@" do_"${action:=all}" diff --git a/tc_build/builder.py b/tc_build/builder.py index 2aac9389e..3963f383c 100644 --- a/tc_build/builder.py +++ b/tc_build/builder.py @@ -32,6 +32,12 @@ def clean_build_folder(self): else: self.folders.build.unlink() + def make_build_folder(self): + if not self.folders.build: + raise RuntimeError('No build folder set?') + + self.folders.build.mkdir(parents=True) + def run_cmd(self, cmd, capture_output=False, cwd=None): if self.show_commands: # Acts sort of like 'set -x' in bash diff --git a/tc_build/llvm.py b/tc_build/llvm.py index 37a90a015..182a20fd7 100644 --- a/tc_build/llvm.py +++ b/tc_build/llvm.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import contextlib import os from pathlib import Path import platform @@ -10,6 +9,7 @@ import time from tc_build.builder import Builder +from tc_build.source import GitSourceManager import tc_build.utils LLVM_VER_FOR_RUNTIMES = 20 @@ -243,9 +243,9 @@ def configure(self): # disable the optimization when compiler-rt is enabled and there is an # install directory. Ideally thin archives should still be usable for # non-compiler-rt projects. - if not (self.folders.install and self.project_is_enabled('compiler-rt')): - self.cmake_defines['CMAKE_CXX_ARCHIVE_CREATE'] = ' DqcT ' - self.cmake_defines['CMAKE_CXX_ARCHIVE_FINISH'] = 'true' + #if not (self.folders.install and self.project_is_enabled('compiler-rt')): + # self.cmake_defines['CMAKE_CXX_ARCHIVE_CREATE'] = ' DqcT ' + #self.cmake_defines['CMAKE_CXX_ARCHIVE_FINISH'] = 'true' if self.tools.ranlib: self.cmake_defines['CMAKE_RANLIB'] = self.tools.ranlib @@ -430,7 +430,11 @@ def configure(self): self.set_llvm_major_version() - distribution_components = [] + distribution_components = [ + 'llvm-config', + 'llvm-headers', + 'llvm-libraries', + ] runtime_distribution_components = [] if llvm_build_tools: distribution_components += [ @@ -572,10 +576,13 @@ class LLVMSlimInstrumentedBuilder(LLVMInstrumentedBuilder, LLVMSlimBuilder): pass -class LLVMSourceManager: +class LLVMSourceManager(GitSourceManager): def __init__(self, repo): - self.repo = repo + super().__init__(repo) + + self._pretty_name = 'LLVM' + self._repo_url = 'https://github.com/llvm/llvm-project.git' def default_projects(self): return ['clang', 'compiler-rt', 'lld', 'polly'] @@ -591,57 +598,3 @@ def default_targets(self): targets.append('LoongArch') return targets - - def download(self, ref, shallow=False): - if self.repo.exists(): - return - - tc_build.utils.print_header('Downloading LLVM') - - git_clone = ['git', 'clone'] - if shallow: - git_clone.append('--depth=1') - if ref != 'main': - git_clone.append('--no-single-branch') - git_clone += ['https://github.com/llvm/llvm-project', self.repo] - - subprocess.run(git_clone, check=True) - - self.git(['checkout', ref]) - - def git(self, cmd, capture_output=False): - return subprocess.run(['git', *cmd], - capture_output=capture_output, - check=True, - cwd=self.repo, - text=True) - - def git_capture(self, cmd): - return self.git(cmd, capture_output=True).stdout.strip() - - def is_shallow(self): - git_dir = self.git_capture(['rev-parse', '--git-dir']) - return Path(git_dir, 'shallow').exists() - - def ref_exists(self, ref): - try: - self.git(['show-branch', ref]) - except subprocess.CalledProcessError: - return False - return True - - def update(self, ref): - tc_build.utils.print_header('Updating LLVM') - - self.git(['fetch', 'origin']) - - if self.is_shallow() and not self.ref_exists(ref): - raise RuntimeError(f"Repo is shallow and supplied ref ('{ref}') does not exist!") - - self.git(['checkout', ref]) - - local_ref = None - with contextlib.suppress(subprocess.CalledProcessError): - local_ref = self.git_capture(['symbolic-ref', '-q', 'HEAD']) - if local_ref and local_ref.startswith('refs/heads/'): - self.git(['pull', '--rebase', 'origin', local_ref.replace('refs/heads/', '')]) diff --git a/tc_build/rust.py b/tc_build/rust.py new file mode 100644 index 000000000..e98d5d83d --- /dev/null +++ b/tc_build/rust.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import subprocess +import time + +from tc_build.builder import Builder +from tc_build.source import GitSourceManager +import tc_build.utils + + +class RustBuilder(Builder): + + def __init__(self): + super().__init__() + + self.llvm_install_folder = None + self.debug = False + self.vendor_string = '' + + def build(self): + if not self.folders.build: + raise RuntimeError('No build folder set for build()?') + if not Path(self.folders.build, 'bootstrap.toml').exists(): + raise RuntimeError('No bootstrap.toml in build folder, run configure()?') + + build_start = time.time() + self.run_cmd([Path(self.folders.source, 'x.py'), 'install'], cwd=self.folders.build) + + tc_build.utils.print_info(f"Build duration: {tc_build.utils.get_duration(build_start)}") + + if self.folders.install: + tc_build.utils.create_gitignore(self.folders.install) + + def configure(self): + if not self.llvm_install_folder: + raise RuntimeError('No LLVM install folder set?') + if not self.folders.source: + raise RuntimeError('No source folder set?') + if not self.folders.build: + raise RuntimeError('No build folder set?') + + # Configure the build + # + # 'codegen-tests' requires '-DLLVM_INSTALL_UTILS=ON'. + install_folder = self.folders.install if self.folders.install else self.folders.build + + # yapf: disable + configure_cmd = [ + Path(self.folders.source, 'configure'), + '--release-description', self.vendor_string, + '--disable-docs', + '--enable-locked-deps', + '--tools', 'cargo,clippy,rustdoc,rustfmt,src', + '--prefix', install_folder, + '--sysconfdir', 'etc', + '--disable-codegen-tests', + '--disable-lld', + '--disable-llvm-bitcode-linker', + '--llvm-root', self.llvm_install_folder, + ] + # yapf: enable + + if self.debug: + configure_cmd.append('--enable-debug') + + self.clean_build_folder() + self.make_build_folder() + self.run_cmd(configure_cmd, cwd=self.folders.build) + + def show_install_info(self): + # Installation folder is optional, show build folder as the + # installation location in that case. + install_folder = self.folders.install if self.folders.install else self.folders.build + if not install_folder: + raise RuntimeError('Installation folder not set?') + if not install_folder.exists(): + raise RuntimeError('Installation folder does not exist, run build()?') + if not (bin_folder := Path(install_folder, 'bin')).exists(): + raise RuntimeError('bin folder does not exist in installation folder, run build()?') + + tc_build.utils.print_header('Rust installation information') + install_info = (f"Toolchain is available at: {install_folder}\n\n" + 'To use, either run:\n\n' + f"\t$ export PATH={bin_folder}:$PATH\n\n" + 'or add:\n\n' + f"\tPATH={bin_folder}:$PATH\n\n" + 'before the command you want to use this toolchain.\n') + print(install_info) + + for tool in ['rustc', 'rustdoc', 'rustfmt', 'clippy-driver', 'cargo']: + if (binary := Path(bin_folder, tool)).exists(): + subprocess.run([binary, '--version', '--verbose'], check=True) + print() + tc_build.utils.flush_std_err_out() + + +class RustSourceManager(GitSourceManager): + + def __init__(self, repo): + super().__init__(repo) + + self._pretty_name = 'Rust' + self._repo_url = 'https://github.com/rust-lang/rust.git' diff --git a/tc_build/source.py b/tc_build/source.py index 624fb7e26..784e0637e 100644 --- a/tc_build/source.py +++ b/tc_build/source.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 +import contextlib import hashlib +from pathlib import Path import re import subprocess @@ -86,3 +88,67 @@ class SourceManager: def __init__(self, location=None): self.location = location self.tarball = Tarball() + + +class GitSourceManager: + + def __init__(self, repo): + self.repo = repo + + # Will be set by derived classes but used here + self._pretty_name = '' + self._repo_url = '' + + def download(self, ref, shallow=False): + if self.repo.exists(): + return + + tc_build.utils.print_header(f"Downloading {self._pretty_name}") + + git_clone = ['git', 'clone'] + if shallow: + git_clone.append('--depth=1') + if ref != 'main': + git_clone.append('--no-single-branch') + git_clone += [self._repo_url, self.repo] + + subprocess.run(git_clone, check=True) + + self.git(['checkout', ref]) + + def git(self, cmd, capture_output=False): + return subprocess.run(['git', *cmd], + capture_output=capture_output, + check=True, + cwd=self.repo, + text=True) + + def git_capture(self, cmd): + return self.git(cmd, capture_output=True).stdout.strip() + + def is_shallow(self): + git_dir = self.git_capture(['rev-parse', '--git-dir']) + return Path(git_dir, 'shallow').exists() + + def ref_exists(self, ref): + try: + self.git(['show-branch', ref]) + except subprocess.CalledProcessError: + return False + return True + + def update(self, ref): + tc_build.utils.print_header(f"Updating {self._pretty_name}") + + self.git(['fetch', 'origin']) + + if self.is_shallow() and not self.ref_exists(ref): + raise RuntimeError(f"Repo is shallow and supplied ref ('{ref}') does not exist!") + + self.git(['checkout', ref]) + + local_ref = None + with contextlib.suppress(subprocess.CalledProcessError): + local_ref = self.git_capture(['symbolic-ref', '-q', 'HEAD']) + if local_ref and local_ref.startswith('refs/heads/'): + self.git(['pull', '--rebase', 'origin', local_ref.replace('refs/heads/', '')])