From ac28cff28f7d979c2b90f737839635f27f048eb4 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 13 Jun 2024 13:03:25 -0700 Subject: [PATCH 1/6] version: update to 2.90.0-rc1 Change-Id: I913852cfb88b3b300ffb1050a91784be659ae66a --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 877e7a20b..1481a97fd 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "version": "2.90.0", + "version": "2.90.0-rc1", "release_name": "Vanilla Ice Cream" } From 74fb2e8c47cbcb3fe9dff417b5ce61550588708f Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 13 Jun 2024 14:03:27 -0700 Subject: [PATCH 2/6] releng: support multiple systems I guess this is kind of important to being able to "release it". Change-Id: Id6f295d0b4944fa1203783a400a246727dbd94b6 --- releng/cli.py | 10 +++++--- releng/create_release.xsh | 15 ++++------- releng/docker_assemble.py | 8 ------ releng/release-jobs.nix | 54 +++++++++++++++++++++++++++++++-------- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/releng/cli.py b/releng/cli.py index 89391e0a7..f78d4b12d 100644 --- a/releng/cli.py +++ b/releng/cli.py @@ -2,16 +2,16 @@ from . import create_release from . import docker from .environment import RelengEnvironment from . import environment -import functools import argparse import sys def do_build(args): if args.target == 'all': - create_release.build_artifacts(no_check_git=args.no_check_git) + create_release.build_artifacts(args.profile, no_check_git=args.no_check_git) elif args.target == 'manual': - eval_result = create_release.eval_jobs() + # n.b. args.profile does nothing here, you will just get the x86_64-linux manual no matter what. + eval_result = create_release.eval_jobs(args.profile) create_release.build_manual(eval_result) else: raise ValueError('invalid target, unreachable') @@ -80,6 +80,10 @@ def main(): build.add_argument('--target', choices=['manual', 'all'], help='Whether to build everything or just the manual') + build.add_argument('--profile', + default='all', + choices=('all', 'x86_64-linux-only'), + help='Which systems to build targets for.') build.set_defaults(cmd=do_build) upload = sps.add_parser( diff --git a/releng/create_release.xsh b/releng/create_release.xsh index b51a3ad23..96b13ae4c 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -27,9 +27,6 @@ RELENG_MSG = "Release created with releng/create_release.xsh" BUILD_CORES = 16 MAX_JOBS = 2 -# TODO -RELEASE_SYSTEMS = ["x86_64-linux"] - def setup_creds(env: RelengEnvironment): key = keys.get_ephemeral_key(env) @@ -82,11 +79,9 @@ def realise(paths: list[str]): nix-store @(args) @(paths) -def eval_jobs(): - nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake '.#release-jobs') - return [x for x in (json.loads(s) for s in nej_output.strip().split('\n')) - if x['system'] in RELEASE_SYSTEMS - ] +def eval_jobs(build_profile): + nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake f'.#release-jobs.{build_profile}') + return [json.loads(s) for s in nej_output.strip().split('\n')] def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]): @@ -295,14 +290,14 @@ def upload_manual(env: RelengEnvironment): aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/ -def build_artifacts(no_check_git=False): +def build_artifacts(build_profile, no_check_git=False): rm -rf release/ if not no_check_git: verify_are_on_tag() git_preconditions() print('[+] Evaluating') - eval_result = eval_jobs() + eval_result = eval_jobs(build_profile) drv_paths = [x['drvPath'] for x in eval_result] print('[+] Building') diff --git a/releng/docker_assemble.py b/releng/docker_assemble.py index ef1d8c4e6..a03ec2766 100644 --- a/releng/docker_assemble.py +++ b/releng/docker_assemble.py @@ -100,14 +100,6 @@ class OCIIndex: } -def docker_architecture_from_nix_system(system: str) -> DockerArchitecture: - MAP = { - 'x86_64-linux': 'amd64', - 'aarch64-linux': 'arm64', - } - return MAP[system] # type: ignore - - @dataclasses.dataclass class TaggingOperation: manifest: OCIIndex diff --git a/releng/release-jobs.nix b/releng/release-jobs.nix index 4db0baed3..49508e4ef 100644 --- a/releng/release-jobs.nix +++ b/releng/release-jobs.nix @@ -3,8 +3,27 @@ let inherit (pkgs) lib; lix = hydraJobs.build.x86_64-linux; - systems = [ "x86_64-linux" ]; - dockerSystems = [ "x86_64-linux" ]; + # This is all so clumsy because we can't use arguments to functions in + # flakes, and certainly not with n-e-j. + profiles = { + # Used for testing + x86_64-linux-only = { + systems = [ "x86_64-linux" ]; + dockerSystems = [ "x86_64-linux" ]; + }; + all = { + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; + dockerSystems = [ + "x86_64-linux" + "aarch64-linux" + ]; + }; + }; doTarball = { @@ -27,7 +46,8 @@ let sha256sum --binary $filename | cut -f1 -d' ' > $out/$basename.sha256 ''; - targets = + targetsFor = + { systems, dockerSystems }: builtins.map (system: { target = hydraJobs.binaryTarball.${system}; targetName = "*.tar.xz"; @@ -44,14 +64,26 @@ let tar -cvzf "$out/lix-${lix.version}-manual.tar.gz" lix-${lix.version}-manual ''; - tarballs = pkgs.runCommand "lix-release-tarballs" { } '' - mkdir -p $out - ${lib.concatMapStringsSep "\n" doTarball targets} - cp ${manualTar}/*.tar.gz $out - cp -r ${lix.doc}/share/doc/nix/manual $out - ''; + tarballsFor = + { systems, dockerSystems }: + pkgs.runCommand "lix-release-tarballs" { } '' + mkdir -p $out + ${lib.concatMapStringsSep "\n" doTarball (targetsFor { + inherit systems dockerSystems; + })} + cp ${manualTar}/*.tar.gz $out + cp -r ${lix.doc}/share/doc/nix/manual $out + ''; in -{ +(builtins.mapAttrs ( + _: + { systems, dockerSystems }: + { + build = lib.filterAttrs (x: _: builtins.elem x systems) hydraJobs.build; + tarballs = tarballsFor { inherit systems dockerSystems; }; + } +) profiles) +// { inherit (hydraJobs) build; - inherit tarballs; + inherit tarballsFor; } From e715e5fd31e0f79eac17c02163bd2035ef7c9166 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 13 Jun 2024 14:36:32 -0700 Subject: [PATCH 3/6] releng: fix logging inside interactive xonsh I don't know when this broke, it seems like it happened since the 24.05 upgrade, so xonsh 0.15. What happened is that xonsh was trying to intercept log output, which explodes if you have the logger survive past one command input. This is, however, impossible to avoid if you are trying to use logging when you import releng from inside xonsh for interactive use! The error below is because the memory handler backing the stdout/stderr of the one command that's just been run was closed after the command completed. Change-Id: I2be642aebf93da9818d08ff8b97c2e72ba5ac581 --- Logging error --- Traceback (most recent call last): File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/logging/__init__.py", line 1113, in emit stream.write(msg + self.terminator) File "/nix/store/34951j60xcsw6zj4v8lsaf491acv0by3-python3-3.11.9-env/lib/python3.11/site-packages/xonsh/base_shell.py", line 183, in write self.mem.write(s) ValueError: I/O operation on closed file. Call stack: File "/nix/store/xgdp1p1gv8ni1awnkzyqasnn6gz5wlvx-xonsh-0.15.1/bin/xonsh", line 8, in sys.exit(main()) File "/nix/store/34951j60xcsw6zj4v8lsaf491acv0by3-python3-3.11.9-env/lib/python3.11/site-packages/xonsh/main.py", line 470, in main sys.exit(main_xonsh(args)) File "/nix/store/34951j60xcsw6zj4v8lsaf491acv0by3-python3-3.11.9-env/lib/python3.11/site-packages/xonsh/main.py", line 514, in main_xonsh shell.shell.cmdloop() File "/nix/store/34951j60xcsw6zj4v8lsaf491acv0by3-python3-3.11.9-env/lib/python3.11/site-packages/xonsh/ptk_shell/shell.py", line 406, in cmd loop line = self.singleline(auto_suggest=auto_suggest) File "/nix/store/34951j60xcsw6zj4v8lsaf491acv0by3-python3-3.11.9-env/lib/python3.11/site-packages/xonsh/ptk_shell/shell.py", line 374, in sin gleline line = self.prompter.prompt(**prompt_args) File "/nix/store/34951j60xcsw6zj4v8lsaf491acv0by3-python3-3.11.9-env/lib/python3.11/site-packages/prompt_toolkit/shortcuts/prompt.py", line 1 026, in prompt return self.app.run( File "/nix/store/34951j60xcsw6zj4v8lsaf491acv0by3-python3-3.11.9-env/lib/python3.11/site-packages/prompt_toolkit/application/application.py", line 1002, in run return asyncio.run(coro) File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/asyncio/runners.py", line 189, in run with Runner(debug=debug) as runner: File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/asyncio/runners.py", line 59, in __enter__ self._lazy_init() File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/asyncio/runners.py", line 137, in _lazy_init self._loop = events.new_event_loop() File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/asyncio/events.py", line 810, in new_event_loop return get_event_loop_policy().new_event_loop() File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/asyncio/events.py", line 699, in new_event_loop return self._loop_factory() File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/asyncio/unix_events.py", line 64, in __init__ super().__init__(selector) File "/nix/store/7hnr99nxrd2aw6lghybqdmkckq60j6l9-python3-3.11.9/lib/python3.11/asyncio/selector_events.py", line 54, in __init__ logger.debug('Using selector: %s', selector.__class__.__name__) Message: 'Using selector: %s' Arguments: ('EpollSelector',) Change-Id: I90959809129aaf96aad4577599031688599ed85e --- releng/__init__.py | 62 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/releng/__init__.py b/releng/__init__.py index 59d1709b9..179ea3e2b 100644 --- a/releng/__init__.py +++ b/releng/__init__.py @@ -1,8 +1,12 @@ from xonsh.main import setup + setup() del setup import logging +import sys + +import xonsh.base_shell from . import environment from . import create_release @@ -13,19 +17,55 @@ from . import docker from . import docker_assemble from . import gitutils -rootLogger = logging.getLogger() -rootLogger.setLevel(logging.DEBUG) -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -fmt = logging.Formatter('{asctime} {levelname} {name}: {message}', - datefmt='%b %d %H:%M:%S', - style='{') +def setup_logging(): + """ + Sets up logging to work properly. The following are intended to work: + - ipython/xonsh configuration files adding log handlers out of band + - Reloading the module in xonsh/ipython not causing Bonus Loggers (which is + why we check if there is already a handler. This also helps the previous + case) + - Importing the releng module from xonsh and poking at it interactively + """ + + LEVELS = { + # Root logger must be DEBUG so that anything else can be DEBUG + None: logging.DEBUG, + # Everything in releng + __name__: logging.DEBUG, + # Log spam caused by prompt_toolkit + 'asyncio': logging.INFO, + } + + for name, level in LEVELS.items(): + logger = logging.getLogger(name) + logger.setLevel(level) + + root_logger = logging.getLogger() + + fmt = logging.Formatter('{asctime} {levelname} {name}: {message}', + datefmt='%b %d %H:%M:%S', + style='{') + + if not any( + isinstance(h, logging.StreamHandler) for h in root_logger.handlers): + stderr = sys.stderr + # XXX: Horrible hack required by the virtual stderr xonsh uses for each entered + # command getting closed after the command is run: we need to pull out + # the real stderr because this survives across multiple command runs. + # + # This only applies when running xonsh in interactive mode and importing releng. + if isinstance(sys.stderr, xonsh.base_shell._TeeStd): + stderr = stderr.std # type: ignore + + hand = logging.StreamHandler(stream=stderr) + hand.set_name('releng root handler') + hand.setFormatter(fmt) + root_logger.addHandler(hand) + + +setup_logging() -if not any(isinstance(h, logging.StreamHandler) for h in rootLogger.handlers): - hand = logging.StreamHandler() - hand.setFormatter(fmt) - rootLogger.addHandler(hand) def reload(): import importlib From 7be0d237e0e88e65be9015c65e4f0fc67d2d9aad Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 13 Jun 2024 15:24:21 -0700 Subject: [PATCH 4/6] releng: fix git checking Change-Id: I82ddd918311b48e596adb807b81221973113fe7a --- releng/gitutils.xsh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/releng/gitutils.xsh b/releng/gitutils.xsh index 3352a6b21..e18b4da5f 100644 --- a/releng/gitutils.xsh +++ b/releng/gitutils.xsh @@ -1,6 +1,8 @@ import subprocess import json +from .version import VERSION + def version_compare(v1: str, v2: str): return json.loads($(nix-instantiate --eval --json --argstr v1 @(v1) --argstr v2 @(v2) --expr '{v1, v2}: builtins.compareVersions v1 v2')) From 16ea19ced8e19d419de2a5ae7b8dca609d5d951e Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 13 Jun 2024 17:04:07 -0700 Subject: [PATCH 5/6] releng: fix upload of multiarch images to forgejo Forgejo appears to immediately delete registry content that is overwritten. This means that we are forced to delete our previous workaround of making a temporary tag and use a new, more absurd workaround of making an entire temporary image that we basically only need to create to get its hash. However, on the plus side, the new workaround doesn't create garbage tags to begin with, which means that we don't have to deal with GitHub not implementing the standardized tag delete endpoint and instead only implementing a proprietary one. Upstream-Bug: https://github.com/containers/skopeo/issues/2354 Change-Id: I220e7ce9a17fd230c38882f12c009a166dcc9336 --- releng/docker.xsh | 29 +++++++++++++++++++---------- releng/docker_assemble.py | 6 +++--- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/releng/docker.xsh b/releng/docker.xsh index 20fb30cd3..13bdd7868 100644 --- a/releng/docker.xsh +++ b/releng/docker.xsh @@ -19,6 +19,7 @@ def check_all_logins(env: RelengEnvironment): check_login(target) def check_login(target: DockerTarget): + log.info('Checking login for %s', target.registry_name) skopeo login @(target.registry_name()) def upload_docker_images(target: DockerTarget, paths: list[Path]): @@ -43,7 +44,23 @@ def upload_docker_images(target: DockerTarget, paths: list[Path]): for path in paths: digest_file = tmp / (path.name + '.digest') - inspection = json.loads($(skopeo inspect docker-archive:@(path))) + tmp_image = tmp / 'tmp-image.tar.gz' + + # insecure-policy: we don't have any signature policy, we are just uploading an image + # + # Absurd: we copy it into an OCI image first so we can get the hash + # we need to upload it untagged, because skopeo has no "don't tag + # this" option. + # The reason for this is that forgejo's container registry throws + # away old versions of tags immediately, so we cannot use a temp + # tag, and it *does* reduce confusion to not upload tags that + # should not be used. + # + # Workaround for: https://github.com/containers/skopeo/issues/2354 + log.info('skopeo copy to temp oci-archive %s', tmp_image) + skopeo --insecure-policy copy --format oci --all --digestfile @(digest_file) docker-archive:@(path) oci-archive:@(tmp_image) + + inspection = json.loads($(skopeo inspect oci-archive:@(tmp_image))) docker_arch = inspection['Architecture'] docker_os = inspection['Os'] @@ -51,21 +68,13 @@ def upload_docker_images(target: DockerTarget, paths: list[Path]): log.info('Pushing image %s for %s to %s', path, docker_arch, target.registry_path) - # insecure-policy: we don't have any signature policy, we are just uploading an image - # We upload to a junk tag, because otherwise it will upload to `latest`, which is undesirable - skopeo --insecure-policy copy --format oci --digestfile @(digest_file) docker-archive:@(path) docker://@(target.registry_path):temp - digest = digest_file.read_text().strip() + skopeo --insecure-policy copy --preserve-digests --all oci-archive:@(tmp_image) f'docker://{target.registry_path}@{digest}' # skopeo doesn't give us the manifest size directly, so we just ask the registry metadata = reg.image_info(target.registry_path, digest) manifests.append(OCIIndexItem(metadata=metadata, architecture=docker_arch, os=docker_os)) - # delete the temp tag, which we only have to create because of skopeo - # limitations anyhow (it seems to not have a way to say "don't tag it, find - # your checksum and put it there") - # FIXME: this is not possible because GitHub only has a proprietary API for it. amazing. 11/10. - # reg.delete_tag(target.registry_path, 'temp') log.info('Pushed images to %r, building a bigger and more menacing manifest from %r with metadata %r', target, manifests, meta) # send the multiarch manifest to each tag diff --git a/releng/docker_assemble.py b/releng/docker_assemble.py index a03ec2766..d5b47c328 100644 --- a/releng/docker_assemble.py +++ b/releng/docker_assemble.py @@ -49,8 +49,8 @@ if DEBUG_REQUESTS: # fix that. Thus, a little bit of homebrew containers code. # # Essentially what we are doing in here is splatting a bunch of images into the -# registry without tagging them (except as "temp", due to podman issues), then -# simply sending a new composite manifest ourselves. +# registry without tagging them (with a silly workaround to skopeo issues), +# then simply sending a new composite manifest ourselves. DockerArchitecture = Literal['amd64'] | Literal['arm64'] MANIFEST_MIME = 'application/vnd.oci.image.manifest.v1+json' @@ -276,7 +276,7 @@ class AuthState: 'Authorization': 'Basic ' + creds }).json() token = resp['token'] - self.token_cache[service] = token + self.token_cache[authority] = token return token def find_credential_for(self, image_path: str): From d5c670ad0175fe2a9279784f2acacae4ba7b3828 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 13 Jun 2024 17:14:06 -0700 Subject: [PATCH 6/6] releng: add sha256 for the manual tarball Whoops. Change-Id: Ic6f8cdcb074d679e9b1fc3323c106cc853328dcc --- releng/release-jobs.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/releng/release-jobs.nix b/releng/release-jobs.nix index 49508e4ef..7f1017002 100644 --- a/releng/release-jobs.nix +++ b/releng/release-jobs.nix @@ -71,7 +71,10 @@ let ${lib.concatMapStringsSep "\n" doTarball (targetsFor { inherit systems dockerSystems; })} - cp ${manualTar}/*.tar.gz $out + ${doTarball { + target = manualTar; + targetName = "lix-*.tar.gz"; + }} cp -r ${lix.doc}/share/doc/nix/manual $out ''; in