From ff95b980d4913e90bc334227f8f3f7b3daf18b36 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 6 Jun 2024 22:28:49 -0700 Subject: [PATCH 1/6] Implement docker upload in the releng tools This uses skopeo to not think about docker daemons. I, however, noticed that the docker image we had would have totally terrible cache hits, so I rewrote it. Fixes: https://git.lix.systems/lix-project/lix/issues/252 Change-Id: I3c5b6c1f3ba0b9dfcac212b2148f390e0cd542b7 --- package.nix | 3 +++ releng/__init__.py | 12 +++++++----- releng/cli.py | 28 ++++++++++++++++++++------ releng/create_release.xsh | 41 +++++++++++++++++++++------------------ releng/docker.xsh | 13 +++++++++++++ releng/environment.py | 32 +++++++++++++++++++++++++++--- 6 files changed, 96 insertions(+), 33 deletions(-) create mode 100644 releng/docker.xsh diff --git a/package.nix b/package.nix index 9c108308d..1ac54dab6 100644 --- a/package.nix +++ b/package.nix @@ -39,6 +39,7 @@ pkg-config, python3, rapidcheck, + skopeo, sqlite, toml11, util-linuxMinimal ? utillinuxMinimal, @@ -447,6 +448,8 @@ stdenv.mkDerivation (finalAttrs: { lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm ++ [ pythonEnv + # docker image tool + skopeo just nixfmt # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above. diff --git a/releng/__init__.py b/releng/__init__.py index 401b8e322..39d2beb51 100644 --- a/releng/__init__.py +++ b/releng/__init__.py @@ -2,11 +2,12 @@ from xonsh.main import setup setup() del setup -from releng import environment -from releng import create_release -from releng import keys -from releng import version -from releng import cli +from . import environment +from . import create_release +from . import keys +from . import version +from . import cli +from . import docker def reload(): import importlib @@ -15,3 +16,4 @@ def reload(): importlib.reload(keys) importlib.reload(version) importlib.reload(cli) + importlib.reload(docker) diff --git a/releng/cli.py b/releng/cli.py index bba50f534..89391e0a7 100644 --- a/releng/cli.py +++ b/releng/cli.py @@ -1,4 +1,8 @@ from . import create_release +from . import docker +from .environment import RelengEnvironment +from . import environment +import functools import argparse import sys @@ -18,13 +22,16 @@ def do_tag(args): no_check_git=args.no_check_git) -def do_upload(args): - create_release.setup_creds() +def do_upload(env: RelengEnvironment, args): + create_release.setup_creds(env) if args.target == 'all': - create_release.upload_artifacts(force_push_tag=args.force_push_tag, - noconfirm=args.noconfirm) + docker.check_all_logins(env) + create_release.upload_artifacts(env, + force_push_tag=args.force_push_tag, + noconfirm=args.noconfirm, + no_check_git=args.no_check_git) elif args.target == 'manual': - create_release.upload_manual() + create_release.upload_manual(env) else: raise ValueError('invalid target, unreachable') @@ -77,6 +84,10 @@ def main(): upload = sps.add_parser( 'upload', help='Upload artifacts to cache and releases bucket') + upload.add_argument( + '--no-check-git', + action='store_true', + help="Don't check git state before uploading. For testing.") upload.add_argument('--force-push-tag', action='store_true', help='Force push the tag. For testing.') @@ -90,7 +101,12 @@ def main(): '--noconfirm', action='store_true', help="Don't ask for confirmation. For testing/automation.") - upload.set_defaults(cmd=do_upload) + upload.add_argument('--environment', + choices=list(environment.ENVIRONMENTS.keys()), + default='staging', + help='Environment to release to') + upload.set_defaults(cmd=lambda args: do_upload( + environment.ENVIRONMENTS[args.environment], args)) args = ap.parse_args() args.cmd(args) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index c57a92b2f..128edb63f 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -7,19 +7,14 @@ import tempfile import hashlib import datetime from . import environment +from .environment import RelengEnvironment from . import keys +from . import docker from .version import VERSION, RELEASE_NAME, MAJOR $RAISE_SUBPROC_ERROR = True $XONSH_SHOW_TRACEBACK = True -RELENG_ENV = environment.STAGING - -RELEASES_BUCKET = RELENG_ENV.releases_bucket -DOCS_BUCKET = RELENG_ENV.docs_bucket -CACHE_STORE = RELENG_ENV.cache_store_uri() -REPO = RELENG_ENV.git_repo - GCROOTS_DIR = Path('./release/gcroots') BUILT_GCROOTS_DIR = Path('./release/gcroots-build') DRVS_TXT = Path('./release/drvs.txt') @@ -35,8 +30,8 @@ MAX_JOBS = 2 RELEASE_SYSTEMS = ["x86_64-linux"] -def setup_creds(): - key = keys.get_ephemeral_key(RELENG_ENV) +def setup_creds(env: RelengEnvironment): + key = keys.get_ephemeral_key(env) $AWS_SECRET_ACCESS_KEY = key.secret_key $AWS_ACCESS_KEY_ID = key.id $AWS_DEFAULT_REGION = 'garage' @@ -102,13 +97,13 @@ def eval_jobs(): ] -def upload_drv_paths_and_outputs(paths: list[str]): +def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]): proc = subprocess.Popen([ 'nix', 'copy', '-v', '--to', - CACHE_STORE, + env.cache_store_uri(), '--stdin', ], stdin=subprocess.PIPE, @@ -250,7 +245,10 @@ def verify_are_on_tag(): assert current_tag == VERSION -def upload_artifacts(noconfirm=False, force_push_tag=False): +def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False): + if not no_check_git: + verify_are_on_tag() + git_preconditions() assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env tree @(ARTIFACTS) @@ -262,16 +260,21 @@ def upload_artifacts(noconfirm=False, force_push_tag=False): print('[+] Upload to cache') with open(DRVS_TXT) as fh: - upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x]) + upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x]) + docker_images = (ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz') + print('[+] Upload docker images') + for image in docker_images: + for target in env.docker_targets: + docker.upload_docker_image(target, image) print('[+] Upload to release bucket') - aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/ + aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/ print('[+] Upload manual') - upload_manual() + upload_manual(env) print('[+] git push tag') - git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}' + git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' def do_tag_merge(force_tag=False, no_check_git=False): @@ -290,7 +293,7 @@ def build_manual(eval_result): cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL) -def upload_manual(): +def upload_manual(env: RelengEnvironment): stable = json.loads($(nix eval --json '.#nix.officialRelease')) if stable: version = MAJOR @@ -298,9 +301,9 @@ def upload_manual(): version = 'nightly' print('[+] aws s3 sync manual') - aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/ + aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/ if stable: - aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/ + aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/ def build_artifacts(no_check_git=False): diff --git a/releng/docker.xsh b/releng/docker.xsh new file mode 100644 index 000000000..1ed2330cf --- /dev/null +++ b/releng/docker.xsh @@ -0,0 +1,13 @@ +from .environment import DockerTarget, RelengEnvironment +from .version import VERSION +from pathlib import Path + +def check_all_logins(env: RelengEnvironment): + for target in env.docker_targets: + check_login(target) + +def check_login(target: DockerTarget): + skopeo login @(target.registry_name()) + +def upload_docker_image(target: DockerTarget, path: Path): + skopeo --insecure-policy copy docker-archive:@(path) docker://@(target.resolve(version=VERSION)) diff --git a/releng/environment.py b/releng/environment.py index 58633d548..e8e7e771e 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -16,6 +16,21 @@ DEFAULT_STORE_URI_BITS = { } +@dataclasses.dataclass +class DockerTarget: + registry_path: str + + def resolve(self, version: str) -> str: + """Applies templates: + - version: the Lix version + """ + return self.registry_path.format(version=version) + + def registry_name(self) -> str: + [a, _, _] = self.registry_path.partition('/') + return a + + @dataclasses.dataclass class RelengEnvironment: name: str @@ -26,22 +41,33 @@ class RelengEnvironment: docs_bucket: str git_repo: str + docker_targets: list[DockerTarget] + def cache_store_uri(self): qs = DEFAULT_STORE_URI_BITS.copy() qs.update(self.cache_store_overlay) return self.cache_bucket + "?" + urllib.parse.urlencode(qs) + STAGING = RelengEnvironment( name='staging', docs_bucket='s3://staging-docs', cache_bucket='s3://staging-cache', - cache_store_overlay={ - 'secret-key': 'staging.key' - }, + cache_store_overlay={'secret-key': 'staging.key'}, releases_bucket='s3://staging-releases', git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', + docker_targets=[ + DockerTarget( + 'git.lix.systems/lix-project/lix-releng-staging:{version}'), + DockerTarget( + 'ghcr.io/lix-project/lix-releng-staging:{version}'), + ], ) +ENVIRONMENTS = { + 'staging': STAGING, +} + @dataclasses.dataclass class S3Credentials: From 9bb7fb8f698d6ef4d42d068a576abd98d36f8239 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 7 Jun 2024 01:35:26 -0700 Subject: [PATCH 2/6] Rewrite docker to be sensible and smaller I have checked the image can build things and inspected `diff -ru` compared to the old image. As far as I can tell it is more or less the same besides the later git change. Layers are now 65MB or less, and we aren't against the maxLayers limit for the broken automatic layering to do anything but shove one store path in a layer (which is good behaviour, actually). This uses nix2container which streams images, so the build time is much shorter. I have also taken the opportunity to, in addition to fixing the 400MB single layer (terrible, and what motivated this in the first place), delete about 200MB of closure size inflicted by git vs gitMinimal causing both perl and python to get into closure. People mostly use this thing for CI, so I don't really think you need advanced git operations, and large git can be added at the user side if really motivated. With love for whichever container developer somewhat ironically assumed that one would not run skopeo in a minimal container that doesn't have a /var/tmp. Fixes: https://git.lix.systems/lix-project/lix/issues/378 Change-Id: Icc3aa20e64446276716fbbb87535fd5b50628010 --- docker.nix | 203 +++++++++++++++++++++++++++------------- flake.lock | 17 ++++ flake.nix | 23 +++-- releng/release-jobs.nix | 2 +- 4 files changed, 168 insertions(+), 77 deletions(-) diff --git a/docker.nix b/docker.nix index cec3a2950..a60fd5763 100644 --- a/docker.nix +++ b/docker.nix @@ -1,5 +1,6 @@ { pkgs ? import { }, + nix2container, lib ? pkgs.lib, name ? "lix", tag ? "latest", @@ -12,26 +13,51 @@ flake-registry ? null, }: let + layerContents = with pkgs; [ + # pulls in glibc and openssl, about 60MB + { contents = [ coreutils-full ]; } + # some stuff that is low in the closure graph and small ish, mostly to make + # incremental lix updates cheaper + { + contents = [ + curl + libxml2 + sqlite + ]; + } + # 50MB of git + { contents = [ gitMinimal ]; } + # 144MB of nixpkgs + { + contents = [ channel ]; + inProfile = false; + } + ]; + + # These packages are left to be auto layered by nix2container, since it is + # less critical that they get layered sensibly and they tend to not be deps + # of anything in particular + autoLayered = with pkgs; [ + bashInteractive + gnutar + gzip + gnugrep + which + less + wget + man + cacert.out + findutils + iana-etc + openssh + nix + ]; + defaultPkgs = - with pkgs; - [ - nix - bashInteractive - coreutils-full - gnutar - gzip - gnugrep - which - curl - less - wget - man - cacert.out - findutils - iana-etc - git - openssh - ] + lib.lists.flatten ( + map (x: if !(x ? inProfile) || x.inProfile then x.contents else [ ]) layerContents + ) + ++ autoLayered ++ extraPkgs; users = @@ -139,16 +165,17 @@ let )) + "\n"; + nixpkgs = pkgs.path; + channel = pkgs.runCommand "channel-nixpkgs" { } '' + mkdir $out + ${lib.optionalString bundleNixpkgs '' + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + ''} + ''; + baseSystem = let - nixpkgs = pkgs.path; - channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } '' - mkdir $out - if [ "$bundleNixpkgs" ]; then - ln -s ${nixpkgs} $out/nixpkgs - echo "[]" > $out/manifest.nix - fi - ''; rootEnv = pkgs.buildPackages.buildEnv { name = "root-profile-env"; paths = defaultPkgs; @@ -187,7 +214,7 @@ let profile = pkgs.buildPackages.runCommand "user-environment" { } '' mkdir $out cp -a ${rootEnv}/* $out/ - ln -s ${manifest} $out/manifest.nix + ln -sf ${manifest} $out/manifest.nix ''; flake-registry-path = if (flake-registry == null) then @@ -236,6 +263,7 @@ let ln -s /nix/var/nix/profiles/share $out/usr/ mkdir -p $out/nix/var/nix/gcroots + ln -s /nix/var/nix/profiles $out/nix/var/nix/gcroots/profiles mkdir $out/tmp @@ -248,14 +276,14 @@ let mkdir -p $out/nix/var/nix/profiles/per-user/root ln -s ${profile} $out/nix/var/nix/profiles/default-1-link - ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default ln -s /nix/var/nix/profiles/default $out/root/.nix-profile ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link - ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels + ln -s /nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels mkdir -p $out/root/.nix-defexpr - ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels + ln -s /nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels echo "${channelURL} ${channelName}" > $out/root/.nix-channels mkdir -p $out/bin $out/usr/bin @@ -273,43 +301,90 @@ let ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName '') ); -in -pkgs.dockerTools.buildLayeredImageWithNixDb { - inherit name tag maxLayers; + layers = builtins.foldl' ( + layersList: el: + let + layer = nix2container.buildLayer { + deps = el.contents; + layers = layersList; + }; + in + layersList ++ [ layer ] + ) [ ] layerContents; - contents = [ baseSystem ]; + image = nix2container.buildImage { - extraCommands = '' - rm -rf nix-support - ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles - ''; - fakeRootCommands = '' - chmod 1777 tmp - chmod 1777 var/tmp - ''; + inherit name tag maxLayers; - config = { - Cmd = [ "/root/.nix-profile/bin/bash" ]; - Env = [ - "USER=root" - "PATH=${ - lib.concatStringsSep ":" [ - "/root/.nix-profile/bin" - "/nix/var/nix/profiles/default/bin" - "/nix/var/nix/profiles/default/sbin" - ] - }" - "MANPATH=${ - lib.concatStringsSep ":" [ - "/root/.nix-profile/share/man" - "/nix/var/nix/profiles/default/share/man" - ] - }" - "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" - "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" - "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" - "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + inherit layers; + + copyToRoot = [ baseSystem ]; + + initializeNixDatabase = true; + + perms = [ + { + path = baseSystem; + regex = "(/var)?/tmp"; + mode = "1777"; + } ]; + + config = { + Cmd = [ "/root/.nix-profile/bin/bash" ]; + Env = [ + "USER=root" + "PATH=${ + lib.concatStringsSep ":" [ + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ] + }" + "MANPATH=${ + lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ] + }" + "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + ]; + }; + }; +in +image +// { + # We don't ship the tarball as the default output because it is a strange thing to want imo + tarball = + pkgs.buildPackages.runCommand "docker-image-tarball-${pkgs.nix.version}" + { + nativeBuildInputs = [ pkgs.buildPackages.bubblewrap ]; + meta.description = "Docker image tarball with Lix for ${pkgs.system}"; + } + '' + mkdir -p $out/nix-support + image=$out/image.tar + # bwrap for foolish temp dir selection code that forces /var/tmp: + # https://github.com/containers/skopeo.git/blob/60ee543f7f7c242f46cc3a7541d9ac8ab1c89168/vendor/github.com/containers/image/v5/internal/tmpdir/tmpdir.go#L15-L18 + mkdir -p $TMPDIR/fake-var/tmp + args=(--unshare-user --bind "$TMPDIR/fake-var" /var) + for dir in /*; do + args+=(--dev-bind "/$dir" "/$dir") + done + bwrap ''${args[@]} -- ${lib.getExe image.copyTo} docker-archive:$image + gzip $image + echo "file binary-dist $image" >> $out/nix-support/hydra-build-products + ''; + meta = image.meta // { + description = "Docker image for Lix. This is built with nix2container; see that project's README for details"; + longDescription = '' + Docker image for Lix, built with nix2container. + To copy it to your docker daemon, nix run .#dockerImage.copyToDockerDaemon + To copy it to podman, nix run .#dockerImage.copyTo containers-storage:lix + ''; }; } diff --git a/flake.lock b/flake.lock index 8e59b2942..436cbcd03 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,22 @@ "type": "github" } }, + "nix2container": { + "flake": false, + "locked": { + "lastModified": 1712990762, + "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1715123187, @@ -67,6 +83,7 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression", "pre-commit-hooks": "pre-commit-hooks" diff --git a/flake.nix b/flake.nix index 8460201b0..6c7abaeec 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,10 @@ url = "github:cachix/git-hooks.nix"; flake = false; }; + nix2container = { + url = "github:nlewo/nix2container"; + flake = false; + }; flake-compat = { url = "github:edolstra/flake-compat"; flake = false; @@ -20,6 +24,7 @@ nixpkgs, nixpkgs-regression, pre-commit-hooks, + nix2container, flake-compat, }: @@ -330,19 +335,13 @@ dockerImage = let pkgs = nixpkgsFor.${system}.native; - image = import ./docker.nix { - inherit pkgs; - tag = pkgs.nix.version; - }; + nix2container' = import nix2container { inherit pkgs system; }; in - pkgs.runCommand "docker-image-tarball-${pkgs.nix.version}" - { meta.description = "Docker image with Lix for ${system}"; } - '' - mkdir -p $out/nix-support - image=$out/image.tar.gz - ln -s ${image} $image - echo "file binary-dist $image" >> $out/nix-support/hydra-build-products - ''; + import ./docker.nix { + inherit pkgs; + nix2container = nix2container'.nix2container; + tag = pkgs.nix.version; + }; } // builtins.listToAttrs ( map (crossSystem: { diff --git a/releng/release-jobs.nix b/releng/release-jobs.nix index e693a3a81..4db0baed3 100644 --- a/releng/release-jobs.nix +++ b/releng/release-jobs.nix @@ -33,7 +33,7 @@ let targetName = "*.tar.xz"; }) systems ++ builtins.map (system: { - target = hydraJobs.dockerImage.${system}; + target = hydraJobs.dockerImage.${system}.tarball; targetName = "image.tar.gz"; rename = "lix-${lix.version}-docker-image-${system}.tar.gz"; }) dockerSystems; From 4392d89eeaf4560bf41e0c914b8f42f2959964d3 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sat, 8 Jun 2024 19:15:56 -0700 Subject: [PATCH 3/6] manual: rewrite the docker guide now that we have images Change-Id: I5bdf47e67059ae4099552750a47ae070dbe094df --- .../src/installation/installing-docker.md | 84 +++++++++---------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 2d7701024..3a5e6ef77 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -1,64 +1,62 @@ # Using Lix within Docker -Currently the Lix project doesn't ship docker images. However, we have the infrastructure to do it, it's just not yet been done. See https://git.lix.systems/lix-project/lix/issues/252 - - From 9aeb314e6a8276d3bd29f968c2baa44d5d19ca37 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sun, 9 Jun 2024 00:27:06 -0700 Subject: [PATCH 4/6] releng: support multiarch docker images If we don't want to have separate registry tags by architecture (EWWWW), we need to be able to build multiarch docker images. This is pretty simple, and just requires making a manifest pointing to each of the component images. I was *going* to just do this API prodding with manifest-tool, but it doesn't support putting metadata on the outer manifest, which is actually kind of a problem because it then doesn't render the metadata on github. So I guess we get a simple little containers API implementation that is 90% auth code. Change-Id: I8bdd118d4cbc13b23224f2fb174b232432686bea --- docker.nix | 27 ++- package.nix | 2 + releng/__init__.py | 18 ++ releng/create_release.xsh | 12 +- releng/docker.xsh | 69 ++++++- releng/docker_assemble.py | 399 ++++++++++++++++++++++++++++++++++++++ releng/environment.py | 26 ++- 7 files changed, 527 insertions(+), 26 deletions(-) create mode 100644 releng/docker_assemble.py diff --git a/docker.nix b/docker.nix index a60fd5763..b23a9637f 100644 --- a/docker.nix +++ b/docker.nix @@ -1,5 +1,7 @@ { pkgs ? import { }, + # Git commit ID, if available + lixRevision ? null, nix2container, lib ? pkgs.lib, name ? "lix", @@ -353,6 +355,23 @@ let "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" ]; + + Labels = { + "org.opencontainers.image.title" = "Lix"; + "org.opencontainers.image.source" = "https://git.lix.systems/lix-project/lix"; + "org.opencontainers.image.vendor" = "Lix project"; + "org.opencontainers.image.version" = pkgs.nix.version; + "org.opencontainers.image.description" = "Minimal Lix container image, with some batteries included."; + } // lib.optionalAttrs (lixRevision != null) { "org.opencontainers.image.revision" = lixRevision; }; + }; + + meta = { + description = "Docker image for Lix. This is built with nix2container; see that project's README for details"; + longDescription = '' + Docker image for Lix, built with nix2container. + To copy it to your docker daemon, nix run .#dockerImage.copyToDockerDaemon + To copy it to podman, nix run .#dockerImage.copyTo containers-storage:lix + ''; }; }; in @@ -379,12 +398,4 @@ image gzip $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - meta = image.meta // { - description = "Docker image for Lix. This is built with nix2container; see that project's README for details"; - longDescription = '' - Docker image for Lix, built with nix2container. - To copy it to your docker daemon, nix run .#dockerImage.copyToDockerDaemon - To copy it to podman, nix run .#dockerImage.copyTo containers-storage:lix - ''; - }; } diff --git a/package.nix b/package.nix index 1ac54dab6..6c0201c7f 100644 --- a/package.nix +++ b/package.nix @@ -417,6 +417,8 @@ stdenv.mkDerivation (finalAttrs: { p: [ p.yapf p.python-frontmatter + p.requests + p.xdg-base-dirs (p.toPythonModule xonsh-unwrapped) ] ); diff --git a/releng/__init__.py b/releng/__init__.py index 39d2beb51..fc23c52f3 100644 --- a/releng/__init__.py +++ b/releng/__init__.py @@ -2,12 +2,29 @@ from xonsh.main import setup setup() del setup +import logging + from . import environment from . import create_release from . import keys from . import version from . import cli from . import docker +from . import docker_assemble + +rootLogger = logging.getLogger() +rootLogger.setLevel(logging.DEBUG) +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +fmt = logging.Formatter('{asctime} {levelname} {name}: {message}', + datefmt='%b %d %H:%M:%S', + style='{') + +if not any(isinstance(h, logging.StreamHandler) for h in rootLogger.handlers): + hand = logging.StreamHandler() + hand.setFormatter(fmt) + rootLogger.addHandler(hand) def reload(): import importlib @@ -17,3 +34,4 @@ def reload(): importlib.reload(version) importlib.reload(cli) importlib.reload(docker) + importlib.reload(docker_assemble) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index 128edb63f..6f4df2142 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -258,15 +258,16 @@ def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False 'I want to release this' ) + docker_images = list((ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz')) + assert docker_images + print('[+] Upload to cache') with open(DRVS_TXT) as fh: upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x]) - docker_images = (ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz') print('[+] Upload docker images') - for image in docker_images: - for target in env.docker_targets: - docker.upload_docker_image(target, image) + for target in env.docker_targets: + docker.upload_docker_images(target, docker_images) print('[+] Upload to release bucket') aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/ @@ -321,7 +322,8 @@ def build_artifacts(no_check_git=False): build_manual(eval_result) with open(DRVS_TXT, 'w') as fh: - fh.write('\n'.join(drv_paths)) + # don't bother putting the release tarballs themselves because they are duplicate and huge + fh.write('\n'.join(x['drvPath'] for x in eval_result if x['attr'] != 'lix-release-tarballs')) make_artifacts_dir(eval_result, ARTIFACTS) print(f'[+] Done! See {ARTIFACTS}') diff --git a/releng/docker.xsh b/releng/docker.xsh index 1ed2330cf..f45a69d27 100644 --- a/releng/docker.xsh +++ b/releng/docker.xsh @@ -1,6 +1,18 @@ -from .environment import DockerTarget, RelengEnvironment -from .version import VERSION +import json +import logging from pathlib import Path +import tempfile + +import requests + +from .environment import DockerTarget, RelengEnvironment +from .version import VERSION, MAJOR +from . import gitutils +from .docker_assemble import Registry, OCIIndex, OCIIndexItem +from . import docker_assemble + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) def check_all_logins(env: RelengEnvironment): for target in env.docker_targets: @@ -9,5 +21,54 @@ def check_all_logins(env: RelengEnvironment): def check_login(target: DockerTarget): skopeo login @(target.registry_name()) -def upload_docker_image(target: DockerTarget, path: Path): - skopeo --insecure-policy copy docker-archive:@(path) docker://@(target.resolve(version=VERSION)) +def upload_docker_images(target: DockerTarget, paths: list[Path]): + if not paths: return + + sess = requests.Session() + sess.headers['User-Agent'] = 'lix-releng' + + tag_names = [DockerTarget.resolve(tag, version=VERSION, major=MAJOR) for tag in target.tags] + + # latest only gets tagged for the current release branch of Lix + if not gitutils.is_maintenance_branch('HEAD'): + tag_names.append('latest') + + meta = {} + + reg = docker_assemble.Registry(sess) + manifests = [] + + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + + for path in paths: + digest_file = tmp / (path.name + '.digest') + inspection = json.loads($(skopeo inspect docker-archive:@(path))) + + docker_arch = inspection['Architecture'] + docker_os = inspection['Os'] + meta = inspection['Labels'] + + log.info('Pushing image %s for %s', path, docker_arch) + + # insecure-policy: we don't have any signature policy, we are just uploading an image + # We upload to a junk tag, because otherwise it will upload to `latest`, which is undesirable + skopeo --insecure-policy copy --format oci --digestfile @(digest_file) docker-archive:@(path) docker://@(target.registry_path):temp + + digest = digest_file.read_text().strip() + + # skopeo doesn't give us the manifest size directly, so we just ask the registry + metadata = reg.image_info(target.registry_path, digest) + + manifests.append(OCIIndexItem(metadata=metadata, architecture=docker_arch, os=docker_os)) + # delete the temp tag, which we only have to create because of skopeo + # limitations anyhow (it seems to not have a way to say "don't tag it, find + # your checksum and put it there") + # FIXME: this is not possible because GitHub only has a proprietary API for it. amazing. 11/10. + # reg.delete_tag(target.registry_path, 'temp') + + log.info('Pushed images, building a bigger and more menacing manifest from %r with metadata %r', manifests, meta) + # send the multiarch manifest to each tag + index = OCIIndex(manifests=manifests, annotations=meta) + for tag in tag_names: + reg.upload_index(target.registry_path, tag, index) diff --git a/releng/docker_assemble.py b/releng/docker_assemble.py new file mode 100644 index 000000000..ef1d8c4e6 --- /dev/null +++ b/releng/docker_assemble.py @@ -0,0 +1,399 @@ +from typing import Any, Literal, Optional +import re +from pathlib import Path +import json +import dataclasses +import time +from urllib.parse import unquote +import urllib.request +import logging + +import requests.auth +import requests +import xdg_base_dirs + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + +DEBUG_REQUESTS = False +if DEBUG_REQUESTS: + urllib3_logger = logging.getLogger('requests.packages.urllib3') + urllib3_logger.setLevel(logging.DEBUG) + urllib3_logger.propagate = True + +# So, there is a bunch of confusing stuff happening in this file. The gist of why it's Like This is: +# +# nix2container does not concern itself with tags (reasonably enough): +# https://github.com/nlewo/nix2container/issues/59 +# +# This is fine. But then we noticed: docker images don't play nice if you have +# multiple architectures you want to abstract over if you don't do special +# things. Those special things are images with manifests containing multiple +# images. +# +# Docker has a data model vaguely analogous to git: you have higher level +# objects referring to a bunch of content-addressed blobs. +# +# A multiarch image is more or less just a manifest that refers to more +# manifests; in OCI it is an Index. +# +# See the API spec here: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#definitions +# And the Index spec here: https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md +# +# skopeo doesn't *know* how to make multiarch *manifests*: +# https://github.com/containers/skopeo/issues/1136 +# +# There is a tool called manifest-tool that is supposed to do this +# (https://github.com/estesp/manifest-tool) but it doesn't support putting in +# annotations on the outer image, and I *really* didn't want to write golang to +# fix that. Thus, a little bit of homebrew containers code. +# +# Essentially what we are doing in here is splatting a bunch of images into the +# registry without tagging them (except as "temp", due to podman issues), then +# simply sending a new composite manifest ourselves. + +DockerArchitecture = Literal['amd64'] | Literal['arm64'] +MANIFEST_MIME = 'application/vnd.oci.image.manifest.v1+json' +INDEX_MIME = 'application/vnd.oci.image.index.v1+json' + + +@dataclasses.dataclass(frozen=True, order=True) +class ImageMetadata: + size: int + digest: str + """sha256:SOMEHEX""" + + +@dataclasses.dataclass(frozen=True, order=True) +class OCIIndexItem: + """Information about an untagged uploaded image.""" + + metadata: ImageMetadata + + architecture: DockerArchitecture + + os: str = 'linux' + + def serialize(self): + return { + 'mediaType': MANIFEST_MIME, + 'size': self.metadata.size, + 'digest': self.metadata.digest, + 'platform': { + 'architecture': self.architecture, + 'os': self.os, + } + } + + +@dataclasses.dataclass(frozen=True) +class OCIIndex: + manifests: list[OCIIndexItem] + + annotations: dict[str, str] + + def serialize(self): + return { + 'schemaVersion': 2, + 'manifests': [item.serialize() for item in sorted(self.manifests)], + 'annotations': self.annotations + } + + +def docker_architecture_from_nix_system(system: str) -> DockerArchitecture: + MAP = { + 'x86_64-linux': 'amd64', + 'aarch64-linux': 'arm64', + } + return MAP[system] # type: ignore + + +@dataclasses.dataclass +class TaggingOperation: + manifest: OCIIndex + tags: list[str] + """Tags this image is uploaded under""" + + +runtime_dir = xdg_base_dirs.xdg_runtime_dir() +config_dir = xdg_base_dirs.xdg_config_home() + +AUTH_FILES = ([runtime_dir / 'containers/auth.json'] if runtime_dir else []) + \ + [config_dir / 'containers/auth.json', Path.home() / '.docker/config.json'] + + +# Copied from Werkzeug https://github.com/pallets/werkzeug/blob/62e3ea45846d06576199a2f8470be7fe44c867c1/src/werkzeug/http.py#L300-L325 +def parse_list_header(value: str) -> list[str]: + """Parse a header value that consists of a list of comma separated items according + to `RFC 9110 `__. + + This extends :func:`urllib.request.parse_http_list` to remove surrounding quotes + from values. + + .. code-block:: python + + parse_list_header('token, "quoted value"') + ['token', 'quoted value'] + + This is the reverse of :func:`dump_header`. + + :param value: The header value to parse. + """ + result = [] + + for item in urllib.request.parse_http_list(value): + if len(item) >= 2 and item[0] == item[-1] == '"': + item = item[1:-1] + + result.append(item) + + return result + + +# https://www.rfc-editor.org/rfc/rfc2231#section-4 +_charset_value_re = re.compile( + r""" + ([\w!#$%&*+\-.^`|~]*)' # charset part, could be empty + [\w!#$%&*+\-.^`|~]*' # don't care about language part, usually empty + ([\w!#$%&'*+\-.^`|~]+) # one or more token chars with percent encoding + """, + re.ASCII | re.VERBOSE, +) + + +# Copied from: https://github.com/pallets/werkzeug/blob/62e3ea45846d06576199a2f8470be7fe44c867c1/src/werkzeug/http.py#L327-L394 +def parse_dict_header(value: str) -> dict[str, str | None]: + """Parse a list header using :func:`parse_list_header`, then parse each item as a + ``key=value`` pair. + + .. code-block:: python + + parse_dict_header('a=b, c="d, e", f') + {"a": "b", "c": "d, e", "f": None} + + This is the reverse of :func:`dump_header`. + + If a key does not have a value, it is ``None``. + + This handles charsets for values as described in + `RFC 2231 `__. Only ASCII, UTF-8, + and ISO-8859-1 charsets are accepted, otherwise the value remains quoted. + + :param value: The header value to parse. + + .. versionchanged:: 3.0 + Passing bytes is not supported. + + .. versionchanged:: 3.0 + The ``cls`` argument is removed. + + .. versionchanged:: 2.3 + Added support for ``key*=charset''value`` encoded items. + + .. versionchanged:: 0.9 + The ``cls`` argument was added. + """ + result: dict[str, str | None] = {} + + for item in parse_list_header(value): + key, has_value, value = item.partition("=") + key = key.strip() + + if not has_value: + result[key] = None + continue + + value = value.strip() + encoding: str | None = None + + if key[-1] == "*": + # key*=charset''value becomes key=value, where value is percent encoded + # adapted from parse_options_header, without the continuation handling + key = key[:-1] + match = _charset_value_re.match(value) + + if match: + # If there is a charset marker in the value, split it off. + encoding, value = match.groups() + assert encoding + encoding = encoding.lower() + + # A safe list of encodings. Modern clients should only send ASCII or UTF-8. + # This list will not be extended further. An invalid encoding will leave the + # value quoted. + if encoding in {"ascii", "us-ascii", "utf-8", "iso-8859-1"}: + # invalid bytes are replaced during unquoting + value = unquote(value, encoding=encoding) + + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + + result[key] = value + + return result + + +def parse_www_authenticate(www_authenticate): + scheme, _, rest = www_authenticate.partition(' ') + scheme = scheme.lower() + rest = rest.strip() + + parsed = parse_dict_header(rest.rstrip('=')) + return parsed + + +class AuthState: + + def __init__(self, auth_files: list[Path] = AUTH_FILES): + self.auth_map: dict[str, str] = {} + for f in auth_files: + self.auth_map.update(AuthState.load_auth_file(f)) + self.token_cache: dict[str, str] = {} + + @staticmethod + def load_auth_file(path: Path) -> dict[str, str]: + if path.exists(): + with path.open() as fh: + try: + json_obj = json.load(fh) + return {k: v['auth'] for k, v in json_obj['auths'].items()} + except (json.JSONDecodeError, KeyError) as e: + log.exception('JSON decode error in %s', path, exc_info=e) + return {} + + def get_token(self, hostname: str) -> Optional[str]: + return self.token_cache.get(hostname) + + def obtain_token(self, session: requests.Session, token_endpoint: str, + scope: str, service: str, image_path: str) -> str: + authority, _, _ = image_path.partition('/') + if tok := self.get_token(authority): + return tok + + creds = self.find_credential_for(image_path) + if not creds: + raise ValueError('No credentials available for ' + image_path) + + resp = session.get(token_endpoint, + params={ + 'client_id': 'lix-releng', + 'scope': scope, + 'service': service, + }, + headers={ + 'Authorization': 'Basic ' + creds + }).json() + token = resp['token'] + self.token_cache[service] = token + return token + + def find_credential_for(self, image_path: str): + trails = image_path.split('/') + for i in range(len(trails)): + prefix = '/'.join(trails[:len(trails) - i]) + if prefix in self.auth_map: + return self.auth_map[prefix] + + return None + + +class RegistryAuthenticator(requests.auth.AuthBase): + """Authenticates to an OCI compliant registry""" + + def __init__(self, auth_state: AuthState, session: requests.Session, + image: str): + self.auth_map: dict[str, str] = {} + self.image = image + self.session = session + self.auth_state = auth_state + + def response_hook(self, r: requests.Response, + **kwargs: Any) -> requests.Response: + if r.status_code == 401: + www_authenticate = r.headers.get('www-authenticate', '').lower() + parsed = parse_www_authenticate(www_authenticate) + assert parsed + + tok = self.auth_state.obtain_token( + self.session, + parsed['realm'], # type: ignore + parsed['scope'], # type: ignore + parsed['service'], # type: ignore + self.image) + + new_req = r.request.copy() + new_req.headers['Authorization'] = 'Bearer ' + tok + + return self.session.send(new_req) + else: + return r + + def __call__(self, + r: requests.PreparedRequest) -> requests.PreparedRequest: + authority, _, _ = self.image.partition('/') + auth_may = self.auth_state.get_token(authority) + + if auth_may: + r.headers['Authorization'] = 'Bearer ' + auth_may + + r.register_hook('response', self.response_hook) + return r + + +class Registry: + + def __init__(self, session: requests.Session): + self.auth_state = AuthState() + self.session = session + + def image_info(self, image_path: str, manifest_id: str) -> ImageMetadata: + authority, _, path = image_path.partition('/') + resp = self.session.head( + f'https://{authority}/v2/{path}/manifests/{manifest_id}', + headers={'Accept': MANIFEST_MIME}, + auth=RegistryAuthenticator(self.auth_state, self.session, + image_path)) + resp.raise_for_status() + return ImageMetadata(int(resp.headers['content-length']), + resp.headers['docker-content-digest']) + + def delete_tag(self, image_path: str, tag: str): + authority, _, path = image_path.partition('/') + resp = self.session.delete( + f'https://{authority}/v2/{path}/manifests/{tag}', + headers={'Content-Type': INDEX_MIME}, + auth=RegistryAuthenticator(self.auth_state, self.session, + image_path)) + resp.raise_for_status() + + def _upload_index(self, image_path: str, tag: str, index: OCIIndex): + authority, _, path = image_path.partition('/') + body = json.dumps(index.serialize(), + separators=(',', ':'), + sort_keys=True) + + resp = self.session.put( + f'https://{authority}/v2/{path}/manifests/{tag}', + data=body, + headers={'Content-Type': INDEX_MIME}, + auth=RegistryAuthenticator(self.auth_state, self.session, + image_path)) + resp.raise_for_status() + + return resp.headers['Location'] + + def upload_index(self, + image_path: str, + tag: str, + index: OCIIndex, + retries=20, + retry_delay=1): + # eventual consistency lmao + for _ in range(retries): + try: + return self._upload_index(image_path, tag, index) + except requests.HTTPError as e: + if e.response.status_code != 404: + raise + + time.sleep(retry_delay) diff --git a/releng/environment.py b/releng/environment.py index e8e7e771e..3d65b2799 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -1,5 +1,5 @@ -import dataclasses import urllib.parse +import dataclasses S3_HOST = 's3.lix.systems' S3_ENDPOINT = 'https://s3.lix.systems' @@ -19,12 +19,19 @@ DEFAULT_STORE_URI_BITS = { @dataclasses.dataclass class DockerTarget: registry_path: str + """Registry path without the tag, e.g. ghcr.io/lix-project/lix""" - def resolve(self, version: str) -> str: - """Applies templates: - - version: the Lix version + tags: list[str] + """List of tags this image should take. There must be at least one.""" + + @staticmethod + def resolve(item: str, version: str, major: str) -> str: """ - return self.registry_path.format(version=version) + Applies templates: + - version: the Lix version e.g. 2.90.0 + - major: the major Lix version e.g. 2.90 + """ + return item.format(version=version, major=major) def registry_name(self) -> str: [a, _, _] = self.registry_path.partition('/') @@ -57,10 +64,11 @@ STAGING = RelengEnvironment( releases_bucket='s3://staging-releases', git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', docker_targets=[ - DockerTarget( - 'git.lix.systems/lix-project/lix-releng-staging:{version}'), - DockerTarget( - 'ghcr.io/lix-project/lix-releng-staging:{version}'), + # FIXME: how do we make sure that latest gets the latest of the *most recent* branch? + DockerTarget('git.lix.systems/lix-project/lix-releng-staging', + tags=['{version}', '{major}']), + DockerTarget('ghcr.io/lix-project/lix-releng-staging', + tags=['{version}', '{major}']), ], ) From ce71d0e9abe347147f50fba4bf11eac97b2a90ef Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sun, 9 Jun 2024 00:50:40 -0700 Subject: [PATCH 5/6] releng: automatically figure out if we should tag latest for docker For example, when releasing from release-2.90, if `main` has a 2.91 tag ancestor, we know that 2.91 was released, so we should *not* tag latest. Change-Id: Ia56b17a2ee03bbec74b7c271c742858c690d450d --- releng/__init__.py | 2 ++ releng/create_release.xsh | 15 +-------------- releng/environment.py | 2 +- releng/gitutils.xsh | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 releng/gitutils.xsh diff --git a/releng/__init__.py b/releng/__init__.py index fc23c52f3..59d1709b9 100644 --- a/releng/__init__.py +++ b/releng/__init__.py @@ -11,6 +11,7 @@ from . import version from . import cli from . import docker from . import docker_assemble +from . import gitutils rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG) @@ -35,3 +36,4 @@ def reload(): importlib.reload(cli) importlib.reload(docker) importlib.reload(docker_assemble) + importlib.reload(gitutils) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index 6f4df2142..373c079bc 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -11,6 +11,7 @@ from .environment import RelengEnvironment from . import keys from . import docker from .version import VERSION, RELEASE_NAME, MAJOR +from .gitutils import verify_are_on_tag, git_preconditions $RAISE_SUBPROC_ERROR = True $XONSH_SHOW_TRACEBACK = True @@ -38,15 +39,6 @@ def setup_creds(env: RelengEnvironment): $AWS_ENDPOINT_URL = environment.S3_ENDPOINT -def git_preconditions(): - # verify there is nothing in index ready to stage - proc = !(git diff-index --quiet --cached HEAD --) - assert proc.rtn == 0 - # verify there is nothing *stageable* and tracked - proc = !(git diff-files --quiet) - assert proc.rtn == 0 - - def official_release_commit_tag(force_tag=False): print('[+] Setting officialRelease in flake.nix and tagging') prev_branch = $(git symbolic-ref --short HEAD).strip() @@ -240,11 +232,6 @@ def prepare_release_notes(): git commit -m @(commit_msg) -def verify_are_on_tag(): - current_tag = $(git describe --tag).strip() - assert current_tag == VERSION - - def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False): if not no_check_git: verify_are_on_tag() diff --git a/releng/environment.py b/releng/environment.py index 3d65b2799..b2278491a 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -64,7 +64,7 @@ STAGING = RelengEnvironment( releases_bucket='s3://staging-releases', git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', docker_targets=[ - # FIXME: how do we make sure that latest gets the latest of the *most recent* branch? + # latest will be auto tagged if appropriate DockerTarget('git.lix.systems/lix-project/lix-releng-staging', tags=['{version}', '{major}']), DockerTarget('ghcr.io/lix-project/lix-releng-staging', diff --git a/releng/gitutils.xsh b/releng/gitutils.xsh new file mode 100644 index 000000000..3352a6b21 --- /dev/null +++ b/releng/gitutils.xsh @@ -0,0 +1,37 @@ +import subprocess +import json + + +def version_compare(v1: str, v2: str): + return json.loads($(nix-instantiate --eval --json --argstr v1 @(v1) --argstr v2 @(v2) --expr '{v1, v2}: builtins.compareVersions v1 v2')) + + +def latest_tag_on_branch(branch: str) -> str: + return $(git describe --abbrev=0 @(branch) e>/dev/null).strip() + + +def is_maintenance_branch(branch: str) -> bool: + try: + main_tag = latest_tag_on_branch('main') + current_tag = latest_tag_on_branch(branch) + + return version_compare(current_tag, main_tag) < 0 + except subprocess.CalledProcessError: + # This is the case before Lix releases 2.90, since main *has* no + # release tag on it. + # FIXME: delete this case after 2.91 + return False + + +def verify_are_on_tag(): + current_tag = $(git describe --tag).strip() + assert current_tag == VERSION + + +def git_preconditions(): + # verify there is nothing in index ready to stage + proc = !(git diff-index --quiet --cached HEAD --) + assert proc.rtn == 0 + # verify there is nothing *stageable* and tracked + proc = !(git diff-files --quiet) + assert proc.rtn == 0 From 82dc712d9312f06e653213d030b6db21529a422f Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sun, 9 Jun 2024 01:26:21 -0700 Subject: [PATCH 6/6] releng: add prod environment, ready for release I am *reasonably* confident that this releng infrastructure can actually build a Lix 2.90 and release it successfully. Let's make it possible to do, and add some cute colours to the confirmation message. Change-Id: I85e498b6fb49ffc5e75c0a72c5e45fb1f69030d3 --- releng/create_release.xsh | 5 ++-- releng/docker.xsh | 4 ++-- releng/environment.py | 50 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index 373c079bc..94c78a83f 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -240,9 +240,10 @@ def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False tree @(ARTIFACTS) + env_part = f'environment {env.name}' not noconfirm and confirm( - f'Would you like to release {ARTIFACTS} as {VERSION}? Type "I want to release this" to confirm\n', - 'I want to release this' + f'Would you like to release {ARTIFACTS} as {VERSION} in {env.colour(env_part)}? Type "I want to release this to {env.name}" to confirm\n', + f'I want to release this to {env.name}' ) docker_images = list((ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz')) diff --git a/releng/docker.xsh b/releng/docker.xsh index f45a69d27..20fb30cd3 100644 --- a/releng/docker.xsh +++ b/releng/docker.xsh @@ -49,7 +49,7 @@ def upload_docker_images(target: DockerTarget, paths: list[Path]): docker_os = inspection['Os'] meta = inspection['Labels'] - log.info('Pushing image %s for %s', path, docker_arch) + log.info('Pushing image %s for %s to %s', path, docker_arch, target.registry_path) # insecure-policy: we don't have any signature policy, we are just uploading an image # We upload to a junk tag, because otherwise it will upload to `latest`, which is undesirable @@ -67,7 +67,7 @@ def upload_docker_images(target: DockerTarget, paths: list[Path]): # FIXME: this is not possible because GitHub only has a proprietary API for it. amazing. 11/10. # reg.delete_tag(target.registry_path, 'temp') - log.info('Pushed images, building a bigger and more menacing manifest from %r with metadata %r', manifests, meta) + log.info('Pushed images to %r, building a bigger and more menacing manifest from %r with metadata %r', target, manifests, meta) # send the multiarch manifest to each tag index = OCIIndex(manifests=manifests, annotations=meta) for tag in tag_names: diff --git a/releng/environment.py b/releng/environment.py index b2278491a..ca8194fe5 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -1,4 +1,8 @@ +from typing import Callable import urllib.parse +import re +import functools +import subprocess import dataclasses S3_HOST = 's3.lix.systems' @@ -41,6 +45,7 @@ class DockerTarget: @dataclasses.dataclass class RelengEnvironment: name: str + colour: Callable[[str], str] cache_store_overlay: dict[str, str] cache_bucket: str @@ -56,8 +61,19 @@ class RelengEnvironment: return self.cache_bucket + "?" + urllib.parse.urlencode(qs) +SGR = '\x1b[' +RED = '31;1m' +GREEN = '32;1m' +RESET = '0m' + + +def sgr(colour: str, text: str) -> str: + return f'{SGR}{colour}{text}{SGR}{RESET}' + + STAGING = RelengEnvironment( name='staging', + colour=functools.partial(sgr, GREEN), docs_bucket='s3://staging-docs', cache_bucket='s3://staging-cache', cache_store_overlay={'secret-key': 'staging.key'}, @@ -72,8 +88,42 @@ STAGING = RelengEnvironment( ], ) +GERRIT_REMOTE_RE = re.compile(r'^ssh://(\w+@)?gerrit.lix.systems:2022/lix$') + + +def guess_gerrit_remote(): + """ + Deals with people having unknown gerrit username. + """ + out = [ + x.split()[1] for x in subprocess.check_output( + ['git', 'remote', '-v']).decode().splitlines() + ] + return next(x for x in out if GERRIT_REMOTE_RE.match(x)) + + +PROD = RelengEnvironment( + name='production', + colour=functools.partial(sgr, RED), + docs_bucket='s3://docs', + cache_bucket='s3://cache', + # FIXME: we should decrypt this with age into a tempdir in the future, but + # the issue is how to deal with the recipients file. For now, we should + # just delete it after doing a release. + cache_store_overlay={'secret-key': 'prod.key'}, + releases_bucket='s3://releases', + git_repo=guess_gerrit_remote(), + docker_targets=[ + # latest will be auto tagged if appropriate + DockerTarget('git.lix.systems/lix-project/lix', + tags=['{version}', '{major}']), + DockerTarget('ghcr.io/lix-project/lix', tags=['{version}', '{major}']), + ], +) + ENVIRONMENTS = { 'staging': STAGING, + 'production': PROD, }