From a7c271d78ccb6127472647cf2501ac0b5bf37825 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 6 Jun 2024 22:28:49 -0700 Subject: [PATCH] yeet Change-Id: I3c5b6c1f3ba0b9dfcac212b2148f390e0cd542b7 --- package.nix | 3 +++ releng/__init__.py | 2 ++ releng/cli.py | 21 ++++++++++++++++----- releng/create_release.xsh | 30 ++++++++++++------------------ releng/environment.py | 30 +++++++++++++++++++++++++++--- 5 files changed, 60 insertions(+), 26 deletions(-) diff --git a/package.nix b/package.nix index 9c108308d..1ac54dab6 100644 --- a/package.nix +++ b/package.nix @@ -39,6 +39,7 @@ pkg-config, python3, rapidcheck, + skopeo, sqlite, toml11, util-linuxMinimal ? utillinuxMinimal, @@ -447,6 +448,8 @@ stdenv.mkDerivation (finalAttrs: { lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm ++ [ pythonEnv + # docker image tool + skopeo just nixfmt # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above. diff --git a/releng/__init__.py b/releng/__init__.py index 401b8e322..f828fe907 100644 --- a/releng/__init__.py +++ b/releng/__init__.py @@ -7,6 +7,7 @@ from releng import create_release from releng import keys from releng import version from releng import cli +from releng import docker def reload(): import importlib @@ -15,3 +16,4 @@ def reload(): importlib.reload(keys) importlib.reload(version) importlib.reload(cli) + importlib.reload(docker) diff --git a/releng/cli.py b/releng/cli.py index bba50f534..b2d1b4965 100644 --- a/releng/cli.py +++ b/releng/cli.py @@ -1,4 +1,8 @@ from . import create_release +from . import docker +from .environment import RelengEnvironment +from . import environment +import functools import argparse import sys @@ -18,13 +22,15 @@ def do_tag(args): no_check_git=args.no_check_git) -def do_upload(args): - create_release.setup_creds() +def do_upload(env: RelengEnvironment, args): + create_release.setup_creds(env) if args.target == 'all': - create_release.upload_artifacts(force_push_tag=args.force_push_tag, + docker.check_all_logins(env) + create_release.upload_artifacts(env, + force_push_tag=args.force_push_tag, noconfirm=args.noconfirm) elif args.target == 'manual': - create_release.upload_manual() + create_release.upload_manual(env) else: raise ValueError('invalid target, unreachable') @@ -90,7 +96,12 @@ def main(): '--noconfirm', action='store_true', help="Don't ask for confirmation. For testing/automation.") - upload.set_defaults(cmd=do_upload) + upload.add_argument('--environment', + choices=list(environment.ENVIRONMENTS.keys()), + default='staging', + help='Environment to release to') + upload.set_defaults(cmd=lambda args: do_upload( + environment.ENVIRONMENTS[args.environment], args)) args = ap.parse_args() args.cmd(args) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index c57a92b2f..604b4a7ef 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -7,19 +7,13 @@ import tempfile import hashlib import datetime from . import environment +from .environment import RelengEnvironment from . import keys from .version import VERSION, RELEASE_NAME, MAJOR $RAISE_SUBPROC_ERROR = True $XONSH_SHOW_TRACEBACK = True -RELENG_ENV = environment.STAGING - -RELEASES_BUCKET = RELENG_ENV.releases_bucket -DOCS_BUCKET = RELENG_ENV.docs_bucket -CACHE_STORE = RELENG_ENV.cache_store_uri() -REPO = RELENG_ENV.git_repo - GCROOTS_DIR = Path('./release/gcroots') BUILT_GCROOTS_DIR = Path('./release/gcroots-build') DRVS_TXT = Path('./release/drvs.txt') @@ -35,8 +29,8 @@ MAX_JOBS = 2 RELEASE_SYSTEMS = ["x86_64-linux"] -def setup_creds(): - key = keys.get_ephemeral_key(RELENG_ENV) +def setup_creds(env: RelengEnvironment): + key = keys.get_ephemeral_key(env) $AWS_SECRET_ACCESS_KEY = key.secret_key $AWS_ACCESS_KEY_ID = key.id $AWS_DEFAULT_REGION = 'garage' @@ -102,13 +96,13 @@ def eval_jobs(): ] -def upload_drv_paths_and_outputs(paths: list[str]): +def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]): proc = subprocess.Popen([ 'nix', 'copy', '-v', '--to', - CACHE_STORE, + env.cache_store_uri(), '--stdin', ], stdin=subprocess.PIPE, @@ -250,7 +244,7 @@ def verify_are_on_tag(): assert current_tag == VERSION -def upload_artifacts(noconfirm=False, force_push_tag=False): +def upload_artifacts(env: RelengEnvironment, noconfirm=False, force_push_tag=False): assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env tree @(ARTIFACTS) @@ -262,16 +256,16 @@ def upload_artifacts(noconfirm=False, force_push_tag=False): print('[+] Upload to cache') with open(DRVS_TXT) as fh: - upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x]) + upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x]) print('[+] Upload to release bucket') - aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/ + aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/ print('[+] Upload manual') upload_manual() print('[+] git push tag') - git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}' + git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' def do_tag_merge(force_tag=False, no_check_git=False): @@ -290,7 +284,7 @@ def build_manual(eval_result): cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL) -def upload_manual(): +def upload_manual(env: RelengEnvironment): stable = json.loads($(nix eval --json '.#nix.officialRelease')) if stable: version = MAJOR @@ -298,9 +292,9 @@ def upload_manual(): version = 'nightly' print('[+] aws s3 sync manual') - aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/ + aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/ if stable: - aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/ + aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/ def build_artifacts(no_check_git=False): diff --git a/releng/environment.py b/releng/environment.py index 58633d548..6e352fabc 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -16,6 +16,21 @@ DEFAULT_STORE_URI_BITS = { } +@dataclasses.dataclass +class DockerTarget: + registry_path: str + + def resolve(self, version: str) -> str: + """Applies templates: + - version: the Lix version + """ + return self.registry_path.format(version=version) + + def registry_name(self) -> str: + [a, _, _] = self.registry_path.partition('/') + return a + + @dataclasses.dataclass class RelengEnvironment: name: str @@ -26,22 +41,31 @@ class RelengEnvironment: docs_bucket: str git_repo: str + docker_targets: list[DockerTarget] + def cache_store_uri(self): qs = DEFAULT_STORE_URI_BITS.copy() qs.update(self.cache_store_overlay) return self.cache_bucket + "?" + urllib.parse.urlencode(qs) + STAGING = RelengEnvironment( name='staging', docs_bucket='s3://staging-docs', cache_bucket='s3://staging-cache', - cache_store_overlay={ - 'secret-key': 'staging.key' - }, + cache_store_overlay={'secret-key': 'staging.key'}, releases_bucket='s3://staging-releases', git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', + docker_targets=[ + DockerTarget( + 'git.lix.systems/lix-project/lix-releng-staging/lix:{version}'), + ], ) +ENVIRONMENTS = { + 'staging': STAGING, +} + @dataclasses.dataclass class S3Credentials: