releng: support multiple systems

I guess this is kind of important to being able to "release it".

Change-Id: Id6f295d0b4944fa1203783a400a246727dbd94b6
This commit is contained in:
jade 2024-06-13 14:03:27 -07:00
parent ac28cff28f
commit 74fb2e8c47
4 changed files with 55 additions and 32 deletions

View file

@ -2,16 +2,16 @@ from . import create_release
from . import docker from . import docker
from .environment import RelengEnvironment from .environment import RelengEnvironment
from . import environment from . import environment
import functools
import argparse import argparse
import sys import sys
def do_build(args): def do_build(args):
if args.target == 'all': if args.target == 'all':
create_release.build_artifacts(no_check_git=args.no_check_git) create_release.build_artifacts(args.profile, no_check_git=args.no_check_git)
elif args.target == 'manual': elif args.target == 'manual':
eval_result = create_release.eval_jobs() # n.b. args.profile does nothing here, you will just get the x86_64-linux manual no matter what.
eval_result = create_release.eval_jobs(args.profile)
create_release.build_manual(eval_result) create_release.build_manual(eval_result)
else: else:
raise ValueError('invalid target, unreachable') raise ValueError('invalid target, unreachable')
@ -80,6 +80,10 @@ def main():
build.add_argument('--target', build.add_argument('--target',
choices=['manual', 'all'], choices=['manual', 'all'],
help='Whether to build everything or just the manual') help='Whether to build everything or just the manual')
build.add_argument('--profile',
default='all',
choices=('all', 'x86_64-linux-only'),
help='Which systems to build targets for.')
build.set_defaults(cmd=do_build) build.set_defaults(cmd=do_build)
upload = sps.add_parser( upload = sps.add_parser(

View file

@ -27,9 +27,6 @@ RELENG_MSG = "Release created with releng/create_release.xsh"
BUILD_CORES = 16 BUILD_CORES = 16
MAX_JOBS = 2 MAX_JOBS = 2
# TODO
RELEASE_SYSTEMS = ["x86_64-linux"]
def setup_creds(env: RelengEnvironment): def setup_creds(env: RelengEnvironment):
key = keys.get_ephemeral_key(env) key = keys.get_ephemeral_key(env)
@ -82,11 +79,9 @@ def realise(paths: list[str]):
nix-store @(args) @(paths) nix-store @(args) @(paths)
def eval_jobs(): def eval_jobs(build_profile):
nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake '.#release-jobs') nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake f'.#release-jobs.{build_profile}')
return [x for x in (json.loads(s) for s in nej_output.strip().split('\n')) return [json.loads(s) for s in nej_output.strip().split('\n')]
if x['system'] in RELEASE_SYSTEMS
]
def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]): def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
@ -295,14 +290,14 @@ def upload_manual(env: RelengEnvironment):
aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/ aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/
def build_artifacts(no_check_git=False): def build_artifacts(build_profile, no_check_git=False):
rm -rf release/ rm -rf release/
if not no_check_git: if not no_check_git:
verify_are_on_tag() verify_are_on_tag()
git_preconditions() git_preconditions()
print('[+] Evaluating') print('[+] Evaluating')
eval_result = eval_jobs() eval_result = eval_jobs(build_profile)
drv_paths = [x['drvPath'] for x in eval_result] drv_paths = [x['drvPath'] for x in eval_result]
print('[+] Building') print('[+] Building')

View file

@ -100,14 +100,6 @@ class OCIIndex:
} }
def docker_architecture_from_nix_system(system: str) -> DockerArchitecture:
MAP = {
'x86_64-linux': 'amd64',
'aarch64-linux': 'arm64',
}
return MAP[system] # type: ignore
@dataclasses.dataclass @dataclasses.dataclass
class TaggingOperation: class TaggingOperation:
manifest: OCIIndex manifest: OCIIndex

View file

@ -3,8 +3,27 @@ let
inherit (pkgs) lib; inherit (pkgs) lib;
lix = hydraJobs.build.x86_64-linux; lix = hydraJobs.build.x86_64-linux;
systems = [ "x86_64-linux" ]; # This is all so clumsy because we can't use arguments to functions in
dockerSystems = [ "x86_64-linux" ]; # flakes, and certainly not with n-e-j.
profiles = {
# Used for testing
x86_64-linux-only = {
systems = [ "x86_64-linux" ];
dockerSystems = [ "x86_64-linux" ];
};
all = {
systems = [
"x86_64-linux"
"aarch64-linux"
"aarch64-darwin"
"x86_64-darwin"
];
dockerSystems = [
"x86_64-linux"
"aarch64-linux"
];
};
};
doTarball = doTarball =
{ {
@ -27,7 +46,8 @@ let
sha256sum --binary $filename | cut -f1 -d' ' > $out/$basename.sha256 sha256sum --binary $filename | cut -f1 -d' ' > $out/$basename.sha256
''; '';
targets = targetsFor =
{ systems, dockerSystems }:
builtins.map (system: { builtins.map (system: {
target = hydraJobs.binaryTarball.${system}; target = hydraJobs.binaryTarball.${system};
targetName = "*.tar.xz"; targetName = "*.tar.xz";
@ -44,14 +64,26 @@ let
tar -cvzf "$out/lix-${lix.version}-manual.tar.gz" lix-${lix.version}-manual tar -cvzf "$out/lix-${lix.version}-manual.tar.gz" lix-${lix.version}-manual
''; '';
tarballs = pkgs.runCommand "lix-release-tarballs" { } '' tarballsFor =
mkdir -p $out { systems, dockerSystems }:
${lib.concatMapStringsSep "\n" doTarball targets} pkgs.runCommand "lix-release-tarballs" { } ''
cp ${manualTar}/*.tar.gz $out mkdir -p $out
cp -r ${lix.doc}/share/doc/nix/manual $out ${lib.concatMapStringsSep "\n" doTarball (targetsFor {
''; inherit systems dockerSystems;
})}
cp ${manualTar}/*.tar.gz $out
cp -r ${lix.doc}/share/doc/nix/manual $out
'';
in in
{ (builtins.mapAttrs (
_:
{ systems, dockerSystems }:
{
build = lib.filterAttrs (x: _: builtins.elem x systems) hydraJobs.build;
tarballs = tarballsFor { inherit systems dockerSystems; };
}
) profiles)
// {
inherit (hydraJobs) build; inherit (hydraJobs) build;
inherit tarballs; inherit tarballsFor;
} }