nrabulinski/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Implement docker upload in the releng tools

Browse source 
This uses skopeo to not think about docker daemons. I, however, noticed
that the docker image we had would have totally terrible cache hits, so
I rewrote it.

Fixes: lix-project/lix#252

Change-Id: I3c5b6c1f3ba0b9dfcac212b2148f390e0cd542b7
This commit is contained in:
jade 2024-06-06 22:28:49 -07:00
parent 98e8475147
commit ff95b980d4
6 changed files with 96 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
package.nix
View file

@ -39,6 +39,7 @@
pkg-config,
python3,
rapidcheck,
skopeo,
sqlite,
toml11,
util-linuxMinimal ? utillinuxMinimal,
@ -447,6 +448,8 @@ stdenv.mkDerivation (finalAttrs: {
lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm
++ [
pythonEnv
# docker image tool
skopeo
just
nixfmt
# Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above.

12
releng/__init__.py
View file

@ -2,11 +2,12 @@ from xonsh.main import setup
setup()
del setup
from releng import environment
from releng import create_release
from releng import keys
from releng import version
from releng import cli
from . import environment
from . import create_release
from . import keys
from . import version
from . import cli
from . import docker
def reload():
import importlib
@ -15,3 +16,4 @@ def reload():
importlib.reload(keys)
importlib.reload(version)
importlib.reload(cli)
importlib.reload(docker)

28
releng/cli.py
View file

@ -1,4 +1,8 @@
from . import create_release
from . import docker
from .environment import RelengEnvironment
from . import environment
import functools
import argparse
import sys
@ -18,13 +22,16 @@ def do_tag(args):
no_check_git=args.no_check_git)
def do_upload(args):
create_release.setup_creds()
def do_upload(env: RelengEnvironment, args):
create_release.setup_creds(env)
if args.target == 'all':
create_release.upload_artifacts(force_push_tag=args.force_push_tag,
noconfirm=args.noconfirm)
docker.check_all_logins(env)
create_release.upload_artifacts(env,
force_push_tag=args.force_push_tag,
noconfirm=args.noconfirm,
no_check_git=args.no_check_git)
elif args.target == 'manual':
create_release.upload_manual()
create_release.upload_manual(env)
else:
raise ValueError('invalid target, unreachable')
@ -77,6 +84,10 @@ def main():
upload = sps.add_parser(
'upload', help='Upload artifacts to cache and releases bucket')
upload.add_argument(
'--no-check-git',
action='store_true',
help="Don't check git state before uploading. For testing.")
upload.add_argument('--force-push-tag',
action='store_true',
help='Force push the tag. For testing.')
@ -90,7 +101,12 @@ def main():
'--noconfirm',
action='store_true',
help="Don't ask for confirmation. For testing/automation.")
upload.set_defaults(cmd=do_upload)
upload.add_argument('--environment',
choices=list(environment.ENVIRONMENTS.keys()),
default='staging',
help='Environment to release to')
upload.set_defaults(cmd=lambda args: do_upload(
environment.ENVIRONMENTS[args.environment], args))
args = ap.parse_args()
args.cmd(args)

41
releng/create_release.xsh
View file

@ -7,19 +7,14 @@ import tempfile
import hashlib
import datetime
from . import environment
from .environment import RelengEnvironment
from . import keys
from . import docker
from .version import VERSION, RELEASE_NAME, MAJOR
$RAISE_SUBPROC_ERROR = True
$XONSH_SHOW_TRACEBACK = True
RELENG_ENV = environment.STAGING
RELEASES_BUCKET = RELENG_ENV.releases_bucket
DOCS_BUCKET = RELENG_ENV.docs_bucket
CACHE_STORE = RELENG_ENV.cache_store_uri()
REPO = RELENG_ENV.git_repo
GCROOTS_DIR = Path('./release/gcroots')
BUILT_GCROOTS_DIR = Path('./release/gcroots-build')
DRVS_TXT = Path('./release/drvs.txt')
@ -35,8 +30,8 @@ MAX_JOBS = 2
RELEASE_SYSTEMS = ["x86_64-linux"]
def setup_creds():
key = keys.get_ephemeral_key(RELENG_ENV)
def setup_creds(env: RelengEnvironment):
key = keys.get_ephemeral_key(env)
$AWS_SECRET_ACCESS_KEY = key.secret_key
$AWS_ACCESS_KEY_ID = key.id
$AWS_DEFAULT_REGION = 'garage'
@ -102,13 +97,13 @@ def eval_jobs():
]
def upload_drv_paths_and_outputs(paths: list[str]):
def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
proc = subprocess.Popen([
'nix',
'copy',
'-v',
'--to',
CACHE_STORE,
env.cache_store_uri(),
'--stdin',
],
stdin=subprocess.PIPE,
@ -250,7 +245,10 @@ def verify_are_on_tag():
assert current_tag == VERSION
def upload_artifacts(noconfirm=False, force_push_tag=False):
def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False):
if not no_check_git:
verify_are_on_tag()
git_preconditions()
assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env
tree @(ARTIFACTS)
@ -262,16 +260,21 @@ def upload_artifacts(noconfirm=False, force_push_tag=False):
print('[+] Upload to cache')
with open(DRVS_TXT) as fh:
upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x])
upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x])
docker_images = (ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz')
print('[+] Upload docker images')
for image in docker_images:
for target in env.docker_targets:
docker.upload_docker_image(target, image)
print('[+] Upload to release bucket')
aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/
aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/
print('[+] Upload manual')
upload_manual()
upload_manual(env)
print('[+] git push tag')
git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}'
git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'
def do_tag_merge(force_tag=False, no_check_git=False):
@ -290,7 +293,7 @@ def build_manual(eval_result):
cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL)
def upload_manual():
def upload_manual(env: RelengEnvironment):
stable = json.loads($(nix eval --json '.#nix.officialRelease'))
if stable:
version = MAJOR
@ -298,9 +301,9 @@ def upload_manual():
version = 'nightly'
print('[+] aws s3 sync manual')
aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/
aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/
if stable:
aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/
aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/
def build_artifacts(no_check_git=False):

13
releng/docker.xsh Normal file
View file

@ -0,0 +1,13 @@
from .environment import DockerTarget, RelengEnvironment
from .version import VERSION
from pathlib import Path
def check_all_logins(env: RelengEnvironment):
for target in env.docker_targets:
check_login(target)
def check_login(target: DockerTarget):
skopeo login @(target.registry_name())
def upload_docker_image(target: DockerTarget, path: Path):
skopeo --insecure-policy copy docker-archive:@(path) docker://@(target.resolve(version=VERSION))

32
releng/environment.py
View file

@ -16,6 +16,21 @@ DEFAULT_STORE_URI_BITS = {
}
@dataclasses.dataclass
class DockerTarget:
registry_path: str
def resolve(self, version: str) -> str:
"""Applies templates:
- version: the Lix version
"""
return self.registry_path.format(version=version)
def registry_name(self) -> str:
[a, _, _] = self.registry_path.partition('/')
return a
@dataclasses.dataclass
class RelengEnvironment:
name: str
@ -26,22 +41,33 @@ class RelengEnvironment:
docs_bucket: str
git_repo: str
docker_targets: list[DockerTarget]
def cache_store_uri(self):
qs = DEFAULT_STORE_URI_BITS.copy()
qs.update(self.cache_store_overlay)
return self.cache_bucket + "?" + urllib.parse.urlencode(qs)
STAGING = RelengEnvironment(
name='staging',
docs_bucket='s3://staging-docs',
cache_bucket='s3://staging-cache',
cache_store_overlay={
'secret-key': 'staging.key'
},
cache_store_overlay={'secret-key': 'staging.key'},
releases_bucket='s3://staging-releases',
git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging',
docker_targets=[
DockerTarget(
'git.lix.systems/lix-project/lix-releng-staging:{version}'),
DockerTarget(
'ghcr.io/lix-project/lix-releng-staging:{version}'),
],
)
ENVIRONMENTS = {
'staging': STAGING,
}
@dataclasses.dataclass
class S3Credentials: