lix/releng/create_release.xsh

312 lines
9.3 KiB
Plaintext
Raw Permalink Normal View History

import json
import subprocess
import itertools
import textwrap
import logging
from pathlib import Path
import tempfile
import hashlib
import datetime
from . import environment
from .environment import RelengEnvironment
from . import keys
from . import docker
from .version import VERSION, RELEASE_NAME, MAJOR, OFFICIAL_RELEASE
from .gitutils import verify_are_on_tag, git_preconditions
from . import release_notes
log = logging.getLogger(__name__)
$RAISE_SUBPROC_ERROR = True
$XONSH_SHOW_TRACEBACK = True
GCROOTS_DIR = Path('./release/gcroots')
BUILT_GCROOTS_DIR = Path('./release/gcroots-build')
DRVS_TXT = Path('./release/drvs.txt')
ARTIFACTS = Path('./release/artifacts')
MANUAL = Path('./release/manual')
RELENG_MSG = "Release created with releng/create_release.xsh"
BUILD_CORES = 16
MAX_JOBS = 2
def setup_creds(env: RelengEnvironment):
key = keys.get_ephemeral_key(env)
$AWS_SECRET_ACCESS_KEY = key.secret_key
$AWS_ACCESS_KEY_ID = key.id
$AWS_DEFAULT_REGION = 'garage'
$AWS_ENDPOINT_URL = environment.S3_ENDPOINT
def official_release_commit_tag(force_tag=False):
print('[+] Setting officialRelease in version.json and tagging')
prev_branch = $(git symbolic-ref --short HEAD).strip()
git switch --detach
# Must be done in two parts due to buffering (opening the file immediately
# would truncate it).
new_version_json = $(jq --indent 4 '.official_release = true' version.json)
with open('version.json', 'w') as fh:
fh.write(new_version_json)
git add version.json
message = f'release: {VERSION} "{RELEASE_NAME}"\n\nRelease produced with releng/create_release.xsh'
git commit -m @(message)
git tag @(['-f'] if force_tag else []) -a -m @(message) @(VERSION)
with open('releng/prev-git-branch.txt', 'w') as fh:
fh.write(prev_branch)
return prev_branch
def merge_to_release(prev_branch):
git switch @(prev_branch)
# Create a merge back into the release branch so that git tools understand
# that the release branch contains the tag, without the release commit
# actually influencing the tree.
merge_msg = textwrap.dedent("""\
release: merge release {VERSION} back to mainline
This merge commit returns to the previous state prior to the release but leaves the tag in the branch history.
{RELENG_MSG}
""").format(VERSION=VERSION, RELENG_MSG=RELENG_MSG)
git merge -m @(merge_msg) -s ours @(VERSION)
def realise(paths: list[str]):
args = [
'--realise',
'--max-jobs',
MAX_JOBS,
'--cores',
BUILD_CORES,
'--log-format',
'bar-with-logs',
'--add-root',
BUILT_GCROOTS_DIR
]
nix-store @(args) @(paths)
def eval_jobs(build_profile):
nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake f'.#release-jobs.{build_profile}')
return [json.loads(s) for s in nej_output.strip().split('\n')]
def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
proc = subprocess.Popen([
'nix',
'copy',
'-v',
'--to',
env.cache_store_uri(),
'--stdin',
],
stdin=subprocess.PIPE,
env=__xonsh__.env.detype(),
)
proc.stdin.write('\n'.join(itertools.chain(paths, x + '^*' for x in paths)).encode())
proc.stdin.close()
rv = proc.wait()
if rv != 0:
raise subprocess.CalledProcessError(rv, proc.args)
def make_manifest(builds_by_system):
def manifest_line(system, out):
return f' {system} = "{out}";'
manifest_text = textwrap.dedent("""\
# This file was generated by releng/create_release.xsh in Lix
{{
{lines}
}}
""").format(lines='\n'.join(manifest_line(s, p) for (s, p) in builds_by_system.items()))
return manifest_text
def make_git_tarball(to: Path):
git archive --verbose --prefix=lix-@(VERSION)/ --format=tar.gz -o @(to) @(VERSION)
def confirm(prompt, expected):
resp = input(prompt)
if resp != expected:
raise ValueError('Unconfirmed')
def sha256_file(f: Path):
hasher = hashlib.sha256()
with open(f, 'rb') as h:
while data := h.read(1024 * 1024):
hasher.update(data)
return hasher.hexdigest()
def extract_builds_by_system(eval_result):
# This could be a dictionary comprehension, but we want to be absolutely
# sure we don't have duplicates.
ret = {}
for attr in eval_result:
if attr['attrPath'][0] != 'build':
continue
assert attr['system'] not in ret
ret[attr['system']] = attr['outputs']['out']
return ret
def make_artifacts_dir(eval_result, d: Path):
d.mkdir(exist_ok=True, parents=True)
version_dir = d / 'lix' / f'lix-{VERSION}'
version_dir.mkdir(exist_ok=True, parents=True)
tarballs_drv = next(p for p in eval_result if p['attr'] == 'tarballs')
cp --no-preserve=mode -r @(tarballs_drv['outputs']['out'])/* @(version_dir)
builds_by_system = extract_builds_by_system(eval_result)
# FIXME: upgrade-nix searches for manifest.nix at root, which is rather annoying
with open(d / 'manifest.nix', 'w') as h:
h.write(make_manifest(builds_by_system))
with open(version_dir / 'manifest.nix', 'w') as h:
h.write(make_manifest(builds_by_system))
print('[+] Make sources tarball')
filename = f'lix-{VERSION}.tar.gz'
git_tarball = version_dir / filename
make_git_tarball(git_tarball)
file_hash = sha256_file(git_tarball)
print(f'Hash: {file_hash}')
with open(version_dir / f'{filename}.sha256', 'w') as h:
h.write(file_hash)
def prepare_release_notes():
rl_path = release_notes.build_release_notes_to_file()
commit_msg = textwrap.dedent("""\
release: release notes for {VERSION}
{RELENG_MSG}
""").format(VERSION=VERSION, RELENG_MSG=RELENG_MSG)
git add @(rl_path) @(release_notes.SUMMARY)
git rm --ignore-unmatch 'doc/manual/rl-next/*.md'
git commit -m @(commit_msg)
def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False):
if not no_check_git:
verify_are_on_tag()
git_preconditions()
assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env
tree @(ARTIFACTS)
env_part = f'environment {env.name}'
not noconfirm and confirm(
f'Would you like to release {ARTIFACTS} as {VERSION} in {env.colour(env_part)}? Type "I want to release this to {env.name}" to confirm\n',
f'I want to release this to {env.name}'
)
docker_images = list((ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz'))
assert docker_images
print('[+] Upload to cache')
with open(DRVS_TXT) as fh:
upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x])
print('[+] Upload docker images')
for target in env.docker_targets:
docker.upload_docker_images(target, docker_images)
print('[+] Upload to release bucket')
aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/
print('[+] Upload manual')
upload_manual(env)
prev_branch = None
try:
with open('releng/prev-git-branch.txt', 'r') as fh:
prev_branch = fh.read().strip()
except FileNotFoundError:
log.warn('Cannot find previous git branch file, skipping pushing git objects')
if prev_branch:
print('[+] git push to the repo')
# We have to push the ref to gerrit for review at least such that the
# commit is known, before we can push it as a tag.
if env.git_repo_is_gerrit:
git push @(env.git_repo) f'{prev_branch}:refs/for/{prev_branch}'
else:
git push @(env.git_repo) f'{prev_branch}:{prev_branch}'
print('[+] git push tag')
git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'
def do_tag_merge(force_tag=False, no_check_git=False):
if not no_check_git:
git_preconditions()
prev_branch = official_release_commit_tag(force_tag=force_tag)
merge_to_release(prev_branch)
git switch --detach @(VERSION)
def build_manual(eval_result):
(drv, manual) = next((x['drvPath'], x['outputs']['doc']) for x in eval_result if x['attr'] == 'build.x86_64-linux')
print('[+] Building manual')
realise([drv])
cp --no-preserve=mode -T -vr @(manual)/share/doc/nix/manual @(MANUAL)
def upload_manual(env: RelengEnvironment):
if OFFICIAL_RELEASE:
version = MAJOR
else:
version = 'nightly'
print('[+] aws s3 sync manual')
aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/
if OFFICIAL_RELEASE:
aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/
def build_artifacts(build_profile, no_check_git=False):
rm -rf release/
if not no_check_git:
verify_are_on_tag()
git_preconditions()
print('[+] Evaluating')
eval_result = eval_jobs(build_profile)
drv_paths = [x['drvPath'] for x in eval_result]
print('[+] Building')
realise(drv_paths)
build_manual(eval_result)
with open(DRVS_TXT, 'w') as fh:
# don't bother putting the release tarballs themselves because they are duplicate and huge
fh.write('\n'.join(x['drvPath'] for x in eval_result if x['attr'] != 'lix-release-tarballs'))
make_artifacts_dir(eval_result, ARTIFACTS)
print(f'[+] Done! See {ARTIFACTS}')