buildbot-nix/buildbot_nix/__init__.py

import json
import multiprocessing
import os
import sys
import uuid
from collections import defaultdict
from collections.abc import Generator
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any

from buildbot.configurators import ConfiguratorBase
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
from buildbot.process import buildstep, logobserver, remotecommand
from buildbot.process.project import Project
from buildbot.process.properties import Interpolate, Properties
from buildbot.process.results import ALL_RESULTS, statusToString
from buildbot.steps.trigger import Trigger
from buildbot.util import asyncSleep
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
from buildbot.www.oauth2 import OAuth2Auth
from buildbot.changes.gerritchangesource import GerritChangeSource

if TYPE_CHECKING:
    from buildbot.process.log import Log

from twisted.internet import defer, threads
from twisted.logger import Logger
from twisted.python.failure import Failure

from .github_projects import (
    slugify_project_name,
)

SKIPPED_BUILDER_NAME = "skipped-builds"

log = Logger()

class LixSystemsOAuth2(OAuth2Auth):
    name = 'Lix'
    faIcon = 'fa-login'
    resourceEndpoint = "https://identity.lix.systems"
    # is passing scope necessary?
    authUri = 'https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth'
    tokenUri = 'https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token'

class BuildbotNixError(Exception):
    pass

@dataclass
class GerritProject:
    # `project` field.
    name: str

class BuildTrigger(Trigger):
    """Dynamic trigger that creates a build for every attribute."""

    def __init__(
        self,
        builds_scheduler: str,
        skipped_builds_scheduler: str,
        jobs: list[dict[str, Any]],
        **kwargs: Any,
    ) -> None:
        if "name" not in kwargs:
            kwargs["name"] = "trigger"
        self.jobs = jobs
        self.config = None
        self.builds_scheduler = builds_scheduler
        self.skipped_builds_scheduler = skipped_builds_scheduler
        Trigger.__init__(
            self,
            waitForFinish=True,
            schedulerNames=[builds_scheduler, skipped_builds_scheduler],
            haltOnFailure=True,
            flunkOnFailure=True,
            sourceStamps=[],
            alwaysUseLatest=False,
            updateSourceStamp=False,
            **kwargs,
        )

    def createTriggerProperties(self, props: Any) -> Any:  # noqa: N802
        return props

    def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]:  # noqa: N802
        build_props = self.build.getProperties()
        repo_name = build_props.getProperty(
            "github.base.repo.full_name",
            build_props.getProperty("github.repository.full_name"),
        )
        project_id = slugify_project_name(repo_name)
        source = f"nix-eval-{project_id}"

        triggered_schedulers = []
        for job in self.jobs:
            attr = job.get("attr", "eval-error")
            name = attr
            if repo_name is not None:
                name = f"github:{repo_name}#checks.{name}"
            else:
                name = f"checks.{name}"
            error = job.get("error")
            props = Properties()
            props.setProperty("virtual_builder_name", name, source)
            props.setProperty("status_name", f"nix-build .#checks.{attr}", source)
            props.setProperty("virtual_builder_tags", "", source)

            if error is not None:
                props.setProperty("error", error, source)
                triggered_schedulers.append((self.skipped_builds_scheduler, props))
                continue

            if job.get("isCached"):
                triggered_schedulers.append((self.skipped_builds_scheduler, props))
                continue

            drv_path = job.get("drvPath")
            system = job.get("system")
            out_path = job.get("outputs", {}).get("out")

            build_props.setProperty(f"{attr}-out_path", out_path, source)
            build_props.setProperty(f"{attr}-drv_path", drv_path, source)

            props.setProperty("attr", attr, source)
            props.setProperty("system", system, source)
            props.setProperty("drv_path", drv_path, source)
            props.setProperty("out_path", out_path, source)
            # we use this to identify builds when running a retry
            props.setProperty("build_uuid", str(uuid.uuid4()), source)

            triggered_schedulers.append((self.builds_scheduler, props))
        return triggered_schedulers

    def getCurrentSummary(self) -> dict[str, str]:  # noqa: N802
        """The original build trigger will the generic builder name `nix-build` in this case, which is not helpful"""
        if not self.triggeredNames:
            return {"step": "running"}
        summary = []
        if self._result_list:
            for status in ALL_RESULTS:
                count = self._result_list.count(status)
                if count:
                    summary.append(
                        f"{self._result_list.count(status)} {statusToString(status, count)}",
                    )
        return {"step": f"({', '.join(summary)})"}


class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
    """Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for
    every attribute.
    """

    def __init__(self, supported_systems: list[str], **kwargs: Any) -> None:
        kwargs = self.setupShellMixin(kwargs)
        super().__init__(**kwargs)
        self.observer = logobserver.BufferLogObserver()
        self.addLogObserver("stdio", self.observer)
        self.supported_systems = supported_systems

    @defer.inlineCallbacks
    def run(self) -> Generator[Any, object, Any]:
        # run nix-eval-jobs --flake .#checks to generate the dict of stages
        cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
        yield self.runCommand(cmd)

        # if the command passes extract the list of stages
        result = cmd.results()
        if result == util.SUCCESS:
            # create a ShellCommand for each stage and add them to the build
            jobs = []

            for line in self.observer.getStdout().split("\n"):
                if line != "":
                    try:
                        job = json.loads(line)
                    except json.JSONDecodeError as e:
                        msg = f"Failed to parse line: {line}"
                        raise BuildbotNixError(msg) from e
                    jobs.append(job)
            build_props = self.build.getProperties()
            repo_name = build_props.getProperty(
                "github.base.repo.full_name",
                build_props.getProperty("github.repository.full_name"),
            )
            project_id = slugify_project_name(repo_name)
            filtered_jobs = []
            for job in jobs:
                system = job.get("system")
                if not system or system in self.supported_systems:  # report eval errors
                    filtered_jobs.append(job)

            self.build.addStepsAfterCurrentStep(
                [
                    BuildTrigger(
                        builds_scheduler=f"{project_id}-nix-build",
                        skipped_builds_scheduler=f"{project_id}-nix-skipped-build",
                        name="build flake",
                        jobs=filtered_jobs,
                    ),
                ],
            )

        return result


# FIXME this leaks memory... but probably not enough that we care
class RetryCounter:
    def __init__(self, retries: int) -> None:
        self.builds: dict[uuid.UUID, int] = defaultdict(lambda: retries)

    def retry_build(self, build_id: uuid.UUID) -> int:
        retries = self.builds[build_id]
        if retries > 1:
            self.builds[build_id] = retries - 1
            return retries
        return 0


# For now we limit this to two. Often this allows us to make the error log
# shorter because we won't see the logs for all previous succeeded builds
RETRY_COUNTER = RetryCounter(retries=2)


class EvalErrorStep(steps.BuildStep):
    """Shows the error message of a failed evaluation."""

    @defer.inlineCallbacks
    def run(self) -> Generator[Any, object, Any]:
        error = self.getProperty("error")
        attr = self.getProperty("attr")
        # show eval error
        error_log: Log = yield self.addLog("nix_error")
        error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
        return util.FAILURE


class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
    """Builds a nix derivation."""

    def __init__(self, **kwargs: Any) -> None:
        kwargs = self.setupShellMixin(kwargs)
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self) -> Generator[Any, object, Any]:
        # run `nix build`
        cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
        yield self.runCommand(cmd)

        res = cmd.results()
        if res == util.FAILURE:
            retries = RETRY_COUNTER.retry_build(self.getProperty("build_uuid"))
            if retries > 0:
                return util.RETRY
        return res


class UpdateBuildOutput(steps.BuildStep):
    """Updates store paths in a public www directory.
    This is useful to prefetch updates without having to evaluate
    on the target machine.
    """

    def __init__(self, path: Path, **kwargs: Any) -> None:
        super().__init__(**kwargs)
        self.path = path

    def run(self) -> Generator[Any, object, Any]:
        props = self.build.getProperties()
        if props.getProperty("branch") != props.getProperty(
            "github.repository.default_branch",
        ):
            return util.SKIPPED

        attr = Path(props.getProperty("attr")).name
        out_path = props.getProperty("out_path")
        # XXX don't hardcode this
        self.path.mkdir(parents=True, exist_ok=True)
        (self.path / attr).write_text(out_path)
        return util.SUCCESS


# The builtin retry mechanism doesn't seem to work for github,
# since github is sometimes not delivering the pull request ref fast enough.
class GitWithRetry(steps.Git):
    @defer.inlineCallbacks
    def run_vc(
        self,
        branch: str,
        revision: str,
        patch: str,
    ) -> Generator[Any, object, Any]:
        retry_counter = 0
        while True:
            try:
                res = yield super().run_vc(branch, revision, patch)
            except Exception as e:  # noqa: BLE001
                retry_counter += 1
                if retry_counter == 3:
                    msg = "Failed to clone"
                    raise BuildbotNixError(msg) from e
                log: Log = yield self.addLog("log")
                yield log.addStderr(f"Retrying git clone (error: {e})\n")
                yield asyncSleep(2 << retry_counter)  # 2, 4, 8
            else:
                return res


def nix_eval_config(
    project: GerritProject,
    gerrit_private_key: str,
    worker_names: list[str],
    supported_systems: list[str],
    eval_lock: util.MasterLock,
    worker_count: int,
    max_memory_size: int,
) -> util.BuilderConfig:
    """Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
    For each evaluated attribute a new build pipeline is started.
    """
    factory = util.BuildFactory()
    # check out the source
    factory.addStep(
        steps.Gerrit(
            repourl="ssh://buildbot@gerrit.lix.systems:2022/lix",
            mode="full",
            retry=[60, 60],
            timeout=3600,
            sshPrivateKey=gerrit_private_key
        ),
    )
    drv_gcroots_dir = util.Interpolate(
        "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/",
    )

    factory.addStep(
        NixEvalCommand(
            env={},
            name="evaluate flake",
            supported_systems=supported_systems,
            command=[
                "nix-eval-jobs",
                "--workers",
                str(worker_count),
                "--max-memory-size",
                str(max_memory_size),
                "--option",
                "accept-flake-config",
                "true",
                "--gc-roots-dir",
                drv_gcroots_dir,
                "--force-recurse",
                "--check-cache-status",
                "--flake",
                ".#checks",
            ],
            haltOnFailure=True,
            locks=[eval_lock.access("exclusive")],
        ),
    )

    factory.addStep(
        steps.ShellCommand(
            name="Cleanup drv paths",
            command=[
                "rm",
                "-rf",
                drv_gcroots_dir,
            ],
        ),
    )

    return util.BuilderConfig(
        name=f"{project.name}/nix-eval",
        workernames=worker_names,
        project=project.name,
        factory=factory,
        properties=dict(status_name="nix-eval"),
    )


@dataclass
class CachixConfig:
    name: str
    signing_key_secret_name: str | None = None
    auth_token_secret_name: str | None = None

    def cachix_env(self) -> dict[str, str]:
        env = {}
        if self.signing_key_secret_name is not None:
            env["CACHIX_SIGNING_KEY"] = util.Secret(self.signing_key_secret_name)
        if self.auth_token_secret_name is not None:
            env["CACHIX_AUTH_TOKEN"] = util.Secret(self.auth_token_secret_name)
        return env


def nix_build_config(
    project: GerritProject,
    worker_names: list[str],
    cachix: CachixConfig | None = None,
    outputs_path: Path | None = None,
) -> util.BuilderConfig:
    """Builds one nix flake attribute."""
    factory = util.BuildFactory()
    factory.addStep(
        NixBuildCommand(
            env={},
            name="Build flake attr",
            command=[
                "nix",
                "build",
                "-L",
                "--option",
                "keep-going",
                "true",
                "--option",
                # stop stuck builds after 20 minutes
                "--max-silent-time",
                str(60 * 20),
                "--accept-flake-config",
                "--out-link",
                util.Interpolate("result-%(prop:attr)s"),
                util.Interpolate("%(prop:drv_path)s^*"),
            ],
            # 3 hours, defaults to 20 minutes
            # We increase this over the default since the build output might end up in a different `nix build`.
            timeout=60 * 60 * 3,
            haltOnFailure=True,
        ),
    )
    if cachix:
        factory.addStep(
            steps.ShellCommand(
                name="Upload cachix",
                env=cachix.cachix_env(),
                command=[
                    "cachix",
                    "push",
                    cachix.name,
                    util.Interpolate("result-%(prop:attr)s"),
                ],
            ),
        )

    factory.addStep(
        steps.ShellCommand(
            name="Register gcroot",
            command=[
                "nix-store",
                "--add-root",
                # FIXME: cleanup old build attributes
                util.Interpolate(
                    "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s",
                ),
                "-r",
                util.Property("out_path"),
            ],
            doStepIf=lambda s: s.getProperty("branch")
            == s.getProperty("github.repository.default_branch"),
        ),
    )
    factory.addStep(
        steps.ShellCommand(
            name="Delete temporary gcroots",
            command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")],
        ),
    )
    if outputs_path is not None:
        factory.addStep(
            UpdateBuildOutput(
                name="Update build output",
                path=outputs_path,
            ),
        )
    return util.BuilderConfig(
        name=f"{project.name}/nix-build",
        project=project.name,
        workernames=worker_names,
        collapseRequests=False,
        env={},
        factory=factory,
    )


def nix_skipped_build_config(
    project: GerritProject,
    worker_names: list[str],
) -> util.BuilderConfig:
    """Dummy builder that is triggered when a build is skipped."""
    factory = util.BuildFactory()
    factory.addStep(
        EvalErrorStep(
            name="Nix evaluation",
            doStepIf=lambda s: s.getProperty("error"),
            hideStepIf=lambda _, s: not s.getProperty("error"),
        ),
    )

    # This is just a dummy step showing the cached build
    factory.addStep(
        steps.BuildStep(
            name="Nix build (cached)",
            doStepIf=lambda _: False,
            hideStepIf=lambda _, s: s.getProperty("error"),
        ),
    )
    return util.BuilderConfig(
        name=f"{project.name}/nix-skipped-build",
        project=project.name,
        workernames=worker_names,
        collapseRequests=False,
        env={},
        factory=factory,
    )


def read_secret_file(secret_name: str) -> str:
    directory = os.environ.get("CREDENTIALS_DIRECTORY")
    if directory is None:
        print("directory not set", file=sys.stderr)
        sys.exit(1)
    return Path(directory).joinpath(secret_name).read_text().rstrip()



def config_for_project(
    config: dict[str, Any],
    project: GerritProject,
    worker_names: list[str],
    nix_supported_systems: list[str],
    nix_eval_worker_count: int,
    nix_eval_max_memory_size: int,
    eval_lock: util.MasterLock,
    cachix: CachixConfig | None = None,
    outputs_path: Path | None = None,
) -> Project:
    config["projects"].append(Project(project.name))
    config["schedulers"].extend(
        [
            # build everything pertaining to a project
            # TODO(raito): will this catch also post-merge? we don't really care about that… do we?
            schedulers.SingleBranchScheduler(
                name=f"{project.name}-changes",
                change_filter=util.ChangeFilter(
                    project=project.name,
                ),
                builderNames=[f"{project.name}/nix-eval"],
            ),
            # this is triggered from `nix-eval`
            schedulers.Triggerable(
                name=f"{project.name}-nix-build",
                builderNames=[f"{project.name}/nix-build"],
            ),
            # this is triggered from `nix-eval` when the build is skipped
            schedulers.Triggerable(
                name=f"{project.name}-nix-skipped-build",
                builderNames=[f"{project.name}/nix-skipped-build"],
            ),
            # allow to manually trigger a nix-build
            schedulers.ForceScheduler(
                name=f"{project.name}-force",
                builderNames=[f"{project.name}/nix-eval"],
                properties=[
                    util.StringParameter(
                        name="project",
                        label="Name of the Gerrit repository.",
                        default=project.name,
                    ),
                ],
            ),
        ],
    )
    gerrit_private_key = None
    with open('/var/lib/buildbot/master/id_gerrit', 'r') as f:
        gerrit_private_key = f.read()

    if gerrit_private_key is None:
        raise RuntimeError('No gerrit private key to fetch the repositories')

    config["builders"].extend(
        [
            # Since all workers run on the same machine, we only assign one of them to do the evaluation.
            # This should prevent exessive memory usage.
            nix_eval_config(
                project,
                gerrit_private_key,
                worker_names,
                supported_systems=nix_supported_systems,
                worker_count=nix_eval_worker_count,
                max_memory_size=nix_eval_max_memory_size,
                eval_lock=eval_lock,
            ),
            nix_build_config(
                project,
                worker_names,
                cachix=cachix,
                outputs_path=outputs_path,
            ),
            nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]),
        ],
    )


class PeriodicWithStartup(schedulers.Periodic):
    def __init__(self, *args: Any, run_on_startup: bool = False, **kwargs: Any) -> None:
        super().__init__(*args, **kwargs)
        self.run_on_startup = run_on_startup

    @defer.inlineCallbacks
    def activate(self) -> Generator[Any, object, Any]:
        if self.run_on_startup:
            yield self.setState("last_build", None)
        yield super().activate()

def gerritReviewCB(builderName, build, result, master, arg):
    if result == util.RETRY:
        return dict()

    message =  "Buildbot finished compiling your patchset\n"
    message += "on configuration: %s\n" % builderName
    message += "The result is: %s\n" % util.Results[result].upper()

    if arg:
        message += "\nFor more details visit:\n"
        message += build['url'] + "\n"

    if result == util.SUCCESS:
        verified = 1
    else:
        verified = -1

    return dict(message=message, labels={'Verified': verified})

def gerritStartCB(builderName, build, arg):
    message = "Buildbot started compiling your patchset\n"
    message += "on configuration: %s\n" % builderName
    message += "See your build here: %s" % build['url']

    return dict(message=message)

def gerritSummaryCB(buildInfoList, results, status, arg):
    success = False
    failure = False

    msgs = []

    for buildInfo in buildInfoList:
        msg = "Builder %(name)s %(resultText)s (%(text)s)" % buildInfo
        link = buildInfo.get('url', None)
        if link:
            msg += " - " + link
        else:
            msg += "."

        msgs.append(msg)

        if buildInfo['result'] == util.SUCCESS:
            success = True
        else:
            failure = True

    if success and not failure:
        verified = 1
    else:
        verified = -1

    return dict(message='\n\n'.join(msgs),
                labels={
                    'Verified': verified
                })

class GerritNixConfigurator(ConfiguratorBase):
    """Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""

    def __init__(
        self,
        # Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
        gerrit_server: str,
        gerrit_user: str,
        gerrit_port: int,
        gerrit_sshkey_path: str,
        url: str,
        nix_supported_systems: list[str],
        nix_eval_worker_count: int | None,
        nix_eval_max_memory_size: int,
        nix_workers_secret_name: str = "buildbot-nix-workers",  # noqa: S107
        cachix: CachixConfig | None = None,
        outputs_path: str | None = None,
    ) -> None:
        super().__init__()
        self.gerrit_server = gerrit_server
        self.gerrit_user = gerrit_user
        self.gerrit_port = gerrit_port
        self.nix_workers_secret_name = nix_workers_secret_name
        self.nix_eval_max_memory_size = nix_eval_max_memory_size
        self.nix_eval_worker_count = nix_eval_worker_count
        self.nix_supported_systems = nix_supported_systems
        self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path)
        self.url = url
        self.cachix = cachix
        if outputs_path is None:
            self.outputs_path = None
        else:
            self.outputs_path = Path(outputs_path)

    def configure(self, config: dict[str, Any]) -> None:
        worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
        worker_names = []

        config.setdefault("projects", [])
        config.setdefault("secretsProviders", [])
        config.setdefault("www", {})

        for item in worker_config:
            cores = item.get("cores", 0)
            for i in range(cores):
                worker_name = f"{item['name']}-{i:03}"
                config["workers"].append(worker.Worker(worker_name, item["pass"]))
                worker_names.append(worker_name)

        eval_lock = util.MasterLock("nix-eval")

        # Configure the Lix project.
        config_for_project(
            config,
            GerritProject(name="lix"),
            worker_names,
            self.nix_supported_systems,
            self.nix_eval_worker_count or multiprocessing.cpu_count(),
            self.nix_eval_max_memory_size,
            eval_lock,
            self.cachix,
            self.outputs_path,
        )

        config["change_source"] = self.gerrit_change_source
        config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
        config["services"].append(
            reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
                                    reviewCB=gerritReviewCB,
                                    reviewArg=self.url,
                                    startCB=gerritStartCB,
                                    startArg=self.url,
                                    summaryCB=gerritSummaryCB,
                                    summaryArg=self.url)
        )

        systemd_secrets = secrets.SecretInAFile(
            dirname=os.environ["CREDENTIALS_DIRECTORY"],
        )
        config["secretsProviders"].append(systemd_secrets)

        config["www"].setdefault("plugins", {})
        config["www"]["plugins"].update(dict(base_react={}))

        if "auth" not in config["www"]:
            config["www"]["auth"] = LixSystemsOAuth2('buildbot', read_secret_file('buildbot-oauth2-secret'), autologin=True)