make nginx output paths optional

This commit is contained in:
Jörg Thalheim 2023-11-04 09:50:23 +01:00
parent fe70af3ad6
commit ecaafe61b4
2 changed files with 45 additions and 23 deletions

View file

@ -235,8 +235,9 @@ class UpdateBuildOutput(steps.BuildStep):
on the target machine. on the target machine.
""" """
def __init__(self, **kwargs: Any) -> None: def __init__(self, path: Path, **kwargs: Any) -> None:
super().__init__(**kwargs) super().__init__(**kwargs)
self.path = path
def run(self) -> Generator[Any, object, Any]: def run(self) -> Generator[Any, object, Any]:
props = self.build.getProperties() props = self.build.getProperties()
@ -247,9 +248,8 @@ class UpdateBuildOutput(steps.BuildStep):
attr = os.path.basename(props.getProperty("attr")) attr = os.path.basename(props.getProperty("attr"))
out_path = props.getProperty("out_path") out_path = props.getProperty("out_path")
# XXX don't hardcode this # XXX don't hardcode this
p = Path("/var/www/buildbot/nix-outputs/") self.path.mkdir(parents=True, exist_ok=True)
os.makedirs(p, exist_ok=True) (self.path / attr).write_text(out_path)
(p / attr).write_text(out_path)
return util.SUCCESS return util.SUCCESS
@ -471,6 +471,7 @@ def nix_build_config(
worker_names: list[str], worker_names: list[str],
has_cachix_auth_token: bool = False, has_cachix_auth_token: bool = False,
has_cachix_signing_key: bool = False, has_cachix_signing_key: bool = False,
outputs_path: Path | None = None,
) -> util.BuilderConfig: ) -> util.BuilderConfig:
""" """
Builds one nix flake attribute. Builds one nix flake attribute.
@ -536,7 +537,13 @@ def nix_build_config(
command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")], command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")],
) )
) )
factory.addStep(UpdateBuildOutput(name="Update build output")) if outputs_path is not None:
factory.addStep(
UpdateBuildOutput(
name="Update build output",
path=outputs_path,
)
)
return util.BuilderConfig( return util.BuilderConfig(
name=f"{project.name}/nix-build", name=f"{project.name}/nix-build",
project=project.name, project=project.name,
@ -578,6 +585,7 @@ def config_for_project(
github: GithubConfig, github: GithubConfig,
nix_supported_systems: list[str], nix_supported_systems: list[str],
nix_eval_max_memory_size: int, nix_eval_max_memory_size: int,
outputs_path: Path | None = None,
) -> Project: ) -> Project:
## get a deterministic jitter for the project ## get a deterministic jitter for the project
# random.seed(project.name) # random.seed(project.name)
@ -666,6 +674,7 @@ def config_for_project(
worker_names, worker_names,
has_cachix_auth_token, has_cachix_auth_token,
has_cachix_signing_key, has_cachix_signing_key,
outputs_path=outputs_path,
), ),
nix_update_flake_config( nix_update_flake_config(
project, project,
@ -689,6 +698,7 @@ class NixConfigurator(ConfiguratorBase):
nix_supported_systems: list[str], nix_supported_systems: list[str],
nix_eval_max_memory_size: int = 4096, nix_eval_max_memory_size: int = 4096,
nix_workers_secret_name: str = "buildbot-nix-workers", nix_workers_secret_name: str = "buildbot-nix-workers",
outputs_path: str | None = None,
) -> None: ) -> None:
super().__init__() super().__init__()
self.nix_workers_secret_name = nix_workers_secret_name self.nix_workers_secret_name = nix_workers_secret_name
@ -697,6 +707,10 @@ class NixConfigurator(ConfiguratorBase):
self.github = github self.github = github
self.url = url self.url = url
self.systemd_credentials_dir = os.environ["CREDENTIALS_DIRECTORY"] self.systemd_credentials_dir = os.environ["CREDENTIALS_DIRECTORY"]
if outputs_path is None:
self.outputs_path = None
else:
self.outputs_path = Path(outputs_path)
def configure(self, config: dict[str, Any]) -> None: def configure(self, config: dict[str, Any]) -> None:
projects = load_projects(self.github.token(), self.github.project_cache_file) projects = load_projects(self.github.token(), self.github.project_cache_file)
@ -734,6 +748,7 @@ class NixConfigurator(ConfiguratorBase):
self.github, self.github,
self.nix_supported_systems, self.nix_supported_systems,
self.nix_eval_max_memory_size, self.nix_eval_max_memory_size,
self.outputs_path,
) )
# Reload github projects # Reload github projects

View file

@ -80,6 +80,13 @@ in
description = "Buildbot domain"; description = "Buildbot domain";
example = "buildbot.numtide.com"; example = "buildbot.numtide.com";
}; };
outputsPath = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "Path where we store the latest build store paths names for nix attributes as text files. This path will be exposed via nginx at \${domain}/nix-outputs";
default = null;
example = "/var/www/buildbot/nix-outputs";
};
}; };
}; };
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
@ -120,6 +127,7 @@ in
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl}, url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize}, nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_supported_systems=${builtins.toJSON cfg.buildSystems}, nix_supported_systems=${builtins.toJSON cfg.buildSystems},
outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
) )
'' ''
]; ];
@ -175,27 +183,26 @@ in
services.nginx.enable = true; services.nginx.enable = true;
services.nginx.virtualHosts.${cfg.domain} = { services.nginx.virtualHosts.${cfg.domain} = {
locations."/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/"; locations = {
locations."/sse" = { "/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/";
"/sse" = {
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse"; proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse";
# proxy buffering will prevent sse to work # proxy buffering will prevent sse to work
extraConfig = "proxy_buffering off;"; extraConfig = "proxy_buffering off;";
}; };
locations."/ws" = { "/ws" = {
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws"; proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws";
proxyWebsockets = true; proxyWebsockets = true;
# raise the proxy timeout for the websocket # raise the proxy timeout for the websocket
extraConfig = "proxy_read_timeout 6000s;"; extraConfig = "proxy_read_timeout 6000s;";
}; };
} // lib.optionalAttrs (cfg.outputsPath != null) {
# In this directory we store the lastest build store paths for nix attributes "/nix-outputs".root = cfg.outputsPath;
locations."/nix-outputs".root = "/var/www/buildbot/"; };
}; };
# Allow buildbot-master to write to this directory # Allow buildbot-master to write to this directory
systemd.tmpfiles.rules = [ systemd.tmpfiles.rules = lib.optional (cfg.outputsPath != null)
"d /var/www/buildbot/nix-outputs 0755 buildbot buildbot - -" "d ${cfg.outputPath} 0755 buildbot buildbot - -";
];
}; };
} }