feat: init project
Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
This commit is contained in:
commit
9c6d4e5f04
5
.env
Normal file
5
.env
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
evaluation_gc_roots_dir="./gc-roots"
|
||||||
|
evaluation_logs_dir="./logs"
|
||||||
|
max_parallel_evaluation=1
|
||||||
|
local_nixpkgs_checkout="/home/raito/dev/github.com/NixOS/nixpkgs"
|
||||||
|
nixpkgs_repo_url="https://cl.forkos.org/nixpkgs"
|
17
.gitignore
vendored
Normal file
17
.gitignore
vendored
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
result
|
||||||
|
.direnv
|
||||||
|
.pre-commit-config.yaml
|
||||||
|
__pycache__
|
||||||
|
.credentials
|
||||||
|
*.jsonl
|
||||||
|
.data_cache
|
||||||
|
nixpkgs
|
||||||
|
result
|
||||||
|
*.zst
|
||||||
|
*.dump
|
||||||
|
*.jsonl
|
||||||
|
*.log
|
||||||
|
# A shallow checkout of nixpkgs
|
||||||
|
nixpkgs/
|
||||||
|
gc-roots/
|
||||||
|
logs/
|
115
default.nix
Normal file
115
default.nix
Normal file
|
@ -0,0 +1,115 @@
|
||||||
|
{
|
||||||
|
sources ? import ./npins,
|
||||||
|
overlay ? import ./nix/overlay.nix,
|
||||||
|
pkgs ? import sources.nixpkgs { overlays = [ overlay ]; },
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
self = rec {
|
||||||
|
inherit (pkgs) python3;
|
||||||
|
|
||||||
|
package = pkgs.evolive;
|
||||||
|
|
||||||
|
pre-commit-check = pkgs.pre-commit-hooks {
|
||||||
|
src = ./.;
|
||||||
|
|
||||||
|
hooks =
|
||||||
|
let
|
||||||
|
pythonExcludes = [
|
||||||
|
"/migrations/" # auto-generated code
|
||||||
|
];
|
||||||
|
in
|
||||||
|
{
|
||||||
|
# Nix setup
|
||||||
|
nixfmt = {
|
||||||
|
enable = true;
|
||||||
|
entry = pkgs.lib.mkForce "${pkgs.lib.getExe pkgs.nixfmt-rfc-style}";
|
||||||
|
};
|
||||||
|
statix = {
|
||||||
|
enable = true;
|
||||||
|
settings.ignore = [ "npins" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
# Python setup
|
||||||
|
ruff = {
|
||||||
|
enable = true;
|
||||||
|
excludes = pythonExcludes;
|
||||||
|
};
|
||||||
|
ruff-format = {
|
||||||
|
enable = true;
|
||||||
|
name = "Format python code with ruff";
|
||||||
|
types = [
|
||||||
|
"text"
|
||||||
|
"python"
|
||||||
|
];
|
||||||
|
entry = "${pkgs.lib.getExe pkgs.ruff} format";
|
||||||
|
excludes = pythonExcludes;
|
||||||
|
};
|
||||||
|
|
||||||
|
pyright =
|
||||||
|
let
|
||||||
|
pyEnv = pkgs.python3.buildEnv.override {
|
||||||
|
extraLibs = pkgs.evolive.propagatedBuildInputs;
|
||||||
|
# Necessary for FastAPI CLI.
|
||||||
|
ignoreCollisions = true;
|
||||||
|
};
|
||||||
|
wrappedPyright = pkgs.runCommand "pyright" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
|
||||||
|
makeWrapper ${pkgs.pyright}/bin/pyright $out \
|
||||||
|
--set PYTHONPATH ${pyEnv}/${pyEnv.sitePackages} \
|
||||||
|
--prefix PATH : ${pyEnv}/bin \
|
||||||
|
--set PYTHONHOME ${pyEnv}
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
{
|
||||||
|
enable = true;
|
||||||
|
entry = pkgs.lib.mkForce (builtins.toString wrappedPyright);
|
||||||
|
excludes = pythonExcludes;
|
||||||
|
};
|
||||||
|
|
||||||
|
# Global setup
|
||||||
|
prettier = {
|
||||||
|
enable = true;
|
||||||
|
excludes = [
|
||||||
|
"\\.min.css$"
|
||||||
|
"\\.html$"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
commitizen.enable = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
shell = pkgs.mkShell {
|
||||||
|
DATA_CACHE_DIRECTORY = toString ./. + "/.data_cache";
|
||||||
|
REDIS_SOCKET_URL = "unix:///run/redis/redis.sock";
|
||||||
|
# `./src/website/tracker/settings.py` by default looks for LOCAL_NIXPKGS_CHECKOUT
|
||||||
|
# in the root of the repo. Make it the default here for local development.
|
||||||
|
LOCAL_NIXPKGS_CHECKOUT = toString ./. + "/nixpkgs";
|
||||||
|
|
||||||
|
packages = [
|
||||||
|
package
|
||||||
|
# (pkgs.python3.buildEnv.override {
|
||||||
|
# extraLibs = with pkgs.python3.pkgs; [ fastapi uvicorn ];
|
||||||
|
# ignoreCollisions = true;
|
||||||
|
# })
|
||||||
|
pkgs.nix-eval-jobs
|
||||||
|
pkgs.commitizen
|
||||||
|
pkgs.npins
|
||||||
|
pkgs.nixfmt-rfc-style
|
||||||
|
pkgs.hivemind
|
||||||
|
];
|
||||||
|
|
||||||
|
shellHook = ''
|
||||||
|
${pre-commit-check.shellHook}
|
||||||
|
|
||||||
|
mkdir -p .credentials
|
||||||
|
export DATABASE_URL=postgres:///evolive
|
||||||
|
export CREDENTIALS_DIRECTORY=${builtins.toString ./.credentials}
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
# tests = import ./nix/tests/vm-basic.nix {
|
||||||
|
# inherit pkgs;
|
||||||
|
# wstModule = module;
|
||||||
|
# };
|
||||||
|
};
|
||||||
|
in
|
||||||
|
self # // self.tests
|
43
nix/overlay.nix
Normal file
43
nix/overlay.nix
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
final: prev:
|
||||||
|
let
|
||||||
|
inherit (final) python3;
|
||||||
|
extraPackages = import ../pkgs {
|
||||||
|
pkgs = prev;
|
||||||
|
inherit python3;
|
||||||
|
};
|
||||||
|
extraPython3Packages = extraPackages.python3Packages;
|
||||||
|
sources = import ../npins;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
python3 = prev.lib.attrsets.recursiveUpdate prev.python3 { pkgs = extraPython3Packages; };
|
||||||
|
|
||||||
|
# go through the motions to make a flake-incompat project use the build
|
||||||
|
# inputs we want
|
||||||
|
pre-commit-hooks = final.callPackage "${sources.pre-commit-hooks}/nix/run.nix" {
|
||||||
|
tools = import "${sources.pre-commit-hooks}/nix/call-tools.nix" final;
|
||||||
|
# wat
|
||||||
|
gitignore-nix-src = {
|
||||||
|
lib = import sources.gitignore { inherit (final) lib; };
|
||||||
|
};
|
||||||
|
isFlakes = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
evolive = python3.pkgs.buildPythonPackage {
|
||||||
|
pname = "evolive";
|
||||||
|
version = "0.1.0";
|
||||||
|
format = "pyproject";
|
||||||
|
|
||||||
|
src = final.nix-gitignore.gitignoreSourcePure [ ../.gitignore ] ../src;
|
||||||
|
|
||||||
|
propagatedBuildInputs = with python3.pkgs; [
|
||||||
|
setuptools
|
||||||
|
setuptools-scm
|
||||||
|
fastapi
|
||||||
|
ipython
|
||||||
|
tqdm
|
||||||
|
aiofiles
|
||||||
|
pydantic-settings
|
||||||
|
uvicorn
|
||||||
|
];
|
||||||
|
};
|
||||||
|
}
|
80
npins/default.nix
Normal file
80
npins/default.nix
Normal file
|
@ -0,0 +1,80 @@
|
||||||
|
# Generated by npins. Do not modify; will be overwritten regularly
|
||||||
|
let
|
||||||
|
data = builtins.fromJSON (builtins.readFile ./sources.json);
|
||||||
|
version = data.version;
|
||||||
|
|
||||||
|
mkSource =
|
||||||
|
spec:
|
||||||
|
assert spec ? type;
|
||||||
|
let
|
||||||
|
path =
|
||||||
|
if spec.type == "Git" then
|
||||||
|
mkGitSource spec
|
||||||
|
else if spec.type == "GitRelease" then
|
||||||
|
mkGitSource spec
|
||||||
|
else if spec.type == "PyPi" then
|
||||||
|
mkPyPiSource spec
|
||||||
|
else if spec.type == "Channel" then
|
||||||
|
mkChannelSource spec
|
||||||
|
else
|
||||||
|
builtins.throw "Unknown source type ${spec.type}";
|
||||||
|
in
|
||||||
|
spec // { outPath = path; };
|
||||||
|
|
||||||
|
mkGitSource =
|
||||||
|
{
|
||||||
|
repository,
|
||||||
|
revision,
|
||||||
|
url ? null,
|
||||||
|
hash,
|
||||||
|
branch ? null,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
assert repository ? type;
|
||||||
|
# At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository
|
||||||
|
# In the latter case, there we will always be an url to the tarball
|
||||||
|
if url != null then
|
||||||
|
(builtins.fetchTarball {
|
||||||
|
inherit url;
|
||||||
|
sha256 = hash;
|
||||||
|
})
|
||||||
|
else
|
||||||
|
assert repository.type == "Git";
|
||||||
|
let
|
||||||
|
urlToName =
|
||||||
|
url: rev:
|
||||||
|
let
|
||||||
|
matched = builtins.match "^.*/([^/]*)(\\.git)?$" repository.url;
|
||||||
|
|
||||||
|
short = builtins.substring 0 7 rev;
|
||||||
|
|
||||||
|
appendShort = if (builtins.match "[a-f0-9]*" rev) != null then "-${short}" else "";
|
||||||
|
in
|
||||||
|
"${if matched == null then "source" else builtins.head matched}${appendShort}";
|
||||||
|
name = urlToName repository.url revision;
|
||||||
|
in
|
||||||
|
builtins.fetchGit {
|
||||||
|
url = repository.url;
|
||||||
|
rev = revision;
|
||||||
|
inherit name;
|
||||||
|
narHash = hash;
|
||||||
|
};
|
||||||
|
|
||||||
|
mkPyPiSource =
|
||||||
|
{ url, hash, ... }:
|
||||||
|
builtins.fetchurl {
|
||||||
|
inherit url;
|
||||||
|
sha256 = hash;
|
||||||
|
};
|
||||||
|
|
||||||
|
mkChannelSource =
|
||||||
|
{ url, hash, ... }:
|
||||||
|
builtins.fetchTarball {
|
||||||
|
inherit url;
|
||||||
|
sha256 = hash;
|
||||||
|
};
|
||||||
|
in
|
||||||
|
if version == 4 then
|
||||||
|
builtins.mapAttrs (_: mkSource) data.pins
|
||||||
|
else
|
||||||
|
throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`"
|
23
npins/sources.json
Normal file
23
npins/sources.json
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
{
|
||||||
|
"pins": {
|
||||||
|
"nixpkgs": {
|
||||||
|
"type": "Channel",
|
||||||
|
"name": "nixpkgs-unstable",
|
||||||
|
"url": "https://releases.nixos.org/nixpkgs/nixpkgs-24.11pre674705.b833ff01a0d6/nixexprs.tar.xz",
|
||||||
|
"hash": "12cda9rvpgjcsxykbcg5cxjaayhibjjabv6svacjc5n5kpcbx5sf"
|
||||||
|
},
|
||||||
|
"pre-commit-hooks": {
|
||||||
|
"type": "Git",
|
||||||
|
"repository": {
|
||||||
|
"type": "GitHub",
|
||||||
|
"owner": "cachix",
|
||||||
|
"repo": "pre-commit-hooks.nix"
|
||||||
|
},
|
||||||
|
"branch": "master",
|
||||||
|
"revision": "4509ca64f1084e73bc7a721b20c669a8d4c5ebe6",
|
||||||
|
"url": "https://github.com/cachix/pre-commit-hooks.nix/archive/4509ca64f1084e73bc7a721b20c669a8d4c5ebe6.tar.gz",
|
||||||
|
"hash": "06cjchycvj4wjzwm8ly1d36s9d7lj3mz1zqnkikvck0chi1psaa3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"version": 4
|
||||||
|
}
|
6
pkgs/default.nix
Normal file
6
pkgs/default.nix
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
{ pkgs, python3 }:
|
||||||
|
|
||||||
|
{
|
||||||
|
# place more custom packages here
|
||||||
|
python3Packages = pkgs.callPackage ./python { inherit python3; };
|
||||||
|
}
|
19
pkgs/python/default.nix
Normal file
19
pkgs/python/default.nix
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
{ pkgs, python3 }:
|
||||||
|
|
||||||
|
let
|
||||||
|
callPackage = pkgs.lib.callPackageWith (
|
||||||
|
pkgs // { inherit python3; } // python3.pkgs // python3Packages
|
||||||
|
);
|
||||||
|
|
||||||
|
python3Packages = mkPackages ./.;
|
||||||
|
mkPackages =
|
||||||
|
dir:
|
||||||
|
with builtins;
|
||||||
|
listToAttrs (
|
||||||
|
map (name: {
|
||||||
|
inherit name;
|
||||||
|
value = callPackage (dir + "/${name}") { };
|
||||||
|
}) (attrNames (readDir dir))
|
||||||
|
);
|
||||||
|
in
|
||||||
|
python3Packages
|
18
pyproject.toml
Normal file
18
pyproject.toml
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
[tool.ruff]
|
||||||
|
target-version = "py311"
|
||||||
|
select = ["E", "F", "I", "N", "U", "ANN"]
|
||||||
|
ignore = [
|
||||||
|
"F403",
|
||||||
|
"E501", # line too long
|
||||||
|
"ANN101", # Missing type annotation for `self` in method
|
||||||
|
"ANN401" # Dynamically typed expressions (typing.Any) are disallowed
|
||||||
|
]
|
||||||
|
exclude = ["src/website/shared/migrations/*.py"] # auto-generated code
|
||||||
|
|
||||||
|
[tool.commitizen]
|
||||||
|
name = "cz_conventional_commits"
|
||||||
|
tag_format = "$version"
|
||||||
|
version_scheme = "pep440"
|
||||||
|
version_provider = "pep621"
|
||||||
|
update_changelog_on_bump = true
|
||||||
|
major_version_zero = true
|
16
src/api/config.py
Normal file
16
src/api/config.py
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
|
|
||||||
|
class Settings(BaseSettings):
|
||||||
|
evaluation_gc_roots_dir: pathlib.Path
|
||||||
|
evaluation_logs_dir: pathlib.Path
|
||||||
|
max_parallel_evaluation: int
|
||||||
|
local_nixpkgs_checkout: pathlib.Path
|
||||||
|
nixpkgs_repo_url: str
|
||||||
|
|
||||||
|
model_config = SettingsConfigDict(env_file=".env")
|
||||||
|
|
||||||
|
|
||||||
|
settings = Settings() # pyright: ignore
|
165
src/api/evaluation.py
Normal file
165
src/api/evaluation.py
Normal file
|
@ -0,0 +1,165 @@
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import pathlib
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from collections.abc import AsyncGenerator
|
||||||
|
|
||||||
|
import aiofiles
|
||||||
|
|
||||||
|
from api.config import settings
|
||||||
|
from api.git import GitRepo
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
SIGSEGV = 137
|
||||||
|
SIGABRT = 134
|
||||||
|
|
||||||
|
|
||||||
|
async def perform_evaluation(
|
||||||
|
working_tree: pathlib.Path, evaluation_log_fd: int, limit: int = 16 * 1024 * 1024
|
||||||
|
) -> asyncio.subprocess.Process:
|
||||||
|
"""
|
||||||
|
This will run `nix-eval-jobs` on the working tree as a nixpkgs
|
||||||
|
input and collect all Hydra jobs, including insecure ones.
|
||||||
|
|
||||||
|
`limit` is the stream reader buffer limit.
|
||||||
|
|
||||||
|
By default, we use 16MB which should give enough time to consume all
|
||||||
|
the chunks in time and buffer enough of the evaluation attributes.
|
||||||
|
|
||||||
|
This will return an asynchronous process you can use to control
|
||||||
|
the execution runtime.
|
||||||
|
"""
|
||||||
|
# TODO(raitobezarius): bring a Nix code generator, that'd be cuter.
|
||||||
|
nixpkgs_config = "{ config = { allowUnfree = true; inHydra = false; allowInsecurePredicate = (_: true); scrubJobs = false; }; };"
|
||||||
|
evaluation_wrapper = f"(import <nixpkgs/pkgs/top-level/release.nix> {{ nixpkgsArgs = {nixpkgs_config} }})"
|
||||||
|
arguments = [
|
||||||
|
"--force-recurse",
|
||||||
|
"--meta",
|
||||||
|
"--repair",
|
||||||
|
"--quiet",
|
||||||
|
"--gc-roots-dir",
|
||||||
|
settings.evaluation_gc_roots_dir,
|
||||||
|
"--expr",
|
||||||
|
evaluation_wrapper,
|
||||||
|
"--include",
|
||||||
|
f"nixpkgs={working_tree}",
|
||||||
|
]
|
||||||
|
return await asyncio.create_subprocess_exec(
|
||||||
|
"nix-eval-jobs",
|
||||||
|
*arguments,
|
||||||
|
limit=limit,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=evaluation_log_fd,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def drain_lines(
|
||||||
|
stream: asyncio.StreamReader, timeout: float = 0.25, max_batch_window: int = 10_000
|
||||||
|
) -> AsyncGenerator[list[bytes], None]:
|
||||||
|
"""
|
||||||
|
This utility will perform an opportunistic line draining
|
||||||
|
operation on a StreamReader, the timeout will be reset
|
||||||
|
every time we obtain another line.
|
||||||
|
"""
|
||||||
|
lines = []
|
||||||
|
eof = False
|
||||||
|
|
||||||
|
class TooManyStuff(BaseException):
|
||||||
|
pass
|
||||||
|
|
||||||
|
while not eof:
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(timeout) as cm:
|
||||||
|
lines.append(await stream.readline())
|
||||||
|
old_deadline = cm.when()
|
||||||
|
assert (
|
||||||
|
old_deadline is not None
|
||||||
|
), "Timeout context does not have timeout!"
|
||||||
|
new_deadline = old_deadline + timeout
|
||||||
|
cm.reschedule(new_deadline)
|
||||||
|
|
||||||
|
if len(lines) >= max_batch_window:
|
||||||
|
raise TooManyStuff
|
||||||
|
except (TimeoutError, TooManyStuff):
|
||||||
|
# No line, we retry.
|
||||||
|
if len(lines) == 0:
|
||||||
|
continue
|
||||||
|
# Last line is EOF, so there won't be more lines.
|
||||||
|
while lines and (
|
||||||
|
lines[-1] == b"" or lines[-1].decode("utf8").strip() == ""
|
||||||
|
):
|
||||||
|
eof = True
|
||||||
|
# Drop the last line.
|
||||||
|
lines = lines[:-1]
|
||||||
|
# If we had lines = ["", ...], we just break immediately, there's nothing to yield anymore.
|
||||||
|
if not lines:
|
||||||
|
break
|
||||||
|
|
||||||
|
yield lines
|
||||||
|
lines = []
|
||||||
|
|
||||||
|
assert eof, "Reached the end of `drain_lines` without EOF!"
|
||||||
|
|
||||||
|
|
||||||
|
async def evaluation_entrypoint(commit_sha1: str) -> AsyncGenerator[list[str], None]:
|
||||||
|
# TODO: relinquish control immediately if there's too many parallel evaluations ongoing.
|
||||||
|
# TODO: take a lock.
|
||||||
|
repo = GitRepo(str(settings.local_nixpkgs_checkout))
|
||||||
|
start = time.time()
|
||||||
|
try:
|
||||||
|
# Pull our local checkout up to that evaluation revision.
|
||||||
|
await repo.update_from_ref(settings.nixpkgs_repo_url, commit_sha1)
|
||||||
|
with tempfile.TemporaryDirectory() as working_tree_path:
|
||||||
|
# Extract a working tree out of it for our needs.
|
||||||
|
evaluation_log_filepath = (
|
||||||
|
settings.evaluation_logs_dir / f"evaluation-{commit_sha1}.log"
|
||||||
|
)
|
||||||
|
async with (
|
||||||
|
repo.extract_working_tree(
|
||||||
|
commit_sha1, working_tree_path
|
||||||
|
) as working_tree,
|
||||||
|
aiofiles.open(evaluation_log_filepath, "w") as eval_log,
|
||||||
|
):
|
||||||
|
# Kickstart the evaluation asynchronously.
|
||||||
|
eval_process = await perform_evaluation(
|
||||||
|
working_tree.path, eval_log.fileno()
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
eval_process.stdout is not None
|
||||||
|
), "Expected a valid `stdout` pipe for the asynchronous evaluation process"
|
||||||
|
|
||||||
|
# The idea here is that we want to match as close as possible
|
||||||
|
# our evaluation speed. So, we read as much lines as possible
|
||||||
|
# and then insert them During the insertion time, more lines
|
||||||
|
# may come in our internal buffer. On the next read, we will
|
||||||
|
# drain them again.
|
||||||
|
# Adding an item in the database takes around 1s max.
|
||||||
|
# So we don't want to wait more than one second for all the lines we can get.
|
||||||
|
count = 0
|
||||||
|
async for lines in drain_lines(eval_process.stdout):
|
||||||
|
yield [line.decode("utf8") for line in lines]
|
||||||
|
count += len(lines)
|
||||||
|
# Wait for `nix-eval-jobs` to exit, at this point,
|
||||||
|
# It should be fairly quick because EOF has been reached.
|
||||||
|
rc = await eval_process.wait()
|
||||||
|
elapsed = time.time() - start
|
||||||
|
if rc in (SIGSEGV, SIGABRT):
|
||||||
|
raise RuntimeError("`nix-eval-jobs` crashed!")
|
||||||
|
elif rc != 0:
|
||||||
|
logger.error(
|
||||||
|
"`nix-eval-jobs` failed to evaluate (non-zero exit status), check the evaluation logs"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"Processed %d derivations in real-time in %f seconds",
|
||||||
|
count,
|
||||||
|
elapsed,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
elapsed = time.time() - start
|
||||||
|
logger.exception(
|
||||||
|
"Failed to run the `nix-eval-job` on revision '%s', marking job as crashed...",
|
||||||
|
commit_sha1,
|
||||||
|
)
|
226
src/api/git.py
Normal file
226
src/api/git.py
Normal file
|
@ -0,0 +1,226 @@
|
||||||
|
# Taken from the security tracker project.
|
||||||
|
import asyncio
|
||||||
|
import itertools
|
||||||
|
import os.path
|
||||||
|
import pathlib
|
||||||
|
from collections.abc import AsyncGenerator
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import IO, Any
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Worktree:
|
||||||
|
path: pathlib.Path
|
||||||
|
revision: str
|
||||||
|
detached: bool
|
||||||
|
prunable: bool
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parse_from_porcelain(
|
||||||
|
cls: type["Worktree"], porcelain: list[str]
|
||||||
|
) -> "Worktree | None":
|
||||||
|
path = porcelain[0].split(" ")[1]
|
||||||
|
if porcelain[1] == "bare":
|
||||||
|
return None
|
||||||
|
|
||||||
|
revision = porcelain[1].split(" ")[1]
|
||||||
|
detached = porcelain[2] == "detached"
|
||||||
|
if len(porcelain) >= 4:
|
||||||
|
prunable = porcelain[3].startswith("prunable")
|
||||||
|
else:
|
||||||
|
prunable = False
|
||||||
|
return Worktree(
|
||||||
|
path=pathlib.Path(path),
|
||||||
|
revision=revision,
|
||||||
|
detached=detached,
|
||||||
|
prunable=prunable,
|
||||||
|
)
|
||||||
|
|
||||||
|
def name(self) -> str:
|
||||||
|
"""
|
||||||
|
By default, `git` uses the basename
|
||||||
|
of the target path to name a worktree.
|
||||||
|
"""
|
||||||
|
return os.path.basename(str(self.path))
|
||||||
|
|
||||||
|
|
||||||
|
class RepositoryError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class GitRepo:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
repo_path: str,
|
||||||
|
stdout: int | IO[Any] | None = None,
|
||||||
|
stderr: int | IO[Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.stdout = stdout
|
||||||
|
self.stderr = stderr
|
||||||
|
self.repo_path = repo_path
|
||||||
|
|
||||||
|
async def execute_git_command(
|
||||||
|
self,
|
||||||
|
cmd: str,
|
||||||
|
stdout: int | IO[Any] | None = None,
|
||||||
|
stderr: int | IO[Any] | None = None,
|
||||||
|
) -> asyncio.subprocess.Process:
|
||||||
|
final_stdout = stdout or self.stdout or asyncio.subprocess.PIPE
|
||||||
|
final_stderr = stderr or self.stderr or asyncio.subprocess.PIPE
|
||||||
|
return await asyncio.create_subprocess_shell(
|
||||||
|
cmd, cwd=self.repo_path, stdout=final_stdout, stderr=final_stderr
|
||||||
|
)
|
||||||
|
|
||||||
|
async def clone(
|
||||||
|
self, repo_clone_url: str, reference_repo_path: str | None = None
|
||||||
|
) -> asyncio.subprocess.Process:
|
||||||
|
"""
|
||||||
|
Clones the repository.
|
||||||
|
If you pass a `reference_repo_path`, the cloning will be much faster.
|
||||||
|
|
||||||
|
Otherwise, it will use a shallow clone, since we can fetch commits manually.
|
||||||
|
"""
|
||||||
|
stdout = self.stdout or asyncio.subprocess.PIPE
|
||||||
|
stderr = self.stderr or asyncio.subprocess.PIPE
|
||||||
|
if reference_repo_path is not None:
|
||||||
|
clone_process = await asyncio.create_subprocess_shell(
|
||||||
|
f"git clone --depth=1 --bare --progress --reference={reference_repo_path} {repo_clone_url} {self.repo_path}",
|
||||||
|
stdout=stdout,
|
||||||
|
stderr=stderr,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
clone_process = await asyncio.create_subprocess_shell(
|
||||||
|
f"git clone --depth=1 --bare --progress {repo_clone_url} {self.repo_path}",
|
||||||
|
stdout=stdout,
|
||||||
|
stderr=stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
return clone_process
|
||||||
|
|
||||||
|
async def worktrees(self) -> list[Worktree]:
|
||||||
|
"""
|
||||||
|
Returns a list of relevant worktrees,
|
||||||
|
e.g. filter out the `bare` worktree.
|
||||||
|
"""
|
||||||
|
process = await self.execute_git_command(
|
||||||
|
"git worktree list -z --porcelain", stdout=asyncio.subprocess.PIPE
|
||||||
|
)
|
||||||
|
stdout, _ = await process.communicate()
|
||||||
|
parts = stdout.split(b"\x00")
|
||||||
|
assert parts[-1] == b"", "Worktrees list are not terminated by a NUL character"
|
||||||
|
parts = [part.decode("utf8") for part in parts]
|
||||||
|
return list(
|
||||||
|
filter(
|
||||||
|
None,
|
||||||
|
[
|
||||||
|
Worktree.parse_from_porcelain(list(group))
|
||||||
|
for k, group in itertools.groupby(parts, lambda x: x == "")
|
||||||
|
if not k
|
||||||
|
],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def update_from_ref(self, repo_clone_url: str, object_sha1: str) -> bool:
|
||||||
|
"""
|
||||||
|
This checks if `object_sha1` is already present in the repo or not.
|
||||||
|
If not, perform a fetch to our remote to obtain it.
|
||||||
|
|
||||||
|
Returns whether this was fetched or already present.
|
||||||
|
"""
|
||||||
|
exists = (
|
||||||
|
await (
|
||||||
|
await self.execute_git_command(
|
||||||
|
f"git cat-file commit {object_sha1}",
|
||||||
|
stderr=asyncio.subprocess.PIPE,
|
||||||
|
)
|
||||||
|
).wait()
|
||||||
|
== 0
|
||||||
|
)
|
||||||
|
if exists:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
locking_problem = True
|
||||||
|
while locking_problem:
|
||||||
|
# We need to acquire a shallow lock here.
|
||||||
|
# TODO: replace me by an async variant.
|
||||||
|
while os.path.exists(os.path.join(self.repo_path, "shallow.lock")):
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
process = await self.execute_git_command(
|
||||||
|
f"git fetch --porcelain --depth=1 {repo_clone_url} {object_sha1}",
|
||||||
|
stderr=asyncio.subprocess.PIPE,
|
||||||
|
)
|
||||||
|
_, stderr = await process.communicate()
|
||||||
|
rc = await process.wait()
|
||||||
|
locking_problem = "shallow" in stderr.decode("utf8")
|
||||||
|
if rc != 0 and not locking_problem:
|
||||||
|
print(stderr)
|
||||||
|
raise RepositoryError(
|
||||||
|
f"failed to fetch {object_sha1} while running `git fetch --depth=1 {repo_clone_url} {object_sha1}`"
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def remove_working_tree(self, name: str) -> bool:
|
||||||
|
"""
|
||||||
|
This deletes the working tree, if it exists.
|
||||||
|
|
||||||
|
Returns `True` if it does exist, otherwise `False`.
|
||||||
|
"""
|
||||||
|
process = await self.execute_git_command(f"git worktree remove {name}")
|
||||||
|
deleted = await process.wait() == 0
|
||||||
|
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
async def prune_working_trees(self) -> bool:
|
||||||
|
"""
|
||||||
|
This prunes all the working trees, if possible.
|
||||||
|
|
||||||
|
Returns `True` if it does get pruned, otherwise `False`.
|
||||||
|
"""
|
||||||
|
process = await self.execute_git_command("git worktree prune")
|
||||||
|
pruned = await process.wait() == 0
|
||||||
|
|
||||||
|
return pruned
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def extract_working_tree(
|
||||||
|
self, commit_sha1: str, target_path: str
|
||||||
|
) -> AsyncGenerator[Worktree, None]:
|
||||||
|
"""
|
||||||
|
This will extract the working tree represented at the reference
|
||||||
|
induced by the object's commit SHA1 into the `target_path`.
|
||||||
|
|
||||||
|
This returns it as an asynchronous context manager.
|
||||||
|
"""
|
||||||
|
path = pathlib.Path(target_path)
|
||||||
|
worktrees = {wt.path: wt for wt in await self.worktrees()}
|
||||||
|
existing_wt = worktrees.get(path)
|
||||||
|
if existing_wt is not None and not existing_wt.prunable:
|
||||||
|
raise RepositoryError(
|
||||||
|
f"failed to perform extraction of the worktree at {target_path} for commit {commit_sha1}, such a worktree already exist!"
|
||||||
|
)
|
||||||
|
if existing_wt is not None and existing_wt.prunable:
|
||||||
|
await self.prune_working_trees()
|
||||||
|
|
||||||
|
process = await self.execute_git_command(
|
||||||
|
f"git worktree add {target_path} {commit_sha1}"
|
||||||
|
)
|
||||||
|
created = await process.wait() == 0
|
||||||
|
|
||||||
|
if not created:
|
||||||
|
raise RepositoryError(
|
||||||
|
f"failed to perform extraction of the worktree at {target_path} for commit {commit_sha1}, cannot create it!"
|
||||||
|
)
|
||||||
|
|
||||||
|
wt = None
|
||||||
|
try:
|
||||||
|
wt = Worktree(
|
||||||
|
path=pathlib.Path(target_path),
|
||||||
|
revision=commit_sha1,
|
||||||
|
detached=True,
|
||||||
|
prunable=False,
|
||||||
|
)
|
||||||
|
yield wt
|
||||||
|
finally:
|
||||||
|
if wt is not None:
|
||||||
|
await self.remove_working_tree(wt.name())
|
28
src/api/main.py
Normal file
28
src/api/main.py
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
from collections.abc import AsyncGenerator
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import FastAPI, Path
|
||||||
|
from fastapi.exceptions import HTTPException
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
|
||||||
|
from api.evaluation import evaluation_entrypoint
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
|
||||||
|
|
||||||
|
async def stream_evaluation(commit_sha1: str) -> AsyncGenerator[bytes, None]:
|
||||||
|
async for lines in evaluation_entrypoint(commit_sha1):
|
||||||
|
for line in lines:
|
||||||
|
yield line.encode("utf8")
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/evaluations/{project_slug}/{revision}")
|
||||||
|
async def submit_evaluation(
|
||||||
|
project_slug: Annotated[str, Path(title="The slug of the project, e.g. nixpkgs")],
|
||||||
|
revision: Annotated[str, Path(title="The SHA1 revision for this repository")],
|
||||||
|
) -> StreamingResponse:
|
||||||
|
if project_slug != "nixpkgs":
|
||||||
|
raise HTTPException(status_code=404, detail="Project not found")
|
||||||
|
|
||||||
|
print("Evaluating nixpkgs at", revision)
|
||||||
|
return StreamingResponse(stream_evaluation(revision))
|
10
src/pyproject.toml
Normal file
10
src/pyproject.toml
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
[project]
|
||||||
|
name = "evolive"
|
||||||
|
version = "0.1.0"
|
||||||
|
|
||||||
|
[setuptools]
|
||||||
|
packages = [ "api" ]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["setuptools", "setuptools-scm"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
Loading…
Reference in a new issue