From 9bb7fb8f698d6ef4d42d068a576abd98d36f8239 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 7 Jun 2024 01:35:26 -0700 Subject: [PATCH] Rewrite docker to be sensible and smaller I have checked the image can build things and inspected `diff -ru` compared to the old image. As far as I can tell it is more or less the same besides the later git change. Layers are now 65MB or less, and we aren't against the maxLayers limit for the broken automatic layering to do anything but shove one store path in a layer (which is good behaviour, actually). This uses nix2container which streams images, so the build time is much shorter. I have also taken the opportunity to, in addition to fixing the 400MB single layer (terrible, and what motivated this in the first place), delete about 200MB of closure size inflicted by git vs gitMinimal causing both perl and python to get into closure. People mostly use this thing for CI, so I don't really think you need advanced git operations, and large git can be added at the user side if really motivated. With love for whichever container developer somewhat ironically assumed that one would not run skopeo in a minimal container that doesn't have a /var/tmp. Fixes: https://git.lix.systems/lix-project/lix/issues/378 Change-Id: Icc3aa20e64446276716fbbb87535fd5b50628010 --- docker.nix | 203 +++++++++++++++++++++++++++------------- flake.lock | 17 ++++ flake.nix | 23 +++-- releng/release-jobs.nix | 2 +- 4 files changed, 168 insertions(+), 77 deletions(-) diff --git a/docker.nix b/docker.nix index cec3a2950..a60fd5763 100644 --- a/docker.nix +++ b/docker.nix @@ -1,5 +1,6 @@ { pkgs ? import { }, + nix2container, lib ? pkgs.lib, name ? "lix", tag ? "latest", @@ -12,26 +13,51 @@ flake-registry ? null, }: let + layerContents = with pkgs; [ + # pulls in glibc and openssl, about 60MB + { contents = [ coreutils-full ]; } + # some stuff that is low in the closure graph and small ish, mostly to make + # incremental lix updates cheaper + { + contents = [ + curl + libxml2 + sqlite + ]; + } + # 50MB of git + { contents = [ gitMinimal ]; } + # 144MB of nixpkgs + { + contents = [ channel ]; + inProfile = false; + } + ]; + + # These packages are left to be auto layered by nix2container, since it is + # less critical that they get layered sensibly and they tend to not be deps + # of anything in particular + autoLayered = with pkgs; [ + bashInteractive + gnutar + gzip + gnugrep + which + less + wget + man + cacert.out + findutils + iana-etc + openssh + nix + ]; + defaultPkgs = - with pkgs; - [ - nix - bashInteractive - coreutils-full - gnutar - gzip - gnugrep - which - curl - less - wget - man - cacert.out - findutils - iana-etc - git - openssh - ] + lib.lists.flatten ( + map (x: if !(x ? inProfile) || x.inProfile then x.contents else [ ]) layerContents + ) + ++ autoLayered ++ extraPkgs; users = @@ -139,16 +165,17 @@ let )) + "\n"; + nixpkgs = pkgs.path; + channel = pkgs.runCommand "channel-nixpkgs" { } '' + mkdir $out + ${lib.optionalString bundleNixpkgs '' + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + ''} + ''; + baseSystem = let - nixpkgs = pkgs.path; - channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } '' - mkdir $out - if [ "$bundleNixpkgs" ]; then - ln -s ${nixpkgs} $out/nixpkgs - echo "[]" > $out/manifest.nix - fi - ''; rootEnv = pkgs.buildPackages.buildEnv { name = "root-profile-env"; paths = defaultPkgs; @@ -187,7 +214,7 @@ let profile = pkgs.buildPackages.runCommand "user-environment" { } '' mkdir $out cp -a ${rootEnv}/* $out/ - ln -s ${manifest} $out/manifest.nix + ln -sf ${manifest} $out/manifest.nix ''; flake-registry-path = if (flake-registry == null) then @@ -236,6 +263,7 @@ let ln -s /nix/var/nix/profiles/share $out/usr/ mkdir -p $out/nix/var/nix/gcroots + ln -s /nix/var/nix/profiles $out/nix/var/nix/gcroots/profiles mkdir $out/tmp @@ -248,14 +276,14 @@ let mkdir -p $out/nix/var/nix/profiles/per-user/root ln -s ${profile} $out/nix/var/nix/profiles/default-1-link - ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default ln -s /nix/var/nix/profiles/default $out/root/.nix-profile ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link - ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels + ln -s /nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels mkdir -p $out/root/.nix-defexpr - ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels + ln -s /nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels echo "${channelURL} ${channelName}" > $out/root/.nix-channels mkdir -p $out/bin $out/usr/bin @@ -273,43 +301,90 @@ let ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName '') ); -in -pkgs.dockerTools.buildLayeredImageWithNixDb { - inherit name tag maxLayers; + layers = builtins.foldl' ( + layersList: el: + let + layer = nix2container.buildLayer { + deps = el.contents; + layers = layersList; + }; + in + layersList ++ [ layer ] + ) [ ] layerContents; - contents = [ baseSystem ]; + image = nix2container.buildImage { - extraCommands = '' - rm -rf nix-support - ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles - ''; - fakeRootCommands = '' - chmod 1777 tmp - chmod 1777 var/tmp - ''; + inherit name tag maxLayers; - config = { - Cmd = [ "/root/.nix-profile/bin/bash" ]; - Env = [ - "USER=root" - "PATH=${ - lib.concatStringsSep ":" [ - "/root/.nix-profile/bin" - "/nix/var/nix/profiles/default/bin" - "/nix/var/nix/profiles/default/sbin" - ] - }" - "MANPATH=${ - lib.concatStringsSep ":" [ - "/root/.nix-profile/share/man" - "/nix/var/nix/profiles/default/share/man" - ] - }" - "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" - "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" - "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" - "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + inherit layers; + + copyToRoot = [ baseSystem ]; + + initializeNixDatabase = true; + + perms = [ + { + path = baseSystem; + regex = "(/var)?/tmp"; + mode = "1777"; + } ]; + + config = { + Cmd = [ "/root/.nix-profile/bin/bash" ]; + Env = [ + "USER=root" + "PATH=${ + lib.concatStringsSep ":" [ + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ] + }" + "MANPATH=${ + lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ] + }" + "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + ]; + }; + }; +in +image +// { + # We don't ship the tarball as the default output because it is a strange thing to want imo + tarball = + pkgs.buildPackages.runCommand "docker-image-tarball-${pkgs.nix.version}" + { + nativeBuildInputs = [ pkgs.buildPackages.bubblewrap ]; + meta.description = "Docker image tarball with Lix for ${pkgs.system}"; + } + '' + mkdir -p $out/nix-support + image=$out/image.tar + # bwrap for foolish temp dir selection code that forces /var/tmp: + # https://github.com/containers/skopeo.git/blob/60ee543f7f7c242f46cc3a7541d9ac8ab1c89168/vendor/github.com/containers/image/v5/internal/tmpdir/tmpdir.go#L15-L18 + mkdir -p $TMPDIR/fake-var/tmp + args=(--unshare-user --bind "$TMPDIR/fake-var" /var) + for dir in /*; do + args+=(--dev-bind "/$dir" "/$dir") + done + bwrap ''${args[@]} -- ${lib.getExe image.copyTo} docker-archive:$image + gzip $image + echo "file binary-dist $image" >> $out/nix-support/hydra-build-products + ''; + meta = image.meta // { + description = "Docker image for Lix. This is built with nix2container; see that project's README for details"; + longDescription = '' + Docker image for Lix, built with nix2container. + To copy it to your docker daemon, nix run .#dockerImage.copyToDockerDaemon + To copy it to podman, nix run .#dockerImage.copyTo containers-storage:lix + ''; }; } diff --git a/flake.lock b/flake.lock index 8e59b2942..436cbcd03 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,22 @@ "type": "github" } }, + "nix2container": { + "flake": false, + "locked": { + "lastModified": 1712990762, + "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1715123187, @@ -67,6 +83,7 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression", "pre-commit-hooks": "pre-commit-hooks" diff --git a/flake.nix b/flake.nix index 8460201b0..6c7abaeec 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,10 @@ url = "github:cachix/git-hooks.nix"; flake = false; }; + nix2container = { + url = "github:nlewo/nix2container"; + flake = false; + }; flake-compat = { url = "github:edolstra/flake-compat"; flake = false; @@ -20,6 +24,7 @@ nixpkgs, nixpkgs-regression, pre-commit-hooks, + nix2container, flake-compat, }: @@ -330,19 +335,13 @@ dockerImage = let pkgs = nixpkgsFor.${system}.native; - image = import ./docker.nix { - inherit pkgs; - tag = pkgs.nix.version; - }; + nix2container' = import nix2container { inherit pkgs system; }; in - pkgs.runCommand "docker-image-tarball-${pkgs.nix.version}" - { meta.description = "Docker image with Lix for ${system}"; } - '' - mkdir -p $out/nix-support - image=$out/image.tar.gz - ln -s ${image} $image - echo "file binary-dist $image" >> $out/nix-support/hydra-build-products - ''; + import ./docker.nix { + inherit pkgs; + nix2container = nix2container'.nix2container; + tag = pkgs.nix.version; + }; } // builtins.listToAttrs ( map (crossSystem: { diff --git a/releng/release-jobs.nix b/releng/release-jobs.nix index e693a3a81..4db0baed3 100644 --- a/releng/release-jobs.nix +++ b/releng/release-jobs.nix @@ -33,7 +33,7 @@ let targetName = "*.tar.xz"; }) systems ++ builtins.map (system: { - target = hydraJobs.dockerImage.${system}; + target = hydraJobs.dockerImage.${system}.tarball; targetName = "image.tar.gz"; rename = "lix-${lix.version}-docker-image-${system}.tar.gz"; }) dockerSystems;