Rewrite docker to be sensible and smaller

I have checked the image can build things and inspected `diff -ru`
compared to the old image. As far as I can tell it is more or less
the same besides the later git change.

Layers are now 65MB or less, and we aren't against the maxLayers limit
for the broken automatic layering to do anything but shove one store
path in a layer (which is good behaviour, actually).

This uses nix2container which streams images, so the build time is much
shorter.

I have also taken the opportunity to, in addition to fixing the 400MB
single layer (terrible, and what motivated this in the first place),
delete about 200MB of closure size inflicted by git vs gitMinimal
causing both perl and python to get into closure.

People mostly use this thing for CI, so I don't really think you need
advanced git operations, and large git can be added at the user side if
really motivated.

With love for whichever container developer somewhat ironically assumed
that one would not run skopeo in a minimal container that doesn't have a
/var/tmp.

Fixes: lix-project/lix#378

Change-Id: Icc3aa20e64446276716fbbb87535fd5b50628010
This commit is contained in:
jade 2024-06-07 01:35:26 -07:00
parent ff95b980d4
commit 9bb7fb8f69
4 changed files with 168 additions and 77 deletions

View file

@ -1,5 +1,6 @@
{ {
pkgs ? import <nixpkgs> { }, pkgs ? import <nixpkgs> { },
nix2container,
lib ? pkgs.lib, lib ? pkgs.lib,
name ? "lix", name ? "lix",
tag ? "latest", tag ? "latest",
@ -12,26 +13,51 @@
flake-registry ? null, flake-registry ? null,
}: }:
let let
defaultPkgs = layerContents = with pkgs; [
with pkgs; # pulls in glibc and openssl, about 60MB
[ { contents = [ coreutils-full ]; }
nix # some stuff that is low in the closure graph and small ish, mostly to make
# incremental lix updates cheaper
{
contents = [
curl
libxml2
sqlite
];
}
# 50MB of git
{ contents = [ gitMinimal ]; }
# 144MB of nixpkgs
{
contents = [ channel ];
inProfile = false;
}
];
# These packages are left to be auto layered by nix2container, since it is
# less critical that they get layered sensibly and they tend to not be deps
# of anything in particular
autoLayered = with pkgs; [
bashInteractive bashInteractive
coreutils-full
gnutar gnutar
gzip gzip
gnugrep gnugrep
which which
curl
less less
wget wget
man man
cacert.out cacert.out
findutils findutils
iana-etc iana-etc
git
openssh openssh
] nix
];
defaultPkgs =
lib.lists.flatten (
map (x: if !(x ? inProfile) || x.inProfile then x.contents else [ ]) layerContents
)
++ autoLayered
++ extraPkgs; ++ extraPkgs;
users = users =
@ -139,16 +165,17 @@ let
)) ))
+ "\n"; + "\n";
baseSystem =
let
nixpkgs = pkgs.path; nixpkgs = pkgs.path;
channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } '' channel = pkgs.runCommand "channel-nixpkgs" { } ''
mkdir $out mkdir $out
if [ "$bundleNixpkgs" ]; then ${lib.optionalString bundleNixpkgs ''
ln -s ${nixpkgs} $out/nixpkgs ln -s ${nixpkgs} $out/nixpkgs
echo "[]" > $out/manifest.nix echo "[]" > $out/manifest.nix
fi ''}
''; '';
baseSystem =
let
rootEnv = pkgs.buildPackages.buildEnv { rootEnv = pkgs.buildPackages.buildEnv {
name = "root-profile-env"; name = "root-profile-env";
paths = defaultPkgs; paths = defaultPkgs;
@ -187,7 +214,7 @@ let
profile = pkgs.buildPackages.runCommand "user-environment" { } '' profile = pkgs.buildPackages.runCommand "user-environment" { } ''
mkdir $out mkdir $out
cp -a ${rootEnv}/* $out/ cp -a ${rootEnv}/* $out/
ln -s ${manifest} $out/manifest.nix ln -sf ${manifest} $out/manifest.nix
''; '';
flake-registry-path = flake-registry-path =
if (flake-registry == null) then if (flake-registry == null) then
@ -236,6 +263,7 @@ let
ln -s /nix/var/nix/profiles/share $out/usr/ ln -s /nix/var/nix/profiles/share $out/usr/
mkdir -p $out/nix/var/nix/gcroots mkdir -p $out/nix/var/nix/gcroots
ln -s /nix/var/nix/profiles $out/nix/var/nix/gcroots/profiles
mkdir $out/tmp mkdir $out/tmp
@ -248,14 +276,14 @@ let
mkdir -p $out/nix/var/nix/profiles/per-user/root mkdir -p $out/nix/var/nix/profiles/per-user/root
ln -s ${profile} $out/nix/var/nix/profiles/default-1-link ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
ln -s /nix/var/nix/profiles/default $out/root/.nix-profile ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels ln -s /nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
mkdir -p $out/root/.nix-defexpr mkdir -p $out/root/.nix-defexpr
ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels ln -s /nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
echo "${channelURL} ${channelName}" > $out/root/.nix-channels echo "${channelURL} ${channelName}" > $out/root/.nix-channels
mkdir -p $out/bin $out/usr/bin mkdir -p $out/bin $out/usr/bin
@ -273,21 +301,35 @@ let
ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName
'') '')
); );
layers = builtins.foldl' (
layersList: el:
let
layer = nix2container.buildLayer {
deps = el.contents;
layers = layersList;
};
in in
pkgs.dockerTools.buildLayeredImageWithNixDb { layersList ++ [ layer ]
) [ ] layerContents;
image = nix2container.buildImage {
inherit name tag maxLayers; inherit name tag maxLayers;
contents = [ baseSystem ]; inherit layers;
extraCommands = '' copyToRoot = [ baseSystem ];
rm -rf nix-support
ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles initializeNixDatabase = true;
'';
fakeRootCommands = '' perms = [
chmod 1777 tmp {
chmod 1777 var/tmp path = baseSystem;
''; regex = "(/var)?/tmp";
mode = "1777";
}
];
config = { config = {
Cmd = [ "/root/.nix-profile/bin/bash" ]; Cmd = [ "/root/.nix-profile/bin/bash" ];
@ -312,4 +354,37 @@ pkgs.dockerTools.buildLayeredImageWithNixDb {
"NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels"
]; ];
}; };
};
in
image
// {
# We don't ship the tarball as the default output because it is a strange thing to want imo
tarball =
pkgs.buildPackages.runCommand "docker-image-tarball-${pkgs.nix.version}"
{
nativeBuildInputs = [ pkgs.buildPackages.bubblewrap ];
meta.description = "Docker image tarball with Lix for ${pkgs.system}";
}
''
mkdir -p $out/nix-support
image=$out/image.tar
# bwrap for foolish temp dir selection code that forces /var/tmp:
# https://github.com/containers/skopeo.git/blob/60ee543f7f7c242f46cc3a7541d9ac8ab1c89168/vendor/github.com/containers/image/v5/internal/tmpdir/tmpdir.go#L15-L18
mkdir -p $TMPDIR/fake-var/tmp
args=(--unshare-user --bind "$TMPDIR/fake-var" /var)
for dir in /*; do
args+=(--dev-bind "/$dir" "/$dir")
done
bwrap ''${args[@]} -- ${lib.getExe image.copyTo} docker-archive:$image
gzip $image
echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
'';
meta = image.meta // {
description = "Docker image for Lix. This is built with nix2container; see that project's README for details";
longDescription = ''
Docker image for Lix, built with nix2container.
To copy it to your docker daemon, nix run .#dockerImage.copyToDockerDaemon
To copy it to podman, nix run .#dockerImage.copyTo containers-storage:lix
'';
};
} }

View file

@ -16,6 +16,22 @@
"type": "github" "type": "github"
} }
}, },
"nix2container": {
"flake": false,
"locked": {
"lastModified": 1712990762,
"narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
"owner": "nlewo",
"repo": "nix2container",
"rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
"type": "github"
},
"original": {
"owner": "nlewo",
"repo": "nix2container",
"type": "github"
}
},
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1715123187, "lastModified": 1715123187,
@ -67,6 +83,7 @@
"root": { "root": {
"inputs": { "inputs": {
"flake-compat": "flake-compat", "flake-compat": "flake-compat",
"nix2container": "nix2container",
"nixpkgs": "nixpkgs", "nixpkgs": "nixpkgs",
"nixpkgs-regression": "nixpkgs-regression", "nixpkgs-regression": "nixpkgs-regression",
"pre-commit-hooks": "pre-commit-hooks" "pre-commit-hooks": "pre-commit-hooks"

View file

@ -8,6 +8,10 @@
url = "github:cachix/git-hooks.nix"; url = "github:cachix/git-hooks.nix";
flake = false; flake = false;
}; };
nix2container = {
url = "github:nlewo/nix2container";
flake = false;
};
flake-compat = { flake-compat = {
url = "github:edolstra/flake-compat"; url = "github:edolstra/flake-compat";
flake = false; flake = false;
@ -20,6 +24,7 @@
nixpkgs, nixpkgs,
nixpkgs-regression, nixpkgs-regression,
pre-commit-hooks, pre-commit-hooks,
nix2container,
flake-compat, flake-compat,
}: }:
@ -330,19 +335,13 @@
dockerImage = dockerImage =
let let
pkgs = nixpkgsFor.${system}.native; pkgs = nixpkgsFor.${system}.native;
image = import ./docker.nix { nix2container' = import nix2container { inherit pkgs system; };
in
import ./docker.nix {
inherit pkgs; inherit pkgs;
nix2container = nix2container'.nix2container;
tag = pkgs.nix.version; tag = pkgs.nix.version;
}; };
in
pkgs.runCommand "docker-image-tarball-${pkgs.nix.version}"
{ meta.description = "Docker image with Lix for ${system}"; }
''
mkdir -p $out/nix-support
image=$out/image.tar.gz
ln -s ${image} $image
echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
'';
} }
// builtins.listToAttrs ( // builtins.listToAttrs (
map (crossSystem: { map (crossSystem: {

View file

@ -33,7 +33,7 @@ let
targetName = "*.tar.xz"; targetName = "*.tar.xz";
}) systems }) systems
++ builtins.map (system: { ++ builtins.map (system: {
target = hydraJobs.dockerImage.${system}; target = hydraJobs.dockerImage.${system}.tarball;
targetName = "image.tar.gz"; targetName = "image.tar.gz";
rename = "lix-${lix.version}-docker-image-${system}.tar.gz"; rename = "lix-${lix.version}-docker-image-${system}.tar.gz";
}) dockerSystems; }) dockerSystems;