From 163a3e591f867eac116b610366ed7cfaa2a7859b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 10 Dec 2012 18:22:46 +0000 Subject: [PATCH] * Cleanup scripts: handle the binary cache. git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@34661 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb --- ReadCache.pm | 21 --------------- delete-old-releases.sh | 17 ++++++------ print-dead-files.pl | 60 ++++++++++++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 43 deletions(-) delete mode 100644 ReadCache.pm diff --git a/ReadCache.pm b/ReadCache.pm deleted file mode 100644 index 9addf02..0000000 --- a/ReadCache.pm +++ /dev/null @@ -1,21 +0,0 @@ -package readcache; -use strict; - -# Read the archive directories. -our %archives; - -sub readDir { - my $dir = shift; - opendir(DIR, "$dir") or die "cannot open `$dir': $!"; - my @as = readdir DIR; - foreach my $archive (@as) { - next unless $archive =~ /^sha256_/ || $archive =~ /\.nar-bsdiff$/ || $archive =~ /\.nar\.bz2$/; - $archives{$archive} = "$dir/$archive"; - } - closedir DIR; -} - -readDir "/data/releases/nars"; -readDir "/data/releases/patches"; - -print STDERR scalar (keys %archives), "\n"; diff --git a/delete-old-releases.sh b/delete-old-releases.sh index 95ae781..71fe413 100755 --- a/delete-old-releases.sh +++ b/delete-old-releases.sh @@ -5,31 +5,32 @@ mkdir -p $trash # Remove garbage temporary directories. find /data/releases/nixos/ /data/releases/nixpkgs/ -maxdepth 1 -name ".tmp*" -mtime +7 | while read rel; do - echo "removing temporary directory $rel" + echo "removing temporary directory $rel" >&2 mv $rel $trash/ done # Remove old NixOS releases. find /data/releases/nixos/ -maxdepth 1 -name "nixos-*pre*" -mtime +30 | sort | while read rel; do if [ -e $rel/keep ]; then - echo "keeping NixOS release $rel" + echo "keeping NixOS release $rel" >&2 continue fi - echo "removing old NixOS release $rel" + echo "removing old NixOS release $rel" >&2 mv $rel $trash/ done # Remove old Nixpkgs releases. find /data/releases/nixpkgs/ -maxdepth 1 -name "nixpkgs-*pre*" -mtime +30 | sort | while read rel; do if [ -e $rel/keep ]; then - echo "keeping Nixpkgs release $rel" + echo "keeping Nixpkgs release $rel" >&2 continue fi - echo "removing old Nixpkgs release $rel" + echo "removing old Nixpkgs release $rel" >&2 mv $rel $trash/ done -# Remove unreferenced NARs/patches. -./print-dead-files.pl /data/releases/patches/all-patches $(find /data/releases -name MANIFEST | grep -v '\.trash' | grep -v '\.tmp') \ -| xargs -d '\n' sh -c 'find "$@" -mtime +50 -print' \ +# Remove unreferenced NARs/patches (but only if they're older than 2 +# weeks, to prevent messing with binary patch generation in progress). +./print-dead-files.pl /data/releases/patches/all-patches $(find /data/releases/nix* /data/releases/patchelf -name MANIFEST | grep -v '\.trash' | grep -v '\.tmp') \ +| xargs -d '\n' sh -c 'find "$@" -mtime +14 -print' \ | xargs -d '\n' mv -v --target-directory=$trash diff --git a/print-dead-files.pl b/print-dead-files.pl index 3b16e53..eca9250 100755 --- a/print-dead-files.pl +++ b/print-dead-files.pl @@ -2,9 +2,10 @@ use strict; use Nix::Manifest; -use ReadCache; use File::Basename; +my $cacheDir = "/data/releases/binary-cache"; + # Read the manifests. my %narFiles; @@ -13,23 +14,31 @@ my %patches; foreach my $manifest (@ARGV) { print STDERR "loading $manifest\n"; if (readManifest($manifest, \%narFiles, \%patches, 1) < 3) { -# die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n"; + warn "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n"; } } # Find the live archives. my %usedFiles; +my %hashParts; -foreach my $narFile (keys %narFiles) { - foreach my $file (@{$narFiles{$narFile}}) { +foreach my $storePath (keys %narFiles) { + $storePath =~ /\/nix\/store\/([a-z0-9]+)/ or die "WRONG: $storePath"; + $hashParts{$1} = 1; + foreach my $file (@{$narFiles{$storePath}}) { $file->{url} =~ /\/([^\/]+)$/; my $basename = $1; die unless defined $basename; #print STDERR "GOT $basename\n"; $usedFiles{$basename} = 1; - print STDERR "missing archive `$basename'\n" - unless defined $readcache::archives{$basename}; + die "$storePath does not have a file hash" unless defined $file->{hash}; + if ($file->{hash} =~ /sha256:(.+)/) { + die unless length($1) == 52; + $usedFiles{"$1.nar.bz2"} = 1; + } + #print STDERR "missing archive `$basename'\n" + # unless defined $readcache::archives{$basename}; } } @@ -40,17 +49,40 @@ foreach my $patch (keys %patches) { die unless defined $basename; #print STDERR "GOT2 $basename\n"; $usedFiles{$basename} = 1; - die "missing archive `$basename'" - unless defined $readcache::archives{$basename}; + #die "missing archive `$basename'" + # unless defined $readcache::archives{$basename}; } } -# Print out the dead archives. -foreach my $archive (keys %readcache::archives) { - next if $archive eq "." || $archive eq ".."; - if (!defined $usedFiles{$archive}) { - my $file = $readcache::archives{$archive}; - print "$file\n"; +sub checkDir { + my ($dir) = @_; + opendir(DIR, "$dir") or die "cannot open `$dir': $!"; + while (readdir DIR) { + next unless $_ =~ /^sha256_/ || $_ =~ /\.nar-bsdiff$/ || $_ =~ /\.nar\.bz2$/; + if (!defined $usedFiles{$_}) { + print "$dir/$_\n"; + } else { + #print STDERR "keeping $dir/$_\n"; + } + + } + closedir DIR; +} + +checkDir("/data/releases/nars"); +checkDir("/data/releases/patches"); +checkDir("$cacheDir/nar"); + +# Look for obsolete narinfo files. +opendir(DIR, $cacheDir) or die; +while (readdir DIR) { + next unless /^(.*)\.narinfo$/; + my $hashPart = $1; + if (!defined $hashParts{$hashPart}) { + print "$cacheDir/$_\n"; + } else { + #print STDERR "keeping $cacheDir/$_\n"; } } +closedir DIR;