diff --git a/default.nix b/default.nix index c93f05a..7e9b7c1 100644 --- a/default.nix +++ b/default.nix @@ -15,7 +15,7 @@ stdenv.mkDerivation { name = "nixos-channel-scripts"; buildInputs = with perlPackages; - [ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json ]; + [ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json boost ]; buildCommand = '' mkdir -p $out/bin diff --git a/mirror-nixos-branch.pl b/mirror-nixos-branch.pl index abaf8fe..28a65b1 100755 --- a/mirror-nixos-branch.pl +++ b/mirror-nixos-branch.pl @@ -17,6 +17,7 @@ my $channelName = $ARGV[0]; my $releaseUrl = $ARGV[1]; +my $useAWS = ($ENV{'AWS_DISABLE'} or "") eq "1"; die "Usage: $0 CHANNEL-NAME RELEASE-URL\n" unless defined $channelName && defined $releaseUrl; @@ -31,19 +32,21 @@ $ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels"; +my $bucket; +if ($useAWS) { + # S3 setup. + my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die; + my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die; -# S3 setup. -my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die; -my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die; + my $s3 = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + host => "s3-eu-west-1.amazonaws.com", + }); -my $s3 = Net::Amazon::S3->new( - { aws_access_key_id => $aws_access_key_id, - aws_secret_access_key => $aws_secret_access_key, - retry => 1, - host => "s3-eu-west-1.amazonaws.com", - }); - -my $bucket = $s3->bucket($bucketName) or die; + $bucket = $s3->bucket($bucketName) or die; +} sub fetch { @@ -80,7 +83,7 @@ sub fetch { die "channel would go back in time from $curRelease to $releaseName, bailing out\n" if $d == 1; } -if ($bucket->head_key("$releasePrefix")) { +if ($useAWS && $bucket->head_key("$releasePrefix")) { print STDERR "release already exists\n"; } else { my $tmpDir = "/data/releases/tmp/release-$channelName/$releaseName"; @@ -96,25 +99,27 @@ sub fetch { } sub downloadFile { - my ($jobName, $dstName) = @_; + my ($jobName, $dstBase) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - my $srcFile = $buildInfo->{buildproducts}->{1}->{path} or die "job '$jobName' lacks a store path"; - $dstName //= basename($srcFile); - my $dstFile = "$tmpDir/" . $dstName; - - my $sha256_expected = $buildInfo->{buildproducts}->{1}->{sha256hash} or die; - - if (! -e $dstFile) { - print STDERR "downloading $srcFile to $dstFile...\n"; - write_file("$dstFile.sha256", "$sha256_expected $dstName"); - system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 - or die "unable to fetch $srcFile\n"; - rename("$dstFile.tmp", $dstFile) or die; - } + my $buildproducts = $buildInfo->{buildproducts}; + for my $product (values %$buildproducts) { + my %product = %$product; + my $srcFile = $product->{path} or die "job '$jobName' lacks a store path"; + my $dstName = $dstBase // basename($srcFile); + my $dstFile = "$tmpDir/" . $dstName; + + my $sha256_expected = $product->{sha256hash} or die "file '$srcFile' lacks an expected hash"; + + if (! -e $dstFile) { + print STDERR "downloading $srcFile to $dstFile...\n"; + write_file("$dstFile.sha256", "$sha256_expected $dstName\n"); + system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 + or die "unable to fetch $srcFile\n"; + rename("$dstFile.tmp", $dstFile) or die; + } - if (-e "$dstFile.sha256") { my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { @@ -134,6 +139,11 @@ sub fetch { #downloadFile("nixos.iso_graphical.i686-linux"); downloadFile("nixos.ova.x86_64-linux"); #downloadFile("nixos.ova.i686-linux"); + # Netboot is currently not included in -small but maybe should be? + # This may fail because of nixos/hydra#580, hence wrapping in eval + # nixos/nixpkgs#44089 works around it, but is not backported to all channels. + eval { downloadFile("nixos.netboot.x86_64-linux"); }; + warn "Could not download netboot files: $@" if $@; } } else { @@ -175,7 +185,7 @@ sub fetch { my $basename = basename $fn; my $key = "$releasePrefix/" . $basename; - unless (defined $bucket->head_key($key)) { + if ($useAWS && ! (defined $bucket->head_key($key))) { print STDERR "mirroring $fn to s3://$bucketName/$key...\n"; $bucket->add_key_filename( $key, $fn, @@ -196,11 +206,13 @@ sub fetch { $html .= ""; - $bucket->add_key($releasePrefix, $html, - { content_type => "text/html" }) - or die $bucket->err . ": " . $bucket->errstr; + if ($useAWS) { + $bucket->add_key($releasePrefix, $html, + { content_type => "text/html" }) + or die $bucket->err . ": " . $bucket->errstr; - File::Path::remove_tree($tmpDir); + File::Path::remove_tree($tmpDir); + } } # Prevent concurrent writes to the channels directory.