From 918d37cc24ad9c46925c35e84e1e6353360b1202 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 19 Jul 2018 11:09:05 +0200 Subject: [PATCH 1/3] Add boost to deps to fix build with nix 2.x --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index c93f05a..7e9b7c1 100644 --- a/default.nix +++ b/default.nix @@ -15,7 +15,7 @@ stdenv.mkDerivation { name = "nixos-channel-scripts"; buildInputs = with perlPackages; - [ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json ]; + [ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json boost ]; buildCommand = '' mkdir -p $out/bin From 0cf0276c5bbfbc028accd3e1b120e7aac774345a Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sat, 21 Jul 2018 13:52:37 +0200 Subject: [PATCH 2/3] Make AWS upload optional This allows testing channel mirroring locally without access to S3 --- mirror-nixos-branch.pl | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/mirror-nixos-branch.pl b/mirror-nixos-branch.pl index abaf8fe..09f98cd 100755 --- a/mirror-nixos-branch.pl +++ b/mirror-nixos-branch.pl @@ -17,6 +17,7 @@ my $channelName = $ARGV[0]; my $releaseUrl = $ARGV[1]; +my $useAWS = ($ENV{'AWS_DISABLE'} or "") eq "1"; die "Usage: $0 CHANNEL-NAME RELEASE-URL\n" unless defined $channelName && defined $releaseUrl; @@ -31,19 +32,21 @@ $ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels"; +my $bucket; +if ($useAWS) { + # S3 setup. + my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die; + my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die; -# S3 setup. -my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die; -my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die; + my $s3 = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + host => "s3-eu-west-1.amazonaws.com", + }); -my $s3 = Net::Amazon::S3->new( - { aws_access_key_id => $aws_access_key_id, - aws_secret_access_key => $aws_secret_access_key, - retry => 1, - host => "s3-eu-west-1.amazonaws.com", - }); - -my $bucket = $s3->bucket($bucketName) or die; + $bucket = $s3->bucket($bucketName) or die; +} sub fetch { @@ -80,7 +83,7 @@ sub fetch { die "channel would go back in time from $curRelease to $releaseName, bailing out\n" if $d == 1; } -if ($bucket->head_key("$releasePrefix")) { +if ($useAWS && $bucket->head_key("$releasePrefix")) { print STDERR "release already exists\n"; } else { my $tmpDir = "/data/releases/tmp/release-$channelName/$releaseName"; @@ -175,7 +178,7 @@ sub fetch { my $basename = basename $fn; my $key = "$releasePrefix/" . $basename; - unless (defined $bucket->head_key($key)) { + if ($useAWS && ! (defined $bucket->head_key($key))) { print STDERR "mirroring $fn to s3://$bucketName/$key...\n"; $bucket->add_key_filename( $key, $fn, @@ -196,11 +199,13 @@ sub fetch { $html .= ""; - $bucket->add_key($releasePrefix, $html, - { content_type => "text/html" }) - or die $bucket->err . ": " . $bucket->errstr; + if ($useAWS) { + $bucket->add_key($releasePrefix, $html, + { content_type => "text/html" }) + or die $bucket->err . ": " . $bucket->errstr; - File::Path::remove_tree($tmpDir); + File::Path::remove_tree($tmpDir); + } } # Prevent concurrent writes to the channels directory. From 96ada149ff312c8206a05aa8687882acbb8963e6 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Wed, 1 Aug 2018 18:28:14 +0200 Subject: [PATCH 3/3] Add netboot files to channel --- mirror-nixos-branch.pl | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/mirror-nixos-branch.pl b/mirror-nixos-branch.pl index 09f98cd..28a65b1 100755 --- a/mirror-nixos-branch.pl +++ b/mirror-nixos-branch.pl @@ -99,25 +99,27 @@ sub fetch { } sub downloadFile { - my ($jobName, $dstName) = @_; + my ($jobName, $dstBase) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - my $srcFile = $buildInfo->{buildproducts}->{1}->{path} or die "job '$jobName' lacks a store path"; - $dstName //= basename($srcFile); - my $dstFile = "$tmpDir/" . $dstName; - - my $sha256_expected = $buildInfo->{buildproducts}->{1}->{sha256hash} or die; - - if (! -e $dstFile) { - print STDERR "downloading $srcFile to $dstFile...\n"; - write_file("$dstFile.sha256", "$sha256_expected $dstName"); - system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 - or die "unable to fetch $srcFile\n"; - rename("$dstFile.tmp", $dstFile) or die; - } + my $buildproducts = $buildInfo->{buildproducts}; + for my $product (values %$buildproducts) { + my %product = %$product; + my $srcFile = $product->{path} or die "job '$jobName' lacks a store path"; + my $dstName = $dstBase // basename($srcFile); + my $dstFile = "$tmpDir/" . $dstName; + + my $sha256_expected = $product->{sha256hash} or die "file '$srcFile' lacks an expected hash"; + + if (! -e $dstFile) { + print STDERR "downloading $srcFile to $dstFile...\n"; + write_file("$dstFile.sha256", "$sha256_expected $dstName\n"); + system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 + or die "unable to fetch $srcFile\n"; + rename("$dstFile.tmp", $dstFile) or die; + } - if (-e "$dstFile.sha256") { my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { @@ -137,6 +139,11 @@ sub fetch { #downloadFile("nixos.iso_graphical.i686-linux"); downloadFile("nixos.ova.x86_64-linux"); #downloadFile("nixos.ova.i686-linux"); + # Netboot is currently not included in -small but maybe should be? + # This may fail because of nixos/hydra#580, hence wrapping in eval + # nixos/nixpkgs#44089 works around it, but is not backported to all channels. + eval { downloadFile("nixos.netboot.x86_64-linux"); }; + warn "Could not download netboot files: $@" if $@; } } else {