Skip to content

Commit d3d33d5

Browse files
edolstraedolstra
committed
Store releases in S3
Fixes #7.
1 parent becf1f2 commit d3d33d5

File tree

3 files changed

+74
-74
lines changed

3 files changed

+74
-74
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@
88

99
$ ./result/bin/mirror-nixos-branch nixos-unstable-small https://hydra.nixos.org/job/nixos/unstable-small/tested/latest-finished
1010

11-
$ ./result/bin/generate-programs-index /data/releases/nixos-files.sqlite /data/releases/nixos/unstable-small/nixos-16.09pre89017.9db1990-tmp/unpack/nixos-16.09pre89017.9db1990/programs.sqlite http://nix-cache.s3.amazonaws.com/ /data/releases/nixos/unstable-small/nixos-16.09pre89017.9db1990-tmp/store-paths /data/releases/nixos/unstable-small/nixos-16.09pre89017.9db1990-tmp/unpack/nixos-16.09pre89017.9db1990/nixpkgs
11+
$ ./result/bin/generate-programs-index /data/releases/nixos-files.sqlite ./programs.sqlite http://nix-cache.s3.amazonaws.com/ /data/releases/nixos/unstable-small/nixos-16.09pre89017.9db1990-tmp/store-paths /data/releases/nixos/unstable-small/nixos-16.09pre89017.9db1990-tmp/unpack/nixos-16.09pre89017.9db1990/nixpkgs

‎default.nix

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,18 @@ with pkgs;
55
let
66

77
# FIXME
8-
nix = lib.overrideDerivation nixUnstable (orig: {
9-
src = /home/eelco/Dev/nix;
10-
});
8+
nix = builtins.storePath /nix/store/b554awsd9vssb936mb32icgpfbyifsai-nix-1.12pre1234_abcdef;
9+
#lib.overrideDerivation nixUnstable (orig: {
10+
# src = /home/eelco/Dev/nix;
11+
#});
1112

1213
in
1314

1415
stdenv.mkDerivation {
1516
name = "nixos-channel-scripts";
1617

1718
buildInputs = with perlPackages;
18-
[ nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite ];
19+
[ nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc ];
1920

2021
buildCommand = ''
2122
mkdir -p $out/bin
@@ -25,6 +26,8 @@ stdenv.mkDerivation {
2526
2627
cp ${./mirror-nixos-branch.pl} $out/bin/mirror-nixos-branch
2728
wrapProgram $out/bin/mirror-nixos-branch --set PERL5LIB $PERL5LIB --prefix PATH : ${wget}/bin:${git}/bin:${nix}/bin:${gnutar}/bin:${xz}/bin:$out/bin
29+
30+
patchShebangs $out/bin
2831
'';
2932

3033
}

‎mirror-nixos-branch.pl

Lines changed: 66 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
#! /run/current-system/sw/bin/perl -w
1+
#! /usr/bin/env perl
22

33
use strict;
4+
use warnings;
45
use Data::Dumper;
56
use Fcntl qw(:flock);
67
use File::Basename;
@@ -9,21 +10,39 @@
910
use JSON::PP;
1011
use LWP::UserAgent;
1112
use List::MoreUtils qw(uniq);
13+
use Net::Amazon::S3;
1214

1315
my $channelName = $ARGV[0];
1416
my $releaseUrl = $ARGV[1];
15-
my $isMainRelease = ($ARGV[2] // 0) eq 1;
1617

17-
die "Usage: $0 CHANNEL-NAME RELEASE-URL [IS-MAIN-RELEASE]\n" unless defined $channelName && defined $releaseUrl;
18+
die "Usage: $0 CHANNEL-NAME RELEASE-URL\n" unless defined $channelName && defined $releaseUrl;
1819

1920
$channelName =~ /^([a-z]+)-(.*)$/ or die;
2021
my $channelDirRel = $channelName eq "nixpkgs-unstable" ? "nixpkgs" : "$1/$2";
21-
my $releasesDir = "/data/releases/$channelDirRel";
22+
23+
24+
# Configuration.
2225
my $channelsDir = "/data/releases/channels";
2326
my $filesCache = "/data/releases/nixos-files.sqlite";
27+
my $bucketName = "nix-releases";
2428

2529
$ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels";
2630

31+
32+
# S3 setup.
33+
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die;
34+
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die;
35+
36+
my $s3 = Net::Amazon::S3->new(
37+
{ aws_access_key_id => $aws_access_key_id,
38+
aws_secret_access_key => $aws_secret_access_key,
39+
retry => 1,
40+
host => "s3-eu-west-1.amazonaws.com",
41+
});
42+
43+
my $bucket = $s3->bucket($bucketName) or die;
44+
45+
2746
sub fetch {
2847
my ($url, $type) = @_;
2948

@@ -42,13 +61,12 @@ sub fetch {
4261
my $releaseName = $releaseInfo->{nixname} or die;
4362
my $evalId = $releaseInfo->{jobsetevals}->[0] or die;
4463
my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
45-
my $releaseDir = "$releasesDir/$releaseName";
46-
4764
my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
65+
my $releasePrefix = "$channelDirRel/$releaseName";
4866

4967
my $rev = $evalInfo->{jobsetevalinputs}->{nixpkgs}->{revision} or die;
5068

51-
print STDERR "release is ‘$releaseName’ (build $releaseId), eval is $evalId, dir is ‘$releaseDir’, Git commit is $rev\n";
69+
print STDERR "release is ‘$releaseName’ (build $releaseId), eval is $evalId, prefix is $releasePrefix, Git commit is $rev\n";
5270

5371
# Guard against the channel going back in time.
5472
my $curReleaseDir = readlink "$channelsDir/$channelName";
@@ -59,10 +77,10 @@ sub fetch {
5977
die "channel would go back in time from $curRelease to $releaseName, bailing out\n" if $d == 1;
6078
}
6179

62-
if (-d $releaseDir) {
80+
if ($bucket->head_key("$releasePrefix/github-link")) {
6381
print STDERR "release already exists\n";
6482
} else {
65-
my $tmpDir = dirname($releaseDir) . "/$releaseName-tmp";
83+
my $tmpDir = "/data/releases/tmp/release-$channelName/$releaseName";
6684
File::Path::make_path($tmpDir);
6785

6886
write_file("$tmpDir/src-url", $evalUrl);
@@ -74,18 +92,6 @@ sub fetch {
7492
write_file("$tmpDir/store-paths", join("\n", uniq(@{$storePaths})) . "\n");
7593
}
7694

77-
# Copy the manual.
78-
my $manualJob = $channelName =~ /nixos/ ? "nixos.manual.x86_64-linux" : "manual";
79-
my $manualDir = $channelName =~ /nixos/ ? "nixos" : "nixpkgs";
80-
if (! -e "$tmpDir/manual") {
81-
my $manualInfo = decode_json(fetch("$evalUrl/job/$manualJob", 'application/json'));
82-
my $manualPath = $manualInfo->{buildoutputs}->{out}->{path} or die;
83-
system("nix-store", "-r", $manualPath) == 0 or die "unable to fetch $manualPath\n";
84-
system("cp", "-rd", "$manualPath/share/doc/$manualDir", "$tmpDir/manual") == 0 or die "unable to copy manual from $manualPath";
85-
system("chmod", "-R", "u+w", "$tmpDir/manual");
86-
symlink("manual.html", "$tmpDir/manual/index.html") unless -e "$tmpDir/manual/index.html";
87-
}
88-
8995
sub downloadFile {
9096
my ($jobName, $dstName) = @_;
9197

@@ -95,22 +101,24 @@ sub fetch {
95101
$dstName //= basename($srcFile);
96102
my $dstFile = "$tmpDir/" . $dstName;
97103

104+
my $sha256_expected = $buildInfo->{buildproducts}->{1}->{sha256hash} or die;
105+
98106
if (! -e $dstFile) {
99107
print STDERR "downloading $srcFile to $dstFile...\n";
108+
write_file("$dstFile.sha256", $sha256_expected);
100109
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
101110
or die "unable to fetch $srcFile\n";
102111
rename("$dstFile.tmp", $dstFile) or die;
103112
}
104113

105-
my $sha256_expected = $buildInfo->{buildproducts}->{1}->{sha256hash} or die;
106-
my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`;
107-
chomp $sha256_actual;
108-
if ($sha256_expected ne $sha256_actual) {
109-
print STDERR "file $dstFile is corrupt\n";
110-
exit 1;
114+
if (-e "$dstFile.sha256") {
115+
my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`;
116+
chomp $sha256_actual;
117+
if ($sha256_expected ne $sha256_actual) {
118+
print STDERR "file $dstFile is corrupt $sha256_expected $sha256_actual\n";
119+
exit 1;
120+
}
111121
}
112-
113-
write_file("$dstFile.sha256", $sha256_expected);
114122
}
115123

116124
if ($channelName =~ /nixos/) {
@@ -129,19 +137,14 @@ sub fetch {
129137
downloadFile("tarball", "nixexprs.tar.xz");
130138
}
131139

132-
# Make "github-link" a redirect to the GitHub history of this
133-
# release.
134-
write_file("$tmpDir/.htaccess",
135-
"Redirect /releases/$channelDirRel/$releaseName/github-link https://github.com/NixOS/nixpkgs-channels/commits/$rev\n");
136-
write_file("$tmpDir/github-link", "");
137-
138140
# Generate the programs.sqlite database and put it in nixexprs.tar.xz.
139-
if ($channelName =~ /nixos/) {
141+
if ($channelName =~ /nixos/ && -e "$tmpDir/store-paths") {
140142
File::Path::make_path("$tmpDir/unpack");
141143
system("tar", "xfJ", "$tmpDir/nixexprs.tar.xz", "-C", "$tmpDir/unpack") == 0 or die;
142144
my $exprDir = glob("$tmpDir/unpack/*");
143145
system("generate-programs-index $filesCache $exprDir/programs.sqlite http://nix-cache.s3.amazonaws.com/ $tmpDir/store-paths $exprDir/nixpkgs") == 0 or die;
144146
system("rm -f $tmpDir/nixexprs.tar.xz $exprDir/programs.sqlite-journal") == 0 or die;
147+
unlink("$tmpDir/nixexprs.tar.xz.sha256");
145148
system("tar", "cfJ", "$tmpDir/nixexprs.tar.xz", "-C", "$tmpDir/unpack", basename($exprDir)) == 0 or die;
146149
system("rm -rf $tmpDir/unpack") == 0 or die;
147150
}
@@ -150,7 +153,29 @@ sub fetch {
150153
system("xz", "$tmpDir/store-paths") == 0 or die;
151154
}
152155

153-
rename($tmpDir, $releaseDir) or die;
156+
# Upload the release to S3.
157+
for my $fn (glob("$tmpDir/*")) {
158+
my $key = "$releasePrefix/" . basename $fn;
159+
unless (defined $bucket->head_key($key)) {
160+
print STDERR "mirroring $fn to s3://$bucketName/$key...\n";
161+
$bucket->add_key_filename(
162+
$key, $fn,
163+
{ content_type => $fn =~ /.sha256|src-url|binary-cache-url|git-revision/ ? "text/plain" : "application/octet-stream" })
164+
or die $bucket->err . $bucket->errstr;
165+
}
166+
}
167+
168+
# Make "github-link" a redirect to the GitHub history of this
169+
# release.
170+
my $link = "https://github.com/NixOS/nixpkgs-channels/commits/$rev";
171+
$bucket->add_key(
172+
"$releasePrefix/github-link", $link,
173+
{ 'x-amz-website-redirect-location' => $link,
174+
content_type => "text/plain"
175+
})
176+
or die $bucket->err . $bucket->errstr;
177+
178+
File::Path::remove_tree($tmpDir);
154179
}
155180

156181
# Prevent concurrent writes to the channels and the Git clone.
@@ -159,13 +184,14 @@ sub fetch {
159184

160185
# Update the channel.
161186
my $htaccess = "$channelsDir/.htaccess-$channelName";
187+
my $target = "https://d3g5gsiof5omrk.cloudfront.net/$releasePrefix";
162188
write_file($htaccess,
163-
"Redirect /channels/$channelName /releases/$channelDirRel/$releaseName\n" .
164-
"Redirect /releases/nixos/channels/$channelName /releases/$channelDirRel/$releaseName\n");
189+
"Redirect /channels/$channelName $target\n" .
190+
"Redirect /releases/nixos/channels/$channelName $target\n");
165191

166192
my $channelLink = "$channelsDir/$channelName";
167193
unlink("$channelLink.tmp");
168-
symlink($releaseDir, "$channelLink.tmp") or die;
194+
write_file("$channelLink.tmp", "$target");
169195
rename("$channelLink.tmp", $channelLink) or die;
170196

171197
system("cat $channelsDir/.htaccess-nix* > $channelsDir/.htaccess.tmp") == 0 or die;
@@ -174,32 +200,3 @@ sub fetch {
174200
# Update the nixpkgs-channels repo.
175201
system("git remote update origin >&2") == 0 or die;
176202
system("git push channels $rev:refs/heads/$channelName >&2") == 0 or die;
177-
178-
# If this is the "main" stable release, generate a .htaccess with some
179-
# symbolic redirects to the latest ISOs.
180-
181-
if ($isMainRelease) {
182-
183-
my $baseURL = "/releases/$channelDirRel/$releaseName";
184-
my $res = "Redirect /releases/nixos/latest $baseURL\n";
185-
186-
sub add {
187-
my ($name, $wildcard) = @_;
188-
my @files = glob "$releaseDir/$wildcard";
189-
die if scalar @files != 1;
190-
my $fn = basename($files[0]);
191-
$res .= "Redirect /releases/nixos/$name $baseURL/$fn\n";
192-
$res .= "Redirect /releases/nixos/$name-sha256 $baseURL/$fn.sha256\n";
193-
}
194-
195-
add("latest-iso-minimal-i686-linux", "nixos-minimal-*-i686-linux.iso");
196-
add("latest-iso-minimal-x86_64-linux", "nixos-minimal-*-x86_64-linux.iso");
197-
#add("latest-iso-graphical-i686-linux", "nixos-graphical-*-i686-linux.iso");
198-
add("latest-iso-graphical-x86_64-linux", "nixos-graphical-*-x86_64-linux.iso");
199-
#add("latest-ova-i686-linux", "nixos-*-i686-linux.ova");
200-
add("latest-ova-x86_64-linux", "nixos-*-x86_64-linux.ova");
201-
202-
my $htaccess2 = "/data/releases/nixos/.htaccess";
203-
write_file("$htaccess2.tmp", $res);
204-
rename("$htaccess2.tmp", $htaccess2) or die;
205-
}

0 commit comments

Comments
 (0)