ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge branch 'binary-cache'

Browse Source
This commit is contained in:
Eelco Dolstra
2016-02-29 18:29:07 +01:00
parent 633381a501 7cd08c7c46
commit bc958c508b
22 changed files with 673 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/lib/Hydra/Controller/Build.pm
View File

@ -64,7 +64,10 @@ sub build_GET {
my $build = $c->stash->{build};
$c->stash->{template} = 'build.tt';
$c->stash->{available} = all { isValidPath($_->path) } $build->buildoutputs->all;
$c->stash->{available} =
($c->config->{store_mode} // "direct") eq "direct"
? all { isValidPath($_->path) } $build->buildoutputs->all
: 1; # FIXME
$c->stash->{drvAvailable} = isValidPath $build->drvpath;
if ($build->finished && $build->iscachedbuild) {
@ -219,7 +222,33 @@ sub download : Chained('buildChain') PathPart {
}
notFound($c, "Build doesn't have a product $productRef.") if !defined $product;
notFound($c, "Build product " . $product->path . " has disappeared.") unless -e $product->path;
if ($product->path !~ /^($Nix::Config::storeDir\/[^\/]+)/) {
die "Invalid store path " . $product->path . ".\n";
}
my $storePath = $1;
# Hack to get downloads to work on binary cache stores: if the
# store path is not available locally, then import it into the
# local store. FIXME: find a better way; this can require an
# unbounded amount of space.
if (!isValidPath($storePath)) {
my $storeMode = $c->config->{store_mode} // "direct";
notFound($c, "File " . $product->path . " has disappeared.")
if $storeMode eq "direct";
my $url =
$storeMode eq "local-binary-cache" ? "file://" . $c->config->{binary_cache_dir} :
$storeMode eq "s3-binary-cache" ? "https://" . $c->config->{binary_cache_s3_bucket} . ".s3.amazonaws.com/" :
die;
my $args = "";
if (defined $c->config->{binary_cache_public_key_file}
&& -r $c->config->{binary_cache_public_key_file})
{
$args = "--option binary-cache-public-keys " . read_file($c->config->{binary_cache_public_key_file});
}
system("nix-store --realise '$storePath' --option extra-binary-caches '$url' $args>/dev/null");
}
notFound($c, "File " . $product->path . " does not exist.") unless -e $product->path;
return $c->res->redirect(defaultUriForProduct($self, $c, $product, @path))
if scalar @path == 0 && ($product->name || $product->defaultpath);

110
src/lib/Hydra/Controller/Root.pm
View File

@ -118,6 +118,22 @@ sub status_GET {
}
sub queue_runner_status :Local :Path('queue-runner-status') :Args(0) :ActionClass('REST') { }
sub queue_runner_status_GET {
my ($self, $c) = @_;
#my $status = from_json($c->model('DB::SystemStatus')->find('queue-runner')->status);
my $status = from_json(`hydra-queue-runner --status`);
if ($?) { $status->{status} = "unknown"; }
my $json = JSON->new->pretty()->canonical();
$c->stash->{template} = 'queue-runner-status.tt';
$c->stash->{status} = $json->encode($status);
$self->status_ok($c, entity => $status);
}
sub machines :Local Args(0) {
my ($self, $c) = @_;
my $machines = getMachines;
@ -241,48 +257,90 @@ sub serialize : ActionClass('Serialize') { }
sub nar :Local :Args(1) {
my ($self, $c, $path) = @_;
$path = ($ENV{NIX_STORE_DIR} || "/nix/store")."/$path";
die if $path =~ /\//;
gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
my $storeMode = $c->config->{store_mode} // "direct";
$c->stash->{current_view} = 'NixNAR';
$c->stash->{storePath} = $path;
if ($storeMode eq "s3-binary-cache") {
notFound($c, "There is no binary cache here.");
}
elsif ($storeMode eq "local-binary-cache") {
my $dir = $c->config->{binary_cache_dir};
$c->serve_static_file($dir . "/nar/" . $path);
}
else {
$path = $Nix::Config::storeDir . "/$path";
gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
$c->stash->{current_view} = 'NixNAR';
$c->stash->{storePath} = $path;
}
}
sub nix_cache_info :Path('nix-cache-info') :Args(0) {
my ($self, $c) = @_;
$c->response->content_type('text/plain');
$c->stash->{plain}->{data} =
#"StoreDir: $Nix::Config::storeDir\n" . # FIXME
"StoreDir: " . ($ENV{NIX_STORE_DIR} || "/nix/store") . "\n" .
"WantMassQuery: 0\n" .
# Give Hydra binary caches a very low priority (lower than the
# static binary cache http://nixos.org/binary-cache).
"Priority: 100\n";
setCacheHeaders($c, 24 * 60 * 60);
$c->forward('Hydra::View::Plain');
my $storeMode = $c->config->{store_mode} // "direct";
if ($storeMode eq "s3-binary-cache") {
notFound($c, "There is no binary cache here.");
}
elsif ($storeMode eq "local-binary-cache") {
my $dir = $c->config->{binary_cache_dir};
$c->serve_static_file($dir . "/nix-cache-info");
}
else {
$c->response->content_type('text/plain');
$c->stash->{plain}->{data} =
"StoreDir: $Nix::Config::storeDir\n" .
"WantMassQuery: 0\n" .
# Give Hydra binary caches a very low priority (lower than the
# static binary cache http://nixos.org/binary-cache).
"Priority: 100\n";
setCacheHeaders($c, 24 * 60 * 60);
$c->forward('Hydra::View::Plain');
}
}
sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) {
my ($self, $c) = @_;
my $hash = $c->req->captures->[0];
die if length($hash) != 32;
my $path = queryPathFromHashPart($hash);
my $storeMode = $c->config->{store_mode} // "direct";
if (!$path) {
$c->response->status(404);
$c->response->content_type('text/plain');
$c->stash->{plain}->{data} = "does not exist\n";
$c->forward('Hydra::View::Plain');
setCacheHeaders($c, 60 * 60);
return;
if ($storeMode eq "s3-binary-cache") {
notFound($c, "There is no binary cache here.");
}
$c->stash->{storePath} = $path;
$c->forward('Hydra::View::NARInfo');
elsif ($storeMode eq "local-binary-cache") {
my $dir = $c->config->{binary_cache_dir};
$c->serve_static_file($dir . "/" . $c->req->captures->[0] . ".narinfo");
}
else {
my $hash = $c->req->captures->[0];
die if length($hash) != 32;
my $path = queryPathFromHashPart($hash);
if (!$path) {
$c->response->status(404);
$c->response->content_type('text/plain');
$c->stash->{plain}->{data} = "does not exist\n";
$c->forward('Hydra::View::Plain');
setCacheHeaders($c, 60 * 60);
return;
}
$c->stash->{storePath} = $path;
$c->forward('Hydra::View::NARInfo');
}
}