diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 9f789978..e9fdb293 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -270,17 +270,24 @@ void State::buildRemote(ref<Store> destStore,
copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
}
- /* Copy the input closure. */
- if (!machine->isLocalhost()) {
+ {
auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
mc1.reset();
MaintainCount<counter> mc2(nrStepsCopyingTo);
+
printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
localStore->printStorePath(step->drvPath), machine->sshName);
auto now1 = std::chrono::steady_clock::now();
- copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true);
+ /* Copy the input closure. */
+ if (machine->isLocalhost()) {
+ StorePathSet closure;
+ destStore->computeFSClosure(inputs, closure);
+ copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
+ } else {
+ copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true);
+ }
auto now2 = std::chrono::steady_clock::now();
diff --git a/t/jobs/dependencies/dependency.nix b/t/jobs/dependencies/dependency.nix
new file mode 100644
index 00000000..f528bdc8
--- /dev/null
+++ b/t/jobs/dependencies/dependency.nix
@@ -0,0 +1,17 @@
+{ exposeUnderlyingJob, exposeDependentJob }:
+with import ../config.nix;
+let
+ underlyingJob = mkDerivation {
+ name = "underlying-job";
+ builder = ../empty-dir-builder.sh;
+ };
+
+ dependentJob = mkDerivation {
+ name = "dependent-job";
+ builder = ../empty-dir-builder.sh;
+ inherit underlyingJob;
+ };
+in
+(if exposeUnderlyingJob then { inherit underlyingJob; } else { }) //
+(if exposeDependentJob then { inherit dependentJob; } else { }) //
+{ }
diff --git a/t/jobs/dependencies/dependentOnly.nix b/t/jobs/dependencies/dependentOnly.nix
new file mode 100644
index 00000000..edf8a912
--- /dev/null
+++ b/t/jobs/dependencies/dependentOnly.nix
@@ -0,0 +1,4 @@
+import ./dependency.nix {
+ exposeUnderlyingJob = false;
+ exposeDependentJob = true;
+}
diff --git a/t/jobs/dependencies/underlyingOnly.nix b/t/jobs/dependencies/underlyingOnly.nix
new file mode 100644
index 00000000..106d17fe
--- /dev/null
+++ b/t/jobs/dependencies/underlyingOnly.nix
@@ -0,0 +1,4 @@
+import ./dependency.nix {
+ exposeUnderlyingJob = true;
+ exposeDependentJob = false;
+}
diff --git a/t/lib/CliRunners.pm b/t/lib/CliRunners.pm
index ddaa34ae..f803c2db 100644
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@@ -37,6 +37,8 @@ sub evalSucceeds {
$jobset->discard_changes; # refresh from DB
if ($res) {
chomp $stdout; chomp $stderr;
+ utf8::decode($stdout) or die "Invalid unicode in stdout.";
+ utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "Evaluation unexpectedly failed for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
print STDERR "STDOUT: $stdout\n" if $stdout ne "";
print STDERR "STDERR: $stderr\n" if $stderr ne "";
@@ -50,6 +52,8 @@ sub evalFails {
$jobset->discard_changes; # refresh from DB
if (!$res) {
chomp $stdout; chomp $stderr;
+ utf8::decode($stdout) or die "Invalid unicode in stdout.";
+ utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "Evaluation unexpectedly succeeded for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
print STDERR "STDOUT: $stdout\n" if $stdout ne "";
print STDERR "STDERR: $stderr\n" if $stderr ne "";
@@ -61,6 +65,8 @@ sub runBuild {
my ($build) = @_;
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));
if ($res) {
+ utf8::decode($stdout) or die "Invalid unicode in stdout.";
+ utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "Queue runner stdout: $stdout\n" if $stdout ne "";
print STDERR "Queue runner stderr: $stderr\n" if $stderr ne "";
}
@@ -70,6 +76,8 @@ sub runBuild {
sub sendNotifications() {
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-notify", "--queued-only"));
if ($res) {
+ utf8::decode($stdout) or die "Invalid unicode in stdout.";
+ utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "hydra notify stdout: $stdout\n" if $stdout ne "";
print STDERR "hydra notify stderr: $stderr\n" if $stderr ne "";
}
diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index 9f9db4f2..b8c254d0 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -69,6 +69,7 @@ sub new {
_db => undef,
db_handle => $pgsql,
tmpdir => $dir,
+ nix_state_dir => "$dir/nix/var/nix",
testdir => abs_path(dirname(__FILE__) . "/.."),
jobsdir => abs_path(dirname(__FILE__) . "/../jobs")
};
@@ -115,6 +116,12 @@ sub jobsdir {
return $self->{jobsdir};
}
+sub nix_state_dir {
+ my ($self) = @_;
+
+ return $self->{nix_state_dir};
+}
+
# Create a jobset, evaluate it, and optionally build the jobs.
#
# In return, you get a hash of all the Builds records, keyed
diff --git a/t/queue-runner/build-locally-with-substitutable-path.t b/t/queue-runner/build-locally-with-substitutable-path.t
new file mode 100644
index 00000000..f335d284
--- /dev/null
+++ b/t/queue-runner/build-locally-with-substitutable-path.t
@@ -0,0 +1,51 @@
+use strict;
+use warnings;
+use Setup;
+use Data::Dumper;
+use Test2::V0;
+my $ctx = test_context(
+ use_external_destination_store => 1
+);
+
+# This test is regarding https://github.com/NixOS/hydra/pull/1126
+#
+# A hydra instance was regularly failing to build derivations with:
+#
+# possibly transient failure building ‘/nix/store/X.drv’ on ‘localhost’:
+# dependency '/nix/store/Y' of '/nix/store/Y.drv' does not exist,
+# and substitution is disabled
+#
+# However it would only fail when building on localhost, and it would only
+# fail if the build output was already in the binary cache.
+#
+# This test replicates this scenario by having two jobs, underlyingJob and
+# dependentJob. dependentJob depends on underlyingJob. We first build
+# underlyingJob and copy it to an external cache. Then forcefully delete
+# the output of underlyingJob, and build dependentJob. In order to pass
+# it must either rebuild underlyingJob or fetch it from the cache.
+
+
+subtest "Building, caching, and then garbage collecting the underlying job" => sub {
+ my $builds = $ctx->makeAndEvaluateJobset(
+ expression => "dependencies/underlyingOnly.nix",
+ build => 1
+ );
+
+ my $path = $builds->{"underlyingJob"}->buildoutputs->find({ name => "out" })->path;
+
+ ok(unlink(Hydra::Helper::Nix::gcRootFor($path)), "Unlinking the GC root for underlying Dependency succeeds");
+
+ (my $ret, my $stdout, my $stderr) = captureStdoutStderr(1, "nix-store", "--delete", $path);
+ is($ret, 0, "Deleting the underlying dependency should succeed");
+};
+
+subtest "Building the dependent job should now succeed, even though we're missing a local dependency" => sub {
+ my $builds = $ctx->makeAndEvaluateJobset(
+ expression => "dependencies/dependentOnly.nix"
+ );
+
+ ok(runBuild($builds->{"dependentJob"}), "building the job should succeed");
+};
+
+
+done_testing;