Distinguish build step states

The web interface now shows whether a build step is connecting,
copying inputs/outputs, building, etc.

src/hydra-queue-runner/build-remote.cc

@@ -122,7 +122,8 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore,
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
     unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
-    RemoteResult & result, std::shared_ptr<ActiveStep> activeStep)
+    RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
+    std::function<void(StepState)> updateStep)
 {
     assert(BuildResult::TimedOut == 8);
 
@@ -140,6 +141,8 @@ void State::buildRemote(ref<Store> destStore,
 
     try {
 
+        updateStep(ssConnecting);
+
         Child child;
         openConnection(machine, tmpDir, logFD.get(), child);
 
@@ -204,6 +207,8 @@ void State::buildRemote(ref<Store> destStore,
            outputs of the input derivations. On Nix > 1.9, we only need to
            copy the immediate sources of the derivation and the required
            outputs of the input derivations. */
+        updateStep(ssSendingInputs);
+
         PathSet inputs;
         BasicDerivation basicDrv(step->drv);
 
@@ -260,6 +265,8 @@ void State::buildRemote(ref<Store> destStore,
         /* Do the build. */
         printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName);
 
+        updateStep(ssBuilding);
+
         if (sendDerivation)
             to << cmdBuildPaths << PathSet({step->drvPath});
         else
@@ -371,6 +378,8 @@ void State::buildRemote(ref<Store> destStore,
 
         /* Copy the output paths. */
         if (/* machine->sshName != "localhost" */ true) {
+            updateStep(ssReceivingOutputs);
+
             MaintainCount<counter> mc(nrStepsCopyingFrom);
 
             auto now1 = std::chrono::steady_clock::now();

src/hydra-queue-runner/builder.cc

@@ -195,10 +195,16 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
             txn.commit();
         }
 
+        auto updateStep = [&](StepState stepState) {
+            pqxx::work txn(*conn);
+            updateBuildStep(txn, buildId, stepNr, stepState);
+            txn.commit();
+        };
+
         /* Do the build. */
         try {
             /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep);
+            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep);
         } catch (NoTokens & e) {
             result.stepStatus = bsNarSizeLimitExceeded;
         } catch (Error & e) {
@@ -213,8 +219,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
             }
         }
 
-        if (result.stepStatus == bsSuccess)
+        if (result.stepStatus == bsSuccess) {
+            updateStep(ssPostProcessing);
             res = getBuildOutput(destStore, ref<FSAccessor>(result.accessor), step->drv);
+        }
 
         result.accessor = 0;
         result.tokens = 0;

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -271,7 +271,7 @@ void State::clearBusy(Connection & conn, time_t stopTime)
 {
     pqxx::work txn(conn);
     txn.parameterized
-        ("update BuildSteps set busy = 0, status = $1, stopTime = $2 where busy = 1")
+        ("update BuildSteps set busy = 0, status = $1, stopTime = $2 where busy != 0")
         ((int) bsAborted)
         (stopTime, stopTime != 0).exec();
     txn.commit();
@@ -317,6 +317,18 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
 }
 
 
+void State::updateBuildStep(pqxx::work & txn, BuildID buildId, unsigned int stepNr, StepState stepState)
+{
+    if (txn.parameterized
+        ("update BuildSteps set busy = $1 where build = $2 and stepnr = $3 and busy != 0 and status is null")
+        ((int) stepState)
+        (buildId)
+        (stepNr)
+        .exec().affected_rows() != 1)
+        throw Error("step %d of build %d is in an unexpected state", stepNr, buildId);
+}
+
+
 void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
     BuildID buildId, unsigned int stepNr, const std::string & machine)
 {
@@ -892,7 +904,7 @@ void State::run(BuildID buildOne)
                 for (auto & step : steps) {
                     printMsg(lvlError, format("cleaning orphaned step %d of build %d") % step.second % step.first);
                     txn.parameterized
-                        ("update BuildSteps set busy = 0, status = $1 where build = $2 and stepnr = $3 and busy = 1")
+                        ("update BuildSteps set busy = 0, status = $1 where build = $2 and stepnr = $3 and busy != 0")
                         ((int) bsAborted)
                         (step.first)
                         (step.second).exec();

src/hydra-queue-runner/state.hh

@@ -41,6 +41,16 @@ typedef enum {
 } BuildStatus;
 
 
+typedef enum {
+    ssPreparing = 1,
+    ssConnecting = 10,
+    ssSendingInputs = 20,
+    ssBuilding = 30,
+    ssReceivingOutputs = 40,
+    ssPostProcessing = 50,
+} StepState;
+
+
 struct RemoteResult
 {
     BuildStatus stepStatus = bsAborted;
@@ -464,6 +474,8 @@ private:
         const std::string & machine, BuildStatus status, const std::string & errorMsg = "",
         BuildID propagatedFrom = 0);
 
+    void updateBuildStep(pqxx::work & txn, BuildID buildId, unsigned int stepNr, StepState stepState);
+
     void finishBuildStep(pqxx::work & txn, const RemoteResult & result, BuildID buildId, unsigned int stepNr,
         const std::string & machine);
 
@@ -518,7 +530,8 @@ private:
         Machine::ptr machine, Step::ptr step,
         unsigned int maxSilentTime, unsigned int buildTimeout,
         unsigned int repeats,
-        RemoteResult & result, std::shared_ptr<ActiveStep> activeStep);
+        RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
+        std::function<void(StepState)> updateStep);
 
     void markSucceededBuild(pqxx::work & txn, Build::ptr build,
         const BuildOutput & res, bool isCachedBuild, time_t startTime, time_t stopTime);