Add support for tracking custom metrics

Builds can now emit metrics that Hydra will store in its database and
render as time series via flot charts. Typical applications are to
keep track of performance indicators, coverage percentages, artifact
sizes, and so on.

For example, a coverage build can emit the coverage percentage as
follows:

  echo "lineCoverage $pct %" > $out/nix-support/hydra-metrics

Graphs of all metrics for a job can be seen at

  http://.../job/<project>/<jobset>/<job>#tabs-charts

Specific metrics are also visible at

  http://.../job/<project>/<jobset>/<job>/metric/<metric>

The latter URL also allows getting the data in JSON format (e.g. via
"curl -H 'Accept: application/json'").

src/hydra-queue-runner/build-result.cc

@@ -7,6 +7,21 @@
 using namespace nix;
 
 
+static std::tuple<bool, string> secureRead(Path fileName)
+{
+    auto fail = std::make_tuple(false, "");
+
+    if (!pathExists(fileName)) return fail;
+
+    try {
+        /* For security, resolve symlinks. */
+        fileName = canonPath(fileName, true);
+        if (!isInStore(fileName)) return fail;
+        return std::make_tuple(true, readFile(fileName));
+    } catch (Error & e) { return fail; }
+}
+
+
 BuildOutput getBuildOutput(std::shared_ptr<StoreAPI> store, const Derivation & drv)
 {
     BuildOutput res;
@@ -40,22 +55,12 @@ BuildOutput getBuildOutput(std::shared_ptr<StoreAPI> store, const Derivation & d
         Path failedFile = output + "/nix-support/failed";
         if (pathExists(failedFile)) res.failed = true;
 
-        Path productsFile = output + "/nix-support/hydra-build-products";
-        if (!pathExists(productsFile)) continue;
+        auto file = secureRead(output + "/nix-support/hydra-build-products");
+        if (!std::get<0>(file)) continue;
+
         explicitProducts = true;
 
-        /* For security, resolve symlinks. */
-        try {
-            productsFile = canonPath(productsFile, true);
-        } catch (Error & e) { continue; }
-        if (!isInStore(productsFile)) continue;
-
-        string contents;
-        try {
-            contents = readFile(productsFile);
-        } catch (Error & e) { continue; }
-
-        for (auto & line : tokenizeString<Strings>(contents, "\n")) {
+        for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {
             BuildProduct product;
 
             Regex::Subs subs;
@@ -122,5 +127,19 @@ BuildOutput getBuildOutput(std::shared_ptr<StoreAPI> store, const Derivation & d
         // FIXME: validate release name
     }
 
+    /* Get metrics. */
+    for (auto & output : outputs) {
+        auto file = secureRead(output + "/nix-support/hydra-metrics");
+        for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {
+            auto fields = tokenizeString<std::vector<std::string>>(line);
+            if (fields.size() < 2) continue;
+            BuildMetric metric;
+            metric.name = fields[0]; // FIXME: validate
+            metric.value = atof(fields[1].c_str()); // FIXME
+            metric.unit = fields.size() >= 3 ? fields[2] : "";
+            res.metrics[metric.name] = metric;
+        }
+    }
+
     return res;
 }

src/hydra-queue-runner/build-result.hh

@@ -15,6 +15,12 @@ struct BuildProduct
     BuildProduct() { }
 };
 
+struct BuildMetric
+{
+    std::string name, unit;
+    double value;
+};
+
 struct BuildOutput
 {
     /* Whether this build has failed with output, i.e., the build
@@ -27,6 +33,8 @@ struct BuildOutput
     unsigned long long closureSize = 0, size = 0;
 
     std::list<BuildProduct> products;
+
+    std::map<std::string, BuildMetric> metrics;
 };
 
 BuildOutput getBuildOutput(std::shared_ptr<nix::StoreAPI> store, const nix::Derivation & drv);

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -238,6 +238,19 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
             (product.defaultPath).exec();
     }
 
+    for (auto & metric : res.metrics) {
+        txn.parameterized
+            ("insert into BuildMetrics (build, name, unit, value, project, jobset, job, timestamp) values ($1, $2, $3, $4, $5, $6, $7, $8)")
+            (build->id)
+            (metric.second.name)
+            (metric.second.unit, metric.second.unit != "")
+            (metric.second.value)
+            (build->projectName)
+            (build->jobsetName)
+            (build->jobName)
+            (build->timestamp).exec();
+    }
+
     nrBuildsDone++;
 }
 

src/hydra-queue-runner/queue-monitor.cc

@@ -64,7 +64,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
     {
         pqxx::work txn(conn);
 
-        auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec();
+        auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec();
 
         for (auto const & row : res) {
             auto builds_(builds.lock());
@@ -76,9 +76,12 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
             auto build = std::make_shared<Build>();
             build->id = id;
             build->drvPath = row["drvPath"].as<string>();
-            build->fullJobName = row["project"].as<string>() + ":" + row["jobset"].as<string>() + ":" + row["job"].as<string>();
+            build->projectName = row["project"].as<string>();
+            build->jobsetName = row["jobset"].as<string>();
+            build->jobName = row["job"].as<string>();
             build->maxSilentTime = row["maxsilent"].as<int>();
             build->buildTimeout = row["timeout"].as<int>();
+            build->timestamp = row["timestamp"].as<time_t>();
 
             newBuilds.emplace(std::make_pair(build->drvPath, build));
         }
@@ -89,7 +92,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
     std::function<void(Build::ptr)> createBuild;
 
     createBuild = [&](Build::ptr build) {
-        printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName);
+        printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());
         nrAdded++;
 
         if (!store->isValidPath(build->drvPath)) {

src/hydra-queue-runner/state.hh

@@ -38,6 +38,7 @@ typedef enum {
     bssFailed = 1,
     bssAborted = 4,
     bssTimedOut = 7,
+    bssCachedFailure = 8,
     bssUnsupported = 9,
     bssBusy = 100, // not stored
 } BuildStepStatus;
@@ -67,12 +68,18 @@ struct Build
     BuildID id;
     nix::Path drvPath;
     std::map<std::string, nix::Path> outputs;
-    std::string fullJobName;
+    std::string projectName, jobsetName, jobName;
+    time_t timestamp;
     unsigned int maxSilentTime, buildTimeout;
 
     std::shared_ptr<Step> toplevel;
 
     std::atomic_bool finishedInDB{false};
+
+    std::string fullJobName()
+    {
+        return projectName + ":" + jobsetName + ":" + jobName;
+    }
 };