ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge pull request #1188 from DeterminateSystems/nix-2.7

Browse Source 
Nix 2.7
This commit is contained in:
Graham Christensen
2022-03-29 15:49:37 -04:00
committed by GitHub
parent 78ef4ae9a5 089da272c7
commit bbb0998699
14 changed files with 74 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
flake.lock generated
View File

@@ -3,16 +3,15 @@
"lowdown-src": {
"flake": false,
"locked": {
"lastModified": 1617481909,
"narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=",
"lastModified": 1633514407,
"narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
"owner": "kristapsdz",
"repo": "lowdown",
"rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d",
"rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
"type": "github"
},
"original": {
"owner": "kristapsdz",
"ref": "VERSION_0_8_4",
"repo": "lowdown",
"type": "github"
}
@@ -20,28 +19,31 @@
"nix": {
"inputs": {
"lowdown-src": "lowdown-src",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs",
"nixpkgs-regression": "nixpkgs-regression"
},
"locked": {
"lastModified": 1628586117,
"narHash": "sha256-8hS4xy7fq3z9XZIMYm4sQi9SzhcYqEJfdbwgDePoWuc=",
"lastModified": 1646680282,
"narHash": "sha256-m8tqCS6uHveDon5GSro5yZor9H+sHeh+v/veF1IGw24=",
"owner": "NixOS",
"repo": "nix",
"rev": "a6ba313a0aac3b6e2fef434cb42d190a0849238e",
"rev": "ffe155abd36366a870482625543f9bf924a58281",
"type": "github"
},
"original": {
"id": "nix",
"type": "indirect"
"owner": "NixOS",
"ref": "2.7.0",
"repo": "nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1624862269,
"narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
"lastModified": 1632864508,
"narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
"rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
"type": "github"
},
"original": {
@@ -50,6 +52,21 @@
"type": "indirect"
}
},
"nixpkgs-regression": {
"locked": {
"lastModified": 1643052045,
"narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "github"
},
"original": {
"id": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "indirect"
}
},
"root": {
"inputs": {
"nix": "nix",

1
flake.nix
View File

@@ -2,6 +2,7 @@
description = "A Nix-based continuous build system";
inputs.nixpkgs.follows = "nix/nixpkgs";
inputs.nix.url = github:NixOS/nix/2.7.0;
outputs = { self, nixpkgs, nix }:
let

16
src/hydra-eval-jobs/hydra-eval-jobs.cc
View File

@@ -63,13 +63,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs
static MyArgs myArgs;
static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute)
static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute)
{
Strings res;
std::function<void(Value & v)> rec;
rec = [&](Value & v) {
state.forceValue(v);
state.forceValue(v, noPos);
if (v.type() == nString)
res.push_back(v.string.s);
else if (v.isList())
@@ -78,7 +78,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const stri
else if (v.type() == nAttrs) {
auto a = v.attrs->find(state.symbols.create(subAttribute));
if (a != v.attrs->end())
res.push_back(state.forceString(*a->value));
res.push_back(std::string(state.forceString(*a->value)));
}
};
@@ -113,7 +113,7 @@ static void worker(
callFlake(state, lockedFlake, *vFlake);
auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
state.forceValue(*vOutputs);
state.forceValue(*vOutputs, noPos);
auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
if (!aHydraJobs)
@@ -157,7 +157,7 @@ static void worker(
if (drv->querySystem() == "unknown")
throw EvalError("derivation must have a 'system' attribute");
auto drvPath = drv->queryDrvPath();
auto drvPath = state.store->printStorePath(drv->requireDrvPath());
nlohmann::json job;
@@ -186,13 +186,13 @@ static void worker(
for (auto & i : context)
if (i.at(0) == '!') {
size_t index = i.find("!", 1);
job["constituents"].push_back(string(i, index + 1));
job["constituents"].push_back(std::string(i, index + 1));
}
state.forceList(*a->value, *a->pos);
for (unsigned int n = 0; n < a->value->listSize(); ++n) {
auto v = a->value->listElems()[n];
state.forceValue(*v);
state.forceValue(*v, noPos);
if (v->type() == nString)
job["namedConstituents"].push_back(state.forceStringNoCtx(*v));
}
@@ -210,7 +210,7 @@ static void worker(
nlohmann::json out;
for (auto & j : outputs)
out[j.first] = j.second;
out[j.first] = state.store->printStorePath(j.second);
job["outputs"] = std::move(out);
reply["job"] = std::move(job);

2
src/hydra-queue-runner/Makefile.am
View File

@@ -2,7 +2,7 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh state.hh db.hh \
hydra-build-result.hh counter.hh state.hh db.hh \
nar-extractor.cc nar-extractor.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

17
src/hydra-queue-runner/build-remote.cc
View File

@@ -5,6 +5,7 @@
#include <sys/stat.h>
#include <fcntl.h>
#include "build-result.hh"
#include "serve-protocol.hh"
#include "state.hh"
#include "util.hh"
@@ -49,7 +50,7 @@ static Strings extraStoreArgs(std::string & machine)
static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child)
{
string pgmName;
std::string pgmName;
Pipe to, from;
to.create();
from.create();
@@ -81,7 +82,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
if (machine->sshPublicHostKey != "") {
Path fileName = tmpDir + "/host-key";
auto p = machine->sshName.find("@");
string host = p != string::npos ? string(machine->sshName, p + 1) : machine->sshName;
std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName;
writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n");
append(argv, {"-oUserKnownHostsFile=" + fileName});
}
@@ -185,8 +186,8 @@ void State::buildRemote(ref<Store> destStore,
{
assert(BuildResult::TimedOut == 8);
string base(step->drvPath.to_string());
result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
std::string base(step->drvPath.to_string());
result.logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2);
AutoDelete autoDelete(result.logFile, false);
createDirs(dirOf(result.logFile));
@@ -249,7 +250,7 @@ void State::buildRemote(ref<Store> destStore,
} catch (EndOfFile & e) {
child.pid.wait();
string s = chomp(readFile(result.logFile));
std::string s = chomp(readFile(result.logFile));
throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
}
@@ -287,9 +288,9 @@ void State::buildRemote(ref<Store> destStore,
this will copy the inputs to the binary cache from the local
store. */
if (localStore != std::shared_ptr<Store>(destStore)) {
StorePathSet closure;
localStore->computeFSClosure(step->drv->inputSrcs, closure);
copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
copyClosure(*localStore, *destStore,
step->drv->inputSrcs,
NoRepair, NoCheckSigs, NoSubstitute);
}
{

4
src/hydra-queue-runner/build-result.cc
View File

@@ -1,4 +1,4 @@
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
#include "util.hh"
#include "fs-accessor.hh"
@@ -78,7 +78,7 @@ BuildOutput getBuildOutput(
product.type = match[1];
product.subtype = match[2];
std::string s(match[3]);
product.path = s[0] == '"' ? string(s, 1, s.size() - 2) : s;
product.path = s[0] == '"' ? std::string(s, 1, s.size() - 2) : s;
product.defaultPath = match[5];
/* Ensure that the path exists and points into the Nix

2
src/hydra-queue-runner/builder.cc
View File

@@ -1,7 +1,7 @@
#include <cmath>
#include "state.hh"
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "finally.hh"
#include "binary-cache-store.hh"

0
src/hydra-queue-runner/build-result.hh → src/hydra-queue-runner/hydra-build-result.hh
View File

18
src/hydra-queue-runner/hydra-queue-runner.cc
View File

@@ -7,7 +7,7 @@
#include <fcntl.h>
#include "state.hh"
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
#include "remote-store.hh"
@@ -87,7 +87,7 @@ void State::parseMachines(const std::string & contents)
}
for (auto line : tokenizeString<Strings>(contents, "\n")) {
line = trim(string(line, 0, line.find('#')));
line = trim(std::string(line, 0, line.find('#')));
auto tokens = tokenizeString<std::vector<std::string>>(line);
if (tokens.size() < 3) continue;
tokens.resize(8);
@@ -95,7 +95,7 @@ void State::parseMachines(const std::string & contents)
auto machine = std::make_shared<Machine>();
machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2] == "-" ? string("") : tokens[2];
machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
if (tokens[3] != "")
machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value();
else
@@ -149,7 +149,7 @@ void State::parseMachines(const std::string & contents)
void State::monitorMachinesFile()
{
string defaultMachinesFile = "/etc/nix/machines";
std::string defaultMachinesFile = "/etc/nix/machines";
auto machinesFiles = tokenizeString<std::vector<Path>>(
getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":");
@@ -191,7 +191,7 @@ void State::monitorMachinesFile()
debug("reloading machines files");
string contents;
std::string contents;
for (auto & machinesFile : machinesFiles) {
try {
contents += readFile(machinesFile);
@@ -308,7 +308,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const StorePath & drvPath, const string & outputName, const StorePath & storePath)
Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath)
{
restart:
auto stepNr = allocBuildStep(txn, build->id);
@@ -683,14 +683,14 @@ void State::showStatus()
auto conn(dbPool.get());
receiver statusDumped(*conn, "status_dumped");
string status;
std::string status;
bool barf = false;
/* Get the last JSON status dump from the database. */
{
pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>();
if (res.size()) status = res[0][0].as<std::string>();
}
if (status != "") {
@@ -710,7 +710,7 @@ void State::showStatus()
{
pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>();
if (res.size()) status = res[0][0].as<std::string>();
}
}

2
src/hydra-queue-runner/nar-extractor.cc
View File

@@ -64,7 +64,7 @@ struct Extractor : ParseSink
}
}
void createSymlink(const Path & path, const string & target) override
void createSymlink(const Path & path, const std::string & target) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
}

18
src/hydra-queue-runner/queue-monitor.cc
View File

@@ -1,5 +1,5 @@
#include "state.hh"
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "globals.hh"
#include <cstring>
@@ -111,12 +111,12 @@ bool State::getQueuedBuilds(Connection & conn,
if (builds_->count(id)) continue;
auto build = std::make_shared<Build>(
localStore->parseStorePath(row["drvPath"].as<string>()));
localStore->parseStorePath(row["drvPath"].as<std::string>()));
build->id = id;
build->jobsetId = row["jobset_id"].as<JobsetID>();
build->projectName = row["project"].as<string>();
build->jobsetName = row["jobset"].as<string>();
build->jobName = row["job"].as<string>();
build->projectName = row["project"].as<std::string>();
build->jobsetName = row["jobset"].as<std::string>();
build->jobName = row["job"].as<std::string>();
build->maxSilentTime = row["maxsilent"].as<int>();
build->buildTimeout = row["timeout"].as<int>();
build->timestamp = row["timestamp"].as<time_t>();
@@ -513,9 +513,9 @@ Step::ptr State::createStep(ref<Store> destStore,
// FIXME: should copy directly from substituter to destStore.
}
StorePathSet closure;
localStore->computeFSClosure({*path}, closure);
copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute);
copyClosure(*localStore, *destStore,
StorePathSet { *path },
NoRepair, CheckSigs, NoSubstitute);
time_t stopTime = time(0);
@@ -620,7 +620,7 @@ void State::processJobsetSharesChange(Connection & conn)
auto res = txn.exec("select project, name, schedulingShares from Jobsets");
for (auto const & row : res) {
auto jobsets_(jobsets.lock());
auto i = jobsets_->find(std::make_pair(row["project"].as<string>(), row["name"].as<string>()));
auto i = jobsets_->find(std::make_pair(row["project"].as<std::string>(), row["name"].as<std::string>()));
if (i == jobsets_->end()) continue;
i->second->setShares(row["schedulingShares"].as<unsigned int>());
}

1
src/hydra-queue-runner/state.hh
View File

@@ -12,6 +12,7 @@
#include "parsed-derivations.hh"
#include "pathlocks.hh"
#include "pool.hh"
#include "build-result.hh"
#include "store-api.hh"
#include "sync.hh"
#include "nar-extractor.hh"

2
src/libhydra/db.hh
View File

@@ -18,7 +18,7 @@ struct Connection : pqxx::connection
std::string upper_prefix = "DBI:Pg:";
if (hasPrefix(s, lower_prefix) || hasPrefix(s, upper_prefix)) {
return concatStringsSep(" ", tokenizeString<Strings>(string(s, lower_prefix.size()), ";"));
return concatStringsSep(" ", tokenizeString<Strings>(std::string(s, lower_prefix.size()), ";"));
}
throw Error("$HYDRA_DBI does not denote a PostgreSQL database");

2
src/libhydra/hydra-config.hh
View File

@@ -17,7 +17,7 @@ struct HydraConfig
if (hydraConfigFile && pathExists(*hydraConfigFile)) {
for (auto line : tokenizeString<Strings>(readFile(*hydraConfigFile), "\n")) {
line = trim(string(line, 0, line.find('#')));
line = trim(std::string(line, 0, line.find('#')));
auto eq = line.find('=');
if (eq == std::string::npos) continue;