From 1698e490d098e57a17cfc18bc478c9500c579ce0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Nov 2008 18:13:55 +0000 Subject: [PATCH] * Support Subversion checkouts. --- src/Hydra/lib/Hydra/Schema.pm | 4 +- src/Hydra/lib/Hydra/Schema/Buildinputs.pm | 4 +- src/Hydra/lib/Hydra/Schema/Buildproducts.pm | 4 +- src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm | 4 +- src/Hydra/lib/Hydra/Schema/Builds.pm | 4 +- .../lib/Hydra/Schema/Buildschedulinginfo.pm | 4 +- src/Hydra/lib/Hydra/Schema/Buildsteps.pm | 4 +- .../lib/Hydra/Schema/Cachedpathinputs.pm | 30 ++++++++++ .../Hydra/Schema/Cachedsubversioninputs.pm | 28 ++++++++++ src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm | 4 +- src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm | 4 +- src/Hydra/lib/Hydra/Schema/Jobsets.pm | 4 +- src/Hydra/lib/Hydra/Schema/Projects.pm | 4 +- src/Hydra/programs/Scheduler.pl | 56 ++++++++++++++++++- src/hydra.sql | 9 +++ 15 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 src/Hydra/lib/Hydra/Schema/Cachedpathinputs.pm create mode 100644 src/Hydra/lib/Hydra/Schema/Cachedsubversioninputs.pm diff --git a/src/Hydra/lib/Hydra/Schema.pm b/src/Hydra/lib/Hydra/Schema.pm index 1158cca5..5acec076 100644 --- a/src/Hydra/lib/Hydra/Schema.pm +++ b/src/Hydra/lib/Hydra/Schema.pm @@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema'; __PACKAGE__->load_classes; -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4eWDgovhuQLEGBA42J9kKA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:woBkuPRoV3DnJmHdcZ2ZIg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildinputs.pm b/src/Hydra/lib/Hydra/Schema/Buildinputs.pm index 205c9f01..48a065ef 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildinputs.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildinputs.pm @@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); __PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/SS7rITRzALAmC9rNn70cQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Mwn4Pp+vduPrAU9s089pAg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildproducts.pm b/src/Hydra/lib/Hydra/Schema/Buildproducts.pm index dbc9f1a9..2bff9d60 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildproducts.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildproducts.pm @@ -33,8 +33,8 @@ __PACKAGE__->set_primary_key("build", "productnr"); __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:j3niyjyAmX10xY1maNh4gA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kzQauEWWXur6WilY/1LcfA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm b/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm index e770c2ff..3c9fc59d 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm @@ -29,8 +29,8 @@ __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/KtAIFiCb8EkAMd6OdHSXA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:85sz3phT3arumqXladRtMQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Builds.pm b/src/Hydra/lib/Hydra/Schema/Builds.pm index bbd93df7..98b1d152 100644 --- a/src/Hydra/lib/Hydra/Schema/Builds.pm +++ b/src/Hydra/lib/Hydra/Schema/Builds.pm @@ -70,8 +70,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ig6LUSp4zaPhrxuzC8gL4Q +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:1fnKUzCj0eAt4HWx0UB3SA __PACKAGE__->has_many(dependents => 'Hydra::Schema::Buildinputs', 'dependency'); diff --git a/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm b/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm index c2dadd7e..6689bdb1 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm @@ -23,8 +23,8 @@ __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QuIbtSr8Q9JwhAcePu3Kuw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+a/kUdHtdjxdAOBSWdUsVA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildsteps.pm b/src/Hydra/lib/Hydra/Schema/Buildsteps.pm index b5e789d8..29804812 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildsteps.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildsteps.pm @@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id", "stepnr"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:sLsyDl8Nu63DWaYDIYBTkg +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:FYosIfp2O9OAolXdzalK+Q # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Cachedpathinputs.pm b/src/Hydra/lib/Hydra/Schema/Cachedpathinputs.pm new file mode 100644 index 00000000..39793c4a --- /dev/null +++ b/src/Hydra/lib/Hydra/Schema/Cachedpathinputs.pm @@ -0,0 +1,30 @@ +package Hydra::Schema::Cachedpathinputs; + +use strict; +use warnings; + +use base 'DBIx::Class'; + +__PACKAGE__->load_components("Core"); +__PACKAGE__->table("CachedPathInputs"); +__PACKAGE__->add_columns( + "srcpath", + { data_type => "text", is_nullable => 0, size => undef }, + "timestamp", + { data_type => "integer", is_nullable => 0, size => undef }, + "lastseen", + { data_type => "integer", is_nullable => 0, size => undef }, + "sha256hash", + { data_type => "text", is_nullable => 0, size => undef }, + "storepath", + { data_type => "text", is_nullable => 0, size => undef }, +); +__PACKAGE__->set_primary_key("srcpath", "sha256hash"); + + +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:KhMAz8E3CDxG2D+zleQEFw + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/Hydra/lib/Hydra/Schema/Cachedsubversioninputs.pm b/src/Hydra/lib/Hydra/Schema/Cachedsubversioninputs.pm new file mode 100644 index 00000000..ad071a4a --- /dev/null +++ b/src/Hydra/lib/Hydra/Schema/Cachedsubversioninputs.pm @@ -0,0 +1,28 @@ +package Hydra::Schema::Cachedsubversioninputs; + +use strict; +use warnings; + +use base 'DBIx::Class'; + +__PACKAGE__->load_components("Core"); +__PACKAGE__->table("CachedSubversionInputs"); +__PACKAGE__->add_columns( + "uri", + { data_type => "text", is_nullable => 0, size => undef }, + "revision", + { data_type => "integer", is_nullable => 0, size => undef }, + "sha256hash", + { data_type => "text", is_nullable => 0, size => undef }, + "storepath", + { data_type => "text", is_nullable => 0, size => undef }, +); +__PACKAGE__->set_primary_key("uri", "revision"); + + +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:S62F6Z2wPT/+NoIApQcBmQ + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm b/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm index a3f31e84..9e7b648d 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm @@ -31,8 +31,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:nL04tddxVrJnpaDHMBWkEQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6N8dRzQi9IYgP5F2rbDtWg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm b/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm index 9d1f910a..4468282e 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm @@ -43,8 +43,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vPNmAznaZ6z3M8eYOwnENg +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QeB1Zh2L3q81y5j+fLcazw # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsets.pm b/src/Hydra/lib/Hydra/Schema/Jobsets.pm index 4a06e0dc..b8ffe740 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsets.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsets.pm @@ -48,8 +48,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:juL3q644z+R8S4ynbp6MMw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6aZCPgci+SLJuoGHcR3boA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Projects.pm b/src/Hydra/lib/Hydra/Schema/Projects.pm index 34a15f8e..90ac9085 100644 --- a/src/Hydra/lib/Hydra/Schema/Projects.pm +++ b/src/Hydra/lib/Hydra/Schema/Projects.pm @@ -30,8 +30,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 17:11:25 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:e+sS5o6q9SX12ik9beTuFQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 18:59:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:meEhwhiWsywgQlrkwOz48A # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/programs/Scheduler.pl b/src/Hydra/programs/Scheduler.pl index 0965b414..e85bb6e7 100644 --- a/src/Hydra/programs/Scheduler.pl +++ b/src/Hydra/programs/Scheduler.pl @@ -105,13 +105,65 @@ sub fetchInput { }; } + elsif ($type eq "svn") { + my $uri = $alt->value; + + my $sha256; + my $storePath; + + # First figure out the last-modified revision of the URI. + my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri], + "|", ["sed", 's/^ *\([0-9]*\).*/\1/']); + my $stdout; my $stderr; + die "cannot get head revision of Subversion repository at `$uri':\n$stderr" + unless IPC::Run::run(@cmd, \$stdout, \$stderr); + my $revision = $stdout; chomp $revision; + die unless $revision =~ /^\d+$/; + + (my $cachedInput) = $db->resultset('Cachedsubversioninputs')->search( + {uri => $uri, revision => $revision}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + } else { + + # Then download this revision into the store. + print "checking out Subversion input ", $input->name, " from $uri revision $revision\n"; + $ENV{"NIX_HASH_ALGO"} = "sha256"; + $ENV{"PRINT_PATH"} = "1"; + (my $res, $stdout, $stderr) = captureStdoutStderr( + "nix-prefetch-svn", $uri, $revision); + die "cannot check out Subversion repository `$uri':\n$stderr" unless $res; + + ($sha256, $storePath) = split ' ', $stdout; + + $db->txn_do(sub { + $db->resultset('Cachedsubversioninputs')->create( + { uri => $uri + , revision => $revision + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } + + $$inputInfo{$input->name} = + { type => $type + , uri => $uri + , storePath => $storePath + , sha256hash => $sha256 + , revision => $revision + }; + } + elsif ($type eq "string") { die unless defined $alt->value; $$inputInfo{$input->name} = {type => $type, value => $alt->value}; } else { - die "input `" . $input->type . "' has unknown type `$type'"; + die "input `" . $input->name . "' has unknown type `$type'"; } } @@ -346,7 +398,7 @@ sub checkJobs { }; if ($@) { print "error evaluating jobset ", $jobset->name, ": $@"; - setProjectError($jobset, $@); + setJobsetError($jobset, $@); } } } diff --git a/src/hydra.sql b/src/hydra.sql index 0f8ee1bb..67ee3e90 100644 --- a/src/hydra.sql +++ b/src/hydra.sql @@ -241,3 +241,12 @@ create table CachedPathInputs ( storePath text not null, primary key (srcPath, sha256hash) ); + + +create table CachedSubversionInputs ( + uri text not null, + revision integer not null, + sha256hash text not null, + storePath text not null, + primary key (uri, revision) +);