Support revision control systems via plugins
This commit is contained in:
Eelco Dolstra
87
src/lib/Hydra/Plugin/BazaarInput.pm
Normal file
87
src/lib/Hydra/Plugin/BazaarInput.pm
Normal file
@ -0,0 +1,87 @@
|
||||
package Hydra::Plugin::BazaarInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use File::Path;
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'bzr'} = 'Bazaar export';
|
||||
$inputTypes->{'bzr-checkout'} = 'Bazaar checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "bzr" && $type ne "bzr-checkout";
|
||||
|
||||
my $uri = $value;
|
||||
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
my $stdout; my $stderr;
|
||||
|
||||
my $cacheDir = getSCMCacheDir . "/bzr";
|
||||
mkpath($cacheDir);
|
||||
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
|
||||
|
||||
if (! -d $clonePath) {
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "branch", $uri, $clonePath);
|
||||
die "error cloning bazaar branch at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
chdir $clonePath or die $!;
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "pull");
|
||||
die "error pulling latest change bazaar branch at `$uri':\n$stderr" if $res;
|
||||
|
||||
# First figure out the last-modified revision of the URI.
|
||||
my @cmd = (["bzr", "revno"], "|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
|
||||
|
||||
IPC::Run::run(@cmd, \$stdout, \$stderr);
|
||||
die "cannot get head revision of Bazaar branch at `$uri':\n$stderr" if $?;
|
||||
my $revision = $stdout; chomp $revision;
|
||||
die unless $revision =~ /^\d+$/;
|
||||
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
|
||||
{uri => $uri, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
|
||||
# Then download this revision into the store.
|
||||
print STDERR "checking out Bazaar input ", $name, " from $uri revision $revision\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
$ENV{"NIX_PREFETCH_BZR_LEAVE_DOT_BZR"} = $type eq "bzr-checkout" ? "1" : "0";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"nix-prefetch-bzr", $clonePath, $revision);
|
||||
die "cannot check out Bazaar branch `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedBazaarInputs')->create(
|
||||
{ uri => $uri
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
148
src/lib/Hydra/Plugin/GitInput.pm
Normal file
148
src/lib/Hydra/Plugin/GitInput.pm
Normal file
@ -0,0 +1,148 @@
|
||||
package Hydra::Plugin::GitInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use File::Path;
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'git'} = 'Git checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "git";
|
||||
|
||||
(my $uri, my $branch, my $deepClone) = split ' ', $value;
|
||||
$branch = defined $branch ? $branch : "master";
|
||||
|
||||
my $timestamp = time;
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
my $cacheDir = getSCMCacheDir . "/git";
|
||||
mkpath($cacheDir);
|
||||
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
|
||||
|
||||
my $stdout = ""; my $stderr = ""; my $res;
|
||||
if (! -d $clonePath) {
|
||||
# Clone everything and fetch the branch.
|
||||
# TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "clone", "--branch", $branch, $uri, $clonePath);
|
||||
die "error cloning git repo at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir
|
||||
|
||||
# This command force the update of the local branch to be in the same as
|
||||
# the remote branch for whatever the repository state is. This command mirror
|
||||
# only one branch of the remote repository.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"git", "fetch", "-fu", "origin", "+$branch:$branch");
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"git", "fetch", "-fu", "origin") if $res;
|
||||
die "error fetching latest change from git repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
("git", "rev-parse", "$branch"));
|
||||
die "error getting revision number of Git branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
my ($revision) = split /\n/, $stdout;
|
||||
die "error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout"
|
||||
unless $revision =~ /^[0-9a-fA-F]+$/;
|
||||
|
||||
my $ref = "refs/heads/$branch";
|
||||
|
||||
# If deepClone is defined, then we look at the content of the repository
|
||||
# to determine if this is a top-git branch.
|
||||
if (defined $deepClone) {
|
||||
|
||||
# Checkout the branch to look at its content.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "checkout", "$branch");
|
||||
die "error checking out Git branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
if (-f ".topdeps") {
|
||||
# This is a TopGit branch. Fetch all the topic branches so
|
||||
# that builders can run "tg patch" and similar.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"tg", "remote", "--populate", "origin");
|
||||
print STDERR "warning: `tg remote --populate origin' failed:\n$stderr" if $res;
|
||||
}
|
||||
}
|
||||
|
||||
# Some simple caching: don't check a uri/branch/revision more than once.
|
||||
# TODO: Fix case where the branch is reset to a previous commit.
|
||||
my $cachedInput ;
|
||||
($cachedInput) = $self->{db}->resultset('CachedGitInputs')->search(
|
||||
{uri => $uri, branch => $branch, revision => $revision},
|
||||
{rows => 1});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
$revision = $cachedInput->revision;
|
||||
} else {
|
||||
# Then download this revision into the store.
|
||||
print STDERR "checking out Git branch $branch from $uri\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
|
||||
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";
|
||||
|
||||
if (defined $deepClone) {
|
||||
# Checked out code often wants to be able to run `git
|
||||
# describe', e.g., code that uses Gnulib's `git-version-gen'
|
||||
# script. Thus, we leave `.git' in there. Same for
|
||||
# Subversion (e.g., libgcrypt's build system uses that.)
|
||||
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
|
||||
|
||||
# Ask for a "deep clone" to allow "git describe" and similar
|
||||
# tools to work. See
|
||||
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569
|
||||
# for a discussion.
|
||||
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
|
||||
}
|
||||
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "nix-prefetch-git", $clonePath, $revision);
|
||||
die "cannot check out Git repository branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedGitInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
# For convenience in producing readable version names, pass the
|
||||
# number of commits in the history of this revision (‘revCount’)
|
||||
# the output of git-describe (‘gitTag’), and the abbreviated
|
||||
# revision (‘shortRev’).
|
||||
my $revCount = `git rev-list $revision | wc -l`; chomp $revCount;
|
||||
die "git rev-list failed" if $? != 0;
|
||||
my $gitTag = `git describe --always $revision`; chomp $gitTag;
|
||||
die "git describe failed" if $? != 0;
|
||||
my $shortRev = `git rev-parse --short $revision`; chomp $shortRev;
|
||||
die "git rev-parse failed" if $? != 0;
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
, revCount => int($revCount)
|
||||
, gitTag => $gitTag
|
||||
, shortRev => $shortRev
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
88
src/lib/Hydra/Plugin/MercurialInput.pm
Normal file
88
src/lib/Hydra/Plugin/MercurialInput.pm
Normal file
@ -0,0 +1,88 @@
|
||||
package Hydra::Plugin::MercurialInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use File::Path;
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'hg'} = 'Mercurial checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "hg";
|
||||
|
||||
(my $uri, my $id) = split ' ', $value;
|
||||
$id = defined $id ? $id : "default";
|
||||
|
||||
# init local hg clone
|
||||
|
||||
my $stdout = ""; my $stderr = "";
|
||||
|
||||
my $cacheDir = getSCMCacheDir . "/hg";
|
||||
mkpath($cacheDir);
|
||||
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
|
||||
|
||||
if (! -d $clonePath) {
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"hg", "clone", $uri, $clonePath);
|
||||
die "error cloning mercurial repo at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
# hg pull + check rev
|
||||
chdir $clonePath or die $!;
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "hg", "pull");
|
||||
die "error pulling latest change mercurial repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
(my $res1, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"hg", "log", "-r", $id, "--template", "{node|short} {rev} {branch}");
|
||||
die "error getting branch and revision of $id from `$uri':\n$stderr" if $res1;
|
||||
|
||||
my ($revision, $revCount, $branch) = split ' ', $stdout;
|
||||
|
||||
my $storePath;
|
||||
my $sha256;
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
|
||||
{uri => $uri, branch => $branch, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
print STDERR "checking out Mercurial input from $uri $branch revision $revision\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"nix-prefetch-hg", $clonePath, $revision);
|
||||
die "cannot check out Mercurial repository `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedHgInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
, revCount => int($revCount)
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
77
src/lib/Hydra/Plugin/PathInput.pm
Normal file
77
src/lib/Hydra/Plugin/PathInput.pm
Normal file
@ -0,0 +1,77 @@
|
||||
package Hydra::Plugin::PathInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use POSIX qw(strftime);
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'path'} = 'Local path';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "path";
|
||||
|
||||
my $uri = $value;
|
||||
|
||||
my $timestamp = time;
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
# Some simple caching: don't check a path more than once every N seconds.
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
|
||||
{srcpath => $uri, lastseen => {">", $timestamp - 30}},
|
||||
{rows => 1, order_by => "lastseen DESC"});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
} else {
|
||||
|
||||
print STDERR "copying input ", $name, " from $uri\n";
|
||||
$storePath = `nix-store --add "$uri"`
|
||||
or die "cannot copy path $uri to the Nix store.\n";
|
||||
chomp $storePath;
|
||||
|
||||
$sha256 = (queryPathInfo($storePath, 0))[1] or die;
|
||||
|
||||
($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
|
||||
{srcpath => $uri, sha256hash => $sha256});
|
||||
|
||||
# Path inputs don't have a natural notion of a "revision", so
|
||||
# we simulate it by using the timestamp that we first saw this
|
||||
# path have this SHA-256 hash. So if the contents of the path
|
||||
# changes, we get a new "revision", but if it doesn't change
|
||||
# (or changes back), we don't get a new "revision".
|
||||
if (!defined $cachedInput) {
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedPathInputs')->update_or_create(
|
||||
{ srcpath => $uri
|
||||
, timestamp => $timestamp
|
||||
, lastseen => $timestamp
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
} else {
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
txn_do($self->{db}, sub {
|
||||
$cachedInput->update({lastseen => time});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
90
src/lib/Hydra/Plugin/SubversionInput.pm
Normal file
90
src/lib/Hydra/Plugin/SubversionInput.pm
Normal file
@ -0,0 +1,90 @@
|
||||
package Hydra::Plugin::SubversionInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use Hydra::Helper::Nix;
|
||||
use IPC::Run;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'svn'} = 'Subversion export';
|
||||
$inputTypes->{'svn-checkout'} = 'Subversion checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "svn" && $type ne "svn-checkout";
|
||||
|
||||
# Allow users to specify a revision number next to the URI.
|
||||
my ($uri, $revision) = split ' ', $value;
|
||||
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
my $stdout; my $stderr;
|
||||
|
||||
unless (defined $revision) {
|
||||
# First figure out the last-modified revision of the URI.
|
||||
my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri],
|
||||
"|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
|
||||
IPC::Run::run(@cmd, \$stdout, \$stderr);
|
||||
die "cannot get head revision of Subversion repository at `$uri':\n$stderr" if $?;
|
||||
$revision = int($stdout); $revision =~ s/\s*([0-9]+)\s*/$1/sm;
|
||||
}
|
||||
|
||||
die unless $revision =~ /^\d+$/;
|
||||
$revision = int($revision);
|
||||
|
||||
# Do we already have this revision in the store?
|
||||
# !!! This needs to take $checkout into account! Otherwise "svn"
|
||||
# and "svn-checkout" inputs can get mixed up.
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
|
||||
{uri => $uri, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
|
||||
# No, do a checkout. The working copy is reused between
|
||||
# invocations to speed things up.
|
||||
my $wcPath = getSCMCacheDir . "/svn/" . sha256_hex($uri) . "/svn-checkout";
|
||||
|
||||
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "svn", "checkout", $uri, "-r", $revision, $wcPath);
|
||||
die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
if ($type eq "svn-checkout") {
|
||||
$storePath = addToStore($wcPath, 1, "sha256");
|
||||
} else {
|
||||
# Hm, if the Nix Perl bindings supported filters in
|
||||
# addToStore(), then we wouldn't need to make a copy here.
|
||||
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
|
||||
(system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed";
|
||||
$storePath = addToStore("$tmpDir/svn-export", 1, "sha256");
|
||||
}
|
||||
|
||||
$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedSubversionInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revNumber => $revision
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
Reference in New Issue
Block a user