add gitea refs

Signed-off-by: ahuston-0 <aliceghuston@gmail.com>
Reviewed-on: https://<censored>/ahuston-0/hydra/pulls/1

doc/manual/src/plugins/README.md

@@ -92,6 +92,23 @@ Sets Gitea CI status
 
 - `gitea_authorization.<repo-owner>`
 
+## Gitea pulls
+
+Create jobs based on open Gitea pull requests
+
+### Configuration options
+
+- `gitea_authorization.<repo-owner>`
+
+## Gitea refs
+
+Hydra plugin for retrieving the list of references (branches or tags) from
+Gitea following a certain naming scheme.
+
+### Configuration options
+
+- `gitea_authorization.<repo-owner>`
+
 ## GitHub pulls
 
 Create jobs based on open GitHub pull requests

nixos-tests.nix

@@ -145,11 +145,25 @@ in
             git -C /tmp/repo add .
             git config --global user.email test@localhost
             git config --global user.name test
+
+            # Create initial commit
             git -C /tmp/repo commit -m 'Initial import'
             git -C /tmp/repo remote add origin gitea@machine:root/repo
-            GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
+            export GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no'
             git -C /tmp/repo push origin master
             git -C /tmp/repo log >&2
+
+            # Create PR branch
+            git -C /tmp/repo checkout -b pr
+            git -C /tmp/repo commit --allow-empty -m 'Additional change'
+            git -C /tmp/repo push origin pr
+            git -C /tmp/repo log >&2
+
+            # Create release branch
+            git -C /tmp/repo checkout -b release/release-1.0
+            git -C /tmp/repo commit --allow-empty -m 'Additional change'
+            git -C /tmp/repo push origin release/release-1.0
+            git -C /tmp/repo log >&2
           '';
 
           scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
@@ -185,7 +199,7 @@ in
             cat >data.json <<EOF
             {
               "description": "Trivial",
-              "checkinterval": "60",
+              "checkinterval": "20",
               "enabled": "1",
               "visible": "1",
               "keepnr": "1",
@@ -199,7 +213,17 @@ in
                 "gitea_repo_name": {"value": "repo", "type": "string"},
                 "gitea_repo_owner": {"value": "root", "type": "string"},
                 "gitea_status_repo": {"value": "git", "type": "string"},
-                "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
+                "gitea_http_url": {"value": "http://localhost:3001", "type": "string"},
+                "pulls": {
+                  "type": "giteapulls",
+                  "value": "localhost:3001 root repo http",
+                  "emailresponsible": false
+                },
+                "releases": {
+                  "type": "gitea_refs",
+                  "value": "localhost:3001 root repo heads http - release",
+                  "emailresponseible": false
+                }
               }
             }
             EOF
@@ -227,15 +251,41 @@ in
           };
 
           smallDrv = pkgs.writeText "jobset.nix" ''
-            { trivial = builtins.derivation {
-                name = "trivial";
+            { pulls, releases, ... }:
+
+            let
+              genDrv = name: builtins.derivation {
+                 inherit name;
                  system = "${system}";
                  builder = "/bin/sh";
                  allowSubstitutes = false;
                  preferLocalBuild = true;
                  args = ["-c" "echo success > $out; exit 0"];
               };
+
+              prs = builtins.fromJSON (builtins.readFile pulls);
+              prJobNames = map (n: "pr-''${n}") (builtins.attrNames prs);
+              prJobset = builtins.listToAttrs (
+                map (
+                  name: {
+                    inherit name;
+                    value = genDrv name;
                   }
+                ) prJobNames
+              );
+              rels = builtins.fromJSON (builtins.readFile releases);
+              relJobNames = builtins.attrNames rels;
+              relJobset = builtins.listToAttrs (
+                map (
+                  name: {
+                    inherit name;
+                    value = genDrv name;
+                  }
+                ) relJobNames
+              );
+            in {
+              trivial = genDrv "trivial";
+            } // prJobset // relJobset
           '';
         in
         ''
@@ -279,18 +329,34 @@ in
               + '|  jq .buildstatus | xargs test 0 -eq'
           )
 
+          machine.sleep(3)
+
           data = machine.succeed(
-              'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
+              'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show master | head -n1 | awk "{print \\$2}")?sort=leastindex" '
               + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
               + f"-H 'Authorization: token ${api_token}'"
           )
 
           response = json.loads(data)
 
-          assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
+          assert len(response) == 2, "Expected exactly two status updates for latest commit (queued, finished)!"
           assert response[0]['status'] == "success", "Expected finished status to be success!"
           assert response[1]['status'] == "pending", "Expected queued status to be pending!"
 
+          # giteapulls test
+
+          machine.succeed(
+              "curl --fail -X POST http://localhost:3001/api/v1/repos/root/repo/pulls "
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+              + ' -d \'{"title":"Test PR", "base":"master", "head": "pr"}\'''
+          )
+
+          machine.wait_until_succeeds(
+              'curl -Lf -s http://localhost:3000/build/2 -H "Accept: application/json" '
+              + '|  jq .buildstatus | xargs test 0 -eq'
+          )
+
           machine.shutdown()
         '';
     });

src/lib/Hydra/Plugin/GiteaPulls.pm

@@ -0,0 +1,84 @@
+# Allow building based on Gitea pull requests.
+#
+# Example input:
+#   "pulls": {
+#     "type": "giteapulls",
+#     "value": "example.com alice repo"
+#     "emailresponsible": false
+#   }
+
+package Hydra::Plugin::GiteaPulls;
+
+use strict;
+use warnings;
+use parent 'Hydra::Plugin';
+use HTTP::Request;
+use LWP::UserAgent;
+use JSON::MaybeXS;
+use Hydra::Helper::CatalystUtils;
+use File::Temp;
+use POSIX qw(strftime);
+
+sub supportedInputTypes {
+    my ($self, $inputTypes) = @_;
+    $inputTypes->{'giteapulls'} = 'Open Gitea Pull Requests';
+}
+
+sub _iterate {
+    my ($url, $auth, $pulls, $ua) = @_;
+
+    my $req = HTTP::Request->new('GET', $url);
+    $req->header('Authorization' => 'token ' . $auth) if defined $auth;
+
+    my $res = $ua->request($req);
+    my $content = $res->decoded_content;
+    die "Error pulling from the gitea pulls API: $content\n"
+	unless $res->is_success;
+
+    my $pulls_list = decode_json $content;
+
+    foreach my $pull (@$pulls_list) {
+	$pulls->{$pull->{number}} = $pull;
+    }
+
+    # TODO Make Link header parsing more robust!!!
+    my @links = split ',', ($res->header("Link") // "");
+    my $next = "";
+    foreach my $link (@links) {
+        my ($url, $rel) = split ";", $link;
+        if (trim($rel) eq 'rel="next"') {
+            $next = substr trim($url), 1, -1;
+            last;
+        }
+    }
+    _iterate($next, $auth, $pulls, $ua) unless $next eq "";
+}
+
+sub fetchInput {
+    my ($self, $type, $name, $value, $project, $jobset) = @_;
+    return undef if $type ne "giteapulls";
+
+    my ($baseUrl, $owner, $repo, $proto) = split ' ', $value;
+    if (not defined $proto) { # the protocol handler is exposed as an option in order to do integration testing
+	$proto = "https"
+    }
+    my $auth = $self->{config}->{gitea_authorization}->{$owner};
+
+    my $ua = LWP::UserAgent->new();
+    my %pulls;
+    _iterate("$proto://$baseUrl/api/v1/repos/$owner/$repo/pulls?limit=100", $auth, \%pulls, $ua);
+
+    my $tempdir = File::Temp->newdir("gitea-pulls" . "XXXXX", TMPDIR => 1);
+    my $filename = "$tempdir/gitea-pulls.json";
+    open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!";
+    print $fh encode_json \%pulls;
+    close $fh;
+
+    my $storePath = trim(`nix-store --add "$filename"`
+        or die "cannot copy path $filename to the Nix store.\n");
+    chomp $storePath;
+    my $timestamp = time;
+    return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) };
+}
+
+1;

src/lib/Hydra/Plugin/GiteaRefs.pm

@@ -0,0 +1,129 @@
+package Hydra::Plugin::GiteaRefs;
+
+use strict;
+use warnings;
+use parent 'Hydra::Plugin';
+use HTTP::Request;
+use LWP::UserAgent;
+use JSON::MaybeXS;
+use Hydra::Helper::CatalystUtils;
+use File::Temp;
+use POSIX qw(strftime);
+
+=head1 NAME
+
+GiteaRefs - Hydra plugin for retrieving the list of references (branches or
+tags) from Gitea following a certain naming scheme
+
+=head1 DESCRIPTION
+
+This plugin reads the list of branches or tags using Gitea's REST API. The name
+of the reference must follow a particular prefix. This list is stored in the
+nix-store and used as an input to declarative jobsets.
+
+=head1 CONFIGURATION
+
+The plugin doesn't require any dedicated configuration block, but it has to
+consult C<gitea_authorization> entry for obtaining the API token. In addition,
+
+The declarative project C<spec.json> file must contains an input such as
+
+   "pulls": {
+     "type": "gitea_refs",
+     "value": "[gitea_hostname] [owner] [repo] heads|tags [scheme] - [prefix]",
+     "emailresponsible": false
+   }
+
+In the above snippet, C<[gitea_hostname]> must be set to the hostname of the
+repository's Gitea instance.
+
+C<[owner]> is the repository owner and C<[repo]> is the repository name. Also
+note a literal C<->, which is placed there for the future use.
+
+C<heads|tags> denotes that one of these two is allowed, that is, the third
+position should hold either the C<heads> or the C<tags> keyword. In case of the former, the plugin
+will fetch all branches, while in case of the latter, it will fetch the tags.
+
+C<scheme> should be set to either https or http, depending on what the Gitea
+host supports.
+
+C<prefix> denotes the prefix the reference name must start with, in order to be
+included.
+
+For example, C<"value": "projects.blender.org blender blender heads https - blender-v/"> refers to
+L<https://projects.blender.org/blender/blender> repository, and will fetch all branches that
+begin with C<blender-v/>.
+
+=head1 USE
+
+The result is stored in the nix-store as a JSON I<map>, where the key is the
+name of the reference, while the value is the complete Gitea response. Thus,
+any of the values listed in
+L<https://docs.gitea.com/api#tag/repository/operation/repoListAllGitRefs> can be
+used to build the git input value in C<jobsets.nix>.
+
+=cut
+
+sub supportedInputTypes {
+    my ($self, $inputTypes) = @_;
+    $inputTypes->{'gitea_refs'} = 'Open Gitea Refs';
+}
+
+sub _iterate {
+    my ($url, $auth, $refs, $ua) = @_;
+    my $req = HTTP::Request->new('GET', $url);
+    $req->header('Accept' => 'application/json');
+    $req->header('Authorization' => $auth) if defined $auth;
+    my $res = $ua->request($req);
+    my $content = $res->decoded_content;
+    die "Error pulling from the gitea refs API: $content\n"
+        unless $res->is_success;
+    my $refs_list = decode_json $content;
+    # TODO Stream out the json instead
+    foreach my $ref (@$refs_list) {
+        my $ref_name = $ref->{ref};
+        $ref_name =~ s,^refs/(?:heads|tags)/,,o;
+        $refs->{$ref_name} = $ref;
+    }
+    # TODO Make Link header parsing more robust!!!
+    my @links = split ',', $res->header("Link");
+    my $next = "";
+    foreach my $link (@links) {
+        my ($url, $rel) = split ";", $link;
+        if (trim($rel) eq 'rel="next"') {
+            $next = substr trim($url), 1, -1;
+            last;
+        }
+    }
+    _iterate($next, $auth, $refs, $ua) unless $next eq "";
+}
+
+sub fetchInput {
+    my ($self, $input_type, $name, $value, $project, $jobset) = @_;
+    return undef if $input_type ne "gitea_refs";
+
+    my ($giteaHostname, $owner, $repo, $type, $scheme, $fut, $prefix) = split ' ', $value;
+    die "type field is neither 'heads' nor 'tags', but '$type'"
+        unless $type eq 'heads' or $type eq 'tags';
+    die "scheme field is neither 'https' nor 'http' but '$scheme'"
+        unless $scheme eq 'https' or $scheme eq 'http';
+
+    my $auth = $self->{config}->{gitea_authorization}->{$owner};
+    my $giteaEndpoint = "$scheme://$giteaHostname";
+    my %refs;
+    my $ua = LWP::UserAgent->new();
+    _iterate("$giteaEndpoint/api/v1/repos/$owner/$repo/git/refs/$type/$prefix?per_page=100", $auth, \%refs, $ua);
+    my $tempdir = File::Temp->newdir("gitea-refs" . "XXXXX", TMPDIR => 1);
+    my $filename = "$tempdir/gitea-refs.json";
+    open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!";
+    print $fh encode_json \%refs;
+    close $fh;
+    system("jq -S . < $filename > $tempdir/gitea-refs-sorted.json");
+    my $storePath = trim(qx{nix-store --add "$tempdir/gitea-refs-sorted.json"}
+        or die "cannot copy path $filename to the Nix store.\n");
+    chomp $storePath;
+    my $timestamp = time;
+    return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) };
+}
+
+1;