diff --git a/flake.nix b/flake.nix
index 4f847bb8..ab830f19 100644
--- a/flake.nix
+++ b/flake.nix
@@ -198,7 +198,14 @@
         tests.api.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           simpleTest {
-            machine = hydraServer;
+            machine = { pkgs, ... }: {
+              imports = [ hydraServer ];
+              # No caching for PathInput plugin, otherwise we get wrong values
+              # (as it has a 30s window where no changes to the file are considered).
+              services.hydra-dev.extraConfig = ''
+                path_input_cache_validity_seconds = 0
+              '';
+            };
             testScript =
               let dbi = "dbi:Pg:dbname=hydra;user=root;"; in
               ''
diff --git a/src/lib/Hydra/Plugin/PathInput.pm b/src/lib/Hydra/Plugin/PathInput.pm
index 0faca46a..c1788428 100644
--- a/src/lib/Hydra/Plugin/PathInput.pm
+++ b/src/lib/Hydra/Plugin/PathInput.pm
@@ -22,9 +22,11 @@ sub fetchInput {
     my $sha256;
     my $storePath;
 
+    my $timeout = $self->{config}->{path_input_cache_validity_seconds} // 30;
+
     # Some simple caching: don't check a path more than once every N seconds.
     (my $cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
-        {srcpath => $uri, lastseen => {">", $timestamp - 30}},
+        {srcpath => $uri, lastseen => {">", $timestamp - $timeout}},
         {rows => 1, order_by => "lastseen DESC"});
 
     if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
diff --git a/tests/api-test.pl b/tests/api-test.pl
index b516f736..b07321ea 100644
--- a/tests/api-test.pl
+++ b/tests/api-test.pl
@@ -1,6 +1,6 @@
 use LWP::UserAgent;
 use JSON;
-use Test::Simple tests => 19;
+use Test::Simple tests => 20;
 
 my $ua = LWP::UserAgent->new;
 $ua->cookie_jar({});
@@ -59,6 +59,7 @@ ok($eval->{hasnewbuilds} == 1, "The first eval of a jobset has new builds");
 
 system("echo >> /run/jobset/default.nix; hydra-eval-jobset sample default");
 my $evals = decode_json(request_json({ uri => '/jobset/sample/default/evals' })->content())->{evals};
+ok(scalar(@$evals) == 2, "Changing a jobset source creates the second evaluation");
 ok($evals->[0]->{jobsetevalinputs}->{"my-src"}->{revision} != $evals->[1]->{jobsetevalinputs}->{"my-src"}->{revision}, "Changing a jobset source changes its revision");
 
 my $build = decode_json(request_json({ uri => "/build/" . $evals->[0]->{builds}->[0] })->content());