diff --git a/Makefile.am b/Makefile.am
index e744cc33..9e06ebdb 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,4 +1,8 @@
-SUBDIRS = src t doc
+SUBDIRS = src doc
+if CAN_DO_CHECK
+  SUBDIRS += t
+endif
+
 BOOTCLEAN_SUBDIRS = $(SUBDIRS)
 DIST_SUBDIRS      = $(SUBDIRS)
 EXTRA_DIST        = hydra-module.nix
diff --git a/configure.ac b/configure.ac
index eec647c3..e5c57d14 100644
--- a/configure.ac
+++ b/configure.ac
@@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store])
 testPath="$(dirname $(type -p expr))"
 AC_SUBST(testPath)
 
-jobsPath="$(realpath ./t/jobs)"
-AC_SUBST(jobsPath)
-
 CXXFLAGS+=" -include nix/config.h"
 
 AC_CONFIG_FILES([
@@ -71,11 +68,22 @@ AC_CONFIG_FILES([
   src/lib/Makefile
   src/root/Makefile
   src/script/Makefile
-  t/Makefile
-  t/jobs/config.nix
-  t/jobs/declarative/project.json
 ])
 
+# Tests might be filtered out
+AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"])
+AM_COND_IF(
+  [CAN_DO_CHECK],
+  [
+    jobsPath="$(realpath ./t/jobs)"
+    AC_SUBST(jobsPath)
+    AC_CONFIG_FILES([
+      t/Makefile
+      t/jobs/config.nix
+      t/jobs/declarative/project.json
+    ])
+  ])
+
 AC_CONFIG_COMMANDS([executable-scripts], [])
 
 AC_CONFIG_HEADER([hydra-config.h])
diff --git a/package.nix b/package.nix
index eff9ed37..73ef9b3f 100644
--- a/package.nix
+++ b/package.nix
@@ -144,9 +144,12 @@ stdenv.mkDerivation (finalAttrs: {
       ./src
       ./doc
       ./hydra-module.nix
-      # TODO only when `doCheck`
-      ./t
+      # These are always needed to appease Automake
+      ./t/Makefile.am
+      ./t/jobs/config.nix.in
+      ./t/jobs/declarative/project.json.in
     ] ++ lib.optionals finalAttrs.doCheck [
+      ./t
       ./.perlcriticrc
       ./.yath.rc
     ]);