From 2613d8721ac9a36df006e760f87d22e482bc67d0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <e.dolstra@tudelft.nl>
Date: Wed, 30 Sep 2009 12:26:51 +0000
Subject: [PATCH] * Don't crawl the channel pages, particularly because the
 "all"   channel can be very large.

---
 src/lib/Hydra/Controller/Root.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index eecb5a91..f29dda90 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -266,6 +266,7 @@ sub robots_txt : Path('robots.txt') {
             , uri_for($controller, 'nar', $bindings, "*")
             , uri_for($controller, 'pkg', $bindings, "*")
             , uri_for($controller, 'nixexprs', $bindings)
+            , uri_for($controller, 'channel_contents', $bindings)
             );
     }