From abf18abb52dbdb409d481854bde24ed880c76f56 Mon Sep 17 00:00:00 2001
From: Ben Bodenmiller <bbodenmiller@hotmail.com>
Date: Sun, 4 Oct 2015 23:39:24 -0700
Subject: [PATCH] allow crawling of commit page but not patch/diffs

Commit page has valuable information that search engines should be allowed to crawl however the .patch and .diff pages have no new information that is not on commit page
---
 public/robots.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/public/robots.txt b/public/robots.txt
index 528f421083e..4f616c7f4c1 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -38,7 +38,8 @@ Disallow: /*/*/edit
 Disallow: /*/*/raw
 Disallow: /*/*/blame
 Disallow: /*/*/commits/*/*
-Disallow: /*/*/commit
+Disallow: /*/*/commit/*.patch
+Disallow: /*/*/commit/*.diff
 Disallow: /*/*/compare
 Disallow: /*/*/branches/new
 Disallow: /*/*/tags/new
-- 
GitLab