From 1938c2c2a96fe9df1259e1cc2e259fef507641f9 Mon Sep 17 00:00:00 2001
From: philip Vissenaekens <philipvissenaekens@atmire.com>
Date: Wed, 4 May 2016 12:07:39 +0200
Subject: [PATCH] DS-2962

---
 .../src/main/webapp/xsl/core/page-structure.xsl       |  2 +-
 .../aspect/discovery/RobotsNoIndexTransformer.java    | 11 +++++++++++
 .../src/main/resources/aspects/Discovery/sitemap.xmap |  4 ++++
 dspace-xmlui/src/main/webapp/static/robots.txt        |  3 +--
 .../themes/Mirage/lib/xsl/core/page-structure.xsl     |  2 +-
 5 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/dspace-xmlui-mirage2/src/main/webapp/xsl/core/page-structure.xsl b/dspace-xmlui-mirage2/src/main/webapp/xsl/core/page-structure.xsl
index cf69f42520..0279f03ef9 100644
--- a/dspace-xmlui-mirage2/src/main/webapp/xsl/core/page-structure.xsl
+++ b/dspace-xmlui-mirage2/src/main/webapp/xsl/core/page-structure.xsl
@@ -177,7 +177,7 @@
                 </xsl:attribute>
             </meta>
 
-            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']">
+            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS'][not(@qualifier)]">
                 <meta name="ROBOTS">
                     <xsl:attribute name="content">
                         <xsl:value-of select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']"/>
diff --git a/dspace-xmlui/src/main/java/org/dspace/app/xmlui/aspect/discovery/RobotsNoIndexTransformer.java b/dspace-xmlui/src/main/java/org/dspace/app/xmlui/aspect/discovery/RobotsNoIndexTransformer.java
index debf7a018c..c224480385 100644
--- a/dspace-xmlui/src/main/java/org/dspace/app/xmlui/aspect/discovery/RobotsNoIndexTransformer.java
+++ b/dspace-xmlui/src/main/java/org/dspace/app/xmlui/aspect/discovery/RobotsNoIndexTransformer.java
@@ -1,3 +1,10 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
 package org.dspace.app.xmlui.aspect.discovery;
 
 import java.io.*;
@@ -10,6 +17,10 @@ import org.dspace.authorize.*;
 import org.xml.sax.*;
 
 /**
+ * Robots can not be blocked from dynamic URLs using robots.txt. This transformer is used instead to add
+ * a meta tag that tells robots not to index the content of a page.
+ * Information about this meta tag can be found here: http://www.robotstxt.org/meta.html
+ *
  * @author philip at atmire.com
  */
 public class RobotsNoIndexTransformer extends AbstractDSpaceTransformer {
diff --git a/dspace-xmlui/src/main/resources/aspects/Discovery/sitemap.xmap b/dspace-xmlui/src/main/resources/aspects/Discovery/sitemap.xmap
index 6a0e3ecfd7..deaa11abf1 100644
--- a/dspace-xmlui/src/main/resources/aspects/Discovery/sitemap.xmap
+++ b/dspace-xmlui/src/main/resources/aspects/Discovery/sitemap.xmap
@@ -159,12 +159,16 @@ and the recent submissions listing
                                 <map:parameter name="javascript.static#1" value="loadJQuery.js"/>
                                 <map:parameter name="javascript.static#2" value="static/js/discovery/search-controls.js"/>
                             </map:transform>
+
+                            <!--adds the "no index" robots meta tag to the page-->
                             <map:transform type="RobotsNoIndexTransformer"/>
                             <map:serialize type="xml"/>
 						</map:match>
 
                         <map:match pattern="handle/*/*/search-filter">
                             <map:transform type="SearchFacetFilter"/>
+
+                            <!--adds the "no index" robots meta tag to the page-->
                             <map:transform type="RobotsNoIndexTransformer"/>
                             <map:serialize type="xml"/>
                         </map:match>
diff --git a/dspace-xmlui/src/main/webapp/static/robots.txt b/dspace-xmlui/src/main/webapp/static/robots.txt
index afccddc3f5..75f133cada 100644
--- a/dspace-xmlui/src/main/webapp/static/robots.txt
+++ b/dspace-xmlui/src/main/webapp/static/robots.txt
@@ -11,9 +11,8 @@ Sitemap: ${dspace.url}/htmlmap
 User-agent: *
 # Disable access to Discovery search and filters
 Disallow: /discover
-Disallow: /handle/${handle.prefix}/*/discover
 Disallow: /search-filter
-Disallow: /handle/${handle.prefix}/*/search-filter
+
 #
 # Optionally uncomment the following line ONLY if sitemaps are working
 # and you have verified that your site is being indexed correctly.
diff --git a/dspace-xmlui/src/main/webapp/themes/Mirage/lib/xsl/core/page-structure.xsl b/dspace-xmlui/src/main/webapp/themes/Mirage/lib/xsl/core/page-structure.xsl
index 0834f50282..d006da292c 100644
--- a/dspace-xmlui/src/main/webapp/themes/Mirage/lib/xsl/core/page-structure.xsl
+++ b/dspace-xmlui/src/main/webapp/themes/Mirage/lib/xsl/core/page-structure.xsl
@@ -168,7 +168,7 @@
               </xsl:attribute>
             </meta>
 
-            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']">
+            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS'][not(@qualifier)]">
                 <meta name="ROBOTS">
                     <xsl:attribute name="content">
                         <xsl:value-of select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']"/>