DS-2962

dspace-xmlui-mirage2/src/main/webapp/xsl/core/page-structure.xsl

@@ -177,7 +177,7 @@
                 </xsl:attribute>
             </meta>
 
-            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']">
+            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS'][not(@qualifier)]">
                 <meta name="ROBOTS">
                     <xsl:attribute name="content">
                         <xsl:value-of select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']"/>

dspace-xmlui/src/main/java/org/dspace/app/xmlui/aspect/discovery/RobotsNoIndexTransformer.java

@@ -1,3 +1,10 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
 package org.dspace.app.xmlui.aspect.discovery;
 
 import java.io.*;
@@ -10,6 +17,10 @@ import org.dspace.authorize.*;
 import org.xml.sax.*;
 
 /**
+ * Robots can not be blocked from dynamic URLs using robots.txt. This transformer is used instead to add
+ * a meta tag that tells robots not to index the content of a page.
+ * Information about this meta tag can be found here: http://www.robotstxt.org/meta.html
+ *
  * @author philip at atmire.com
  */
 public class RobotsNoIndexTransformer extends AbstractDSpaceTransformer {

dspace-xmlui/src/main/resources/aspects/Discovery/sitemap.xmap

@@ -159,12 +159,16 @@ and the recent submissions listing
                                 <map:parameter name="javascript.static#1" value="loadJQuery.js"/>
                                 <map:parameter name="javascript.static#2" value="static/js/discovery/search-controls.js"/>
                             </map:transform>
+
+                            <!--adds the "no index" robots meta tag to the page-->
                             <map:transform type="RobotsNoIndexTransformer"/>
                             <map:serialize type="xml"/>
 						</map:match>
 
                         <map:match pattern="handle/*/*/search-filter">
                             <map:transform type="SearchFacetFilter"/>
+
+                            <!--adds the "no index" robots meta tag to the page-->
                             <map:transform type="RobotsNoIndexTransformer"/>
                             <map:serialize type="xml"/>
                         </map:match>

dspace-xmlui/src/main/webapp/static/robots.txt

@@ -11,9 +11,8 @@ Sitemap: ${dspace.url}/htmlmap
 User-agent: *
 # Disable access to Discovery search and filters
 Disallow: /discover
-Disallow: /handle/${handle.prefix}/*/discover
 Disallow: /search-filter
-Disallow: /handle/${handle.prefix}/*/search-filter
+
 #
 # Optionally uncomment the following line ONLY if sitemaps are working
 # and you have verified that your site is being indexed correctly.

dspace-xmlui/src/main/webapp/themes/Mirage/lib/xsl/core/page-structure.xsl

@@ -168,7 +168,7 @@
               </xsl:attribute>
             </meta>
 
-            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']">
+            <xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS'][not(@qualifier)]">
                 <meta name="ROBOTS">
                     <xsl:attribute name="content">
                         <xsl:value-of select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='ROBOTS']"/>