[DS-1683] Add spell checker to discovery

dspace-api/src/main/java/org/dspace/discovery/DiscoverQuery.java

@@ -22,6 +22,7 @@ public class DiscoverQuery {
     private List<String> filterQueries;
     private int DSpaceObjectFilter = -1;
     private List<String> fieldPresentQueries;
+    private boolean spellCheck;
 
     private int start = 0;
     private int maxResults = -1;
@@ -264,4 +265,12 @@ public class DiscoverQuery {
     {
         this.hitHighlighting.put(hitHighlighting.getField(), hitHighlighting);
     }
+
+    public boolean isSpellCheck() {
+        return spellCheck;
+    }
+
+    public void setSpellCheck(boolean spellCheck) {
+        this.spellCheck = spellCheck;
+    }
 }

dspace-api/src/main/java/org/dspace/discovery/DiscoverResult.java

@@ -27,6 +27,7 @@ public class DiscoverResult {
     private int maxResults = -1;
     private int searchTime;
     private Map<String, DSpaceObjectHighlightResult> highlightedResults;
+    private String spellCheckQuery;
 
 
     public DiscoverResult() {
@@ -150,6 +151,14 @@ public class DiscoverResult {
         }
     }
 
+    public String getSpellCheckQuery() {
+        return spellCheckQuery;
+    }
+
+    public void setSpellCheckQuery(String spellCheckQuery) {
+        this.spellCheckQuery = spellCheckQuery;
+    }
+
     public static final class DSpaceObjectHighlightResult
     {
         private DSpaceObject dso;

dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java

@@ -59,10 +59,7 @@ import org.apache.solr.client.solrj.util.ClientUtils;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.FacetParams;
-import org.apache.solr.common.params.HighlightParams;
-import org.apache.solr.common.params.MoreLikeThisParams;
+import org.apache.solr.common.params.*;
 import org.apache.solr.common.util.NamedList;
 import org.dspace.content.Bitstream;
 import org.dspace.content.Bundle;
@@ -1574,6 +1571,12 @@ public class SolrServiceImpl implements SearchService, IndexingService {
 		}
 
         solrQuery.setQuery(query);
+        if(discoveryQuery.isSpellCheck())
+        {
+            solrQuery.setParam(SpellingParams.SPELLCHECK_Q, query);
+            solrQuery.setParam(SpellingParams.SPELLCHECK_COLLATE, Boolean.TRUE);
+            solrQuery.setParam("spellcheck", Boolean.TRUE);
+        }
 
         if (!includeWithdrawn)
         {
@@ -1852,6 +1855,15 @@ public class SolrServiceImpl implements SearchService, IndexingService {
                     }
                 }
             }
+
+            if(solrQueryResponse.getSpellCheckResponse() != null)
+            {
+                String recommendedQuery = solrQueryResponse.getSpellCheckResponse().getCollatedResult();
+                if(StringUtils.isNotBlank(recommendedQuery))
+                {
+                    result.setSpellCheckQuery(recommendedQuery);
+                }
+            }
         }
 
         return result;

dspace-api/src/main/java/org/dspace/discovery/SolrServiceSpellIndexingPlugin.java

@@ -0,0 +1,29 @@
+package org.dspace.discovery;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.dspace.content.DCValue;
+import org.dspace.content.DSpaceObject;
+import org.dspace.content.Item;
+import org.dspace.core.Context;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: kevin
+ * Date: 03/10/13
+ * Time: 15:06
+ * To change this template use File | Settings | File Templates.
+ */
+public class SolrServiceSpellIndexingPlugin implements SolrServiceIndexPlugin {
+
+    @Override
+    public void additionalIndex(Context context, DSpaceObject dso, SolrInputDocument document) {
+        if(dso instanceof Item){
+            DCValue[] dcValues = ((Item) dso).getMetadata(Item.ANY, Item.ANY, Item.ANY, Item.ANY);
+            for (DCValue dcValue : dcValues) {
+                document.addField("a_spell", dcValue.value);
+
+            }
+        }
+
+    }
+}

dspace-api/src/main/java/org/dspace/discovery/configuration/DiscoveryConfiguration.java

@@ -39,6 +39,7 @@ public class DiscoveryConfiguration implements InitializingBean{
     private String id;
     private DiscoveryHitHighlightingConfiguration hitHighlightingConfiguration;
     private DiscoveryMoreLikeThisConfiguration moreLikeThisConfiguration;
+    private boolean spellCheckEnabled;
 
     public String getId() {
         return id;
@@ -122,6 +123,14 @@ public class DiscoveryConfiguration implements InitializingBean{
         return moreLikeThisConfiguration;
     }
 
+    public boolean isSpellCheckEnabled() {
+        return spellCheckEnabled;
+    }
+
+    public void setSpellCheckEnabled(boolean spellCheckEnabled) {
+        this.spellCheckEnabled = spellCheckEnabled;
+    }
+
     /**
      * After all the properties are set check that the sidebar facets are a subset of our search filters
      *

dspace-xmlui/src/main/java/org/dspace/app/xmlui/aspect/discovery/AbstractSearch.java

@@ -330,24 +330,7 @@ public abstract class AbstractSearch extends AbstractDSpaceTransformer implement
             Map<String, String> parameters = new HashMap<String, String>();
             parameters.put("page", "{pageNum}");
             String pageURLMask = generateURL(parameters);
-            Map<String, String[]> filterQueryParams = getParameterFilterQueries();
-            if(filterQueryParams != null)
-            {
-                StringBuilder maskBuilder = new StringBuilder(pageURLMask);
-                for (String filterQueryParam : filterQueryParams.keySet())
-                {
-                    String[] filterQueryValues = filterQueryParams.get(filterQueryParam);
-                    if(filterQueryValues != null)
-                    {
-                        for (String filterQueryValue : filterQueryValues)
-                        {
-                            maskBuilder.append("&").append(filterQueryParam).append("=").append(filterQueryValue);
-                        }
-                    }
-                }
-
-                pageURLMask = maskBuilder.toString();
-            }
+            pageURLMask = addFilterQueriesToUrl(pageURLMask);
 
             results.setMaskedPagination(itemsTotal, firstItemIndex,
                     lastItemIndex, currentPage, pagesTotal, pageURLMask);
@@ -418,6 +401,28 @@ public abstract class AbstractSearch extends AbstractDSpaceTransformer implement
         //}// Empty query
     }
 
+    protected String addFilterQueriesToUrl(String pageURLMask) {
+        Map<String, String[]> filterQueryParams = getParameterFilterQueries();
+        if(filterQueryParams != null)
+        {
+            StringBuilder maskBuilder = new StringBuilder(pageURLMask);
+            for (String filterQueryParam : filterQueryParams.keySet())
+            {
+                String[] filterQueryValues = filterQueryParams.get(filterQueryParam);
+                if(filterQueryValues != null)
+                {
+                    for (String filterQueryValue : filterQueryValues)
+                    {
+                        maskBuilder.append("&").append(filterQueryParam).append("=").append(filterQueryValue);
+                    }
+                }
+            }
+
+            pageURLMask = maskBuilder.toString();
+        }
+        return pageURLMask;
+    }
+
     /**
      * Render the given item, all metadata is added to the given list, which metadata will be rendered where depends on the xsl
      * @param dspaceObjectsList a list of DSpace objects
@@ -818,6 +823,8 @@ public abstract class AbstractSearch extends AbstractDSpaceTransformer implement
             }
         }
 
+        queryArgs.setSpellCheck(discoveryConfiguration.isSpellCheckEnabled());
+
         this.queryResults = SearchUtils.getSearchService().search(context, scope, queryArgs);
     }
 

dspace-xmlui/src/main/java/org/dspace/app/xmlui/aspect/discovery/SimpleSearch.java

@@ -77,6 +77,7 @@ public class SimpleSearch extends AbstractSearch implements CacheableProcessingC
     private static final Message T_filter_notequals = message("xmlui.Discovery.SimpleSearch.filter.notequals");
     private static final Message T_filter_authority = message("xmlui.Discovery.SimpleSearch.filter.authority");
     private static final Message T_filter_notauthority = message("xmlui.Discovery.SimpleSearch.filter.notauthority");
+    private static final Message T_did_you_mean = message("xmlui.Discovery.SimpleSearch.did_you_mean");
 
     private SearchService searchService = null;
 
@@ -145,6 +146,12 @@ public class SimpleSearch extends AbstractSearch implements CacheableProcessingC
         Text text = searchBoxItem.addText("query");
         text.setValue(queryString);
         searchBoxItem.addButton("submit", "search-icon").setValue(T_go);
+        if(queryResults != null && StringUtils.isNotBlank(queryResults.getSpellCheckQuery()))
+        {
+            Item didYouMeanItem = searchList.addItem("did-you-mean", "didYouMean");
+            didYouMeanItem.addContent(T_did_you_mean);
+            didYouMeanItem.addXref(getSuggestUrl(queryResults.getSpellCheckQuery()), queryResults.getSpellCheckQuery(), "didYouMean");
+        }
 
         DSpaceObject dso = HandleUtil.obtainHandle(objectModel);
         DiscoveryConfiguration discoveryConfiguration = SearchUtils.getDiscoveryConfiguration(dso);
@@ -295,7 +302,7 @@ public class SimpleSearch extends AbstractSearch implements CacheableProcessingC
     protected String generateURL(Map<String, String> parameters)
             throws UIException {
         String query = getQuery();
-        if (!"".equals(query))
+        if (!"".equals(query) && parameters.get("query") == null)
         {
             parameters.put("query", encodeForURL(query));
         }
@@ -382,4 +389,10 @@ public class SimpleSearch extends AbstractSearch implements CacheableProcessingC
             }
         }
     }
+
+    protected String getSuggestUrl(String newQuery) throws UIException {
+        Map parameters = new HashMap();
+        parameters.put("query", newQuery);
+        return addFilterQueriesToUrl(generateURL(parameters));
+    }
 }

dspace-xmlui/src/main/resources/aspects/Discovery/i18n/messages.xml

@@ -148,4 +148,6 @@
     <message key="xmlui.Discovery.AbstractSearch.head2">Communities or Collections matching your query</message>
    	<message key="xmlui.Discovery.AbstractSearch.head3">Items matching your query</message>
 
+   	<message key="xmlui.Discovery.SimpleSearch.did_you_mean">Did you mean: </message>
+
 </catalogue>

dspace-xmlui/src/main/webapp/themes/Classic/lib/style.css

@@ -1003,3 +1003,11 @@ div#aspect_discovery_SimpleSearch_div_search a.previous-page-link {
 }
 
 /* End discovery layout DSpace 3.x*/
+
+.didYouMean{
+    font-size: 18px;
+}
+
+.didYouMean a{
+    font-weight: bold;
+}

dspace-xmlui/src/main/webapp/themes/Mirage/lib/css/style.css

@@ -1373,3 +1373,11 @@ table.discovery-filters th.new-filter-header
 .searchTime{
     color: #999999;
 }
+
+.didYouMean{
+    font-size: 18px;
+}
+
+.didYouMean a{
+    font-weight: bold;
+}

dspace-xmlui/src/main/webapp/themes/Reference/lib/style.css

@@ -1449,4 +1449,12 @@ ul.vocabulary div.vocabulary-node-icon.vocabulary-closed{
 div.vocabulary-container li.error{
     color: #c22121;
 }
-/* Controlled vocabulary support css END*/
+/* Controlled vocabulary support css END*/
+
+.didYouMean{
+    font-size: 18px;
+}
+
+.didYouMean a{
+    font-weight: bold;
+}

dspace/config/spring/api/discovery.xml

@@ -23,6 +23,7 @@
     <context:annotation-config /> <!-- allows us to use spring annotations in beans -->
 
     <bean id="solrServiceResourceIndexPlugin" class="org.dspace.discovery.SolrServiceResourceRestrictionPlugin" scope="prototype"/>
+    <bean id="SolrServiceSpellIndexingPlugin" class="org.dspace.discovery.SolrServiceSpellIndexingPlugin" scope="prototype"/>
 
     <alias name="solrServiceResourceIndexPlugin" alias="org.dspace.discovery.SolrServiceResourceRestrictionPlugin"/>
 
@@ -170,21 +171,23 @@
                 <!--When altering this list also alter the "xmlui.Discovery.RelatedItems.help" key as it describes
                 the metadata fields below-->
                 <property name="similarityMetadataFields">
-            <list>
+                    <list>
                         <value>dc.title</value>
-                <value>dc.contributor.author</value>
-                <value>dc.creator</value>
+                        <value>dc.contributor.author</value>
+                        <value>dc.creator</value>
                         <value>dc.subject</value>
-            </list>
-        </property>
+                    </list>
+                </property>
                 <!--The minimum number of matching terms across the metadata fields above before an item is found as related -->
                 <property name="minTermFrequency" value="5"/>
                 <!--The maximum number of related items displayed-->
                 <property name="max" value="3"/>
                 <!--The minimum word length below which words will be ignored-->
                 <property name="minWordLength" value="5"/>
-    </bean>
+            </bean>
         </property>
+        <!-- When true a "did you mean" example will be displayed, value can be true or false -->
+        <property name="spellCheckEnabled" value="true"/>
     </bean>
 
 

dspace/solr/search/conf/schema.xml

@@ -455,6 +455,31 @@
             <filter class="solr.TrimFilterFactory" />
         </analyzer>
     </fieldType>
+
+      <!--
+       SpellCheck analysis config based off of http://wiki.apache.org/solr/
+       SpellCheckingAnalysis
+       -->
+      <fieldType name="textSpell" class="solr.TextField"
+                 positionIncrementGap="100" stored="false" multiValued="true">
+          <analyzer type="index">
+              <tokenizer class="solr.StandardTokenizerFactory"/>
+              <filter class="solr.LowerCaseFilterFactory"/>
+              <filter class="solr.SynonymFilterFactory"
+                      synonyms="synonyms.txt" ignoreCase="true"
+                      expand="true"/>
+              <filter class="solr.StopFilterFactory" ignoreCase="true"
+                      words="stopwords.txt"/>
+              <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+          </analyzer>
+          <analyzer type="query">
+              <tokenizer class="solr.StandardTokenizerFactory"/>
+              <filter class="solr.LowerCaseFilterFactory"/>
+              <filter class="solr.StopFilterFactory" ignoreCase="true"
+                      words="stopwords.txt"/>
+              <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+                </analyzer>
+      </fieldType>
  </types>
 
 
@@ -517,6 +542,9 @@
     <field name="location.comm" type="lowerCaseSort" indexed="true" stored="true" multiValued="true" required="false" omitNorms="true" />
     <field name="location.coll" type="lowerCaseSort" indexed="true" stored="true" multiValued="true" required="false" omitNorms="true" />
 
+    <field name="a_spell" type="textSpell" />
+    <copyField source="fulltext" dest="a_spell" />
+
     <!-- used by the DSpace Discovery Solr Indexer to track the last time a document was indexed -->
    <field name="SolrIndexer.lastIndexed" type="date" indexed="true" stored="true" default="NOW" multiValued="false" omitNorms="true" />
 

dspace/solr/search/conf/solrconfig.xml

@@ -811,6 +811,9 @@
        <int name="rows">10</int>
        <str name="df">search_text</str>
      </lst>
+      <arr name="last-components">
+          <str>spellcheck</str>
+      </arr>
     <!-- In addition to defaults, "appends" params can be specified
          to identify values which should be appended to the list of
          multi-val params from the query (or the existing "defaults").
@@ -862,7 +865,9 @@
          <str>nameOfCustomComponent1</str>
          <str>nameOfCustomComponent2</str>
        </arr>
+
       -->
+
     </requestHandler>
 
   <!-- A request handler that returns indented JSON by default -->
@@ -1232,10 +1237,14 @@
 
     <str name="queryAnalyzerFieldType">textSpell</str>
 
+
     <lst name="spellchecker">
+      <str name="classname">solr.IndexBasedSpellChecker</str>
       <str name="name">default</str>
-      <str name="field">name</str>
+      <str name="field">a_spell</str>
       <str name="spellcheckIndexDir">./spellchecker</str>
+      <str name="buildOnCommit">true</str>
+      <str name="spellcheck.onlyMorePopular">false</str>
     </lst>
 
     <!-- a spellchecker that uses a different distance measure