Merge pull request #3224 from jygaulier/PHRAS-2831_port-truncation_4.1

PHRAS-2831 #comment merge port- truncation 4.1 
PHRAS-2802 #comment merge truncation with dot

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php

@@ -45,6 +45,9 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
                 foreach ($context->localizeField($field) as $f) {
                     $index_fields[] = $f;
                 }
+                foreach ($context->truncationField($field) as $f) {
+                    $index_fields[] = $f;
+                }
             }
             if (!$index_fields) {
                 return null;

lib/Alchemy/Phrasea/SearchEngine/Elastic/Index.php

@@ -90,6 +90,16 @@ class Index
                     // TODO Maybe replace nfkc_normalizer + asciifolding with icu_folding
                     'filter' => ['nfkc_normalizer', 'asciifolding']
                 ],
+                'truncation_analyzer' => [
+                    'type'      => 'custom',
+                    'tokenizer' => 'truncation_tokenizer',
+                    'filter'    => ['lowercase', 'stop', 'kstem']
+                ],
+                'truncation_analyzer#search' => [
+                    'type'      => 'custom',
+                    'tokenizer' => 'truncation_tokenizer',
+                    'filter'    => ['lowercase', 'stop', 'kstem']
+                ],
                 // Lang specific
                 'fr_full' => [
                     'type' => 'custom',
@@ -145,6 +155,12 @@ class Index
                 ]
             ],
             'tokenizer' => [
+                'truncation_tokenizer' => [
+                    "type" => "edgeNGram",
+                    "min_gram" => "2",
+                    "max_gram" => "15",
+                    "token_chars" => [ "letter", "digit", "punctuation", "symbol" ]
+                ],
                 'thesaurus_path' => [
                     'type' => 'path_hierarchy'
                 ]

lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/StringFieldMapping.php

@@ -55,8 +55,13 @@ class StringFieldMapping extends ComplexFieldMapping
     {
         $child = new StringFieldMapping('light');
         $child->setAnalyzer('general_light');
-
         $this->addChild($child);
+
+        $child = new StringFieldMapping('truncated');
+        $child->setAnalyzer('truncation_analyzer', 'indexing');
+        $child->setAnalyzer('truncation_analyzer#search', 'searching');
+        $this->addChild($child);
+
         $this->addLocalizedChildren($locales);
 
         return $this;

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php

@@ -9,6 +9,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
 use Alchemy\Phrasea\SearchEngine\Elastic\AST\Field as ASTField;
 use Alchemy\Phrasea\SearchEngine\Elastic\AST\Flag;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
+use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
 
 /**
  * @todo Check for private fields and only search on them if allowed
@@ -23,13 +24,23 @@ class QueryContext
     private $queryLocale;
     /** @var array */
     private $fields;
+    /** @var  SearchEngineOptions */
+    private $options;
 
-    public function __construct(Structure $structure, array $locales, $queryLocale, array $fields = null)
+    /**
+     * @param SearchEngineOptions|null $options
+     * @param Structure $structure
+     * @param array $locales
+     * @param $queryLocale
+     * @param array $fields
+     */
+    public function __construct($options, Structure $structure, array $locales, $queryLocale, array $fields = null)
     {
         $this->structure = $structure;
         $this->locales = $locales;
         $this->queryLocale = $queryLocale;
         $this->fields = $fields;
+        $this->options = $options;
     }
 
     public function narrowToFields(array $fields)
@@ -43,7 +54,7 @@ class QueryContext
             }
         }
 
-        return new static($this->structure, $this->locales, $this->queryLocale, $fields);
+        return new static($this->options, $this->structure, $this->locales, $this->queryLocale, $fields);
     }
 
     /**
@@ -131,6 +142,16 @@ class QueryContext
         return $ret;
     }
 
+    public function truncationField(Field $field)
+    {
+        if($this->options && $this->options->useTruncation()) {
+            return [sprintf('%s.truncated', $field->getIndexField())];
+        }
+        else {
+            return [];
+        }
+    }
+
     private function localizeFieldName($field)
     {
         $fields = array();

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php

@@ -23,7 +23,7 @@ class QueryContextFactory
             ? $this->getLimitedStructure($options)
             : $this->structure;
 
-        $context = new QueryContext($structure, $this->locales, $this->current_locale);
+        $context = new QueryContext($options, $structure, $this->locales, $this->current_locale);
 
         if ($options) {
             $fields = $this->getSearchedFields($options);

lib/Alchemy/Phrasea/SearchEngine/SearchEngineOptions.php

@@ -71,6 +71,8 @@ class SearchEngineOptions
     protected $i18n;
     /** @var bool */
     protected $stemming = true;
+    /** @var bool */
+    protected $use_truncation = false;
     /** @var string */
     protected $sort_by;
 
@@ -105,7 +107,8 @@ class SearchEngineOptions
         'sort_ord',
         'business_fields',
         'max_results',
-        'first_result'
+        'first_result',
+        'use_truncation',
     ];
 
     /**
@@ -217,6 +220,29 @@ class SearchEngineOptions
         return $this;
     }
 
+    /**
+     * Tells whether to use truncation or not
+     *
+     * @param  boolean             $boolean
+     * @return $this
+     */
+    public function setUseTruncation($boolean)
+    {
+        $this->use_truncation = !!$boolean;
+
+        return $this;
+    }
+
+    /**
+     * Return wheter the use of truncation is enabled or not
+     *
+     * @return boolean
+     */
+    public function useTruncation()
+    {
+        return $this->use_truncation;
+    }
+
     /**
      * Return wheter the use of stemming is enabled or not
      *
@@ -542,6 +568,8 @@ class SearchEngineOptions
         $options->setFields($databoxFields);
         $options->setDateFields($databoxDateFields);
 
+        $options->setUseTruncation((Boolean) $request->get('truncation'));
+
         return $options;
     }
 
@@ -628,6 +656,7 @@ class SearchEngineOptions
                 }
             },
             'stemming' => $optionSetter('setStemming'),
+            'use_truncation' => $optionSetter('setUseTruncation'),
             'date_fields' => function ($value, SearchEngineOptions $options) use ($fieldNormalizer) {
                 $options->setDateFields($fieldNormalizer($value));
             },

tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php

@@ -48,6 +48,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context->getUnrestrictedFields()->willReturn([$field]);
         $query_context->getPrivateFields()->willReturn([]);
         $query_context->localizeField($field)->willReturn(['foo.fr', 'foo.en']);
+        $query_context->truncationField($field)->willReturn([]);
 
         $node = new TextNode('bar', new Context('baz'));
         $query = $node->buildQuery($query_context->reveal());
@@ -80,12 +81,18 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context
             ->localizeField($public_field)
             ->willReturn(['foo.fr', 'foo.en']);
+        $query_context
+            ->truncationField($public_field)
+            ->willReturn([]);
         $query_context
             ->getPrivateFields()
             ->willReturn([$private_field]);
         $query_context
             ->localizeField($private_field)
             ->willReturn(['private_caption.bar.fr', 'private_caption.bar.en']);
+        $query_context
+            ->truncationField($private_field)
+            ->willReturn([]);
 
         $node = new TextNode('baz');
         $query = $node->buildQuery($query_context->reveal());
@@ -136,6 +143,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context->getUnrestrictedFields()->willReturn([$field]);
         $query_context->getPrivateFields()->willReturn([]);
         $query_context->localizeField($field)->willReturn(['foo.fr', 'foo.en']);
+        $query_context->truncationField($field)->willReturn([]);
 
         $node = new TextNode('bar');
         $node->setConcepts([
@@ -180,12 +188,18 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context
             ->localizeField($public_field)
             ->willReturn(['foo.fr', 'foo.en']);
+        $query_context
+            ->truncationField($public_field)
+            ->willReturn([]);
         $query_context
             ->getPrivateFields()
             ->willReturn([$private_field]);
         $query_context
             ->localizeField($private_field)
             ->willReturn(['private_caption.bar.fr', 'private_caption.bar.en']);
+        $query_context
+            ->truncationField($private_field)
+            ->willReturn([]);
 
         $node = new TextNode('baz');
         $node->setConcepts([

tests/Alchemy/Tests/Phrasea/SearchEngine/Search/QueryContextTest.php

@@ -18,7 +18,7 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase
     {
         $structure = $this->prophesize(Structure::class)->reveal();
         $available_locales = ['ab', 'cd', 'ef'];
-        $context = new QueryContext($structure, $available_locales, 'fr');
+        $context = new QueryContext(null, $structure, $available_locales, 'fr');
         $narrowed = $context->narrowToFields(['some_field']);
         $this->assertEquals(['some_field'], $narrowed->getFields());
     }
@@ -33,10 +33,10 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase
             'bar' => $bar_field
         ]);
 
-        $context = new QueryContext($structure->reveal(), [], 'fr');
+        $context = new QueryContext(null, $structure->reveal(), [], 'fr');
         $this->assertEquals([$foo_field, $bar_field], $context->getUnrestrictedFields());
 
-        $narrowed_context = new QueryContext($structure->reveal(), [], 'fr', ['foo']);
+        $narrowed_context = new QueryContext(null, $structure->reveal(), [], 'fr', ['foo']);
         $this->assertEquals([$foo_field], $narrowed_context->getUnrestrictedFields());
     }
 
@@ -50,10 +50,10 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase
             'bar' => $bar_field
         ]);
 
-        $context = new QueryContext($structure->reveal(), [], 'fr');
+        $context = new QueryContext(null, $structure->reveal(), [], 'fr');
         $this->assertEquals([$foo_field, $bar_field], $context->getPrivateFields());
 
-        $narrowed_context = new QueryContext($structure->reveal(), [], 'fr', ['foo']);
+        $narrowed_context = new QueryContext(null, $structure->reveal(), [], 'fr', ['foo']);
         $this->assertEquals([$foo_field], $narrowed_context->getPrivateFields());
     }
 }