diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php index 4bc98ec6b9..bf239ffbd1 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php @@ -45,6 +45,9 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface foreach ($context->localizeField($field) as $f) { $index_fields[] = $f; } + foreach ($context->truncationField($field) as $f) { + $index_fields[] = $f; + } } if (!$index_fields) { return null; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Index.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Index.php index dd5d942f78..9671b83226 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Index.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Index.php @@ -90,6 +90,16 @@ class Index // TODO Maybe replace nfkc_normalizer + asciifolding with icu_folding 'filter' => ['nfkc_normalizer', 'asciifolding'] ], + 'truncation_analyzer' => [ + 'type' => 'custom', + 'tokenizer' => 'truncation_tokenizer', + 'filter' => ['lowercase', 'stop', 'kstem'] + ], + 'truncation_analyzer#search' => [ + 'type' => 'custom', + 'tokenizer' => 'truncation_tokenizer', + 'filter' => ['lowercase', 'stop', 'kstem'] + ], // Lang specific 'fr_full' => [ 'type' => 'custom', @@ -145,6 +155,12 @@ class Index ] ], 'tokenizer' => [ + 'truncation_tokenizer' => [ + "type" => "edgeNGram", + "min_gram" => "2", + "max_gram" => "15", + "token_chars" => [ "letter", "digit", "punctuation", "symbol" ] + ], 'thesaurus_path' => [ 'type' => 'path_hierarchy' ] diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/StringFieldMapping.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/StringFieldMapping.php index b9022b2238..4f7c9d360d 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/StringFieldMapping.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/StringFieldMapping.php @@ -55,8 +55,13 @@ class StringFieldMapping extends ComplexFieldMapping { $child = new StringFieldMapping('light'); $child->setAnalyzer('general_light'); - $this->addChild($child); + + $child = new StringFieldMapping('truncated'); + $child->setAnalyzer('truncation_analyzer', 'indexing'); + $child->setAnalyzer('truncation_analyzer#search', 'searching'); + $this->addChild($child); + $this->addLocalizedChildren($locales); return $this; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php index cd65ed8d1d..69baf9f5bf 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php @@ -9,6 +9,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field; use Alchemy\Phrasea\SearchEngine\Elastic\AST\Field as ASTField; use Alchemy\Phrasea\SearchEngine\Elastic\AST\Flag; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure; +use Alchemy\Phrasea\SearchEngine\SearchEngineOptions; /** * @todo Check for private fields and only search on them if allowed @@ -23,13 +24,23 @@ class QueryContext private $queryLocale; /** @var array */ private $fields; + /** @var SearchEngineOptions */ + private $options; - public function __construct(Structure $structure, array $locales, $queryLocale, array $fields = null) + /** + * @param SearchEngineOptions|null $options + * @param Structure $structure + * @param array $locales + * @param $queryLocale + * @param array $fields + */ + public function __construct($options, Structure $structure, array $locales, $queryLocale, array $fields = null) { $this->structure = $structure; $this->locales = $locales; $this->queryLocale = $queryLocale; $this->fields = $fields; + $this->options = $options; } public function narrowToFields(array $fields) @@ -43,7 +54,7 @@ class QueryContext } } - return new static($this->structure, $this->locales, $this->queryLocale, $fields); + return new static($this->options, $this->structure, $this->locales, $this->queryLocale, $fields); } /** @@ -131,6 +142,16 @@ class QueryContext return $ret; } + public function truncationField(Field $field) + { + if($this->options && $this->options->useTruncation()) { + return [sprintf('%s.truncated', $field->getIndexField())]; + } + else { + return []; + } + } + private function localizeFieldName($field) { $fields = array(); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php index f199434677..9443c79655 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php @@ -23,7 +23,7 @@ class QueryContextFactory ? $this->getLimitedStructure($options) : $this->structure; - $context = new QueryContext($structure, $this->locales, $this->current_locale); + $context = new QueryContext($options, $structure, $this->locales, $this->current_locale); if ($options) { $fields = $this->getSearchedFields($options); diff --git a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineOptions.php b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineOptions.php index f52fbcdf21..5e942b1adf 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineOptions.php @@ -71,6 +71,8 @@ class SearchEngineOptions protected $i18n; /** @var bool */ protected $stemming = true; + /** @var bool */ + protected $use_truncation = false; /** @var string */ protected $sort_by; @@ -105,7 +107,8 @@ class SearchEngineOptions 'sort_ord', 'business_fields', 'max_results', - 'first_result' + 'first_result', + 'use_truncation', ]; /** @@ -217,6 +220,29 @@ class SearchEngineOptions return $this; } + /** + * Tells whether to use truncation or not + * + * @param boolean $boolean + * @return $this + */ + public function setUseTruncation($boolean) + { + $this->use_truncation = !!$boolean; + + return $this; + } + + /** + * Return wheter the use of truncation is enabled or not + * + * @return boolean + */ + public function useTruncation() + { + return $this->use_truncation; + } + /** * Return wheter the use of stemming is enabled or not * @@ -542,6 +568,8 @@ class SearchEngineOptions $options->setFields($databoxFields); $options->setDateFields($databoxDateFields); + $options->setUseTruncation((Boolean) $request->get('truncation')); + return $options; } @@ -628,6 +656,7 @@ class SearchEngineOptions } }, 'stemming' => $optionSetter('setStemming'), + 'use_truncation' => $optionSetter('setUseTruncation'), 'date_fields' => function ($value, SearchEngineOptions $options) use ($fieldNormalizer) { $options->setDateFields($fieldNormalizer($value)); }, diff --git a/tests/Alchemy/Tests/Phrasea/SearchEngine/Search/QueryContextTest.php b/tests/Alchemy/Tests/Phrasea/SearchEngine/Search/QueryContextTest.php index e45a69b87b..268cd917c5 100644 --- a/tests/Alchemy/Tests/Phrasea/SearchEngine/Search/QueryContextTest.php +++ b/tests/Alchemy/Tests/Phrasea/SearchEngine/Search/QueryContextTest.php @@ -18,7 +18,7 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase { $structure = $this->prophesize(Structure::class)->reveal(); $available_locales = ['ab', 'cd', 'ef']; - $context = new QueryContext($structure, $available_locales, 'fr'); + $context = new QueryContext(null, $structure, $available_locales, 'fr'); $narrowed = $context->narrowToFields(['some_field']); $this->assertEquals(['some_field'], $narrowed->getFields()); } @@ -33,10 +33,10 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase 'bar' => $bar_field ]); - $context = new QueryContext($structure->reveal(), [], 'fr'); + $context = new QueryContext(null, $structure->reveal(), [], 'fr'); $this->assertEquals([$foo_field, $bar_field], $context->getUnrestrictedFields()); - $narrowed_context = new QueryContext($structure->reveal(), [], 'fr', ['foo']); + $narrowed_context = new QueryContext(null, $structure->reveal(), [], 'fr', ['foo']); $this->assertEquals([$foo_field], $narrowed_context->getUnrestrictedFields()); } @@ -50,7 +50,7 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase 'bar' => $bar_field ]); - $context = new QueryContext($structure->reveal(), [], 'fr'); + $context = new QueryContext(null, $structure->reveal(), [], 'fr'); $this->assertEquals([$foo_field, $bar_field], $context->getPrivateFields()); $narrowed_context = new QueryContext($structure->reveal(), [], 'fr', ['foo']);