From f8fb8837cb6282537faecaa5d09eb730fbfe3ba2 Mon Sep 17 00:00:00 2001 From: mike-esokia Date: Wed, 23 May 2018 17:00:15 +0400 Subject: [PATCH 1/2] wip restore hightlights and multivalue seperator --- .../Core/Provider/TwigServiceProvider.php | 5 +- .../Phrasea/Twig/PhraseanetExtension.php | 120 +++++++++++------- templates/web/common/macros.html.twig | 10 +- templates/web/prod/results/record.html.twig | 3 +- 4 files changed, 82 insertions(+), 56 deletions(-) diff --git a/lib/Alchemy/Phrasea/Core/Provider/TwigServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/TwigServiceProvider.php index 4dddd878ad..9e4507ba8d 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/TwigServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/TwigServiceProvider.php @@ -130,17 +130,18 @@ class TwigServiceProvider implements ServiceProviderInterface $twig->addFilter(new \Twig_SimpleFilter('parseColor', function (\Twig_Environment $twig, $string) use ($app) { $re = '/^(.*)\[#([0-9a-fA-F]{6})]$/m'; $stringArr = explode(';', $string); + foreach ($stringArr as $key => $value) { preg_match_all($re, trim($value), $matches); if ($matches && $matches[1] != null && $matches[2] != null) { $colorCode = '#' . $matches[2][0]; $colorName = $matches[1][0]; - $stringArr[$key] = '' . $colorName . '
'; + $stringArr[$key] = '' . $colorName . ''; } } - return implode('', $stringArr); + return implode('; ', $stringArr); }, ['needs_environment' => true, 'is_safe' => ['html']])); $twig->addFilter(new \Twig_SimpleFilter('bounce', diff --git a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php index b9c92f5015..ffc872ec95 100644 --- a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php +++ b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php @@ -47,12 +47,36 @@ class PhraseanetExtension extends \Twig_Extension new \Twig_SimpleFunction('record_flags', array($this, 'getRecordFlags')), new \Twig_SimpleFunction('border_checker_from_fqcn', array($this, 'getCheckerFromFQCN')), new \Twig_SimpleFunction('caption_field', array($this, 'getCaptionField')), + new \Twig_SimpleFunction('caption_field_label', array($this, 'getCaptionFieldLabel')), new \Twig_SimpleFunction('caption_field_order', array($this, 'getCaptionFieldOrder')), new \Twig_SimpleFunction('flag_slugify', array(Flag::class, 'normalizeName')), ); } + /** + * get localized field's label + * @param RecordInterface $record + * @param $fieldName + * @return string - the name label + */ + public function getCaptionFieldLabel(RecordInterface $record, $fieldName) + { + if ($record) { + /** @var \appbox $appbox */ + $appbox = $this->app['phraseanet.appbox']; + $databox = $appbox->get_databox($record->getDataboxId()); + foreach ($databox->get_meta_structure() as $meta) { + /** @var \databox_field $meta */ + if ($meta->get_name() === $fieldName) { + return $meta->get_label($this->app['locale']); + } + } + } + + return ''; + } + public function getCaptionField(RecordInterface $record, $field, $value) { if ($record instanceof ElasticsearchRecord) { @@ -99,6 +123,31 @@ class PhraseanetExtension extends \Twig_Extension return $orders[$databoxId][$orderKey]; } + /** + * @param \databox $databox + * @return array + */ + private function retrieveDataboxFieldOrderings(\databox $databox) + { + $publicOrder = []; + $businessOrder = []; + + foreach ($databox->get_meta_structure() as $field) { + $fieldName = $field->get_name(); + + if (!$field->isBusiness()) { + $publicOrder[] = $fieldName; + } + + $businessOrder[] = $fieldName; + }; + + return [ + 'public' => $publicOrder, + 'business' => $businessOrder, + ]; + } + public function getRecordFlags(RecordInterface $record) { $recordStatuses = []; @@ -132,24 +181,6 @@ class PhraseanetExtension extends \Twig_Extension return $recordStatuses; } - public function isGrantedOnDatabox($databoxId, $rights) - { - if (false === ($this->app->getAuthenticatedUser() instanceof User)) { - - return false; - } - - $rights = (array) $rights; - foreach ($rights as $right) { - if (false === $this->app->getAclForUser($this->app->getAuthenticatedUser())->has_right_on_sbas($databoxId, $right)) { - - return false; - } - } - - return true; - } - /** * returns true if user is authenticated and has all the passed rights on the base * todo : wtf $rights is an array since it's never called with more than 1 right in it ? @@ -177,6 +208,24 @@ class PhraseanetExtension extends \Twig_Extension return true; } + public function isGrantedOnDatabox($databoxId, $rights) + { + if (false === ($this->app->getAuthenticatedUser() instanceof User)) { + + return false; + } + + $rights = (array)$rights; + foreach ($rights as $right) { + if (false === $this->app->getAclForUser($this->app->getAuthenticatedUser())->has_right_on_sbas($databoxId, $right)) { + + return false; + } + } + + return true; + } + public function getCollectionLogo($baseId) { if (false === $this->app['filesystem']->exists(sprintf('%s/config/minilogos/%s', $this->app['root.path'], $baseId))) { @@ -242,11 +291,6 @@ class PhraseanetExtension extends \Twig_Extension return $this->getSubdefUrl($record, 'thumbnail'); } - public function getThumbnailGifUrl(RecordInterface $record) - { - return $this->getSubdefUrl($record, 'thumbnailgif'); - } - public function getSubdefUrl(RecordInterface $record, $subdefName) { /** @var StaticMode $staticMode */ @@ -279,6 +323,11 @@ class PhraseanetExtension extends \Twig_Extension return $path; } + public function getThumbnailGifUrl(RecordInterface $record) + { + return $this->getSubdefUrl($record, 'thumbnailgif'); + } + public function getSubdefSize(RecordInterface $record, $subdefName) { $ret = null; @@ -325,29 +374,4 @@ class PhraseanetExtension extends \Twig_Extension { return 'phraseanet'; } - - /** - * @param \databox $databox - * @return array - */ - private function retrieveDataboxFieldOrderings(\databox $databox) - { - $publicOrder = []; - $businessOrder = []; - - foreach ($databox->get_meta_structure() as $field) { - $fieldName = $field->get_name(); - - if (!$field->isBusiness()) { - $publicOrder[] = $fieldName; - } - - $businessOrder[] = $fieldName; - }; - - return [ - 'public' => $publicOrder, - 'business' => $businessOrder, - ]; - } } diff --git a/templates/web/common/macros.html.twig b/templates/web/common/macros.html.twig index 1683aa98c3..9d5ed046bc 100644 --- a/templates/web/common/macros.html.twig +++ b/templates/web/common/macros.html.twig @@ -93,9 +93,9 @@ {% set terms = [] %} {% for data in field.values %} {% if data.from_thesaurus and bounceable %} - {% set value = data.value|e|bounce(field.name, data.qjs, field.sbas_id) %} + {% set value = data.value|e|bounce(field.name, data.qjs, field.sbas_id)|parseColor %} {% else %} - {% set value = data.value|e %} + {% set value = data.value|e|parseColor %} {% endif %} {% set terms = [value]|merge(terms) %} {% endfor %} @@ -126,9 +126,9 @@ {% macro caption(record, can_see_business, display_exif, limitedWidth = false) %}
- {% for field in record.get_caption().get_highlight_fields(null, can_see_business) %} -
{{ field.label_name }}
-
{{ _self.caption_value(field, bounceable|default(true))|highlight|linkify|parseColor }}
+ {% for name, value in record.getCaption(caption_field_order(record, can_see_business)) %} +
{{ caption_field_label(record, name) }}
+
{{ caption_field(record, name, value)|e|highlight|linkify }}
{% endfor %}
{% if display_exif|default(true) and app.getAuthenticator().user is not none and user_setting('technical_display') == 'group' %} diff --git a/templates/web/prod/results/record.html.twig b/templates/web/prod/results/record.html.twig index 8f8dccbe1e..6b99d304c7 100644 --- a/templates/web/prod/results/record.html.twig +++ b/templates/web/prod/results/record.html.twig @@ -24,7 +24,8 @@ {% set can_see_business = granted_on_collection(record.baseId, [constant('\\ACL::CANMODIFRECORD')]) %}
From 04a0f56b4d4cd15889e2df9c1b2f09e8ac5cb595 Mon Sep 17 00:00:00 2001 From: mike-esokia Date: Wed, 23 May 2018 17:15:56 +0400 Subject: [PATCH 2/2] Add PHRAS-1539 --- .../Elastic/ElasticSearchEngine.php | 414 +++++++++--------- .../Elastic/ElasticsearchRecordHydrator.php | 5 +- templates/web/common/macros.html.twig | 2 +- templates/web/prod/results/record.html.twig | 3 +- 4 files changed, 215 insertions(+), 209 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 2c900c9954..cf263206c4 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -201,6 +201,11 @@ class ElasticSearchEngine implements SearchEngineInterface $this->notImplemented(); } + private function notImplemented() + { + throw new LogicException('Not implemented'); + } + /** * {@inheritdoc} */ @@ -241,11 +246,6 @@ class ElasticSearchEngine implements SearchEngineInterface $this->notImplemented(); } - private function notImplemented() - { - throw new LogicException('Not implemented'); - } - /** * {@inheritdoc} */ @@ -330,131 +330,6 @@ class ElasticSearchEngine implements SearchEngineInterface ); } - private function buildHighlightRules(QueryContext $context) - { - $highlighted_fields = []; - foreach ($context->getHighlightedFields() as $field) { - switch ($field->getType()) { - case FieldMapping::TYPE_STRING: - $index_field = $field->getIndexField(); - $raw_index_field = $field->getIndexField(true); - $highlighted_fields[$index_field] = [ - // Requires calling Mapping::enableTermVectors() on this field mapping - 'matched_fields' => [$index_field, $raw_index_field], - 'type' => 'fvh' - ]; - break; - case FieldMapping::TYPE_FLOAT: - case FieldMapping::TYPE_DOUBLE: - case FieldMapping::TYPE_INTEGER: - case FieldMapping::TYPE_LONG: - case FieldMapping::TYPE_SHORT: - case FieldMapping::TYPE_BYTE: - continue; - case FieldMapping::TYPE_DATE: - default: - continue; - } - } - - return [ - 'pre_tags' => ['[[em]]'], - 'post_tags' => ['[[/em]]'], - 'order' => 'score', - 'fields' => $highlighted_fields - ]; - } - - /** - * {@inheritdoc} - */ - public function autocomplete($query, SearchEngineOptions $options) - { - $params = $this->createCompletionParams($query, $options); - - $res = $this->client->suggest($params); - - $ret = [ - 'text' => [], - 'byField' => [] - ]; - foreach(array_keys($params['body']) as $fname) { - $t = []; - foreach($res[$fname] as $suggest) { // don't know why there is a sub-array level - foreach($suggest['options'] as $option) { - $text = $option['text']; - if(!in_array($text, $ret['text'])) { - $ret['text'][] = $text; - } - $t[] = [ - 'label' => $text, - 'query' => $fname.':'.$text - ]; - } - } - if(!empty($t)) { - $ret['byField'][$fname] = $t; - } - } - - return $ret; - } - - /** - * {@inheritdoc} - */ - public function resetCache() - { - } - - /** - * {@inheritdoc} - */ - public function clearCache() - { - } - - /** - * {@inheritdoc} - */ - public function clearAllCache(\DateTime $date = null) - { - } - - private function createCompletionParams($query, SearchEngineOptions $options) - { - $body = []; - $context = [ - 'record_type' => $options->getSearchType() === SearchEngineOptions::RECORD_RECORD ? - SearchEngineInterface::GEM_TYPE_RECORD : SearchEngineInterface::GEM_TYPE_STORY - ]; - - $base_ids = $options->getBasesIds(); - if (count($base_ids) > 0) { - $context['base_id'] = $base_ids; - } - - $search_context = $this->context_factory->createContext($options); - $fields = $search_context->getUnrestrictedFields(); - foreach($fields as $field) { - if($field->getType() == FieldMapping::TYPE_STRING) { - $k = '' . $field->getName(); - $body[$k] = [ - 'text' => $query, - 'completion' => [ - 'field' => "caption." . $field->getName() . ".suggest", - 'context' => &$context - ] - ]; - } - } - - return [ - 'index' => $this->indexName, - 'body' => $body - ]; - } - private function createRecordQueryParams($ESQuery, SearchEngineOptions $options, \record_adapter $record = null) { $params = [ @@ -483,66 +358,28 @@ class ElasticSearchEngine implements SearchEngineInterface return $params; } - private function getAggregationQueryParams(SearchEngineOptions $options) + private function createSortQueryParams(SearchEngineOptions $options) { - $aggs = []; - // technical aggregates (enable + optional limit) - foreach (ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { - $size = $this->options->getAggregableFieldLimit($k); - if ($size !== databox_field::FACET_DISABLED) { - if ($size === databox_field::FACET_NO_LIMIT) { - $size = ESField::FACET_NO_LIMIT; - } - $agg = [ - 'terms' => [ - 'field' => $f['field'], - 'size' => $size - ] - ]; - $aggs[$k] = $agg; - } - } - // fields aggregates - $structure = $this->context_factory->getLimitedStructure($options); - foreach ($structure->getFacetFields() as $name => $field) { - // 2015-05-26 (mdarse) Removed databox filtering. - // It was already done by the ACL filter in the query scope, so no - // document that shouldn't be displayed can go this far. - $agg = [ - 'terms' => [ - 'field' => $field->getIndexField(true), - 'size' => $field->getFacetValuesLimit() - ] - ]; - $aggs[$name] = AggregationHelper::wrapPrivateFieldAggregation($field, $agg); - } - return $aggs; - } + $sort = []; - private function createACLFilters(SearchEngineOptions $options) - { - // No ACLs if no user - if (false === $this->app->getAuthenticator()->isAuthenticated()) { - return []; + if ($options->getSortBy() === null || $options->getSortBy() === SearchEngineOptions::SORT_RELEVANCE) { + $sort['_score'] = $options->getSortOrder(); + } + elseif ($options->getSortBy() === SearchEngineOptions::SORT_CREATED_ON) { + $sort['created_on'] = $options->getSortOrder(); + } + elseif ($options->getSortBy() === 'recordid') { + $sort['record_id'] = $options->getSortOrder(); + } + else { + $sort[sprintf('caption.%s', $options->getSortBy())] = $options->getSortOrder(); } - $acl = $this->app->getAclForUser($this->app->getAuthenticatedUser()); - - $grantedCollections = array_keys($acl->get_granted_base([\ACL::ACTIF])); - - if (count($grantedCollections) === 0) { - return ['bool' => ['must_not' => ['match_all' => new \stdClass()]]]; + if (!array_key_exists('record_id', $sort)) { + $sort['record_id'] = $options->getSortOrder(); } - $appbox = $this->app['phraseanet.appbox']; - - $flagNamesMap = $this->getFlagsKey($appbox); - // Get flags rules - $flagRules = $this->getFlagsRules($appbox, $acl, $grantedCollections); - // Get intersection between collection ACLs and collection chosen by end user - $aclRules = $this->getACLsByCollection($flagRules, $flagNamesMap); - - return $this->buildACLsFilters($aclRules, $options); + return $sort; } private function createQueryFilters(SearchEngineOptions $options) @@ -600,27 +437,6 @@ class ElasticSearchEngine implements SearchEngineInterface return $filters; } - private function createSortQueryParams(SearchEngineOptions $options) - { - $sort = []; - - if ($options->getSortBy() === null || $options->getSortBy() === SearchEngineOptions::SORT_RELEVANCE) { - $sort['_score'] = $options->getSortOrder(); - } elseif ($options->getSortBy() === SearchEngineOptions::SORT_CREATED_ON) { - $sort['created_on'] = $options->getSortOrder(); - } elseif ($options->getSortBy() === 'recordid') { - $sort['record_id'] = $options->getSortOrder(); - } else { - $sort[sprintf('caption.%s', $options->getSortBy())] = $options->getSortOrder(); - } - - if (! array_key_exists('record_id', $sort)) { - $sort['record_id'] = $options->getSortOrder(); - } - - return $sort; - } - private function getFlagsKey(\appbox $appbox) { $flags = []; @@ -635,6 +451,32 @@ class ElasticSearchEngine implements SearchEngineInterface return $flags; } + private function createACLFilters(SearchEngineOptions $options) + { + // No ACLs if no user + if (false === $this->app->getAuthenticator()->isAuthenticated()) { + return []; + } + + $acl = $this->app->getAclForUser($this->app->getAuthenticatedUser()); + + $grantedCollections = array_keys($acl->get_granted_base([\ACL::ACTIF])); + + if (count($grantedCollections) === 0) { + return ['bool' => ['must_not' => ['match_all' => new \stdClass()]]]; + } + + $appbox = $this->app['phraseanet.appbox']; + + $flagNamesMap = $this->getFlagsKey($appbox); + // Get flags rules + $flagRules = $this->getFlagsRules($appbox, $acl, $grantedCollections); + // Get intersection between collection ACLs and collection chosen by end user + $aclRules = $this->getACLsByCollection($flagRules, $flagNamesMap); + + return $this->buildACLsFilters($aclRules, $options); + } + private function getFlagsRules(\appbox $appbox, \ACL $acl, array $collections) { $rules = []; @@ -766,4 +608,166 @@ class ElasticSearchEngine implements SearchEngineInterface return []; } } + + private function buildHighlightRules(QueryContext $context) + { + $highlighted_fields = []; + foreach ($context->getHighlightedFields() as $field) { + switch ($field->getType()) { + case FieldMapping::TYPE_STRING: + $index_field = $field->getIndexField(); + $raw_index_field = $field->getIndexField(true); + $highlighted_fields[$index_field . ".light"] = [ + // Requires calling Mapping::enableTermVectors() on this field mapping +// 'matched_fields' => [$index_field, $raw_index_field], + 'type' => 'fvh', + ]; + break; + case FieldMapping::TYPE_FLOAT: + case FieldMapping::TYPE_DOUBLE: + case FieldMapping::TYPE_INTEGER: + case FieldMapping::TYPE_LONG: + case FieldMapping::TYPE_SHORT: + case FieldMapping::TYPE_BYTE: + continue; + case FieldMapping::TYPE_DATE: + default: + continue; + } + } + + return [ + 'pre_tags' => ['[[em]]'], + 'post_tags' => ['[[/em]]'], + 'order' => 'score', + 'fields' => $highlighted_fields + ]; + } + + private function getAggregationQueryParams(SearchEngineOptions $options) + { + $aggs = []; + // technical aggregates (enable + optional limit) + foreach (ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { + $size = $this->options->getAggregableFieldLimit($k); + if ($size !== databox_field::FACET_DISABLED) { + if ($size === databox_field::FACET_NO_LIMIT) { + $size = ESField::FACET_NO_LIMIT; + } + $agg = [ + 'terms' => [ + 'field' => $f['field'], + 'size' => $size + ] + ]; + $aggs[$k] = $agg; + } + } + // fields aggregates + $structure = $this->context_factory->getLimitedStructure($options); + foreach ($structure->getFacetFields() as $name => $field) { + // 2015-05-26 (mdarse) Removed databox filtering. + // It was already done by the ACL filter in the query scope, so no + // document that shouldn't be displayed can go this far. + $agg = [ + 'terms' => [ + 'field' => $field->getIndexField(true), + 'size' => $field->getFacetValuesLimit() + ] + ]; + $aggs[$name] = AggregationHelper::wrapPrivateFieldAggregation($field, $agg); + } + + return $aggs; + } + + /** + * {@inheritdoc} + */ + public function autocomplete($query, SearchEngineOptions $options) + { + $params = $this->createCompletionParams($query, $options); + + $res = $this->client->suggest($params); + + $ret = [ + 'text' => [], + 'byField' => [] + ]; + foreach (array_keys($params['body']) as $fname) { + $t = []; + foreach ($res[$fname] as $suggest) { // don't know why there is a sub-array level + foreach ($suggest['options'] as $option) { + $text = $option['text']; + if (!in_array($text, $ret['text'])) { + $ret['text'][] = $text; + } + $t[] = [ + 'label' => $text, + 'query' => $fname . ':' . $text + ]; + } + } + if (!empty($t)) { + $ret['byField'][$fname] = $t; + } + } + + return $ret; + } + + private function createCompletionParams($query, SearchEngineOptions $options) + { + $body = []; + $context = [ + 'record_type' => $options->getSearchType() === SearchEngineOptions::RECORD_RECORD ? + SearchEngineInterface::GEM_TYPE_RECORD : SearchEngineInterface::GEM_TYPE_STORY + ]; + + $base_ids = $options->getBasesIds(); + if (count($base_ids) > 0) { + $context['base_id'] = $base_ids; + } + + $search_context = $this->context_factory->createContext($options); + $fields = $search_context->getUnrestrictedFields(); + foreach ($fields as $field) { + if ($field->getType() == FieldMapping::TYPE_STRING) { + $k = '' . $field->getName(); + $body[$k] = [ + 'text' => $query, + 'completion' => [ + 'field' => "caption." . $field->getName() . ".suggest", + 'context' => &$context + ] + ]; + } + } + + return [ + 'index' => $this->indexName, + 'body' => $body + ]; + } + + /** + * {@inheritdoc} + */ + public function resetCache() + { + } + + /** + * {@inheritdoc} + */ + public function clearCache() + { + } + + /** + * {@inheritdoc} + */ + public function clearAllCache(\DateTime $date = null) + { + } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchRecordHydrator.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchRecordHydrator.php index 5a49303b9f..55640dcf22 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchRecordHydrator.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchRecordHydrator.php @@ -32,7 +32,10 @@ class ElasticsearchRecordHydrator if (substr($key, 0, strlen($prefix)) == $prefix) { $key = substr($key, strlen($prefix)); } - $highlight[$key] = $value; + if (substr($key, -6) == '.light') { + $key = substr($key, 0, strlen($key) - 6); + $highlight[$key] = $value; + } } $record = new ElasticsearchRecord(); diff --git a/templates/web/common/macros.html.twig b/templates/web/common/macros.html.twig index 9d5ed046bc..28a0a31f81 100644 --- a/templates/web/common/macros.html.twig +++ b/templates/web/common/macros.html.twig @@ -128,7 +128,7 @@
{% for name, value in record.getCaption(caption_field_order(record, can_see_business)) %}
{{ caption_field_label(record, name) }}
-
{{ caption_field(record, name, value)|e|highlight|linkify }}
+
{{ caption_field(record, name, value)|e|highlight|linkify|parseColor }}
{% endfor %}
{% if display_exif|default(true) and app.getAuthenticator().user is not none and user_setting('technical_display') == 'group' %} diff --git a/templates/web/prod/results/record.html.twig b/templates/web/prod/results/record.html.twig index 6b99d304c7..bccf31775e 100644 --- a/templates/web/prod/results/record.html.twig +++ b/templates/web/prod/results/record.html.twig @@ -24,8 +24,7 @@ {% set can_see_business = granted_on_collection(record.baseId, [constant('\\ACL::CANMODIFRECORD')]) %}