From e36b0785ee95af22c023d2e9342a1f872dc80ec1 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Fri, 30 Oct 2020 10:59:41 +0100 Subject: [PATCH 01/12] - add some simple cache - disable check of conf/searchengine/type - faster construction of concept-paths - pass strucure to es as factory (useless, to be reverted) - add stopwatch (log into file, disabled for now) --- .../Provider/SearchEngineServiceProvider.php | 30 +++++++--- .../Elastic/ElasticSearchEngine.php | 19 +++++-- .../Phrasea/SearchEngine/Elastic/Indexer.php | 10 ++-- .../Elastic/Search/QueryContextFactory.php | 25 ++++++++- .../SearchEngine/Elastic/Structure/Field.php | 35 ++++++------ .../Elastic/Structure/GlobalStructure.php | 8 ++- .../SearchEngine/Elastic/Thesaurus/Helper.php | 56 ++++++++++++++----- lib/Alchemy/Phrasea/Utilities/Stopwatch.php | 10 ++++ lib/classes/databox.php | 30 ++++++---- 9 files changed, 160 insertions(+), 63 deletions(-) diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index 8968ef5f0a..735bb90aa0 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -32,6 +32,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus; +use Alchemy\Phrasea\Utilities\Stopwatch; use Elasticsearch\ClientBuilder; use Hoa\Compiler; use Hoa\File; @@ -72,29 +73,42 @@ class SearchEngineServiceProvider implements ServiceProviderInterface return new SearchEngineLogger($app); }); + $app['search_engine.structure.factory'] = $app->protect(function () use ($app) { + return $app['search_engine.structure']; + }); + $app['search_engine'] = $app->share(function ($app) { - $type = $app['conf']->get(['main', 'search-engine', 'type']); - if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) { - throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type)); - } + $stopwatch = new Stopwatch("se"); + // $type = $app['conf']->get(['main', 'search-engine', 'type']); + // if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) { + // throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type)); + // } + // $stopwatch->lap("se.conf"); /** @var ElasticsearchOptions $options */ $options = $app['elasticsearch.options']; - return new ElasticSearchEngine( + $r = new ElasticSearchEngine( $app, + // $app['search_engine.structure.factory'], $app['search_engine.structure'], $app['elasticsearch.client'], $app['query_context.factory'], $app['elasticsearch.facets_response.factory'], $options ); + $stopwatch->lap("se.new"); + $stopwatch->log(); + return $r; }); $app['search_engine.structure'] = $app->share(function (\Alchemy\Phrasea\Application $app) { + $stopwatch = new Stopwatch("se.structure"); $databoxes = $app->getDataboxes(); - - return GlobalStructure::createFromDataboxes($databoxes); + $stopwatch ->lap('get db'); + $r = GlobalStructure::createFromDataboxes($databoxes); + $stopwatch->log(); + return $r; }); $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { @@ -268,7 +282,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface $app['query_context.factory'] = $app->share(function ($app) { return new QueryContextFactory( - $app['search_engine.structure'], + $app['search_engine.structure.factory'], array_keys($app['locales.available']), $app['locale'] ); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 64e444fc08..6638dc4366 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -61,13 +61,13 @@ class ElasticSearchEngine implements SearchEngineInterface /** * @param Application $app - * @param Structure $structure + * @param Structure|callable $structure * @param Client $client * @param QueryContextFactory $context_factory - * @param callable $facetsResponseFactory + * @param Closure $facetsResponseFactory * @param ElasticsearchOptions $options */ - public function __construct(Application $app, Structure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) + public function __construct(Application $app, $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) { $this->app = $app; $this->structure = $structure; @@ -77,7 +77,18 @@ class ElasticSearchEngine implements SearchEngineInterface $this->options = $options; $this->indexName = $options->getIndexName(); + } + /** + * @return Structure + */ + public function getStructure() + { + if (!($this->structure instanceof Structure)) { + $this->structure = call_user_func($this->structure); + } + + return $this->structure; } public function getIndexName() @@ -129,7 +140,7 @@ class ElasticSearchEngine implements SearchEngineInterface public function getAvailableDateFields() { // TODO Use limited structure - return array_keys($this->structure->getDateFields()); + return array_keys($this->getStructure()->getDateFields()); } /** diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php index bd1471c37a..55b65389f4 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php @@ -279,11 +279,11 @@ class Indexer ); // Optimize index - $this->client->indices()->optimize( - [ - 'index' => $this->index->getName() - ] - ); +// $this->client->indices()->optimize( +// [ +// 'index' => $this->index->getName() +// ] +// ); $event = $stopwatch->stop('populate'); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php index 9443c79655..b04ae6f0d7 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php @@ -10,18 +10,37 @@ class QueryContextFactory { private $structure; - public function __construct(Structure $structure, array $locales, $current_locale) + /** + * QueryContextFactory constructor. + * @param Structure|callable $structure + * @param array $locales + * @param $current_locale + */ + public function __construct($structure, array $locales, $current_locale) { $this->structure = $structure; $this->locales = $locales; $this->current_locale = $current_locale; } + /** + * @return Structure + */ + public function getStructure() + { + if (!($this->structure instanceof Structure)) { + $this->structure = call_user_func($this->structure); + } + + return $this->structure; + } + + public function createContext(SearchEngineOptions $options = null) { $structure = $options ? $this->getLimitedStructure($options) - : $this->structure; + : $this->getStructure(); $context = new QueryContext($options, $structure, $this->locales, $this->current_locale); @@ -46,6 +65,6 @@ class QueryContextFactory public function getLimitedStructure(SearchEngineOptions $options) { - return new LimitedStructure($this->structure, $options); + return new LimitedStructure($this->getStructure(), $options); } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php index 5333da5ab4..c0e4a4ce82 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php @@ -108,26 +108,25 @@ class Field implements Typed { $this->name = (string) $name; $this->type = $type; - $this->databox_id = \igorw\get_in($options, ['databox_id'], 0); - $this->is_searchable = \igorw\get_in($options, ['searchable'], true); - $this->is_private = \igorw\get_in($options, ['private'], false); - $this->facet = \igorw\get_in($options, ['facet']); - $this->thesaurus_roots = \igorw\get_in($options, ['thesaurus_roots'], null); - $this->generate_cterms = \igorw\get_in($options, ['generate_cterms'], false); - $this->used_by_collections = \igorw\get_in($options, ['used_by_collections'], []); - - Assertion::boolean($this->is_searchable); - Assertion::boolean($this->is_private); - - if ($this->facet !== self::FACET_DISABLED) { - Assertion::integer($this->facet); + if(1) { + $this->databox_id = \igorw\get_in($options, ['databox_id'], 0); + $this->is_searchable = \igorw\get_in($options, ['searchable'], true); + $this->is_private = \igorw\get_in($options, ['private'], false); + $this->facet = \igorw\get_in($options, ['facet']); + $this->thesaurus_roots = \igorw\get_in($options, ['thesaurus_roots'], null); + $this->generate_cterms = \igorw\get_in($options, ['generate_cterms'], false); + $this->used_by_collections = \igorw\get_in($options, ['used_by_collections'], []); } - - if ($this->thesaurus_roots !== null) { - Assertion::allIsInstanceOf($this->thesaurus_roots, Concept::class); + else { + // todo: this is faster code, but need to fix unit-tests to pass all options + $this->databox_id = $options['databox_id']; + $this->is_searchable = $options['searchable']; + $this->is_private = $options['private']; + $this->facet = $options['facet']; + $this->thesaurus_roots = $options['thesaurus_roots']; + $this->generate_cterms = $options['generate_cterms']; + $this->used_by_collections = $options['used_by_collections']; } - - Assertion::allScalar($this->used_by_collections); } public function withOptions(array $options) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php index 0793c91c7b..9a527839d7 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php @@ -4,6 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; +use Alchemy\Phrasea\Utilities\Stopwatch; use Assert\Assertion; use DomainException; @@ -56,6 +57,7 @@ final class GlobalStructure implements Structure $fields = []; $flags = []; + $stopwatch = new Stopwatch("globalStructure"); foreach ($databoxes as $databox) { if($what & self::STRUCTURE_WITH_FIELDS) { foreach ($databox->get_meta_structure() as $fieldStructure) { @@ -69,8 +71,12 @@ final class GlobalStructure implements Structure } } } + $stopwatch->lap('loop0'); + $r = new self($fields, $flags, MetadataHelper::createTags()); - return new self($fields, $flags, MetadataHelper::createTags()); + $stopwatch->log(); + + return $r; } /** diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Thesaurus/Helper.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Thesaurus/Helper.php index 7c4ebba7f7..ed2d468e60 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Thesaurus/Helper.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Thesaurus/Helper.php @@ -31,26 +31,54 @@ class Helper $nodes = $xpath->query($expression); $concepts = []; + /** @var DOMElement $node */ foreach ($nodes as $node) { - $me_and_parents = array_merge([$node], self::getElementAncestors($node)); + if(1) { + $me_and_parents = array_merge([$node], self::getElementAncestors($node)); - $path_segments = []; + $path_segments = []; - foreach ($me_and_parents as $me_or_parent) { - if (!Navigator::isConcept($me_or_parent)) { - // Silently skips invalid targeted nodes - break; + foreach ($me_and_parents as $me_or_parent) { + if (!Navigator::isConcept($me_or_parent)) { + // Silently skips invalid targeted nodes + break; + } + + $path_segments[] = self::conceptPathSegment($me_or_parent); } - $path_segments[] = self::conceptPathSegment($me_or_parent); + // Concept paths are have databox identifier at root level + $concepts[] = new Concept(sprintf( + '/%d/%s', + $databox->get_sbas_id(), + implode('/', array_reverse($path_segments)) + )); + } + else { + $path = ''; + // go up thru parents + while ($node) { + $v = null; + for ($n = $node->firstChild; $n; $n = $n->nextSibling) { + if ($n->nodeType === XML_ELEMENT_NODE && $n->nodeName === 'sy') { + if ($v === null) { + $v = $n->getAttribute('v'); + continue; + } + if ($n->getAttribute('lng') === 'en') { + $v = $n->getAttribute('v'); + break; + } + } + } + if ($v !== null) { + $path = '/' . $v . $path; + } + $node = $node->parentNode; + } + $path = '/' . $databox->get_sbas_id() . $path; + $concepts[] = new Concept($path); } - - // Concept paths are have databox identifier at root level - $concepts[] = new Concept(sprintf( - '/%d/%s', - $databox->get_sbas_id(), - implode('/', array_reverse($path_segments)) - )); } return $concepts; diff --git a/lib/Alchemy/Phrasea/Utilities/Stopwatch.php b/lib/Alchemy/Phrasea/Utilities/Stopwatch.php index ca5ebcf65c..16e700457b 100644 --- a/lib/Alchemy/Phrasea/Utilities/Stopwatch.php +++ b/lib/Alchemy/Phrasea/Utilities/Stopwatch.php @@ -62,9 +62,19 @@ class Stopwatch */ public function getLapses() { + $this->stop(); + $this->_lapses['_total'] = ($this->_stop - $this->_start) * 1000.0; return $this->_lapses; } + public function log() + { + // file_put_contents('/var/alchemy/Phraseanet/tmp/phraseanet.log', + // $this->_name . "\n" . var_export($this->getLapses(), true) . "\n\n", + // FILE_APPEND + // ); + } + /** * return all lapses as a "Server-Timing" header value * diff --git a/lib/classes/databox.php b/lib/classes/databox.php index 3377695e39..60baf05a1f 100644 --- a/lib/classes/databox.php +++ b/lib/classes/databox.php @@ -581,13 +581,17 @@ class databox extends base implements ThumbnailedElement */ public function get_collection_unique_ids() { - $collectionsIds = []; + static $_r = null; - foreach ($this->get_collections() as $collection) { - $collectionsIds[] = $collection->get_base_id(); + if($_r === null) { + $_r = []; + + foreach ($this->get_collections() as $collection) { + $_r[] = $collection->get_base_id(); + } } - return $collectionsIds; + return $_r; } /** @@ -595,13 +599,19 @@ class databox extends base implements ThumbnailedElement */ public function get_collections() { - /** @var CollectionRepositoryRegistry $repositoryRegistry */ - $repositoryRegistry = $this->app['repo.collections-registry']; - $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id()); + static $_r = null; - return array_filter($repository->findAll(), function (collection $collection) { - return $collection->is_active(); - }); + if($_r === null) { + /** @var CollectionRepositoryRegistry $repositoryRegistry */ + $repositoryRegistry = $this->app['repo.collections-registry']; + $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id()); + + $_r = array_filter($repository->findAll(), function (collection $collection) { + return $collection->is_active(); + }); + } + + return $_r; } /** From 341f0764124043446e42b41bfcdd60e44b432dd8 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Fri, 30 Oct 2020 12:46:19 +0100 Subject: [PATCH 02/12] - fix : better simple cache --- lib/classes/databox.php | 20 +++++++++----------- tests/files/cestlafete.jpg | Bin 31664 -> 31664 bytes 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/classes/databox.php b/lib/classes/databox.php index 60baf05a1f..b4414fe3f1 100644 --- a/lib/classes/databox.php +++ b/lib/classes/databox.php @@ -579,39 +579,37 @@ class databox extends base implements ThumbnailedElement /** * @return int[] */ + private $_collection_unique_ids = null; public function get_collection_unique_ids() { - static $_r = null; - - if($_r === null) { - $_r = []; + if($this->_collection_unique_ids === null) { + $this->_collection_unique_ids = []; foreach ($this->get_collections() as $collection) { - $_r[] = $collection->get_base_id(); + $this->_collection_unique_ids[] = $collection->get_base_id(); } } - return $_r; + return $this->_collection_unique_ids; } /** * @return collection[] */ + private $_collections = null; public function get_collections() { - static $_r = null; - - if($_r === null) { + if($this->_collections === null) { /** @var CollectionRepositoryRegistry $repositoryRegistry */ $repositoryRegistry = $this->app['repo.collections-registry']; $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id()); - $_r = array_filter($repository->findAll(), function (collection $collection) { + $this->_collections = array_filter($repository->findAll(), function (collection $collection) { return $collection->is_active(); }); } - return $_r; + return $this->_collections; } /** diff --git a/tests/files/cestlafete.jpg b/tests/files/cestlafete.jpg index c0e8cd7f753059b09da50ccb8ebe769b3aa8319f..9339b63c60a33ddf02756222684e5ac14b2601ad 100755 GIT binary patch delta 19 Zcmdn+opA#YU1nr4)Uz-9y2!Q|q delta 19 Zcmdn+opA#YU1nr4&@(jHe4X)TB>-8#2z3Ae From de0f05ad9fe82a36ce42ff62de5f4240f3eddd96 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Mon, 2 Nov 2020 17:37:27 +0100 Subject: [PATCH 03/12] - fix : wip to run tests --- lib/classes/ACL.php | 6 ++++++ lib/classes/collection.php | 6 ++++++ lib/classes/databox.php | 31 +++++++++++++++++++++++++------ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/classes/ACL.php b/lib/classes/ACL.php index 570b917e25..500f8457b8 100644 --- a/lib/classes/ACL.php +++ b/lib/classes/ACL.php @@ -1248,9 +1248,12 @@ class ACL implements cache_cacheableInterface } } + $this->app->getApplicationBox()->get_databox($sbas_id)->clearCache(databox::CACHE_COLLECTIONS); } $stmt_ins->closeCursor(); + $this->delete_data_from_cache(self::CACHE_RIGHTS_SBAS); + // $this->delete_data_from_cache(self::CACHE_GLOBAL_RIGHTS); return $this; } @@ -1365,7 +1368,10 @@ class ACL implements cache_cacheableInterface $stmt->execute([':sbas_id' => $sbas_id, ':usr_id' => $this->user->getId()]); $stmt->closeCursor(); + $this->app->getApplicationBox()->get_databox($sbas_id)->clearCache(databox::CACHE_COLLECTIONS); + $this->delete_data_from_cache(self::CACHE_RIGHTS_SBAS); + // $this->delete_data_from_cache(self::CACHE_GLOBAL_RIGHTS); $this->app['dispatcher']->dispatch( AclEvents::RIGHTS_TO_SBAS_CHANGED, diff --git a/lib/classes/collection.php b/lib/classes/collection.php index 000144a2e3..276098fc1d 100644 --- a/lib/classes/collection.php +++ b/lib/classes/collection.php @@ -597,6 +597,9 @@ class collection implements ThumbnailedElement, cache_cacheableInterface $this->getReferenceRepository()->save($this->reference); $this->collectionRepositoryRegistry->purgeRegistry(); + // clear the trivial cache of databox->get_collections() + $this->get_databox()->clearCache(databox::CACHE_COLLECTIONS); + cache_databox::update($this->app, $this->databox->get_sbas_id(), 'structure'); $this->dispatch(CollectionEvents::DISABLED, new DisabledEvent($this)); @@ -614,6 +617,9 @@ class collection implements ThumbnailedElement, cache_cacheableInterface $this->getReferenceRepository()->save($this->reference); $this->collectionRepositoryRegistry->purgeRegistry(); + // clear the trivial cache of databox->get_collections() + $this->get_databox()->clearCache(databox::CACHE_COLLECTIONS); + cache_databox::update($this->app, $this->databox->get_sbas_id(), 'structure'); $this->dispatch(CollectionEvents::ENABLED, new EnabledEvent($this)); diff --git a/lib/classes/databox.php b/lib/classes/databox.php index b4414fe3f1..51a92b6c1f 100644 --- a/lib/classes/databox.php +++ b/lib/classes/databox.php @@ -576,10 +576,24 @@ class databox extends base implements ThumbnailedElement parent::delete_data_from_cache($option); } + /* + * trivial cache to speed-up get_collections() which does sql + */ + private $_collection_unique_ids = null; + private $_collections = null; + + public function clearCache($what) + { + switch($what) { + case self::CACHE_COLLECTIONS: + $this->_collection_unique_ids = $this->_collections = null; + break; + } + } + /** * @return int[] */ - private $_collection_unique_ids = null; public function get_collection_unique_ids() { if($this->_collection_unique_ids === null) { @@ -596,7 +610,6 @@ class databox extends base implements ThumbnailedElement /** * @return collection[] */ - private $_collections = null; public function get_collections() { if($this->_collections === null) { @@ -604,14 +617,20 @@ class databox extends base implements ThumbnailedElement $repositoryRegistry = $this->app['repo.collections-registry']; $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id()); - $this->_collections = array_filter($repository->findAll(), function (collection $collection) { - return $collection->is_active(); - }); + // $this->_collections = array_filter($repository->findAll(), function (collection $collection) { + // return $collection->is_active(); + // }); + $this->_collections = $repository->findAll(); } - return $this->_collections; + // return $this->_collections; + + return array_filter($this->_collections, function (collection $collection) { + return $collection->is_active(); + }); } + /** * @return collection|null */ From ca250c547ebda46aabf92eefde7cc4e10a7ec66b Mon Sep 17 00:00:00 2001 From: jygaulier Date: Mon, 2 Nov 2020 18:05:36 +0100 Subject: [PATCH 04/12] - fix : wip --- lib/classes/databox.php | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/classes/databox.php b/lib/classes/databox.php index 51a92b6c1f..e7d4c4e2d3 100644 --- a/lib/classes/databox.php +++ b/lib/classes/databox.php @@ -617,17 +617,12 @@ class databox extends base implements ThumbnailedElement $repositoryRegistry = $this->app['repo.collections-registry']; $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id()); - // $this->_collections = array_filter($repository->findAll(), function (collection $collection) { - // return $collection->is_active(); - // }); - $this->_collections = $repository->findAll(); + $this->_collections = array_filter($repository->findAll(), function (collection $collection) { + return $collection->is_active(); + }); } - // return $this->_collections; - - return array_filter($this->_collections, function (collection $collection) { - return $collection->is_active(); - }); + return $this->_collections; } From 4d961e2dbe9814eb3547f3012b9f43029dd643bd Mon Sep 17 00:00:00 2001 From: jygaulier Date: Wed, 4 Nov 2020 11:35:36 +0100 Subject: [PATCH 05/12] - fix : added cache to es "globalstructure" construction --- .../Provider/SearchEngineServiceProvider.php | 43 ++++++--- .../Elastic/ElasticSearchEngine.php | 9 +- .../Elastic/Structure/GlobalStructure.php | 8 +- .../SearchEngine/SearchEngineStructure.php | 88 +++++++++++++++++++ 4 files changed, 129 insertions(+), 19 deletions(-) create mode 100644 lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index 735bb90aa0..00f01645a0 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -32,6 +32,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus; +use Alchemy\Phrasea\SearchEngine\SearchEngineStructure; use Alchemy\Phrasea\Utilities\Stopwatch; use Elasticsearch\ClientBuilder; use Hoa\Compiler; @@ -44,11 +45,14 @@ use Silex\ServiceProviderInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\HttpKernel\KernelEvents; +use \Alchemy\Phrasea\Application as PhraseaApplication; + class SearchEngineServiceProvider implements ServiceProviderInterface { public function register(Application $app) { + /** @var PhraseaApplication $app */ $this->registerElasticSearchClient($app); $this->registerQueryParser($app); $this->registerIndexer($app); @@ -60,22 +64,22 @@ class SearchEngineServiceProvider implements ServiceProviderInterface } /** - * @param Application $app + * @param PhraseaApplication $app * @return Application */ - private function registerSearchEngine(Application $app) + private function registerSearchEngine(PhraseaApplication $app) { $app['phraseanet.SE'] = function ($app) { return $app['search_engine']; }; - $app['phraseanet.SE.logger'] = $app->share(function (Application $app) { + $app['phraseanet.SE.logger'] = $app->share(function (PhraseaApplication $app) { return new SearchEngineLogger($app); }); - $app['search_engine.structure.factory'] = $app->protect(function () use ($app) { - return $app['search_engine.structure']; - }); +// $app['search_engine.structure.factory'] = $app->protect(function () use ($app) { +// return $app['search_engine.structure']; +// }); $app['search_engine'] = $app->share(function ($app) { $stopwatch = new Stopwatch("se"); @@ -88,10 +92,15 @@ class SearchEngineServiceProvider implements ServiceProviderInterface /** @var ElasticsearchOptions $options */ $options = $app['elasticsearch.options']; + /** @var SearchEngineStructure $seStructure */ + //$seStructure = $app['search_engine.structure2']; + + $stopwatch->lap("se.options"); $r = new ElasticSearchEngine( $app, // $app['search_engine.structure.factory'], - $app['search_engine.structure'], + // $app['search_engine.structure'], + $app['search_engine.structure2'], $app['elasticsearch.client'], $app['query_context.factory'], $app['elasticsearch.facets_response.factory'], @@ -102,7 +111,15 @@ class SearchEngineServiceProvider implements ServiceProviderInterface return $r; }); - $app['search_engine.structure'] = $app->share(function (\Alchemy\Phrasea\Application $app) { + $app['search_engine.structure2'] = $app->share(function (PhraseaApplication $app) { + $stopwatch = new Stopwatch("se.structure2"); + $seStructure = new SearchEngineStructure($app['cache']); + $globalStructure = $seStructure->getGlobalStructureFromDataboxes($app->getDataboxes()); + $stopwatch->log(); + return $globalStructure; + }); + + $app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) { $stopwatch = new Stopwatch("se.structure"); $databoxes = $app->getDataboxes(); $stopwatch ->lap('get db'); @@ -112,7 +129,11 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { - return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.structure']); + /** @var SearchEngineStructure $seStructure */ + // $seStructure = $app['search_engine.structure2']; + // $globalStructure = $seStructure->getGlobalStructureFromDataboxes($app->getDataboxes()); + // return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.structure']); + return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.structure2']); }); return $app; @@ -282,7 +303,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface $app['query_context.factory'] = $app->share(function ($app) { return new QueryContextFactory( - $app['search_engine.structure.factory'], + $app['search_engine.structure2'], array_keys($app['locales.available']), $app['locale'] ); @@ -303,7 +324,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['query_visitor.factory'] = $app->protect(function () use ($app) { - return new QueryVisitor($app['search_engine.structure']); + return new QueryVisitor($app['search_engine.structure2']); }); $app['query_compiler'] = $app->share(function ($app) { diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 6638dc4366..eac703fc21 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -21,11 +21,13 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field AS ESField; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag; +use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure; use Alchemy\Phrasea\SearchEngine\SearchEngineInterface; use Alchemy\Phrasea\SearchEngine\SearchEngineOptions; use Alchemy\Phrasea\SearchEngine\SearchEngineResult; use Alchemy\Phrasea\Exception\RuntimeException; +use Alchemy\Phrasea\SearchEngine\SearchEngineStructure; use Alchemy\Phrasea\Utilities\Stopwatch; use Closure; use Doctrine\Common\Collections\ArrayCollection; @@ -61,16 +63,17 @@ class ElasticSearchEngine implements SearchEngineInterface /** * @param Application $app - * @param Structure|callable $structure + * @param GlobalStructure $structure2 * @param Client $client * @param QueryContextFactory $context_factory * @param Closure $facetsResponseFactory * @param ElasticsearchOptions $options */ - public function __construct(Application $app, $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) + public function __construct(Application $app, GlobalStructure $structure2, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) { $this->app = $app; - $this->structure = $structure; + // $this->structure = $structure; + $this->structure = $structure2; $this->client = $client; $this->context_factory = $context_factory; $this->facetsResponseFactory = $facetsResponseFactory; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php index 9a527839d7..8c9e670f8d 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php @@ -57,7 +57,7 @@ final class GlobalStructure implements Structure $fields = []; $flags = []; - $stopwatch = new Stopwatch("globalStructure"); + $stopwatch = new Stopwatch("globalStructure2"); foreach ($databoxes as $databox) { if($what & self::STRUCTURE_WITH_FIELDS) { foreach ($databox->get_meta_structure() as $fieldStructure) { @@ -79,11 +79,12 @@ final class GlobalStructure implements Structure return $r; } + /** * @param \databox $databox * @return GlobalStructure */ - public static function createFromDatabox(\databox $databox) + public static function unused_createFromDatabox(\databox $databox) { $fields = []; $flags = []; @@ -107,9 +108,6 @@ final class GlobalStructure implements Structure */ public function __construct(array $fields = [], array $flags = [], array $metadata_tags = []) { - Assertion::allIsInstanceOf($fields, Field::class); - Assertion::allIsInstanceOf($flags, Flag::class); - Assertion::allIsInstanceOf($metadata_tags, Tag::class); foreach ($fields as $field) { $this->add($field); diff --git a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php new file mode 100644 index 0000000000..1e2d302e21 --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php @@ -0,0 +1,88 @@ +cache = $cache; + } + + /** + * @param databox[] $databoxes + * @param int $what bitmask of what should be included in this structure, in fields, ... + * + * @return GlobalStructure + */ + public function getGlobalStructureFromDataboxes(array $databoxes, $what = Structure::WITH_EVERYTHING) + { + $fields = []; + $flags = []; + + $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes"); + foreach ($databoxes as $databox) { + + // we will store both FIELDS and FLAGS in the same cache entry : + // it's small data and $what=WITH_EVERYTHING allways + $k = $this->getCacheKey("FieldsAndFlags", $databox); + try { + $data = $this->cache->get($k); + } + catch(\Exception $e) { + $data = false; + } + if($data === false) { + $data = [ + 'fields' => [], + 'flags' => [] + ]; + foreach ($databox->get_meta_structure() as $fieldStructure) { + $data['fields'][] = Field::createFromLegacyField($fieldStructure, $what); + } + foreach ($databox->getStatusStructure() as $status) { + $data['flags'][] = Flag::createFromLegacyStatus($status); + } + $this->cache->save($k, $data); + } + + if($what & Structure::STRUCTURE_WITH_FIELDS) { + $fields = array_merge($fields, $data['fields']); + } + + if($what & Structure::STRUCTURE_WITH_FLAGS) { + $flags = array_merge($fields, $data['flags']); + } + } + $stopwatch->lap('loop0'); + $r = new GlobalStructure($fields, $flags, MetadataHelper::createTags()); + + $stopwatch->log(); + + return $r; + } + + private function getCacheKey($what, databox $db) + { + return "es_db-" . $db->get_sbas_id() . '_' . $what; + } +} \ No newline at end of file From 00bcc1c8386daf65dada2798c725b8a176176b6b Mon Sep 17 00:00:00 2001 From: aina esokia Date: Thu, 5 Nov 2020 11:37:45 +0300 Subject: [PATCH 06/12] PHRAS-3255 fix edit position mapGL #time : 2h --- Phraseanet-production-client/dist/production.js | 2 +- Phraseanet-production-client/dist/production.min.js | 2 +- .../components/geolocalisation/providers/markerGLCollection.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Phraseanet-production-client/dist/production.js b/Phraseanet-production-client/dist/production.js index e678641ac2..3cb3e0dc69 100644 --- a/Phraseanet-production-client/dist/production.js +++ b/Phraseanet-production-client/dist/production.js @@ -49654,7 +49654,7 @@ var markerGLCollection = function markerGLCollection(services) { lng: cachedGeoJson.features[0].geometry.coordinates[0], lat: cachedGeoJson.features[0].geometry.coordinates[1] }; - marker.feature = marker.features[0]; + marker.feature = cachedGeoJson.features[0]; eventEmitter.emit('markerChange', { marker: marker, position: marker.lngLat }); }); diff --git a/Phraseanet-production-client/dist/production.min.js b/Phraseanet-production-client/dist/production.min.js index e678641ac2..3cb3e0dc69 100644 --- a/Phraseanet-production-client/dist/production.min.js +++ b/Phraseanet-production-client/dist/production.min.js @@ -49654,7 +49654,7 @@ var markerGLCollection = function markerGLCollection(services) { lng: cachedGeoJson.features[0].geometry.coordinates[0], lat: cachedGeoJson.features[0].geometry.coordinates[1] }; - marker.feature = marker.features[0]; + marker.feature = cachedGeoJson.features[0]; eventEmitter.emit('markerChange', { marker: marker, position: marker.lngLat }); }); diff --git a/Phraseanet-production-client/src/components/geolocalisation/providers/markerGLCollection.js b/Phraseanet-production-client/src/components/geolocalisation/providers/markerGLCollection.js index 30d02fa689..b867e80317 100644 --- a/Phraseanet-production-client/src/components/geolocalisation/providers/markerGLCollection.js +++ b/Phraseanet-production-client/src/components/geolocalisation/providers/markerGLCollection.js @@ -91,7 +91,7 @@ const markerGLCollection = (services) => { lng: cachedGeoJson.features[0].geometry.coordinates[0], lat: cachedGeoJson.features[0].geometry.coordinates[1] }; - marker.feature = marker.features[0]; + marker.feature = cachedGeoJson.features[0]; eventEmitter.emit('markerChange', {marker, position: marker.lngLat}); }); From 5ae105d9c24696242ef6c7b69407d2816529404f Mon Sep 17 00:00:00 2001 From: jygaulier Date: Thu, 5 Nov 2020 11:26:19 +0100 Subject: [PATCH 07/12] - fix : error during populate - add : clear structure cache when structure (field or sb) is changed --- lib/Alchemy/Phrasea/Application.php | 2 + .../Controller/Admin/RootController.php | 14 +++++- .../Admin/SearchEngineController.php | 2 +- .../Api/V3/V3SearchRawController.php | 4 +- .../Subscriber/StructureChangeSubscriber.php | 48 ++++++++++++++++++ .../Provider/SearchEngineServiceProvider.php | 49 ++++++------------- .../Indexer/Record/Hydrator/FlagHydrator.php | 3 ++ .../SearchEngine/Elastic/Structure/Field.php | 11 ++++- .../SearchEngine/SearchEngineStructure.php | 12 +++-- 9 files changed, 105 insertions(+), 40 deletions(-) create mode 100644 lib/Alchemy/Phrasea/Core/Event/Subscriber/StructureChangeSubscriber.php diff --git a/lib/Alchemy/Phrasea/Application.php b/lib/Alchemy/Phrasea/Application.php index a37c78339c..ac9fff0b68 100644 --- a/lib/Alchemy/Phrasea/Application.php +++ b/lib/Alchemy/Phrasea/Application.php @@ -25,6 +25,7 @@ use Alchemy\Phrasea\Core\Event\Subscriber\FeedEntrySubscriber; use Alchemy\Phrasea\Core\Event\Subscriber\LazaretSubscriber; use Alchemy\Phrasea\Core\Event\Subscriber\PhraseaInstallSubscriber; use Alchemy\Phrasea\Core\Event\Subscriber\RegistrationSubscriber; +use Alchemy\Phrasea\Core\Event\Subscriber\StructureChangeSubscriber; use Alchemy\Phrasea\Core\Event\Subscriber\ValidationSubscriber; use Alchemy\Phrasea\Core\Event\Subscriber\WebhookUserEventSubscriber; use Alchemy\Phrasea\Core\MetaProvider\DatabaseMetaProvider; @@ -748,6 +749,7 @@ class Application extends SilexApplication $dispatcher->addSubscriber(new LazaretSubscriber($app)); $dispatcher->addSubscriber(new ValidationSubscriber($app)); $dispatcher->addSubscriber(new WebhookUserEventSubscriber($app)); + $dispatcher->addSubscriber(new StructureChangeSubscriber($app['search_engine.structure'])); return $dispatcher; }) diff --git a/lib/Alchemy/Phrasea/Controller/Admin/RootController.php b/lib/Alchemy/Phrasea/Controller/Admin/RootController.php index 295633f4dc..b2ac52fe7b 100644 --- a/lib/Alchemy/Phrasea/Controller/Admin/RootController.php +++ b/lib/Alchemy/Phrasea/Controller/Admin/RootController.php @@ -11,6 +11,9 @@ namespace Alchemy\Phrasea\Controller\Admin; use Alchemy\Phrasea\Controller\Controller; +use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvents; +use Alchemy\Phrasea\Core\Event\Record\Structure\StatusBitEvent; +use Alchemy\Phrasea\Core\Event\Record\Structure\StatusBitUpdatedEvent; use Alchemy\Phrasea\Exception\SessionNotFound; use Alchemy\Phrasea\Status\StatusStructureProviderInterface; use Symfony\Component\HttpFoundation\Request; @@ -234,6 +237,8 @@ class RootController extends Controller $error = true; } + $this->dispatchEvent(RecordStructureEvents::STATUS_BIT_DELETED, new StatusBitUpdatedEvent($databox, $bit, [])); + return $this->app->json(['success' => !$error]); } @@ -350,9 +355,16 @@ class RootController extends Controller } } + $this->dispatchEvent(RecordStructureEvents::STATUS_BIT_UPDATED, new StatusBitUpdatedEvent($databox, $bit, [])); + return $this->app->redirectPath('database_display_statusbit', ['databox_id' => $databox_id, 'success' => 1]); } - + + private function dispatchEvent($eventName, StatusBitEvent $event = null) + { + $this->app['dispatcher']->dispatch($eventName, $event); + } + /** * @param string $section * @return array diff --git a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php index 04762b3aef..b94176945c 100644 --- a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php +++ b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php @@ -110,7 +110,7 @@ class SearchEngineController extends Controller private function getConfigurationForm(ElasticsearchOptions $options) { /** @var GlobalStructure $g */ - $g = $this->app['search_engine.structure']; + $g = $this->app['search_engine.global_structure']; return $this->app->form(new ElasticsearchSettingsFormType($g, $options), $options, [ 'action' => $this->app->url('admin_searchengine_form'), diff --git a/lib/Alchemy/Phrasea/Controller/Api/V3/V3SearchRawController.php b/lib/Alchemy/Phrasea/Controller/Api/V3/V3SearchRawController.php index e9d6c36f8f..246351bd56 100644 --- a/lib/Alchemy/Phrasea/Controller/Api/V3/V3SearchRawController.php +++ b/lib/Alchemy/Phrasea/Controller/Api/V3/V3SearchRawController.php @@ -6,6 +6,7 @@ use Alchemy\Phrasea\Application\Helper\DispatcherAware; use Alchemy\Phrasea\Application\Helper\JsonBodyAware; use Alchemy\Phrasea\Controller\Api\Result; use Alchemy\Phrasea\Controller\Controller; +use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine; use Alchemy\Phrasea\SearchEngine\SearchEngineInterface; use Alchemy\Phrasea\SearchEngine\SearchEngineOptions; use Alchemy\Phrasea\Utilities\Stopwatch; @@ -60,7 +61,8 @@ class V3SearchRawController extends Controller } /** - * @return SearchEngineInterface + * // return SearchEngineInterface + * @return ElasticSearchEngine */ private function getSearchEngine() { diff --git a/lib/Alchemy/Phrasea/Core/Event/Subscriber/StructureChangeSubscriber.php b/lib/Alchemy/Phrasea/Core/Event/Subscriber/StructureChangeSubscriber.php new file mode 100644 index 0000000000..67e89632e4 --- /dev/null +++ b/lib/Alchemy/Phrasea/Core/Event/Subscriber/StructureChangeSubscriber.php @@ -0,0 +1,48 @@ +searchEngineStructure = $se1; + } + + public static function getSubscribedEvents() + { + return [ + RecordStructureEvents::FIELD_UPDATED => 'onStructureChange', + RecordStructureEvents::FIELD_DELETED => 'onStructureChange', + RecordStructureEvents::STATUS_BIT_UPDATED => 'onStructureChange', + RecordStructureEvents::STATUS_BIT_DELETED => 'onStructureChange', + ]; + } + + /** + * clears the cached translated versions of phr fields;sb to es fields;sb (createFromLegacy...) + * this was cached per db + * + * @param RecordStructureEvent $event + */ + public function onStructureChange(RecordStructureEvent $event) + { + $this->searchEngineStructure->deleteFromCache($event->getDatabox()); + } +} diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index 00f01645a0..8764a7bc2b 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -77,10 +77,6 @@ class SearchEngineServiceProvider implements ServiceProviderInterface return new SearchEngineLogger($app); }); -// $app['search_engine.structure.factory'] = $app->protect(function () use ($app) { -// return $app['search_engine.structure']; -// }); - $app['search_engine'] = $app->share(function ($app) { $stopwatch = new Stopwatch("se"); // $type = $app['conf']->get(['main', 'search-engine', 'type']); @@ -92,15 +88,10 @@ class SearchEngineServiceProvider implements ServiceProviderInterface /** @var ElasticsearchOptions $options */ $options = $app['elasticsearch.options']; - /** @var SearchEngineStructure $seStructure */ - //$seStructure = $app['search_engine.structure2']; - $stopwatch->lap("se.options"); $r = new ElasticSearchEngine( $app, - // $app['search_engine.structure.factory'], - // $app['search_engine.structure'], - $app['search_engine.structure2'], + $app['search_engine.global_structure'], $app['elasticsearch.client'], $app['query_context.factory'], $app['elasticsearch.facets_response.factory'], @@ -111,29 +102,21 @@ class SearchEngineServiceProvider implements ServiceProviderInterface return $r; }); - $app['search_engine.structure2'] = $app->share(function (PhraseaApplication $app) { - $stopwatch = new Stopwatch("se.structure2"); - $seStructure = new SearchEngineStructure($app['cache']); - $globalStructure = $seStructure->getGlobalStructureFromDataboxes($app->getDataboxes()); + $app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) { + return new SearchEngineStructure($app['cache']); + }); + + $app['search_engine.global_structure'] = $app->share(function (PhraseaApplication $app) { + $stopwatch = new Stopwatch("se.global_structure"); + /** @var SearchEngineStructure $seStructure */ + $s = $app['search_engine.structure']; + $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes()); $stopwatch->log(); return $globalStructure; }); - $app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) { - $stopwatch = new Stopwatch("se.structure"); - $databoxes = $app->getDataboxes(); - $stopwatch ->lap('get db'); - $r = GlobalStructure::createFromDataboxes($databoxes); - $stopwatch->log(); - return $r; - }); - - $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { - /** @var SearchEngineStructure $seStructure */ - // $seStructure = $app['search_engine.structure2']; - // $globalStructure = $seStructure->getGlobalStructureFromDataboxes($app->getDataboxes()); - // return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.structure']); - return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.structure2']); + $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { + return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.global_structure']); }); return $app; @@ -151,7 +134,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['elasticsearch.index.record'] = $app->share(function ($app) { - return new Indexer\RecordIndex($app['search_engine.structure'], array_keys($app['locales.available'])); + return new Indexer\RecordIndex($app['search_engine.global_structure'], array_keys($app['locales.available'])); }); $app['elasticsearch.index.term'] = $app->share(function ($app) { @@ -185,7 +168,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface $app['elasticsearch.record_helper'], $app['elasticsearch.options'], $app, - 'search_engine.structure', + 'search_engine.global_structure', 'thesaurus' ); }); @@ -303,7 +286,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface $app['query_context.factory'] = $app->share(function ($app) { return new QueryContextFactory( - $app['search_engine.structure2'], + $app['search_engine.global_structure'], array_keys($app['locales.available']), $app['locale'] ); @@ -324,7 +307,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['query_visitor.factory'] = $app->protect(function () use ($app) { - return new QueryVisitor($app['search_engine.structure2']); + return new QueryVisitor($app['search_engine.global_structure']); }); $app['query_compiler'] = $app->share(function ($app) { diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/FlagHydrator.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/FlagHydrator.php index d6205319f2..97807d9bfe 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/FlagHydrator.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/FlagHydrator.php @@ -20,6 +20,9 @@ use databox; class FlagHydrator implements HydratorInterface { + /** @var array */ + private $field_names_map; + public function __construct(Structure $structure, databox $databox) { $this->field_names_map = self::buildFieldNamesMap($structure, $databox); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php index c0e4a4ce82..26ce40228b 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php @@ -7,6 +7,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper; +use Alchemy\Phrasea\Utilities\Stopwatch; use Assert\Assertion; use databox_field; @@ -54,6 +55,8 @@ class Field implements Typed public static function createFromLegacyField(databox_field $field, $with = Structure::WITH_EVERYTHING) { + // $stopwatch = new Stopwatch("createFromLegacyField.".$field->get_name()); + $type = self::getTypeFromLegacy($field); $databox = $field->get_databox(); @@ -62,7 +65,10 @@ class Field implements Typed // Thesaurus concept inference $xpath = $field->get_tbranch(); if (!empty($xpath)) { + // $stopwatch->lap('before findConceptsByXPath'); $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath); + //$stopwatch->lap('after findConceptsByXPath'); + } } @@ -77,7 +83,7 @@ class Field implements Typed } } - return new self($field->get_name(), $type, [ + $r = new self($field->get_name(), $type, [ 'databox_id' => $databox->get_sbas_id(), 'searchable' => $field->is_indexable(), 'private' => $field->isBusiness(), @@ -86,6 +92,9 @@ class Field implements Typed 'generate_cterms' => $field->get_generate_cterms(), 'used_by_collections' => $databox->get_collection_unique_ids() ]); + + // $stopwatch->log(); + return $r; } private static function getTypeFromLegacy(databox_field $field) diff --git a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php index 1e2d302e21..9e1a7a3aa8 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php @@ -42,8 +42,8 @@ class SearchEngineStructure $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes"); foreach ($databoxes as $databox) { - // we will store both FIELDS and FLAGS in the same cache entry : - // it's small data and $what=WITH_EVERYTHING allways + // we will cache both FIELDS and FLAGS in the same entry : + // it's small data and $what seems always WITH_EVERYTHING $k = $this->getCacheKey("FieldsAndFlags", $databox); try { $data = $this->cache->get($k); @@ -70,7 +70,7 @@ class SearchEngineStructure } if($what & Structure::STRUCTURE_WITH_FLAGS) { - $flags = array_merge($fields, $data['flags']); + $flags = array_merge($flags, $data['flags']); } } $stopwatch->lap('loop0'); @@ -81,6 +81,12 @@ class SearchEngineStructure return $r; } + public function deleteFromCache($databox) + { + $k = $this->getCacheKey("FieldsAndFlags", $databox); + $this->cache->delete($k); + } + private function getCacheKey($what, databox $db) { return "es_db-" . $db->get_sbas_id() . '_' . $what; From 3a34c3c017dbaead54280b4bbe622ef723e81309 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Thu, 5 Nov 2020 12:02:55 +0100 Subject: [PATCH 08/12] - fix (tryout) : system:uninstall crashed on circleci. --- lib/Alchemy/Phrasea/Application.php | 1 - .../Phrasea/Core/Provider/PhraseaEventServiceProvider.php | 6 ++++++ lib/Alchemy/Phrasea/Utilities/Stopwatch.php | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/Alchemy/Phrasea/Application.php b/lib/Alchemy/Phrasea/Application.php index ac9fff0b68..fd6eb315c7 100644 --- a/lib/Alchemy/Phrasea/Application.php +++ b/lib/Alchemy/Phrasea/Application.php @@ -749,7 +749,6 @@ class Application extends SilexApplication $dispatcher->addSubscriber(new LazaretSubscriber($app)); $dispatcher->addSubscriber(new ValidationSubscriber($app)); $dispatcher->addSubscriber(new WebhookUserEventSubscriber($app)); - $dispatcher->addSubscriber(new StructureChangeSubscriber($app['search_engine.structure'])); return $dispatcher; }) diff --git a/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php index 76d34c72b7..e40781eed2 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php @@ -11,6 +11,7 @@ namespace Alchemy\Phrasea\Core\Provider; +use Alchemy\Phrasea\Core\Event\Subscriber\StructureChangeSubscriber; use Alchemy\Phrasea\Core\LazyLocator; use Alchemy\Phrasea\Core\Event\Subscriber\ContentNegotiationSubscriber; use Alchemy\Phrasea\Core\Event\Subscriber\CookiesDisablerSubscriber; @@ -59,6 +60,10 @@ class PhraseaEventServiceProvider implements ServiceProviderInterface return new RecordEditSubscriber(new LazyLocator($app, 'phraseanet.appbox')); }); + $app['phraseanet.structure-change-subscriber'] = $app->share(function (Application $app) { + return new StructureChangeSubscriber($app['search_engine.structure']); + }); + $app['dispatcher'] = $app->share( $app->extend('dispatcher', function (EventDispatcherInterface $dispatcher, Application $app) { $dispatcher->addSubscriber($app['phraseanet.logout-subscriber']); @@ -68,6 +73,7 @@ class PhraseaEventServiceProvider implements ServiceProviderInterface $dispatcher->addSubscriber($app['phraseanet.cookie-disabler-subscriber']); $dispatcher->addSubscriber($app['phraseanet.session-manager-subscriber']); $dispatcher->addSubscriber($app['phraseanet.record-edit-subscriber']); + $dispatcher->addSubscriber($app['phraseanet.structure-change-subscriber']); return $dispatcher; }) diff --git a/lib/Alchemy/Phrasea/Utilities/Stopwatch.php b/lib/Alchemy/Phrasea/Utilities/Stopwatch.php index 16e700457b..5b6f9001c0 100644 --- a/lib/Alchemy/Phrasea/Utilities/Stopwatch.php +++ b/lib/Alchemy/Phrasea/Utilities/Stopwatch.php @@ -69,10 +69,10 @@ class Stopwatch public function log() { - // file_put_contents('/var/alchemy/Phraseanet/tmp/phraseanet.log', - // $this->_name . "\n" . var_export($this->getLapses(), true) . "\n\n", - // FILE_APPEND - // ); + file_put_contents('/var/alchemy/Phraseanet/tmp/phraseanet.log', + $this->_name . "\n" . var_export($this->getLapses(), true) . "\n\n", + FILE_APPEND + ); } /** From b59a3bffcd7c4fcd82ebe01978e7569c9bdd685c Mon Sep 17 00:00:00 2001 From: jygaulier Date: Thu, 5 Nov 2020 14:58:57 +0100 Subject: [PATCH 09/12] - fix : crashed when trying to uninstall a not yet installed phr (circle-ci tests does that) --- .../Core/Provider/PhraseaEventServiceProvider.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php index e40781eed2..96e382120e 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php @@ -60,10 +60,6 @@ class PhraseaEventServiceProvider implements ServiceProviderInterface return new RecordEditSubscriber(new LazyLocator($app, 'phraseanet.appbox')); }); - $app['phraseanet.structure-change-subscriber'] = $app->share(function (Application $app) { - return new StructureChangeSubscriber($app['search_engine.structure']); - }); - $app['dispatcher'] = $app->share( $app->extend('dispatcher', function (EventDispatcherInterface $dispatcher, Application $app) { $dispatcher->addSubscriber($app['phraseanet.logout-subscriber']); @@ -73,7 +69,11 @@ class PhraseaEventServiceProvider implements ServiceProviderInterface $dispatcher->addSubscriber($app['phraseanet.cookie-disabler-subscriber']); $dispatcher->addSubscriber($app['phraseanet.session-manager-subscriber']); $dispatcher->addSubscriber($app['phraseanet.record-edit-subscriber']); - $dispatcher->addSubscriber($app['phraseanet.structure-change-subscriber']); + + // if phr is not yet installed, don't use non-existing service 'search_engine.structure' + if($app->offsetExists('search_engine.structure')) { + $dispatcher->addSubscriber(new StructureChangeSubscriber($app['search_engine.structure'])); + } return $dispatcher; }) From f3259fef16165223de407d7aea7fe79808ad753d Mon Sep 17 00:00:00 2001 From: jygaulier Date: Thu, 5 Nov 2020 15:11:56 +0100 Subject: [PATCH 10/12] - fix : disable forgotten debug stopwatch --- lib/Alchemy/Phrasea/Utilities/Stopwatch.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Alchemy/Phrasea/Utilities/Stopwatch.php b/lib/Alchemy/Phrasea/Utilities/Stopwatch.php index 5b6f9001c0..4b3c2cb95f 100644 --- a/lib/Alchemy/Phrasea/Utilities/Stopwatch.php +++ b/lib/Alchemy/Phrasea/Utilities/Stopwatch.php @@ -69,10 +69,10 @@ class Stopwatch public function log() { - file_put_contents('/var/alchemy/Phraseanet/tmp/phraseanet.log', - $this->_name . "\n" . var_export($this->getLapses(), true) . "\n\n", - FILE_APPEND - ); +// file_put_contents('/var/alchemy/Phraseanet/tmp/phraseanet.log', +// $this->_name . "\n" . var_export($this->getLapses(), true) . "\n\n", +// FILE_APPEND +// ); } /** From 21519c460be2d31b2dd146dfb9ac6f61d73e88a7 Mon Sep 17 00:00:00 2001 From: aina esokia Date: Thu, 5 Nov 2020 17:18:29 +0300 Subject: [PATCH 11/12] PHRAS-3258 fix map displayed #time 3h --- Phraseanet-production-client/dist/production.js | 8 ++++---- Phraseanet-production-client/dist/production.min.js | 8 ++++---- .../src/components/geolocalisation/providers/mapbox.js | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Phraseanet-production-client/dist/production.js b/Phraseanet-production-client/dist/production.js index 3cb3e0dc69..7683ad4ba5 100644 --- a/Phraseanet-production-client/dist/production.js +++ b/Phraseanet-production-client/dist/production.js @@ -2273,16 +2273,16 @@ var leafletMap = function leafletMap(services) { if (!shouldUseMapboxGl()) { L.mapbox.accessToken = activeProvider.accessToken; - map = L.mapbox.map(mapUID, 'mapbox.streets', mapOptions); + map = L.mapbox.map(mapUID, _underscore2.default, mapOptions); shouldUpdateZoom = false; map.setView(activeProvider.defaultPosition, activeProvider.defaultZoom); if (searchable) { map.addControl(L.mapbox.geocoderControl('mapbox.places')); } var layers = { - Streets: L.mapbox.tileLayer('mapbox.streets'), - Outdoors: L.mapbox.tileLayer('mapbox.outdoors'), - Satellite: L.mapbox.tileLayer('mapbox.satellite') + Streets: L.mapbox.styleLayer('mapbox://styles/mapbox/streets-v11'), + Outdoors: L.mapbox.styleLayer('mapbox://styles/mapbox/outdoors-v11'), + Satellite: L.mapbox.styleLayer('mapbox://styles/mapbox/satellite-v9') }; layers.Streets.addTo(map); diff --git a/Phraseanet-production-client/dist/production.min.js b/Phraseanet-production-client/dist/production.min.js index 3cb3e0dc69..7683ad4ba5 100644 --- a/Phraseanet-production-client/dist/production.min.js +++ b/Phraseanet-production-client/dist/production.min.js @@ -2273,16 +2273,16 @@ var leafletMap = function leafletMap(services) { if (!shouldUseMapboxGl()) { L.mapbox.accessToken = activeProvider.accessToken; - map = L.mapbox.map(mapUID, 'mapbox.streets', mapOptions); + map = L.mapbox.map(mapUID, _underscore2.default, mapOptions); shouldUpdateZoom = false; map.setView(activeProvider.defaultPosition, activeProvider.defaultZoom); if (searchable) { map.addControl(L.mapbox.geocoderControl('mapbox.places')); } var layers = { - Streets: L.mapbox.tileLayer('mapbox.streets'), - Outdoors: L.mapbox.tileLayer('mapbox.outdoors'), - Satellite: L.mapbox.tileLayer('mapbox.satellite') + Streets: L.mapbox.styleLayer('mapbox://styles/mapbox/streets-v11'), + Outdoors: L.mapbox.styleLayer('mapbox://styles/mapbox/outdoors-v11'), + Satellite: L.mapbox.styleLayer('mapbox://styles/mapbox/satellite-v9') }; layers.Streets.addTo(map); diff --git a/Phraseanet-production-client/src/components/geolocalisation/providers/mapbox.js b/Phraseanet-production-client/src/components/geolocalisation/providers/mapbox.js index 281d9b1c9b..19078feb96 100644 --- a/Phraseanet-production-client/src/components/geolocalisation/providers/mapbox.js +++ b/Phraseanet-production-client/src/components/geolocalisation/providers/mapbox.js @@ -169,16 +169,16 @@ const leafletMap = (services) => { if (!shouldUseMapboxGl()) { L.mapbox.accessToken = activeProvider.accessToken; - map = L.mapbox.map(mapUID, 'mapbox.streets', mapOptions); + map = L.mapbox.map(mapUID, _, mapOptions); shouldUpdateZoom = false; map.setView(activeProvider.defaultPosition, activeProvider.defaultZoom); if (searchable) { map.addControl(L.mapbox.geocoderControl('mapbox.places')); } var layers = { - Streets: L.mapbox.tileLayer('mapbox.streets'), - Outdoors: L.mapbox.tileLayer('mapbox.outdoors'), - Satellite: L.mapbox.tileLayer('mapbox.satellite') + Streets: L.mapbox.styleLayer('mapbox://styles/mapbox/streets-v11'), + Outdoors: L.mapbox.styleLayer('mapbox://styles/mapbox/outdoors-v11'), + Satellite: L.mapbox.styleLayer('mapbox://styles/mapbox/satellite-v9') }; layers.Streets.addTo(map); From cfd385498d65613e4c79da7362ae033c949bdab4 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Thu, 5 Nov 2020 19:53:00 +0100 Subject: [PATCH 12/12] - add : cache on es "tags" - removed : variable globalstructure construction (WITH_FIELDS, WITH_FLAGS...) (was unused) - cleanup (stopwatches etc) --- .../Controller/Prod/QueryController.php | 5 +- .../Provider/SearchEngineServiceProvider.php | 47 +++++++-------- .../Elastic/ElasticSearchEngine.php | 11 +--- .../SearchEngine/Elastic/Structure/Field.php | 41 ++++--------- .../Elastic/Structure/GlobalStructure.php | 57 ------------------- .../Elastic/Structure/Structure.php | 6 -- .../SearchEngine/SearchEngineStructure.php | 55 +++++++++++------- 7 files changed, 74 insertions(+), 148 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php index d4830859c2..3285b9d1f2 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php @@ -67,10 +67,7 @@ class QueryController extends Controller $word = StringHelper::crlfNormalize($word); $options = SearchEngineOptions::fromRequest($this->app, $request); - $search_engine_structure = GlobalStructure::createFromDataboxes( - $this->app->getDataboxes(), - Structure::WITH_EVERYTHING & ~(Structure::STRUCTURE_WITH_FLAGS | Structure::FIELD_WITH_FACETS | Structure::FIELD_WITH_THESAURUS) - ); + $search_engine_structure = $this->app['search_engine.global_structure']; $query_context_factory = new QueryContextFactory( $search_engine_structure, diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index 8764a7bc2b..05b5a90325 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -18,8 +18,6 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Index; use Alchemy\Phrasea\SearchEngine\Elastic\IndexLocator; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryVisitor; use Alchemy\Phrasea\SearchEngine\SearchEngineLogger; -use Alchemy\Phrasea\Exception\InvalidArgumentException; -use Alchemy\Phrasea\SearchEngine\SearchEngineInterface; use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer; use Alchemy\Phrasea\SearchEngine\Elastic\IndexerSubscriber; @@ -30,10 +28,9 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\Escaper; use Alchemy\Phrasea\SearchEngine\Elastic\Search\FacetsResponse; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler; -use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus; use Alchemy\Phrasea\SearchEngine\SearchEngineStructure; -use Alchemy\Phrasea\Utilities\Stopwatch; +// use Alchemy\Phrasea\Utilities\Stopwatch; use Elasticsearch\ClientBuilder; use Hoa\Compiler; use Hoa\File; @@ -78,28 +75,29 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['search_engine'] = $app->share(function ($app) { - $stopwatch = new Stopwatch("se"); - // $type = $app['conf']->get(['main', 'search-engine', 'type']); - // if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) { - // throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type)); - // } - // $stopwatch->lap("se.conf"); +// $stopwatch = new Stopwatch("se"); - /** @var ElasticsearchOptions $options */ - $options = $app['elasticsearch.options']; +// $stopwatch->lap("se.options"); +// $r = new ElasticSearchEngine( +// $app, +// $app['search_engine.global_structure'], +// $app['elasticsearch.client'], +// $app['query_context.factory'], +// $app['elasticsearch.facets_response.factory'], +// $app['elasticsearch.options'] +// ); +// $stopwatch->lap("se.new"); +// $stopwatch->log(); +// return $r; - $stopwatch->lap("se.options"); - $r = new ElasticSearchEngine( + return new ElasticSearchEngine( $app, $app['search_engine.global_structure'], $app['elasticsearch.client'], $app['query_context.factory'], $app['elasticsearch.facets_response.factory'], - $options + $app['elasticsearch.options'] ); - $stopwatch->lap("se.new"); - $stopwatch->log(); - return $r; }); $app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) { @@ -107,12 +105,15 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['search_engine.global_structure'] = $app->share(function (PhraseaApplication $app) { - $stopwatch = new Stopwatch("se.global_structure"); - /** @var SearchEngineStructure $seStructure */ +// $stopwatch = new Stopwatch("se.global_structure"); + /** @var SearchEngineStructure $s */ $s = $app['search_engine.structure']; - $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes()); - $stopwatch->log(); - return $globalStructure; + +// $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes()); +// $stopwatch->log(); +// return $globalStructure; + + return $s->getGlobalStructureFromDataboxes($app->getDataboxes()); }); $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index eac703fc21..529237b787 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -63,17 +63,16 @@ class ElasticSearchEngine implements SearchEngineInterface /** * @param Application $app - * @param GlobalStructure $structure2 + * @param GlobalStructure $structure * @param Client $client * @param QueryContextFactory $context_factory * @param Closure $facetsResponseFactory * @param ElasticsearchOptions $options */ - public function __construct(Application $app, GlobalStructure $structure2, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) + public function __construct(Application $app, GlobalStructure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) { $this->app = $app; - // $this->structure = $structure; - $this->structure = $structure2; + $this->structure = $structure; $this->client = $client; $this->context_factory = $context_factory; $this->facetsResponseFactory = $facetsResponseFactory; @@ -87,10 +86,6 @@ class ElasticSearchEngine implements SearchEngineInterface */ public function getStructure() { - if (!($this->structure instanceof Structure)) { - $this->structure = call_user_func($this->structure); - } - return $this->structure; } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php index 26ce40228b..63ba164e34 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php @@ -4,11 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper; -use Alchemy\Phrasea\Utilities\Stopwatch; -use Assert\Assertion; use databox_field; /** @@ -53,37 +49,26 @@ class Field implements Typed private $used_by_collections; - public static function createFromLegacyField(databox_field $field, $with = Structure::WITH_EVERYTHING) + public static function createFromLegacyField(databox_field $field) { - // $stopwatch = new Stopwatch("createFromLegacyField.".$field->get_name()); - $type = self::getTypeFromLegacy($field); $databox = $field->get_databox(); + // Thesaurus concept inference $roots = null; - if(($with & Structure::FIELD_WITH_THESAURUS) && $type === FieldMapping::TYPE_STRING) { - // Thesaurus concept inference - $xpath = $field->get_tbranch(); - if (!empty($xpath)) { - // $stopwatch->lap('before findConceptsByXPath'); - $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath); - //$stopwatch->lap('after findConceptsByXPath'); - - } + if($type === FieldMapping::TYPE_STRING && !empty($xpath = $field->get_tbranch())) { + $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath); } - $facet = self::FACET_DISABLED; - if($with & Structure::FIELD_WITH_FACETS) { - // Facet (enable + optional limit) - $facet = $field->getFacetValuesLimit(); - if ($facet === databox_field::FACET_DISABLED) { - $facet = self::FACET_DISABLED; - } elseif ($facet === databox_field::FACET_NO_LIMIT) { - $facet = self::FACET_NO_LIMIT; - } + // Facet (enable + optional limit) + $facet = $field->getFacetValuesLimit(); + if ($facet === databox_field::FACET_DISABLED) { + $facet = self::FACET_DISABLED; + } elseif ($facet === databox_field::FACET_NO_LIMIT) { + $facet = self::FACET_NO_LIMIT; } - $r = new self($field->get_name(), $type, [ + return new self($field->get_name(), $type, [ 'databox_id' => $databox->get_sbas_id(), 'searchable' => $field->is_indexable(), 'private' => $field->isBusiness(), @@ -92,9 +77,6 @@ class Field implements Typed 'generate_cterms' => $field->get_generate_cterms(), 'used_by_collections' => $databox->get_collection_unique_ids() ]); - - // $stopwatch->log(); - return $r; } private static function getTypeFromLegacy(databox_field $field) @@ -278,4 +260,5 @@ class Field implements Typed 'used_by_collections' => $used_by_collections ]); } + } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php index 8c9e670f8d..6ffb8cc8ed 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php @@ -3,9 +3,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; -use Alchemy\Phrasea\Utilities\Stopwatch; -use Assert\Assertion; use DomainException; final class GlobalStructure implements Structure @@ -15,7 +12,6 @@ final class GlobalStructure implements Structure */ private $fields = array(); - /** * @var Field[][] */ @@ -46,59 +42,6 @@ final class GlobalStructure implements Structure */ private $metadata_tags = array(); - /** - * @param \databox[] $databoxes - * @param int $what bitmask of what should be included in this structure, in fields, ... - * - * @return GlobalStructure - */ - public static function createFromDataboxes(array $databoxes, $what = self::WITH_EVERYTHING) - { - $fields = []; - $flags = []; - - $stopwatch = new Stopwatch("globalStructure2"); - foreach ($databoxes as $databox) { - if($what & self::STRUCTURE_WITH_FIELDS) { - foreach ($databox->get_meta_structure() as $fieldStructure) { - $fields[] = Field::createFromLegacyField($fieldStructure, $what); - } - } - - if($what & self::STRUCTURE_WITH_FLAGS) { - foreach ($databox->getStatusStructure() as $status) { - $flags[] = Flag::createFromLegacyStatus($status); - } - } - } - $stopwatch->lap('loop0'); - $r = new self($fields, $flags, MetadataHelper::createTags()); - - $stopwatch->log(); - - return $r; - } - - - /** - * @param \databox $databox - * @return GlobalStructure - */ - public static function unused_createFromDatabox(\databox $databox) - { - $fields = []; - $flags = []; - - foreach ($databox->get_meta_structure() as $fieldStructure) { - $fields[] = Field::createFromLegacyField($fieldStructure); - } - - foreach ($databox->getStatusStructure() as $status) { - $flags[] = Flag::createFromLegacyStatus($status); - } - - return new self($fields, $flags, MetadataHelper::createTags()); - } /** * GlobalStructure constructor. diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php index 44d58e9d5f..72462488e2 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php @@ -12,12 +12,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; interface Structure { - const STRUCTURE_WITH_FIELDS = 0x10; - const STRUCTURE_WITH_FLAGS = 0x20; - const FIELD_WITH_THESAURUS = 0x01; - const FIELD_WITH_FACETS = 0x02; - const WITH_EVERYTHING = 0xFF; - /** * @return Field[] */ diff --git a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php index 9e1a7a3aa8..ac983280f8 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php @@ -9,8 +9,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\MetadataHelper; -use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure; -use Alchemy\Phrasea\Utilities\Stopwatch; +// use Alchemy\Phrasea\Utilities\Stopwatch; use databox; class SearchEngineStructure @@ -30,20 +29,18 @@ class SearchEngineStructure /** * @param databox[] $databoxes - * @param int $what bitmask of what should be included in this structure, in fields, ... * * @return GlobalStructure */ - public function getGlobalStructureFromDataboxes(array $databoxes, $what = Structure::WITH_EVERYTHING) + public function getGlobalStructureFromDataboxes(array $databoxes) { $fields = []; $flags = []; - $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes"); +// $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes"); foreach ($databoxes as $databox) { - // we will cache both FIELDS and FLAGS in the same entry : - // it's small data and $what seems always WITH_EVERYTHING + // we will cache both FIELDS and FLAGS in the same entry : it's small data $k = $this->getCacheKey("FieldsAndFlags", $databox); try { $data = $this->cache->get($k); @@ -57,7 +54,7 @@ class SearchEngineStructure 'flags' => [] ]; foreach ($databox->get_meta_structure() as $fieldStructure) { - $data['fields'][] = Field::createFromLegacyField($fieldStructure, $what); + $data['fields'][] = Field::createFromLegacyField($fieldStructure); } foreach ($databox->getStatusStructure() as $status) { $data['flags'][] = Flag::createFromLegacyStatus($status); @@ -65,20 +62,29 @@ class SearchEngineStructure $this->cache->save($k, $data); } - if($what & Structure::STRUCTURE_WITH_FIELDS) { - $fields = array_merge($fields, $data['fields']); - } - - if($what & Structure::STRUCTURE_WITH_FLAGS) { - $flags = array_merge($flags, $data['flags']); - } + $fields = array_merge($fields, $data['fields']); + $flags = array_merge($flags, $data['flags']); } - $stopwatch->lap('loop0'); - $r = new GlobalStructure($fields, $flags, MetadataHelper::createTags()); +// $stopwatch->lap('loop0'); - $stopwatch->log(); + // tags does not depends on db + $k = $this->getCacheKey("Tags"); + try { + $tags = $this->cache->get($k); + } + catch(\Exception $e) { + $tags = false; + } + if($tags === false) { + $this->cache->save($k, ($tags = MetadataHelper::createTags())); + // nb : tags is a hardcoded list, we don't need to clear this cache + } - return $r; +// $r = new GlobalStructure($fields, $flags, $tags); +// $stopwatch->log(); +// return $r; + + return new GlobalStructure($fields, $flags, $tags); } public function deleteFromCache($databox) @@ -87,8 +93,15 @@ class SearchEngineStructure $this->cache->delete($k); } - private function getCacheKey($what, databox $db) + /** + * build a cache key to store es data, related to a db or not (if db is null) + * + * @param $what + * @param databox|null $db + * @return string + */ + private function getCacheKey($what, databox $db = null) { - return "es_db-" . $db->get_sbas_id() . '_' . $what; + return 'es' . ($db ? ('_db-'.$db->get_sbas_id()) : '') . '_' . $what; } } \ No newline at end of file