From cfd385498d65613e4c79da7362ae033c949bdab4 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Thu, 5 Nov 2020 19:53:00 +0100 Subject: [PATCH] - add : cache on es "tags" - removed : variable globalstructure construction (WITH_FIELDS, WITH_FLAGS...) (was unused) - cleanup (stopwatches etc) --- .../Controller/Prod/QueryController.php | 5 +- .../Provider/SearchEngineServiceProvider.php | 47 +++++++-------- .../Elastic/ElasticSearchEngine.php | 11 +--- .../SearchEngine/Elastic/Structure/Field.php | 41 ++++--------- .../Elastic/Structure/GlobalStructure.php | 57 ------------------- .../Elastic/Structure/Structure.php | 6 -- .../SearchEngine/SearchEngineStructure.php | 55 +++++++++++------- 7 files changed, 74 insertions(+), 148 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php index d4830859c2..3285b9d1f2 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php @@ -67,10 +67,7 @@ class QueryController extends Controller $word = StringHelper::crlfNormalize($word); $options = SearchEngineOptions::fromRequest($this->app, $request); - $search_engine_structure = GlobalStructure::createFromDataboxes( - $this->app->getDataboxes(), - Structure::WITH_EVERYTHING & ~(Structure::STRUCTURE_WITH_FLAGS | Structure::FIELD_WITH_FACETS | Structure::FIELD_WITH_THESAURUS) - ); + $search_engine_structure = $this->app['search_engine.global_structure']; $query_context_factory = new QueryContextFactory( $search_engine_structure, diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index 8764a7bc2b..05b5a90325 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -18,8 +18,6 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Index; use Alchemy\Phrasea\SearchEngine\Elastic\IndexLocator; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryVisitor; use Alchemy\Phrasea\SearchEngine\SearchEngineLogger; -use Alchemy\Phrasea\Exception\InvalidArgumentException; -use Alchemy\Phrasea\SearchEngine\SearchEngineInterface; use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer; use Alchemy\Phrasea\SearchEngine\Elastic\IndexerSubscriber; @@ -30,10 +28,9 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\Escaper; use Alchemy\Phrasea\SearchEngine\Elastic\Search\FacetsResponse; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler; -use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus; use Alchemy\Phrasea\SearchEngine\SearchEngineStructure; -use Alchemy\Phrasea\Utilities\Stopwatch; +// use Alchemy\Phrasea\Utilities\Stopwatch; use Elasticsearch\ClientBuilder; use Hoa\Compiler; use Hoa\File; @@ -78,28 +75,29 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['search_engine'] = $app->share(function ($app) { - $stopwatch = new Stopwatch("se"); - // $type = $app['conf']->get(['main', 'search-engine', 'type']); - // if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) { - // throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type)); - // } - // $stopwatch->lap("se.conf"); +// $stopwatch = new Stopwatch("se"); - /** @var ElasticsearchOptions $options */ - $options = $app['elasticsearch.options']; +// $stopwatch->lap("se.options"); +// $r = new ElasticSearchEngine( +// $app, +// $app['search_engine.global_structure'], +// $app['elasticsearch.client'], +// $app['query_context.factory'], +// $app['elasticsearch.facets_response.factory'], +// $app['elasticsearch.options'] +// ); +// $stopwatch->lap("se.new"); +// $stopwatch->log(); +// return $r; - $stopwatch->lap("se.options"); - $r = new ElasticSearchEngine( + return new ElasticSearchEngine( $app, $app['search_engine.global_structure'], $app['elasticsearch.client'], $app['query_context.factory'], $app['elasticsearch.facets_response.factory'], - $options + $app['elasticsearch.options'] ); - $stopwatch->lap("se.new"); - $stopwatch->log(); - return $r; }); $app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) { @@ -107,12 +105,15 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['search_engine.global_structure'] = $app->share(function (PhraseaApplication $app) { - $stopwatch = new Stopwatch("se.global_structure"); - /** @var SearchEngineStructure $seStructure */ +// $stopwatch = new Stopwatch("se.global_structure"); + /** @var SearchEngineStructure $s */ $s = $app['search_engine.structure']; - $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes()); - $stopwatch->log(); - return $globalStructure; + +// $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes()); +// $stopwatch->log(); +// return $globalStructure; + + return $s->getGlobalStructureFromDataboxes($app->getDataboxes()); }); $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index eac703fc21..529237b787 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -63,17 +63,16 @@ class ElasticSearchEngine implements SearchEngineInterface /** * @param Application $app - * @param GlobalStructure $structure2 + * @param GlobalStructure $structure * @param Client $client * @param QueryContextFactory $context_factory * @param Closure $facetsResponseFactory * @param ElasticsearchOptions $options */ - public function __construct(Application $app, GlobalStructure $structure2, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) + public function __construct(Application $app, GlobalStructure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) { $this->app = $app; - // $this->structure = $structure; - $this->structure = $structure2; + $this->structure = $structure; $this->client = $client; $this->context_factory = $context_factory; $this->facetsResponseFactory = $facetsResponseFactory; @@ -87,10 +86,6 @@ class ElasticSearchEngine implements SearchEngineInterface */ public function getStructure() { - if (!($this->structure instanceof Structure)) { - $this->structure = call_user_func($this->structure); - } - return $this->structure; } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php index 26ce40228b..63ba164e34 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php @@ -4,11 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper; -use Alchemy\Phrasea\Utilities\Stopwatch; -use Assert\Assertion; use databox_field; /** @@ -53,37 +49,26 @@ class Field implements Typed private $used_by_collections; - public static function createFromLegacyField(databox_field $field, $with = Structure::WITH_EVERYTHING) + public static function createFromLegacyField(databox_field $field) { - // $stopwatch = new Stopwatch("createFromLegacyField.".$field->get_name()); - $type = self::getTypeFromLegacy($field); $databox = $field->get_databox(); + // Thesaurus concept inference $roots = null; - if(($with & Structure::FIELD_WITH_THESAURUS) && $type === FieldMapping::TYPE_STRING) { - // Thesaurus concept inference - $xpath = $field->get_tbranch(); - if (!empty($xpath)) { - // $stopwatch->lap('before findConceptsByXPath'); - $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath); - //$stopwatch->lap('after findConceptsByXPath'); - - } + if($type === FieldMapping::TYPE_STRING && !empty($xpath = $field->get_tbranch())) { + $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath); } - $facet = self::FACET_DISABLED; - if($with & Structure::FIELD_WITH_FACETS) { - // Facet (enable + optional limit) - $facet = $field->getFacetValuesLimit(); - if ($facet === databox_field::FACET_DISABLED) { - $facet = self::FACET_DISABLED; - } elseif ($facet === databox_field::FACET_NO_LIMIT) { - $facet = self::FACET_NO_LIMIT; - } + // Facet (enable + optional limit) + $facet = $field->getFacetValuesLimit(); + if ($facet === databox_field::FACET_DISABLED) { + $facet = self::FACET_DISABLED; + } elseif ($facet === databox_field::FACET_NO_LIMIT) { + $facet = self::FACET_NO_LIMIT; } - $r = new self($field->get_name(), $type, [ + return new self($field->get_name(), $type, [ 'databox_id' => $databox->get_sbas_id(), 'searchable' => $field->is_indexable(), 'private' => $field->isBusiness(), @@ -92,9 +77,6 @@ class Field implements Typed 'generate_cterms' => $field->get_generate_cterms(), 'used_by_collections' => $databox->get_collection_unique_ids() ]); - - // $stopwatch->log(); - return $r; } private static function getTypeFromLegacy(databox_field $field) @@ -278,4 +260,5 @@ class Field implements Typed 'used_by_collections' => $used_by_collections ]); } + } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php index 8c9e670f8d..6ffb8cc8ed 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php @@ -3,9 +3,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; -use Alchemy\Phrasea\Utilities\Stopwatch; -use Assert\Assertion; use DomainException; final class GlobalStructure implements Structure @@ -15,7 +12,6 @@ final class GlobalStructure implements Structure */ private $fields = array(); - /** * @var Field[][] */ @@ -46,59 +42,6 @@ final class GlobalStructure implements Structure */ private $metadata_tags = array(); - /** - * @param \databox[] $databoxes - * @param int $what bitmask of what should be included in this structure, in fields, ... - * - * @return GlobalStructure - */ - public static function createFromDataboxes(array $databoxes, $what = self::WITH_EVERYTHING) - { - $fields = []; - $flags = []; - - $stopwatch = new Stopwatch("globalStructure2"); - foreach ($databoxes as $databox) { - if($what & self::STRUCTURE_WITH_FIELDS) { - foreach ($databox->get_meta_structure() as $fieldStructure) { - $fields[] = Field::createFromLegacyField($fieldStructure, $what); - } - } - - if($what & self::STRUCTURE_WITH_FLAGS) { - foreach ($databox->getStatusStructure() as $status) { - $flags[] = Flag::createFromLegacyStatus($status); - } - } - } - $stopwatch->lap('loop0'); - $r = new self($fields, $flags, MetadataHelper::createTags()); - - $stopwatch->log(); - - return $r; - } - - - /** - * @param \databox $databox - * @return GlobalStructure - */ - public static function unused_createFromDatabox(\databox $databox) - { - $fields = []; - $flags = []; - - foreach ($databox->get_meta_structure() as $fieldStructure) { - $fields[] = Field::createFromLegacyField($fieldStructure); - } - - foreach ($databox->getStatusStructure() as $status) { - $flags[] = Flag::createFromLegacyStatus($status); - } - - return new self($fields, $flags, MetadataHelper::createTags()); - } /** * GlobalStructure constructor. diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php index 44d58e9d5f..72462488e2 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php @@ -12,12 +12,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; interface Structure { - const STRUCTURE_WITH_FIELDS = 0x10; - const STRUCTURE_WITH_FLAGS = 0x20; - const FIELD_WITH_THESAURUS = 0x01; - const FIELD_WITH_FACETS = 0x02; - const WITH_EVERYTHING = 0xFF; - /** * @return Field[] */ diff --git a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php index 9e1a7a3aa8..ac983280f8 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php @@ -9,8 +9,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\MetadataHelper; -use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure; -use Alchemy\Phrasea\Utilities\Stopwatch; +// use Alchemy\Phrasea\Utilities\Stopwatch; use databox; class SearchEngineStructure @@ -30,20 +29,18 @@ class SearchEngineStructure /** * @param databox[] $databoxes - * @param int $what bitmask of what should be included in this structure, in fields, ... * * @return GlobalStructure */ - public function getGlobalStructureFromDataboxes(array $databoxes, $what = Structure::WITH_EVERYTHING) + public function getGlobalStructureFromDataboxes(array $databoxes) { $fields = []; $flags = []; - $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes"); +// $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes"); foreach ($databoxes as $databox) { - // we will cache both FIELDS and FLAGS in the same entry : - // it's small data and $what seems always WITH_EVERYTHING + // we will cache both FIELDS and FLAGS in the same entry : it's small data $k = $this->getCacheKey("FieldsAndFlags", $databox); try { $data = $this->cache->get($k); @@ -57,7 +54,7 @@ class SearchEngineStructure 'flags' => [] ]; foreach ($databox->get_meta_structure() as $fieldStructure) { - $data['fields'][] = Field::createFromLegacyField($fieldStructure, $what); + $data['fields'][] = Field::createFromLegacyField($fieldStructure); } foreach ($databox->getStatusStructure() as $status) { $data['flags'][] = Flag::createFromLegacyStatus($status); @@ -65,20 +62,29 @@ class SearchEngineStructure $this->cache->save($k, $data); } - if($what & Structure::STRUCTURE_WITH_FIELDS) { - $fields = array_merge($fields, $data['fields']); - } - - if($what & Structure::STRUCTURE_WITH_FLAGS) { - $flags = array_merge($flags, $data['flags']); - } + $fields = array_merge($fields, $data['fields']); + $flags = array_merge($flags, $data['flags']); } - $stopwatch->lap('loop0'); - $r = new GlobalStructure($fields, $flags, MetadataHelper::createTags()); +// $stopwatch->lap('loop0'); - $stopwatch->log(); + // tags does not depends on db + $k = $this->getCacheKey("Tags"); + try { + $tags = $this->cache->get($k); + } + catch(\Exception $e) { + $tags = false; + } + if($tags === false) { + $this->cache->save($k, ($tags = MetadataHelper::createTags())); + // nb : tags is a hardcoded list, we don't need to clear this cache + } - return $r; +// $r = new GlobalStructure($fields, $flags, $tags); +// $stopwatch->log(); +// return $r; + + return new GlobalStructure($fields, $flags, $tags); } public function deleteFromCache($databox) @@ -87,8 +93,15 @@ class SearchEngineStructure $this->cache->delete($k); } - private function getCacheKey($what, databox $db) + /** + * build a cache key to store es data, related to a db or not (if db is null) + * + * @param $what + * @param databox|null $db + * @return string + */ + private function getCacheKey($what, databox $db = null) { - return "es_db-" . $db->get_sbas_id() . '_' . $what; + return 'es' . ($db ? ('_db-'.$db->get_sbas_id()) : '') . '_' . $what; } } \ No newline at end of file