hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-15 14:03:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

- add : cache on es "tags"

Browse Source 
- removed : variable globalstructure construction (WITH_FIELDS, WITH_FLAGS...) (was unused)
- cleanup (stopwatches etc)
This commit is contained in:
jygaulier
2020-11-05 19:53:00 +01:00
parent 98e0d6ee72
commit cfd385498d
7 changed files with 74 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/Alchemy/Phrasea/Controller/Prod/QueryController.php
View File

@@ -67,10 +67,7 @@ class QueryController extends Controller
$word = StringHelper::crlfNormalize($word); $word = StringHelper::crlfNormalize($word);
$options = SearchEngineOptions::fromRequest($this->app, $request); $options = SearchEngineOptions::fromRequest($this->app, $request);
$search_engine_structure = GlobalStructure::createFromDataboxes( $search_engine_structure = $this->app['search_engine.global_structure'];
$this->app->getDataboxes(),
Structure::WITH_EVERYTHING & ~(Structure::STRUCTURE_WITH_FLAGS | Structure::FIELD_WITH_FACETS | Structure::FIELD_WITH_THESAURUS)
);
$query_context_factory = new QueryContextFactory( $query_context_factory = new QueryContextFactory(
$search_engine_structure, $search_engine_structure,

47
lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php
View File

@@ -18,8 +18,6 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Index;
use Alchemy\Phrasea\SearchEngine\Elastic\IndexLocator; use Alchemy\Phrasea\SearchEngine\Elastic\IndexLocator;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryVisitor; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryVisitor;
use Alchemy\Phrasea\SearchEngine\SearchEngineLogger; use Alchemy\Phrasea\SearchEngine\SearchEngineLogger;
use Alchemy\Phrasea\Exception\InvalidArgumentException;
use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine; use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine;
use Alchemy\Phrasea\SearchEngine\Elastic\Indexer; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer;
use Alchemy\Phrasea\SearchEngine\Elastic\IndexerSubscriber; use Alchemy\Phrasea\SearchEngine\Elastic\IndexerSubscriber;
@@ -30,10 +28,9 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\Escaper;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\FacetsResponse; use Alchemy\Phrasea\SearchEngine\Elastic\Search\FacetsResponse;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus;
use Alchemy\Phrasea\SearchEngine\SearchEngineStructure; use Alchemy\Phrasea\SearchEngine\SearchEngineStructure;
use Alchemy\Phrasea\Utilities\Stopwatch; // use Alchemy\Phrasea\Utilities\Stopwatch;
use Elasticsearch\ClientBuilder; use Elasticsearch\ClientBuilder;
use Hoa\Compiler; use Hoa\Compiler;
use Hoa\File; use Hoa\File;
@@ -78,28 +75,29 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
}); });
$app['search_engine'] = $app->share(function ($app) { $app['search_engine'] = $app->share(function ($app) {
$stopwatch = new Stopwatch("se"); // $stopwatch = new Stopwatch("se");
// $type = $app['conf']->get(['main', 'search-engine', 'type']);
// if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) {
// throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type));
// }
// $stopwatch->lap("se.conf");
/** @var ElasticsearchOptions $options */ // $stopwatch->lap("se.options");
$options = $app['elasticsearch.options']; // $r = new ElasticSearchEngine(
// $app,
// $app['search_engine.global_structure'],
// $app['elasticsearch.client'],
// $app['query_context.factory'],
// $app['elasticsearch.facets_response.factory'],
// $app['elasticsearch.options']
// );
// $stopwatch->lap("se.new");
// $stopwatch->log();
// return $r;
$stopwatch->lap("se.options"); return new ElasticSearchEngine(
$r = new ElasticSearchEngine(
$app, $app,
$app['search_engine.global_structure'], $app['search_engine.global_structure'],
$app['elasticsearch.client'], $app['elasticsearch.client'],
$app['query_context.factory'], $app['query_context.factory'],
$app['elasticsearch.facets_response.factory'], $app['elasticsearch.facets_response.factory'],
$options $app['elasticsearch.options']
); );
$stopwatch->lap("se.new");
$stopwatch->log();
return $r;
}); });
$app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) { $app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) {
@@ -107,12 +105,15 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
}); });
$app['search_engine.global_structure'] = $app->share(function (PhraseaApplication $app) { $app['search_engine.global_structure'] = $app->share(function (PhraseaApplication $app) {
$stopwatch = new Stopwatch("se.global_structure"); // $stopwatch = new Stopwatch("se.global_structure");
/** @var SearchEngineStructure $seStructure */ /** @var SearchEngineStructure $s */
$s = $app['search_engine.structure']; $s = $app['search_engine.structure'];
$globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes());
$stopwatch->log(); // $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes());
return $globalStructure; // $stopwatch->log();
// return $globalStructure;
return $s->getGlobalStructureFromDataboxes($app->getDataboxes());
}); });
$app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) {

11
lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php
View File

@@ -63,17 +63,16 @@ class ElasticSearchEngine implements SearchEngineInterface
/** /**
* @param Application $app * @param Application $app
* @param GlobalStructure $structure2 * @param GlobalStructure $structure
* @param Client $client * @param Client $client
* @param QueryContextFactory $context_factory * @param QueryContextFactory $context_factory
* @param Closure $facetsResponseFactory * @param Closure $facetsResponseFactory
* @param ElasticsearchOptions $options * @param ElasticsearchOptions $options
*/ */
public function __construct(Application $app, GlobalStructure $structure2, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options) public function __construct(Application $app, GlobalStructure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options)
{ {
$this->app = $app; $this->app = $app;
// $this->structure = $structure; $this->structure = $structure;
$this->structure = $structure2;
$this->client = $client; $this->client = $client;
$this->context_factory = $context_factory; $this->context_factory = $context_factory;
$this->facetsResponseFactory = $facetsResponseFactory; $this->facetsResponseFactory = $facetsResponseFactory;
@@ -87,10 +86,6 @@ class ElasticSearchEngine implements SearchEngineInterface
*/ */
public function getStructure() public function getStructure()
{ {
if (!($this->structure instanceof Structure)) {
$this->structure = call_user_func($this->structure);
}
return $this->structure; return $this->structure;
} }

41
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php
View File

@@ -4,11 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException;
use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper;
use Alchemy\Phrasea\Utilities\Stopwatch;
use Assert\Assertion;
use databox_field; use databox_field;
/** /**
@@ -53,37 +49,26 @@ class Field implements Typed
private $used_by_collections; private $used_by_collections;
public static function createFromLegacyField(databox_field $field, $with = Structure::WITH_EVERYTHING) public static function createFromLegacyField(databox_field $field)
{ {
// $stopwatch = new Stopwatch("createFromLegacyField.".$field->get_name());
$type = self::getTypeFromLegacy($field); $type = self::getTypeFromLegacy($field);
$databox = $field->get_databox(); $databox = $field->get_databox();
// Thesaurus concept inference
$roots = null; $roots = null;
if(($with & Structure::FIELD_WITH_THESAURUS) && $type === FieldMapping::TYPE_STRING) { if($type === FieldMapping::TYPE_STRING && !empty($xpath = $field->get_tbranch())) {
// Thesaurus concept inference $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath);
$xpath = $field->get_tbranch();
if (!empty($xpath)) {
// $stopwatch->lap('before findConceptsByXPath');
$roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath);
//$stopwatch->lap('after findConceptsByXPath');
}
} }
$facet = self::FACET_DISABLED; // Facet (enable + optional limit)
if($with & Structure::FIELD_WITH_FACETS) { $facet = $field->getFacetValuesLimit();
// Facet (enable + optional limit) if ($facet === databox_field::FACET_DISABLED) {
$facet = $field->getFacetValuesLimit(); $facet = self::FACET_DISABLED;
if ($facet === databox_field::FACET_DISABLED) { } elseif ($facet === databox_field::FACET_NO_LIMIT) {
$facet = self::FACET_DISABLED; $facet = self::FACET_NO_LIMIT;
} elseif ($facet === databox_field::FACET_NO_LIMIT) {
$facet = self::FACET_NO_LIMIT;
}
} }
$r = new self($field->get_name(), $type, [ return new self($field->get_name(), $type, [
'databox_id' => $databox->get_sbas_id(), 'databox_id' => $databox->get_sbas_id(),
'searchable' => $field->is_indexable(), 'searchable' => $field->is_indexable(),
'private' => $field->isBusiness(), 'private' => $field->isBusiness(),
@@ -92,9 +77,6 @@ class Field implements Typed
'generate_cterms' => $field->get_generate_cterms(), 'generate_cterms' => $field->get_generate_cterms(),
'used_by_collections' => $databox->get_collection_unique_ids() 'used_by_collections' => $databox->get_collection_unique_ids()
]); ]);
// $stopwatch->log();
return $r;
} }
private static function getTypeFromLegacy(databox_field $field) private static function getTypeFromLegacy(databox_field $field)
@@ -278,4 +260,5 @@ class Field implements Typed
'used_by_collections' => $used_by_collections 'used_by_collections' => $used_by_collections
]); ]);
} }
} }

57
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php
View File

@@ -3,9 +3,6 @@
namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
use Alchemy\Phrasea\Utilities\Stopwatch;
use Assert\Assertion;
use DomainException; use DomainException;
final class GlobalStructure implements Structure final class GlobalStructure implements Structure
@@ -15,7 +12,6 @@ final class GlobalStructure implements Structure
*/ */
private $fields = array(); private $fields = array();
/** /**
* @var Field[][] * @var Field[][]
*/ */
@@ -46,59 +42,6 @@ final class GlobalStructure implements Structure
*/ */
private $metadata_tags = array(); private $metadata_tags = array();
/**
* @param \databox[] $databoxes
* @param int $what bitmask of what should be included in this structure, in fields, ...
*
* @return GlobalStructure
*/
public static function createFromDataboxes(array $databoxes, $what = self::WITH_EVERYTHING)
{
$fields = [];
$flags = [];
$stopwatch = new Stopwatch("globalStructure2");
foreach ($databoxes as $databox) {
if($what & self::STRUCTURE_WITH_FIELDS) {
foreach ($databox->get_meta_structure() as $fieldStructure) {
$fields[] = Field::createFromLegacyField($fieldStructure, $what);
}
}
if($what & self::STRUCTURE_WITH_FLAGS) {
foreach ($databox->getStatusStructure() as $status) {
$flags[] = Flag::createFromLegacyStatus($status);
}
}
}
$stopwatch->lap('loop0');
$r = new self($fields, $flags, MetadataHelper::createTags());
$stopwatch->log();
return $r;
}
/**
* @param \databox $databox
* @return GlobalStructure
*/
public static function unused_createFromDatabox(\databox $databox)
{
$fields = [];
$flags = [];
foreach ($databox->get_meta_structure() as $fieldStructure) {
$fields[] = Field::createFromLegacyField($fieldStructure);
}
foreach ($databox->getStatusStructure() as $status) {
$flags[] = Flag::createFromLegacyStatus($status);
}
return new self($fields, $flags, MetadataHelper::createTags());
}
/** /**
* GlobalStructure constructor. * GlobalStructure constructor.

6
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php
View File

@@ -12,12 +12,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
interface Structure interface Structure
{ {
const STRUCTURE_WITH_FIELDS = 0x10;
const STRUCTURE_WITH_FLAGS = 0x20;
const FIELD_WITH_THESAURUS = 0x01;
const FIELD_WITH_FACETS = 0x02;
const WITH_EVERYTHING = 0xFF;
/** /**
* @return Field[] * @return Field[]
*/ */

55
lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php
View File

@@ -9,8 +9,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\MetadataHelper; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\MetadataHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure; // use Alchemy\Phrasea\Utilities\Stopwatch;
use Alchemy\Phrasea\Utilities\Stopwatch;
use databox; use databox;
class SearchEngineStructure class SearchEngineStructure
@@ -30,20 +29,18 @@ class SearchEngineStructure
/** /**
* @param databox[] $databoxes * @param databox[] $databoxes
* @param int $what bitmask of what should be included in this structure, in fields, ...
* *
* @return GlobalStructure * @return GlobalStructure
*/ */
public function getGlobalStructureFromDataboxes(array $databoxes, $what = Structure::WITH_EVERYTHING) public function getGlobalStructureFromDataboxes(array $databoxes)
{ {
$fields = []; $fields = [];
$flags = []; $flags = [];
$stopwatch = new Stopwatch("getGlobalStructureFromDataboxes"); // $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes");
foreach ($databoxes as $databox) { foreach ($databoxes as $databox) {
// we will cache both FIELDS and FLAGS in the same entry : // we will cache both FIELDS and FLAGS in the same entry : it's small data
// it's small data and $what seems always WITH_EVERYTHING
$k = $this->getCacheKey("FieldsAndFlags", $databox); $k = $this->getCacheKey("FieldsAndFlags", $databox);
try { try {
$data = $this->cache->get($k); $data = $this->cache->get($k);
@@ -57,7 +54,7 @@ class SearchEngineStructure
'flags' => [] 'flags' => []
]; ];
foreach ($databox->get_meta_structure() as $fieldStructure) { foreach ($databox->get_meta_structure() as $fieldStructure) {
$data['fields'][] = Field::createFromLegacyField($fieldStructure, $what); $data['fields'][] = Field::createFromLegacyField($fieldStructure);
} }
foreach ($databox->getStatusStructure() as $status) { foreach ($databox->getStatusStructure() as $status) {
$data['flags'][] = Flag::createFromLegacyStatus($status); $data['flags'][] = Flag::createFromLegacyStatus($status);
@@ -65,20 +62,29 @@ class SearchEngineStructure
$this->cache->save($k, $data); $this->cache->save($k, $data);
} }
if($what & Structure::STRUCTURE_WITH_FIELDS) { $fields = array_merge($fields, $data['fields']);
$fields = array_merge($fields, $data['fields']); $flags = array_merge($flags, $data['flags']);
}
if($what & Structure::STRUCTURE_WITH_FLAGS) {
$flags = array_merge($flags, $data['flags']);
}
} }
$stopwatch->lap('loop0'); // $stopwatch->lap('loop0');
$r = new GlobalStructure($fields, $flags, MetadataHelper::createTags());
$stopwatch->log(); // tags does not depends on db
$k = $this->getCacheKey("Tags");
try {
$tags = $this->cache->get($k);
}
catch(\Exception $e) {
$tags = false;
}
if($tags === false) {
$this->cache->save($k, ($tags = MetadataHelper::createTags()));
// nb : tags is a hardcoded list, we don't need to clear this cache
}
return $r; // $r = new GlobalStructure($fields, $flags, $tags);
// $stopwatch->log();
// return $r;
return new GlobalStructure($fields, $flags, $tags);
} }
public function deleteFromCache($databox) public function deleteFromCache($databox)
@@ -87,8 +93,15 @@ class SearchEngineStructure
$this->cache->delete($k); $this->cache->delete($k);
} }
private function getCacheKey($what, databox $db) /**
* build a cache key to store es data, related to a db or not (if db is null)
*
* @param $what
* @param databox|null $db
* @return string
*/
private function getCacheKey($what, databox $db = null)
{ {
return "es_db-" . $db->get_sbas_id() . '_' . $what; return 'es' . ($db ? ('_db-'.$db->get_sbas_id()) : '') . '_' . $what;
} }
} }