hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-15 22:13:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

- add : cache on es "tags"

Browse Source 
- removed : variable globalstructure construction (WITH_FIELDS, WITH_FLAGS...) (was unused)
- cleanup (stopwatches etc)
This commit is contained in:
jygaulier
2020-11-05 19:53:00 +01:00
parent 98e0d6ee72
commit cfd385498d
7 changed files with 74 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/Alchemy/Phrasea/Controller/Prod/QueryController.php
View File

@@ -67,10 +67,7 @@ class QueryController extends Controller
$word = StringHelper::crlfNormalize($word);
$options = SearchEngineOptions::fromRequest($this->app, $request);
$search_engine_structure = GlobalStructure::createFromDataboxes(
$this->app->getDataboxes(),
Structure::WITH_EVERYTHING & ~(Structure::STRUCTURE_WITH_FLAGS | Structure::FIELD_WITH_FACETS | Structure::FIELD_WITH_THESAURUS)
);
$search_engine_structure = $this->app['search_engine.global_structure'];
$query_context_factory = new QueryContextFactory(
$search_engine_structure,

47
lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php
View File

@@ -18,8 +18,6 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Index;
use Alchemy\Phrasea\SearchEngine\Elastic\IndexLocator;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryVisitor;
use Alchemy\Phrasea\SearchEngine\SearchEngineLogger;
use Alchemy\Phrasea\Exception\InvalidArgumentException;
use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine;
use Alchemy\Phrasea\SearchEngine\Elastic\Indexer;
use Alchemy\Phrasea\SearchEngine\Elastic\IndexerSubscriber;
@@ -30,10 +28,9 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\Escaper;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\FacetsResponse;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus;
use Alchemy\Phrasea\SearchEngine\SearchEngineStructure;
use Alchemy\Phrasea\Utilities\Stopwatch;
// use Alchemy\Phrasea\Utilities\Stopwatch;
use Elasticsearch\ClientBuilder;
use Hoa\Compiler;
use Hoa\File;
@@ -78,28 +75,29 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
});
$app['search_engine'] = $app->share(function ($app) {
$stopwatch = new Stopwatch("se");
// $type = $app['conf']->get(['main', 'search-engine', 'type']);
// if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) {
// throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type));
// }
// $stopwatch->lap("se.conf");
// $stopwatch = new Stopwatch("se");
/** @var ElasticsearchOptions $options */
$options = $app['elasticsearch.options'];
// $stopwatch->lap("se.options");
// $r = new ElasticSearchEngine(
// $app,
// $app['search_engine.global_structure'],
// $app['elasticsearch.client'],
// $app['query_context.factory'],
// $app['elasticsearch.facets_response.factory'],
// $app['elasticsearch.options']
// );
// $stopwatch->lap("se.new");
// $stopwatch->log();
// return $r;
$stopwatch->lap("se.options");
$r = new ElasticSearchEngine(
return new ElasticSearchEngine(
$app,
$app['search_engine.global_structure'],
$app['elasticsearch.client'],
$app['query_context.factory'],
$app['elasticsearch.facets_response.factory'],
$options
$app['elasticsearch.options']
);
$stopwatch->lap("se.new");
$stopwatch->log();
return $r;
});
$app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) {
@@ -107,12 +105,15 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
});
$app['search_engine.global_structure'] = $app->share(function (PhraseaApplication $app) {
$stopwatch = new Stopwatch("se.global_structure");
/** @var SearchEngineStructure $seStructure */
// $stopwatch = new Stopwatch("se.global_structure");
/** @var SearchEngineStructure $s */
$s = $app['search_engine.structure'];
$globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes());
$stopwatch->log();
return $globalStructure;
// $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes());
// $stopwatch->log();
// return $globalStructure;
return $s->getGlobalStructureFromDataboxes($app->getDataboxes());
});
$app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) {

11
lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php
View File

@@ -63,17 +63,16 @@ class ElasticSearchEngine implements SearchEngineInterface
/**
* @param Application $app
* @param GlobalStructure $structure2
* @param GlobalStructure $structure
* @param Client $client
* @param QueryContextFactory $context_factory
* @param Closure $facetsResponseFactory
* @param ElasticsearchOptions $options
*/
public function __construct(Application $app, GlobalStructure $structure2, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options)
public function __construct(Application $app, GlobalStructure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options)
{
$this->app = $app;
// $this->structure = $structure;
$this->structure = $structure2;
$this->structure = $structure;
$this->client = $client;
$this->context_factory = $context_factory;
$this->facetsResponseFactory = $facetsResponseFactory;
@@ -87,10 +86,6 @@ class ElasticSearchEngine implements SearchEngineInterface
*/
public function getStructure()
{
if (!($this->structure instanceof Structure)) {
$this->structure = call_user_func($this->structure);
}
return $this->structure;
}

41
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php
View File

@@ -4,11 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException;
use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper;
use Alchemy\Phrasea\Utilities\Stopwatch;
use Assert\Assertion;
use databox_field;
/**
@@ -53,37 +49,26 @@ class Field implements Typed
private $used_by_collections;
public static function createFromLegacyField(databox_field $field, $with = Structure::WITH_EVERYTHING)
public static function createFromLegacyField(databox_field $field)
{
// $stopwatch = new Stopwatch("createFromLegacyField.".$field->get_name());
$type = self::getTypeFromLegacy($field);
$databox = $field->get_databox();
// Thesaurus concept inference
$roots = null;
if(($with & Structure::FIELD_WITH_THESAURUS) && $type === FieldMapping::TYPE_STRING) {
// Thesaurus concept inference
$xpath = $field->get_tbranch();
if (!empty($xpath)) {
// $stopwatch->lap('before findConceptsByXPath');
$roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath);
//$stopwatch->lap('after findConceptsByXPath');
}
if($type === FieldMapping::TYPE_STRING && !empty($xpath = $field->get_tbranch())) {
$roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath);
}
$facet = self::FACET_DISABLED;
if($with & Structure::FIELD_WITH_FACETS) {
// Facet (enable + optional limit)
$facet = $field->getFacetValuesLimit();
if ($facet === databox_field::FACET_DISABLED) {
$facet = self::FACET_DISABLED;
} elseif ($facet === databox_field::FACET_NO_LIMIT) {
$facet = self::FACET_NO_LIMIT;
}
// Facet (enable + optional limit)
$facet = $field->getFacetValuesLimit();
if ($facet === databox_field::FACET_DISABLED) {
$facet = self::FACET_DISABLED;
} elseif ($facet === databox_field::FACET_NO_LIMIT) {
$facet = self::FACET_NO_LIMIT;
}
$r = new self($field->get_name(), $type, [
return new self($field->get_name(), $type, [
'databox_id' => $databox->get_sbas_id(),
'searchable' => $field->is_indexable(),
'private' => $field->isBusiness(),
@@ -92,9 +77,6 @@ class Field implements Typed
'generate_cterms' => $field->get_generate_cterms(),
'used_by_collections' => $databox->get_collection_unique_ids()
]);
// $stopwatch->log();
return $r;
}
private static function getTypeFromLegacy(databox_field $field)
@@ -278,4 +260,5 @@ class Field implements Typed
'used_by_collections' => $used_by_collections
]);
}
}

57
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php
View File

@@ -3,9 +3,6 @@
namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
use Alchemy\Phrasea\Utilities\Stopwatch;
use Assert\Assertion;
use DomainException;
final class GlobalStructure implements Structure
@@ -15,7 +12,6 @@ final class GlobalStructure implements Structure
*/
private $fields = array();
/**
* @var Field[][]
*/
@@ -46,59 +42,6 @@ final class GlobalStructure implements Structure
*/
private $metadata_tags = array();
/**
* @param \databox[] $databoxes
* @param int $what bitmask of what should be included in this structure, in fields, ...
*
* @return GlobalStructure
*/
public static function createFromDataboxes(array $databoxes, $what = self::WITH_EVERYTHING)
{
$fields = [];
$flags = [];
$stopwatch = new Stopwatch("globalStructure2");
foreach ($databoxes as $databox) {
if($what & self::STRUCTURE_WITH_FIELDS) {
foreach ($databox->get_meta_structure() as $fieldStructure) {
$fields[] = Field::createFromLegacyField($fieldStructure, $what);
}
}
if($what & self::STRUCTURE_WITH_FLAGS) {
foreach ($databox->getStatusStructure() as $status) {
$flags[] = Flag::createFromLegacyStatus($status);
}
}
}
$stopwatch->lap('loop0');
$r = new self($fields, $flags, MetadataHelper::createTags());
$stopwatch->log();
return $r;
}
/**
* @param \databox $databox
* @return GlobalStructure
*/
public static function unused_createFromDatabox(\databox $databox)
{
$fields = [];
$flags = [];
foreach ($databox->get_meta_structure() as $fieldStructure) {
$fields[] = Field::createFromLegacyField($fieldStructure);
}
foreach ($databox->getStatusStructure() as $status) {
$flags[] = Flag::createFromLegacyStatus($status);
}
return new self($fields, $flags, MetadataHelper::createTags());
}
/**
* GlobalStructure constructor.

6
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php
View File

@@ -12,12 +12,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
interface Structure
{
const STRUCTURE_WITH_FIELDS = 0x10;
const STRUCTURE_WITH_FLAGS = 0x20;
const FIELD_WITH_THESAURUS = 0x01;
const FIELD_WITH_FACETS = 0x02;
const WITH_EVERYTHING = 0xFF;
/**
* @return Field[]
*/

55
lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php
View File

@@ -9,8 +9,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\MetadataHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
use Alchemy\Phrasea\Utilities\Stopwatch;
// use Alchemy\Phrasea\Utilities\Stopwatch;
use databox;
class SearchEngineStructure
@@ -30,20 +29,18 @@ class SearchEngineStructure
/**
* @param databox[] $databoxes
* @param int $what bitmask of what should be included in this structure, in fields, ...
*
* @return GlobalStructure
*/
public function getGlobalStructureFromDataboxes(array $databoxes, $what = Structure::WITH_EVERYTHING)
public function getGlobalStructureFromDataboxes(array $databoxes)
{
$fields = [];
$flags = [];
$stopwatch = new Stopwatch("getGlobalStructureFromDataboxes");
// $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes");
foreach ($databoxes as $databox) {
// we will cache both FIELDS and FLAGS in the same entry :
// it's small data and $what seems always WITH_EVERYTHING
// we will cache both FIELDS and FLAGS in the same entry : it's small data
$k = $this->getCacheKey("FieldsAndFlags", $databox);
try {
$data = $this->cache->get($k);
@@ -57,7 +54,7 @@ class SearchEngineStructure
'flags' => []
];
foreach ($databox->get_meta_structure() as $fieldStructure) {
$data['fields'][] = Field::createFromLegacyField($fieldStructure, $what);
$data['fields'][] = Field::createFromLegacyField($fieldStructure);
}
foreach ($databox->getStatusStructure() as $status) {
$data['flags'][] = Flag::createFromLegacyStatus($status);
@@ -65,20 +62,29 @@ class SearchEngineStructure
$this->cache->save($k, $data);
}
if($what & Structure::STRUCTURE_WITH_FIELDS) {
$fields = array_merge($fields, $data['fields']);
}
if($what & Structure::STRUCTURE_WITH_FLAGS) {
$flags = array_merge($flags, $data['flags']);
}
$fields = array_merge($fields, $data['fields']);
$flags = array_merge($flags, $data['flags']);
}
$stopwatch->lap('loop0');
$r = new GlobalStructure($fields, $flags, MetadataHelper::createTags());
// $stopwatch->lap('loop0');
$stopwatch->log();
// tags does not depends on db
$k = $this->getCacheKey("Tags");
try {
$tags = $this->cache->get($k);
}
catch(\Exception $e) {
$tags = false;
}
if($tags === false) {
$this->cache->save($k, ($tags = MetadataHelper::createTags()));
// nb : tags is a hardcoded list, we don't need to clear this cache
}
return $r;
// $r = new GlobalStructure($fields, $flags, $tags);
// $stopwatch->log();
// return $r;
return new GlobalStructure($fields, $flags, $tags);
}
public function deleteFromCache($databox)
@@ -87,8 +93,15 @@ class SearchEngineStructure
$this->cache->delete($k);
}
private function getCacheKey($what, databox $db)
/**
* build a cache key to store es data, related to a db or not (if db is null)
*
* @param $what
* @param databox|null $db
* @return string
*/
private function getCacheKey($what, databox $db = null)
{
return "es_db-" . $db->get_sbas_id() . '_' . $what;
return 'es' . ($db ? ('_db-'.$db->get_sbas_id()) : '') . '_' . $what;
}
}