- add some simple cache

- disable check of conf/searchengine/type - faster construction of concept-paths - pass strucure to es as factory (useless, to be reverted) - add stopwatch (log into file, disabled for now)
2025-10-15 22:13:13 +00:00 · 2020-10-30 10:59:41 +01:00
parent 7abd2399b4
commit e36b0785ee
9 changed files with 160 additions and 63 deletions
--- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php
+++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php
@@ -32,6 +32,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus;
+use Alchemy\Phrasea\Utilities\Stopwatch;
 use Elasticsearch\ClientBuilder;
 use Hoa\Compiler;
 use Hoa\File;
@@ -72,29 +73,42 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
            return new SearchEngineLogger($app);
        });

+        $app['search_engine.structure.factory'] = $app->protect(function () use ($app) {
+            return $app['search_engine.structure'];
+        });
+
        $app['search_engine'] = $app->share(function ($app) {
-            $type = $app['conf']->get(['main', 'search-engine', 'type']);
-            if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) {
-                throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type));
-            }
+            $stopwatch = new Stopwatch("se");
+            // $type = $app['conf']->get(['main', 'search-engine', 'type']);
+            // if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) {
+            //     throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type));
+            // }
+            // $stopwatch->lap("se.conf");

            /** @var ElasticsearchOptions $options */
            $options = $app['elasticsearch.options'];

-            return new ElasticSearchEngine(
+            $r = new ElasticSearchEngine(
                $app,
+                // $app['search_engine.structure.factory'],
                $app['search_engine.structure'],
                $app['elasticsearch.client'],
                $app['query_context.factory'],
                $app['elasticsearch.facets_response.factory'],
                $options
            );
+            $stopwatch->lap("se.new");
+            $stopwatch->log();
+            return $r;
        });

        $app['search_engine.structure'] = $app->share(function (\Alchemy\Phrasea\Application $app) {
+            $stopwatch = new Stopwatch("se.structure");
            $databoxes = $app->getDataboxes();
-
-            return GlobalStructure::createFromDataboxes($databoxes);
+            $stopwatch ->lap('get db');
+            $r = GlobalStructure::createFromDataboxes($databoxes);
+            $stopwatch->log();
+            return $r;
        });

        $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) {
@@ -268,7 +282,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface

        $app['query_context.factory'] = $app->share(function ($app) {
            return new QueryContextFactory(
-                $app['search_engine.structure'],
+                $app['search_engine.structure.factory'],
                array_keys($app['locales.available']),
                $app['locale']
            );
--- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php
+++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php
@@ -61,13 +61,13 @@ class ElasticSearchEngine implements SearchEngineInterface

    /**
     * @param Application $app
-     * @param Structure $structure
+     * @param Structure|callable $structure
     * @param Client $client
     * @param QueryContextFactory $context_factory
-     * @param callable $facetsResponseFactory
+     * @param Closure $facetsResponseFactory
     * @param ElasticsearchOptions $options
     */
-    public function __construct(Application $app, Structure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options)
+    public function __construct(Application $app, $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options)
    {
        $this->app = $app;
        $this->structure = $structure;
@@ -77,7 +77,18 @@ class ElasticSearchEngine implements SearchEngineInterface
        $this->options = $options;

        $this->indexName = $options->getIndexName();
+    }

+    /**
+     * @return Structure
+     */
+    public function getStructure()
+    {
+        if (!($this->structure instanceof Structure)) {
+            $this->structure = call_user_func($this->structure);
+        }
+
+        return $this->structure;
    }

    public function getIndexName()
@@ -129,7 +140,7 @@ class ElasticSearchEngine implements SearchEngineInterface
    public function getAvailableDateFields()
    {
        // TODO Use limited structure
-        return array_keys($this->structure->getDateFields());
+        return array_keys($this->getStructure()->getDateFields());
    }

    /**
--- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php
+++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php
@@ -279,11 +279,11 @@ class Indexer
        );

        // Optimize index
-        $this->client->indices()->optimize(
-            [
-                'index' => $this->index->getName()
-            ]
-        );
+//        $this->client->indices()->optimize(
+//            [
+//                'index' => $this->index->getName()
+//            ]
+//        );

        $event = $stopwatch->stop('populate');

--- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php
+++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php
@@ -10,18 +10,37 @@ class QueryContextFactory
 {
    private $structure;

-    public function __construct(Structure $structure, array $locales, $current_locale)
+    /**
+     * QueryContextFactory constructor.
+     * @param Structure|callable $structure
+     * @param array $locales
+     * @param $current_locale
+     */
+    public function __construct($structure, array $locales, $current_locale)
    {
        $this->structure = $structure;
        $this->locales = $locales;
        $this->current_locale = $current_locale;
    }

+    /**
+     * @return Structure
+     */
+    public function getStructure()
+    {
+        if (!($this->structure instanceof Structure)) {
+            $this->structure = call_user_func($this->structure);
+        }
+
+        return $this->structure;
+    }
+
+
    public function createContext(SearchEngineOptions $options = null)
    {
        $structure = $options
            ? $this->getLimitedStructure($options)
-            : $this->structure;
+            : $this->getStructure();

        $context = new QueryContext($options, $structure, $this->locales, $this->current_locale);

@@ -46,6 +65,6 @@ class QueryContextFactory

    public function getLimitedStructure(SearchEngineOptions $options)
    {
-        return new LimitedStructure($this->structure, $options);
+        return new LimitedStructure($this->getStructure(), $options);
    }
 }
--- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php
+++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php
@@ -108,26 +108,25 @@ class Field implements Typed
    {
        $this->name = (string) $name;
        $this->type = $type;
-        $this->databox_id      = \igorw\get_in($options, ['databox_id'], 0);
-        $this->is_searchable   = \igorw\get_in($options, ['searchable'], true);
-        $this->is_private      = \igorw\get_in($options, ['private'], false);
-        $this->facet           = \igorw\get_in($options, ['facet']);
-        $this->thesaurus_roots = \igorw\get_in($options, ['thesaurus_roots'], null);
-        $this->generate_cterms  = \igorw\get_in($options, ['generate_cterms'], false);
-        $this->used_by_collections = \igorw\get_in($options, ['used_by_collections'], []);
-
-        Assertion::boolean($this->is_searchable);
-        Assertion::boolean($this->is_private);
-
-        if ($this->facet !== self::FACET_DISABLED) {
-            Assertion::integer($this->facet);
+        if(1) {
+            $this->databox_id = \igorw\get_in($options, ['databox_id'], 0);
+            $this->is_searchable = \igorw\get_in($options, ['searchable'], true);
+            $this->is_private = \igorw\get_in($options, ['private'], false);
+            $this->facet = \igorw\get_in($options, ['facet']);
+            $this->thesaurus_roots = \igorw\get_in($options, ['thesaurus_roots'], null);
+            $this->generate_cterms = \igorw\get_in($options, ['generate_cterms'], false);
+            $this->used_by_collections = \igorw\get_in($options, ['used_by_collections'], []);
        }
-
-        if ($this->thesaurus_roots !== null) {
-            Assertion::allIsInstanceOf($this->thesaurus_roots, Concept::class);
+        else {
+            // todo: this is faster code, but need to fix unit-tests to pass all options
+            $this->databox_id = $options['databox_id'];
+            $this->is_searchable = $options['searchable'];
+            $this->is_private = $options['private'];
+            $this->facet = $options['facet'];
+            $this->thesaurus_roots = $options['thesaurus_roots'];
+            $this->generate_cterms = $options['generate_cterms'];
+            $this->used_by_collections = $options['used_by_collections'];
        }
-
-        Assertion::allScalar($this->used_by_collections);
    }

    public function withOptions(array $options)
--- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php
+++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php
@@ -4,6 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;

 use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
 use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
+use Alchemy\Phrasea\Utilities\Stopwatch;
 use Assert\Assertion;
 use DomainException;

@@ -56,6 +57,7 @@ final class GlobalStructure implements Structure
        $fields = [];
        $flags = [];

+        $stopwatch = new Stopwatch("globalStructure");
        foreach ($databoxes as $databox) {
            if($what & self::STRUCTURE_WITH_FIELDS) {
                foreach ($databox->get_meta_structure() as $fieldStructure) {
@@ -69,8 +71,12 @@ final class GlobalStructure implements Structure
                }
            }
        }
+        $stopwatch->lap('loop0');
+        $r = new self($fields, $flags, MetadataHelper::createTags());

-        return new self($fields, $flags, MetadataHelper::createTags());
+        $stopwatch->log();
+
+        return $r;
    }

    /**
--- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Thesaurus/Helper.php
+++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Thesaurus/Helper.php
@@ -31,26 +31,54 @@ class Helper
        $nodes = $xpath->query($expression);
        $concepts = [];

+        /** @var DOMElement $node */
        foreach ($nodes as $node) {
-            $me_and_parents = array_merge([$node], self::getElementAncestors($node));
+            if(1) {
+                $me_and_parents = array_merge([$node], self::getElementAncestors($node));

-            $path_segments = [];
+                $path_segments = [];

-            foreach ($me_and_parents as $me_or_parent) {
-                if (!Navigator::isConcept($me_or_parent)) {
-                    // Silently skips invalid targeted nodes
-                    break;
+                foreach ($me_and_parents as $me_or_parent) {
+                    if (!Navigator::isConcept($me_or_parent)) {
+                        // Silently skips invalid targeted nodes
+                        break;
+                    }
+
+                    $path_segments[] = self::conceptPathSegment($me_or_parent);
                }

-                $path_segments[] = self::conceptPathSegment($me_or_parent);
+                // Concept paths are have databox identifier at root level
+                $concepts[] = new Concept(sprintf(
+                    '/%d/%s',
+                    $databox->get_sbas_id(),
+                    implode('/', array_reverse($path_segments))
+                ));
+            }
+            else {
+                $path = '';
+                // go up thru parents
+                while ($node) {
+                    $v = null;
+                    for ($n = $node->firstChild; $n; $n = $n->nextSibling) {
+                        if ($n->nodeType === XML_ELEMENT_NODE && $n->nodeName === 'sy') {
+                            if ($v === null) {
+                                $v = $n->getAttribute('v');
+                                continue;
+                            }
+                            if ($n->getAttribute('lng') === 'en') {
+                                $v = $n->getAttribute('v');
+                                break;
+                            }
+                        }
+                    }
+                    if ($v !== null) {
+                        $path = '/' . $v . $path;
+                    }
+                    $node = $node->parentNode;
+                }
+                $path = '/' . $databox->get_sbas_id() . $path;
+                $concepts[] = new Concept($path);
            }
-
-            // Concept paths are have databox identifier at root level
-            $concepts[] = new Concept(sprintf(
-                '/%d/%s',
-                $databox->get_sbas_id(),
-                implode('/', array_reverse($path_segments))
-            ));
        }

        return $concepts;
--- a/lib/Alchemy/Phrasea/Utilities/Stopwatch.php
+++ b/lib/Alchemy/Phrasea/Utilities/Stopwatch.php
@@ -62,9 +62,19 @@ class Stopwatch
     */
    public function getLapses()
    {
+        $this->stop();
+        $this->_lapses['_total'] = ($this->_stop - $this->_start) * 1000.0;
        return $this->_lapses;
    }

+    public function log()
+    {
+        // file_put_contents('/var/alchemy/Phraseanet/tmp/phraseanet.log',
+        //    $this->_name . "\n" . var_export($this->getLapses(), true) . "\n\n",
+        //    FILE_APPEND
+        // );
+    }
+
    /**
     * return all lapses as a "Server-Timing" header value
     *
--- a/lib/classes/databox.php
+++ b/lib/classes/databox.php
@@ -581,13 +581,17 @@ class databox extends base implements ThumbnailedElement
     */
    public function get_collection_unique_ids()
    {
-        $collectionsIds = [];
+        static $_r = null;

-        foreach ($this->get_collections() as $collection) {
-            $collectionsIds[] = $collection->get_base_id();
+        if($_r === null) {
+            $_r = [];
+
+            foreach ($this->get_collections() as $collection) {
+                $_r[] = $collection->get_base_id();
+            }
        }

-        return $collectionsIds;
+        return $_r;
    }

    /**
@@ -595,13 +599,19 @@ class databox extends base implements ThumbnailedElement
     */
    public function get_collections()
    {
-        /** @var CollectionRepositoryRegistry $repositoryRegistry */
-        $repositoryRegistry = $this->app['repo.collections-registry'];
-        $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id());
+        static $_r = null;

-        return array_filter($repository->findAll(), function (collection $collection) {
-            return $collection->is_active();
-        });
+        if($_r === null) {
+            /** @var CollectionRepositoryRegistry $repositoryRegistry */
+            $repositoryRegistry = $this->app['repo.collections-registry'];
+            $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id());
+
+            $_r = array_filter($repository->findAll(), function (collection $collection) {
+                return $collection->is_active();
+            });
+        }
+
+        return $_r;
    }

    /**