Merge pull request #3637 from alchemy-fr/PHRAS-3174_api-v3-records_4.1

PHRAS-3174 Merge api v3,  structure are now in redis cache, adding searchraw endpoint, without permalink.

lib/Alchemy/Phrasea/Application.php

@@ -25,6 +25,7 @@ use Alchemy\Phrasea\Core\Event\Subscriber\FeedEntrySubscriber;
 use Alchemy\Phrasea\Core\Event\Subscriber\LazaretSubscriber;
 use Alchemy\Phrasea\Core\Event\Subscriber\PhraseaInstallSubscriber;
 use Alchemy\Phrasea\Core\Event\Subscriber\RegistrationSubscriber;
+use Alchemy\Phrasea\Core\Event\Subscriber\StructureChangeSubscriber;
 use Alchemy\Phrasea\Core\Event\Subscriber\ValidationSubscriber;
 use Alchemy\Phrasea\Core\Event\Subscriber\WebhookUserEventSubscriber;
 use Alchemy\Phrasea\Core\MetaProvider\DatabaseMetaProvider;

lib/Alchemy/Phrasea/Controller/Admin/RootController.php

@@ -11,6 +11,9 @@
 namespace Alchemy\Phrasea\Controller\Admin;
 
 use Alchemy\Phrasea\Controller\Controller;
+use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvents;
+use Alchemy\Phrasea\Core\Event\Record\Structure\StatusBitEvent;
+use Alchemy\Phrasea\Core\Event\Record\Structure\StatusBitUpdatedEvent;
 use Alchemy\Phrasea\Exception\SessionNotFound;
 use Alchemy\Phrasea\Status\StatusStructureProviderInterface;
 use Symfony\Component\HttpFoundation\Request;
@@ -234,6 +237,8 @@ class RootController extends Controller
             $error = true;
         }
 
+        $this->dispatchEvent(RecordStructureEvents::STATUS_BIT_DELETED, new StatusBitUpdatedEvent($databox, $bit, []));
+
         return $this->app->json(['success' => !$error]);
     }
     
@@ -350,9 +355,16 @@ class RootController extends Controller
             }
         }
 
+        $this->dispatchEvent(RecordStructureEvents::STATUS_BIT_UPDATED, new StatusBitUpdatedEvent($databox, $bit, []));
+
         return $this->app->redirectPath('database_display_statusbit', ['databox_id' => $databox_id, 'success' => 1]);
     }
-    
+
+    private function dispatchEvent($eventName, StatusBitEvent $event = null)
+    {
+        $this->app['dispatcher']->dispatch($eventName, $event);
+    }
+
     /**
      * @param string $section
      * @return array

lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php

@@ -110,7 +110,7 @@ class SearchEngineController extends Controller
     private function getConfigurationForm(ElasticsearchOptions $options)
     {
         /** @var GlobalStructure $g */
-        $g = $this->app['search_engine.structure'];
+        $g = $this->app['search_engine.global_structure'];
 
         return $this->app->form(new ElasticsearchSettingsFormType($g, $options), $options, [
             'action' => $this->app->url('admin_searchengine_form'),

lib/Alchemy/Phrasea/Controller/Api/V3/V3SearchRawController.php

@@ -6,6 +6,7 @@ use Alchemy\Phrasea\Application\Helper\DispatcherAware;
 use Alchemy\Phrasea\Application\Helper\JsonBodyAware;
 use Alchemy\Phrasea\Controller\Api\Result;
 use Alchemy\Phrasea\Controller\Controller;
+use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine;
 use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
 use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
 use Alchemy\Phrasea\Utilities\Stopwatch;
@@ -60,7 +61,8 @@ class V3SearchRawController extends Controller
     }
 
     /**
-     * @return SearchEngineInterface
+     * // return SearchEngineInterface
+     * @return ElasticSearchEngine
      */
     private function getSearchEngine()
     {

lib/Alchemy/Phrasea/Controller/Prod/QueryController.php

@@ -67,10 +67,7 @@ class QueryController extends Controller
         $word = StringHelper::crlfNormalize($word);
         $options = SearchEngineOptions::fromRequest($this->app, $request);
 
-        $search_engine_structure = GlobalStructure::createFromDataboxes(
+        $search_engine_structure = $this->app['search_engine.global_structure'];
-            $this->app->getDataboxes(),
-            Structure::WITH_EVERYTHING & ~(Structure::STRUCTURE_WITH_FLAGS | Structure::FIELD_WITH_FACETS | Structure::FIELD_WITH_THESAURUS)
-        );
 
         $query_context_factory = new QueryContextFactory(
             $search_engine_structure,

lib/Alchemy/Phrasea/Core/Event/Subscriber/StructureChangeSubscriber.php

@@ -0,0 +1,48 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Core\Event\Subscriber;
+
+use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvent;
+use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvents;
+use Alchemy\Phrasea\SearchEngine\SearchEngineStructure;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+class StructureChangeSubscriber implements EventSubscriberInterface
+{
+    private $searchEngineStructure;
+
+    public function __construct(SearchEngineStructure $se1)
+    {
+        $this->searchEngineStructure = $se1;
+    }
+
+    public static function getSubscribedEvents()
+    {
+        return [
+            RecordStructureEvents::FIELD_UPDATED => 'onStructureChange',
+            RecordStructureEvents::FIELD_DELETED => 'onStructureChange',
+            RecordStructureEvents::STATUS_BIT_UPDATED => 'onStructureChange',
+            RecordStructureEvents::STATUS_BIT_DELETED => 'onStructureChange',
+        ];
+    }
+
+    /**
+     * clears the cached translated versions of phr fields;sb to es fields;sb (createFromLegacy...)
+     * this was cached per db
+     *
+     * @param RecordStructureEvent $event
+     */
+    public function onStructureChange(RecordStructureEvent $event)
+    {
+        $this->searchEngineStructure->deleteFromCache($event->getDatabox());
+    }
+}

lib/Alchemy/Phrasea/Core/Provider/PhraseaEventServiceProvider.php

@@ -11,6 +11,7 @@
 
 namespace Alchemy\Phrasea\Core\Provider;
 
+use Alchemy\Phrasea\Core\Event\Subscriber\StructureChangeSubscriber;
 use Alchemy\Phrasea\Core\LazyLocator;
 use Alchemy\Phrasea\Core\Event\Subscriber\ContentNegotiationSubscriber;
 use Alchemy\Phrasea\Core\Event\Subscriber\CookiesDisablerSubscriber;
@@ -69,6 +70,11 @@ class PhraseaEventServiceProvider implements ServiceProviderInterface
                 $dispatcher->addSubscriber($app['phraseanet.session-manager-subscriber']);
                 $dispatcher->addSubscriber($app['phraseanet.record-edit-subscriber']);
 
+                // if phr is not yet installed, don't use non-existing service 'search_engine.structure'
+                if($app->offsetExists('search_engine.structure')) {
+                    $dispatcher->addSubscriber(new StructureChangeSubscriber($app['search_engine.structure']));
+                }
+
                 return $dispatcher;
             })
         );

lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php

@@ -18,8 +18,6 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Index;
 use Alchemy\Phrasea\SearchEngine\Elastic\IndexLocator;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryVisitor;
 use Alchemy\Phrasea\SearchEngine\SearchEngineLogger;
-use Alchemy\Phrasea\Exception\InvalidArgumentException;
-use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
 use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer;
 use Alchemy\Phrasea\SearchEngine\Elastic\IndexerSubscriber;
@@ -30,8 +28,9 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\Escaper;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\FacetsResponse;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler;
-use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus;
+use Alchemy\Phrasea\SearchEngine\SearchEngineStructure;
+// use Alchemy\Phrasea\Utilities\Stopwatch;
 use Elasticsearch\ClientBuilder;
 use Hoa\Compiler;
 use Hoa\File;
@@ -43,11 +42,14 @@ use Silex\ServiceProviderInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 use Symfony\Component\HttpKernel\KernelEvents;
 
+use \Alchemy\Phrasea\Application as PhraseaApplication;
+
 
 class SearchEngineServiceProvider implements ServiceProviderInterface
 {
     public function register(Application $app)
     {
+        /** @var PhraseaApplication $app */
         $this->registerElasticSearchClient($app);
         $this->registerQueryParser($app);
         $this->registerIndexer($app);
@@ -59,46 +61,63 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
     }
 
     /**
-     * @param Application $app
+     * @param  PhraseaApplication $app
      * @return Application
      */
-    private function registerSearchEngine(Application $app)
+    private function registerSearchEngine(PhraseaApplication $app)
     {
         $app['phraseanet.SE'] = function ($app) {
             return $app['search_engine'];
         };
 
-        $app['phraseanet.SE.logger'] = $app->share(function (Application $app) {
+        $app['phraseanet.SE.logger'] = $app->share(function (PhraseaApplication $app) {
             return new SearchEngineLogger($app);
         });
 
         $app['search_engine'] = $app->share(function ($app) {
-            $type = $app['conf']->get(['main', 'search-engine', 'type']);
+//            $stopwatch = new Stopwatch("se");
-            if ($type !== SearchEngineInterface::TYPE_ELASTICSEARCH) {
-                throw new InvalidArgumentException(sprintf('Invalid search engine type "%s".', $type));
-            }
 
-            /** @var ElasticsearchOptions $options */
+//            $stopwatch->lap("se.options");
-            $options = $app['elasticsearch.options'];
+//            $r = new ElasticSearchEngine(
+//                $app,
+//                $app['search_engine.global_structure'],
+//                $app['elasticsearch.client'],
+//                $app['query_context.factory'],
+//                $app['elasticsearch.facets_response.factory'],
+//                $app['elasticsearch.options']
+//            );
+//            $stopwatch->lap("se.new");
+//            $stopwatch->log();
+//            return $r;
 
             return new ElasticSearchEngine(
                 $app,
-                $app['search_engine.structure'],
+                $app['search_engine.global_structure'],
                 $app['elasticsearch.client'],
                 $app['query_context.factory'],
                 $app['elasticsearch.facets_response.factory'],
-                $options
+                $app['elasticsearch.options']
             );
         });
 
-        $app['search_engine.structure'] = $app->share(function (\Alchemy\Phrasea\Application $app) {
+        $app['search_engine.structure'] = $app->share(function (PhraseaApplication $app) {
-            $databoxes = $app->getDataboxes();
+            return new SearchEngineStructure($app['cache']);
-
-            return GlobalStructure::createFromDataboxes($databoxes);
         });
 
-        $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) {
+        $app['search_engine.global_structure'] = $app->share(function (PhraseaApplication $app) {
-            return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.structure']);
+//            $stopwatch = new Stopwatch("se.global_structure");
+            /** @var SearchEngineStructure $s */
+            $s = $app['search_engine.structure'];
+
+//            $globalStructure = $s->getGlobalStructureFromDataboxes($app->getDataboxes());
+//            $stopwatch->log();
+//            return $globalStructure;
+
+            return $s->getGlobalStructureFromDataboxes($app->getDataboxes());
+        });
+
+       $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) {
+            return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.global_structure']);
         });
 
         return $app;
@@ -116,7 +135,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
         });
 
         $app['elasticsearch.index.record'] = $app->share(function ($app) {
-            return new Indexer\RecordIndex($app['search_engine.structure'], array_keys($app['locales.available']));
+            return new Indexer\RecordIndex($app['search_engine.global_structure'], array_keys($app['locales.available']));
         });
 
         $app['elasticsearch.index.term'] = $app->share(function ($app) {
@@ -150,7 +169,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
                 $app['elasticsearch.record_helper'],
                 $app['elasticsearch.options'],
                 $app,
-                'search_engine.structure',
+                'search_engine.global_structure',
                 'thesaurus'
             );
         });
@@ -268,7 +287,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
 
         $app['query_context.factory'] = $app->share(function ($app) {
             return new QueryContextFactory(
-                $app['search_engine.structure'],
+                $app['search_engine.global_structure'],
                 array_keys($app['locales.available']),
                 $app['locale']
             );
@@ -289,7 +308,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
         });
 
         $app['query_visitor.factory'] = $app->protect(function () use ($app) {
-            return new QueryVisitor($app['search_engine.structure']);
+            return new QueryVisitor($app['search_engine.global_structure']);
         });
 
         $app['query_compiler'] = $app->share(function ($app) {

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php

@@ -21,11 +21,13 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field AS ESField;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
 use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
 use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
 use Alchemy\Phrasea\SearchEngine\SearchEngineResult;
 use Alchemy\Phrasea\Exception\RuntimeException;
+use Alchemy\Phrasea\SearchEngine\SearchEngineStructure;
 use Alchemy\Phrasea\Utilities\Stopwatch;
 use Closure;
 use Doctrine\Common\Collections\ArrayCollection;
@@ -61,13 +63,13 @@ class ElasticSearchEngine implements SearchEngineInterface
 
     /**
      * @param Application $app
-     * @param Structure $structure
+     * @param GlobalStructure $structure
      * @param Client $client
      * @param QueryContextFactory $context_factory
-     * @param callable $facetsResponseFactory
+     * @param Closure $facetsResponseFactory
      * @param ElasticsearchOptions $options
      */
-    public function __construct(Application $app, Structure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options)
+    public function __construct(Application $app, GlobalStructure $structure, Client $client, QueryContextFactory $context_factory, Closure $facetsResponseFactory, ElasticsearchOptions $options)
     {
         $this->app = $app;
         $this->structure = $structure;
@@ -77,7 +79,14 @@ class ElasticSearchEngine implements SearchEngineInterface
         $this->options = $options;
 
         $this->indexName = $options->getIndexName();
+    }
 
+    /**
+     * @return Structure
+     */
+    public function getStructure()
+    {
+        return $this->structure;
     }
 
     public function getIndexName()
@@ -129,7 +138,7 @@ class ElasticSearchEngine implements SearchEngineInterface
     public function getAvailableDateFields()
     {
         // TODO Use limited structure
-        return array_keys($this->structure->getDateFields());
+        return array_keys($this->getStructure()->getDateFields());
     }
 
     /**

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php

@@ -279,11 +279,11 @@ class Indexer
         );
 
         // Optimize index
-        $this->client->indices()->optimize(
+//        $this->client->indices()->optimize(
-            [
+//            [
-                'index' => $this->index->getName()
+//                'index' => $this->index->getName()
-            ]
+//            ]
-        );
+//        );
 
         $event = $stopwatch->stop('populate');
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/FlagHydrator.php

@@ -20,6 +20,9 @@ use databox;
 
 class FlagHydrator implements HydratorInterface
 {
+    /** @var array */
+    private $field_names_map;
+
     public function __construct(Structure $structure, databox $databox)
     {
         $this->field_names_map = self::buildFieldNamesMap($structure, $databox);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php

@@ -10,18 +10,37 @@ class QueryContextFactory
 {
     private $structure;
 
-    public function __construct(Structure $structure, array $locales, $current_locale)
+    /**
+     * QueryContextFactory constructor.
+     * @param Structure|callable $structure
+     * @param array $locales
+     * @param $current_locale
+     */
+    public function __construct($structure, array $locales, $current_locale)
     {
         $this->structure = $structure;
         $this->locales = $locales;
         $this->current_locale = $current_locale;
     }
 
+    /**
+     * @return Structure
+     */
+    public function getStructure()
+    {
+        if (!($this->structure instanceof Structure)) {
+            $this->structure = call_user_func($this->structure);
+        }
+
+        return $this->structure;
+    }
+
+
     public function createContext(SearchEngineOptions $options = null)
     {
         $structure = $options
             ? $this->getLimitedStructure($options)
-            : $this->structure;
+            : $this->getStructure();
 
         $context = new QueryContext($options, $structure, $this->locales, $this->current_locale);
 
@@ -46,6 +65,6 @@ class QueryContextFactory
 
     public function getLimitedStructure(SearchEngineOptions $options)
     {
-        return new LimitedStructure($this->structure, $options);
+        return new LimitedStructure($this->getStructure(), $options);
     }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php

@@ -4,10 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException;
 use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
-use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
-use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper;
-use Assert\Assertion;
 use databox_field;
 
 /**
@@ -52,29 +49,23 @@ class Field implements Typed
 
     private $used_by_collections;
 
-    public static function createFromLegacyField(databox_field $field, $with = Structure::WITH_EVERYTHING)
+    public static function createFromLegacyField(databox_field $field)
     {
         $type = self::getTypeFromLegacy($field);
         $databox = $field->get_databox();
 
+        // Thesaurus concept inference
         $roots = null;
-        if(($with & Structure::FIELD_WITH_THESAURUS) && $type === FieldMapping::TYPE_STRING) {
+        if($type === FieldMapping::TYPE_STRING && !empty($xpath = $field->get_tbranch())) {
-            // Thesaurus concept inference
+            $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath);
-            $xpath = $field->get_tbranch();
-            if (!empty($xpath)) {
-                $roots = ThesaurusHelper::findConceptsByXPath($databox, $xpath);
-            }
         }
 
-        $facet = self::FACET_DISABLED;
+        // Facet (enable + optional limit)
-        if($with & Structure::FIELD_WITH_FACETS) {
+        $facet = $field->getFacetValuesLimit();
-            // Facet (enable + optional limit)
+        if ($facet === databox_field::FACET_DISABLED) {
-            $facet = $field->getFacetValuesLimit();
+            $facet = self::FACET_DISABLED;
-            if ($facet === databox_field::FACET_DISABLED) {
+        } elseif ($facet === databox_field::FACET_NO_LIMIT) {
-                $facet = self::FACET_DISABLED;
+            $facet = self::FACET_NO_LIMIT;
-            } elseif ($facet === databox_field::FACET_NO_LIMIT) {
-                $facet = self::FACET_NO_LIMIT;
-            }
         }
 
         return new self($field->get_name(), $type, [
@@ -108,26 +99,25 @@ class Field implements Typed
     {
         $this->name = (string) $name;
         $this->type = $type;
-        $this->databox_id      = \igorw\get_in($options, ['databox_id'], 0);
+        if(1) {
-        $this->is_searchable   = \igorw\get_in($options, ['searchable'], true);
+            $this->databox_id = \igorw\get_in($options, ['databox_id'], 0);
-        $this->is_private      = \igorw\get_in($options, ['private'], false);
+            $this->is_searchable = \igorw\get_in($options, ['searchable'], true);
-        $this->facet           = \igorw\get_in($options, ['facet']);
+            $this->is_private = \igorw\get_in($options, ['private'], false);
-        $this->thesaurus_roots = \igorw\get_in($options, ['thesaurus_roots'], null);
+            $this->facet = \igorw\get_in($options, ['facet']);
-        $this->generate_cterms  = \igorw\get_in($options, ['generate_cterms'], false);
+            $this->thesaurus_roots = \igorw\get_in($options, ['thesaurus_roots'], null);
-        $this->used_by_collections = \igorw\get_in($options, ['used_by_collections'], []);
+            $this->generate_cterms = \igorw\get_in($options, ['generate_cterms'], false);
-
+            $this->used_by_collections = \igorw\get_in($options, ['used_by_collections'], []);
-        Assertion::boolean($this->is_searchable);
-        Assertion::boolean($this->is_private);
-
-        if ($this->facet !== self::FACET_DISABLED) {
-            Assertion::integer($this->facet);
         }
-
+        else {
-        if ($this->thesaurus_roots !== null) {
+            // todo: this is faster code, but need to fix unit-tests to pass all options
-            Assertion::allIsInstanceOf($this->thesaurus_roots, Concept::class);
+            $this->databox_id = $options['databox_id'];
+            $this->is_searchable = $options['searchable'];
+            $this->is_private = $options['private'];
+            $this->facet = $options['facet'];
+            $this->thesaurus_roots = $options['thesaurus_roots'];
+            $this->generate_cterms = $options['generate_cterms'];
+            $this->used_by_collections = $options['used_by_collections'];
         }
-
-        Assertion::allScalar($this->used_by_collections);
     }
 
     public function withOptions(array $options)
@@ -270,4 +260,5 @@ class Field implements Typed
             'used_by_collections' => $used_by_collections
         ]);
     }
+
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php

@@ -3,8 +3,6 @@
 namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
-use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
-use Assert\Assertion;
 use DomainException;
 
 final class GlobalStructure implements Structure
@@ -14,7 +12,6 @@ final class GlobalStructure implements Structure
      */
     private $fields = array();
 
-
     /**
      * @var Field[][]
      */
@@ -45,53 +42,6 @@ final class GlobalStructure implements Structure
      */
     private $metadata_tags = array();
 
-    /**
-     * @param \databox[] $databoxes
-     * @param int $what    bitmask of what should be included in this structure, in fields, ...
-     *
-     * @return GlobalStructure
-     */
-    public static function createFromDataboxes(array $databoxes, $what = self::WITH_EVERYTHING)
-    {
-        $fields = [];
-        $flags = [];
-
-        foreach ($databoxes as $databox) {
-            if($what & self::STRUCTURE_WITH_FIELDS) {
-                foreach ($databox->get_meta_structure() as $fieldStructure) {
-                    $fields[] = Field::createFromLegacyField($fieldStructure, $what);
-                }
-            }
-
-            if($what & self::STRUCTURE_WITH_FLAGS) {
-                foreach ($databox->getStatusStructure() as $status) {
-                    $flags[] = Flag::createFromLegacyStatus($status);
-                }
-            }
-        }
-
-        return new self($fields, $flags, MetadataHelper::createTags());
-    }
-
-    /**
-     * @param \databox $databox
-     * @return GlobalStructure
-     */
-    public static function createFromDatabox(\databox $databox)
-    {
-        $fields = [];
-        $flags = [];
-
-        foreach ($databox->get_meta_structure() as $fieldStructure) {
-            $fields[] = Field::createFromLegacyField($fieldStructure);
-        }
-
-        foreach ($databox->getStatusStructure() as $status) {
-            $flags[] = Flag::createFromLegacyStatus($status);
-        }
-
-        return new self($fields, $flags, MetadataHelper::createTags());
-    }
 
     /**
      * GlobalStructure constructor.
@@ -101,9 +51,6 @@ final class GlobalStructure implements Structure
      */
     public function __construct(array $fields = [], array $flags = [], array $metadata_tags = [])
     {
-        Assertion::allIsInstanceOf($fields, Field::class);
-        Assertion::allIsInstanceOf($flags, Flag::class);
-        Assertion::allIsInstanceOf($metadata_tags, Tag::class);
 
         foreach ($fields as $field) {
             $this->add($field);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php

@@ -12,12 +12,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
 
 interface Structure
 {
-    const STRUCTURE_WITH_FIELDS = 0x10;
-    const STRUCTURE_WITH_FLAGS  = 0x20;
-    const FIELD_WITH_THESAURUS  = 0x01;
-    const FIELD_WITH_FACETS     = 0x02;
-    const WITH_EVERYTHING       = 0xFF;
-
     /**
      * @return Field[]
      */

lib/Alchemy/Phrasea/SearchEngine/Elastic/Thesaurus/Helper.php

@@ -31,26 +31,54 @@ class Helper
         $nodes = $xpath->query($expression);
         $concepts = [];
 
+        /** @var DOMElement $node */
         foreach ($nodes as $node) {
-            $me_and_parents = array_merge([$node], self::getElementAncestors($node));
+            if(1) {
+                $me_and_parents = array_merge([$node], self::getElementAncestors($node));
 
-            $path_segments = [];
+                $path_segments = [];
 
-            foreach ($me_and_parents as $me_or_parent) {
+                foreach ($me_and_parents as $me_or_parent) {
-                if (!Navigator::isConcept($me_or_parent)) {
+                    if (!Navigator::isConcept($me_or_parent)) {
-                    // Silently skips invalid targeted nodes
+                        // Silently skips invalid targeted nodes
-                    break;
+                        break;
+                    }
+
+                    $path_segments[] = self::conceptPathSegment($me_or_parent);
                 }
 
-                $path_segments[] = self::conceptPathSegment($me_or_parent);
+                // Concept paths are have databox identifier at root level
+                $concepts[] = new Concept(sprintf(
+                    '/%d/%s',
+                    $databox->get_sbas_id(),
+                    implode('/', array_reverse($path_segments))
+                ));
+            }
+            else {
+                $path = '';
+                // go up thru parents
+                while ($node) {
+                    $v = null;
+                    for ($n = $node->firstChild; $n; $n = $n->nextSibling) {
+                        if ($n->nodeType === XML_ELEMENT_NODE && $n->nodeName === 'sy') {
+                            if ($v === null) {
+                                $v = $n->getAttribute('v');
+                                continue;
+                            }
+                            if ($n->getAttribute('lng') === 'en') {
+                                $v = $n->getAttribute('v');
+                                break;
+                            }
+                        }
+                    }
+                    if ($v !== null) {
+                        $path = '/' . $v . $path;
+                    }
+                    $node = $node->parentNode;
+                }
+                $path = '/' . $databox->get_sbas_id() . $path;
+                $concepts[] = new Concept($path);
             }
-
-            // Concept paths are have databox identifier at root level
-            $concepts[] = new Concept(sprintf(
-                '/%d/%s',
-                $databox->get_sbas_id(),
-                implode('/', array_reverse($path_segments))
-            ));
         }
 
         return $concepts;

lib/Alchemy/Phrasea/SearchEngine/SearchEngineStructure.php

@@ -0,0 +1,107 @@
+<?php
+
+
+namespace Alchemy\Phrasea\SearchEngine;
+
+
+use Alchemy\Phrasea\Cache\Cache;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\MetadataHelper;
+// use Alchemy\Phrasea\Utilities\Stopwatch;
+use databox;
+
+class SearchEngineStructure
+{
+
+    /** @var Cache */
+    private  $cache;
+
+    /**
+     * SearchEngineStructure constructor.
+     * @param Cache $cache
+     */
+    public function __construct(Cache $cache)
+    {
+        $this->cache = $cache;
+    }
+
+    /**
+     * @param databox[] $databoxes
+     *
+     * @return GlobalStructure
+     */
+    public function getGlobalStructureFromDataboxes(array $databoxes)
+    {
+        $fields = [];
+        $flags = [];
+
+//        $stopwatch = new Stopwatch("getGlobalStructureFromDataboxes");
+        foreach ($databoxes as $databox) {
+
+            // we will cache both FIELDS and FLAGS in the same entry : it's small data
+            $k = $this->getCacheKey("FieldsAndFlags", $databox);
+            try {
+                $data = $this->cache->get($k);
+            }
+            catch(\Exception $e) {
+                $data = false;
+            }
+            if($data === false) {
+                $data = [
+                    'fields' => [],
+                    'flags'  => []
+                ];
+                foreach ($databox->get_meta_structure() as $fieldStructure) {
+                    $data['fields'][] = Field::createFromLegacyField($fieldStructure);
+                }
+                foreach ($databox->getStatusStructure() as $status) {
+                    $data['flags'][] = Flag::createFromLegacyStatus($status);
+                }
+                $this->cache->save($k, $data);
+            }
+
+            $fields = array_merge($fields, $data['fields']);
+            $flags  = array_merge($flags, $data['flags']);
+        }
+//        $stopwatch->lap('loop0');
+
+        // tags does not depends on db
+        $k = $this->getCacheKey("Tags");
+        try {
+            $tags = $this->cache->get($k);
+        }
+        catch(\Exception $e) {
+            $tags = false;
+        }
+        if($tags === false) {
+            $this->cache->save($k, ($tags = MetadataHelper::createTags()));
+            // nb : tags is a hardcoded list, we don't need to clear this cache
+        }
+
+//        $r = new GlobalStructure($fields, $flags, $tags);
+//        $stopwatch->log();
+//        return $r;
+
+        return new GlobalStructure($fields, $flags, $tags);
+    }
+
+    public function deleteFromCache($databox)
+    {
+        $k = $this->getCacheKey("FieldsAndFlags", $databox);
+        $this->cache->delete($k);
+    }
+
+    /**
+     * build a cache key to store es data, related to a db or not (if db is null)
+     *
+     * @param $what
+     * @param databox|null $db
+     * @return string
+     */
+    private function getCacheKey($what, databox $db = null)
+    {
+        return 'es' . ($db ? ('_db-'.$db->get_sbas_id()) : '') . '_' . $what;
+    }
+}

lib/Alchemy/Phrasea/Utilities/Stopwatch.php

@@ -62,9 +62,19 @@ class Stopwatch
      */
     public function getLapses()
     {
+        $this->stop();
+        $this->_lapses['_total'] = ($this->_stop - $this->_start) * 1000.0;
         return $this->_lapses;
     }
 
+    public function log()
+    {
+//         file_put_contents('/var/alchemy/Phraseanet/tmp/phraseanet.log',
+//            $this->_name . "\n" . var_export($this->getLapses(), true) . "\n\n",
+//            FILE_APPEND
+//         );
+    }
+
     /**
      * return all lapses as a "Server-Timing" header value
      *

lib/classes/ACL.php

@@ -1248,9 +1248,12 @@ class ACL implements cache_cacheableInterface
 
                 }
             }
+            $this->app->getApplicationBox()->get_databox($sbas_id)->clearCache(databox::CACHE_COLLECTIONS);
         }
         $stmt_ins->closeCursor();
+
         $this->delete_data_from_cache(self::CACHE_RIGHTS_SBAS);
+        // $this->delete_data_from_cache(self::CACHE_GLOBAL_RIGHTS);
 
         return $this;
     }
@@ -1365,7 +1368,10 @@ class ACL implements cache_cacheableInterface
         $stmt->execute([':sbas_id' => $sbas_id, ':usr_id' => $this->user->getId()]);
         $stmt->closeCursor();
 
+        $this->app->getApplicationBox()->get_databox($sbas_id)->clearCache(databox::CACHE_COLLECTIONS);
+
         $this->delete_data_from_cache(self::CACHE_RIGHTS_SBAS);
+        // $this->delete_data_from_cache(self::CACHE_GLOBAL_RIGHTS);
 
         $this->app['dispatcher']->dispatch(
             AclEvents::RIGHTS_TO_SBAS_CHANGED,

lib/classes/collection.php

@@ -597,6 +597,9 @@ class collection implements ThumbnailedElement, cache_cacheableInterface
         $this->getReferenceRepository()->save($this->reference);
         $this->collectionRepositoryRegistry->purgeRegistry();
 
+        // clear the trivial cache of databox->get_collections()
+        $this->get_databox()->clearCache(databox::CACHE_COLLECTIONS);
+
         cache_databox::update($this->app, $this->databox->get_sbas_id(), 'structure');
         
 	    $this->dispatch(CollectionEvents::DISABLED, new DisabledEvent($this));
@@ -614,6 +617,9 @@ class collection implements ThumbnailedElement, cache_cacheableInterface
         $this->getReferenceRepository()->save($this->reference);
         $this->collectionRepositoryRegistry->purgeRegistry();
 
+        // clear the trivial cache of databox->get_collections()
+        $this->get_databox()->clearCache(databox::CACHE_COLLECTIONS);
+
         cache_databox::update($this->app, $this->databox->get_sbas_id(), 'structure');
         
 	    $this->dispatch(CollectionEvents::ENABLED, new EnabledEvent($this));

lib/classes/databox.php

@@ -576,18 +576,35 @@ class databox extends base implements ThumbnailedElement
         parent::delete_data_from_cache($option);
     }
 
+    /*
+     * trivial cache to speed-up get_collections() which does sql
+     */
+    private $_collection_unique_ids = null;
+    private $_collections = null;
+
+    public function clearCache($what)
+    {
+        switch($what) {
+            case self::CACHE_COLLECTIONS:
+                $this->_collection_unique_ids = $this->_collections = null;
+                break;
+        }
+    }
+
     /**
      * @return int[]
      */
     public function get_collection_unique_ids()
     {
-        $collectionsIds = [];
+        if($this->_collection_unique_ids === null) {
+            $this->_collection_unique_ids = [];
 
-        foreach ($this->get_collections() as $collection) {
+            foreach ($this->get_collections() as $collection) {
-            $collectionsIds[] = $collection->get_base_id();
+                $this->_collection_unique_ids[] = $collection->get_base_id();
+            }
         }
 
-        return $collectionsIds;
+        return $this->_collection_unique_ids;
     }
 
     /**
@@ -595,15 +612,20 @@ class databox extends base implements ThumbnailedElement
      */
     public function get_collections()
     {
-        /** @var CollectionRepositoryRegistry $repositoryRegistry */
+        if($this->_collections === null) {
-        $repositoryRegistry = $this->app['repo.collections-registry'];
+            /** @var CollectionRepositoryRegistry $repositoryRegistry */
-        $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id());
+            $repositoryRegistry = $this->app['repo.collections-registry'];
+            $repository = $repositoryRegistry->getRepositoryByDatabox($this->get_sbas_id());
 
-        return array_filter($repository->findAll(), function (collection $collection) {
+           $this->_collections = array_filter($repository->findAll(), function (collection $collection) {
-            return $collection->is_active();
+               return $collection->is_active();
-        });
+           });
+        }
+
+        return $this->_collections;
     }
 
+
     /**
      * @return collection|null
      */