porting PHRAS-1578/PHRAS-1579/PHRAS-1621/PHRAS-1675/PHRAS-1404/PHRAS-1336 to 4.1

lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php

@@ -36,12 +36,14 @@ class SearchEngineController extends Controller
             return $this->app->redirectPath('admin_searchengine_form');
         }
 
-        return $this->render('admin/search-engine/elastic-search.html.twig', [
+        return $this->render('admin/search-engine/search-engine-settings.html.twig', [
             'form' => $form->createView(),
             'indexer' => $this->app['elasticsearch.indexer']
         ]);
     }
 
+
+
     public function dropIndexAction(Request $request)
     {
         $indexer = $this->app['elasticsearch.indexer'];
@@ -87,4 +89,28 @@ class SearchEngineController extends Controller
             'action' => $this->app->url('admin_searchengine_form'),
         ]);
     }
+
+    /**
+     * @param Request $request
+     * @return \Symfony\Component\HttpFoundation\JsonResponse
+     */
+    public function getSettingFromIndexAction(Request $request)
+    {
+        if (!$request->isXmlHttpRequest()) {
+            $this->app->abort(400);
+        }
+        $indexer = $this->app['elasticsearch.indexer'];
+        $index = $request->get('index');
+        if (!$indexer->indexExists() || is_null($index))
+        {
+            return $this->app->json([
+                'success' => false,
+                'message' => $this->app->trans('An error occurred'),
+            ]);
+        }
+        return $this->app->json([
+            'success' => true,
+            'response' => $indexer->getSettings(['index' => $index])
+        ]);
+    }
 }

lib/Alchemy/Phrasea/Controller/Prod/QueryController.php

@@ -15,6 +15,7 @@ use Alchemy\Phrasea\Cache\Exception;
 use Alchemy\Phrasea\Collection\Reference\CollectionReference;
 use Alchemy\Phrasea\Controller\Controller;
 use Alchemy\Phrasea\Core\Configuration\DisplaySettingService;
+use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
 use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine;
@@ -280,11 +281,12 @@ class QueryController extends Controller
             $json['parsed_query'] = $result->getEngineQuery();
             /** End debug */
 
-            $fieldLabels = [
-                'Base_Name' => $this->app->trans('prod::facet:base_label'),
-                'Collection_Name' => $this->app->trans('prod::facet:collection_label'),
-                'Type_Name' => $this->app->trans('prod::facet:doctype_label'),
-            ];
+            $fieldLabels = [];
+            // add technical fields
+            foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
+                $fieldLabels[$k] = $this->app->trans($f['label']);
+            }
+            // add databox fields
             foreach ($this->app->getDataboxes() as $databox) {
                 foreach ($databox->get_meta_structure() as $field) {
                     if (!isset($fieldLabels[$field->get_name()])) {

lib/Alchemy/Phrasea/ControllerProvider/Admin/SearchEngine.php

@@ -43,6 +43,9 @@ class SearchEngine implements ControllerProviderInterface, ServiceProviderInterf
         $controllers->post('/create_index', 'controller.admin.search-engine:createIndexAction')
             ->bind("admin_searchengine_create_index");
 
+        $controllers->get('/setting_from_index', 'controller.admin.search-engine:getSettingFromIndexAction')
+            ->bind('admin_searchengine_setting_from_index');
+
         $controllers->match('/', 'controller.admin.search-engine:formConfigurationPanelAction')
             ->method('GET|POST')
             ->bind('admin_searchengine_form');

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php

@@ -19,6 +19,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\FacetsResponse;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryCompiler;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field AS ESField;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Flag;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
 use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
@@ -29,6 +30,7 @@ use Closure;
 use Doctrine\Common\Collections\ArrayCollection;
 use Alchemy\Phrasea\Model\Entities\FeedEntry;
 use Alchemy\Phrasea\Application;
+use databox_field;
 use Elasticsearch\Client;
 
 class ElasticSearchEngine implements SearchEngineInterface
@@ -487,33 +489,36 @@ class ElasticSearchEngine implements SearchEngineInterface
     private function getAggregationQueryParams(SearchEngineOptions $options)
     {
         $aggs = [];
-
-        // We always want a collection facet right now
-        $collection_facet_agg = array();
-        $collection_facet_agg['terms']['field'] = 'collection_name';
-        $aggs['Collection_Name'] = $collection_facet_agg;
-
-        // We always want a base facet right now
-        $base_facet_agg = array();
-        $base_facet_agg['terms']['field'] = 'databox_name';
-        $aggs['Base_Name'] = $base_facet_agg;
-
-        // We always want a type facet right now
-        $base_facet_agg = array();
-        $base_facet_agg['terms']['field'] = 'type';
-        $aggs['Type_Name'] = $base_facet_agg;
-
+        // technical aggregates (enable + optional limit)
+        foreach (ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
+            $size = $this->options->getAggregableFieldLimit($k);
+            if ($size !== databox_field::FACET_DISABLED) {
+                if ($size === databox_field::FACET_NO_LIMIT) {
+                    $size = ESField::FACET_NO_LIMIT;
+                }
+                $agg = [
+                    'terms' => [
+                        'field' => $f['field'],
+                        'size'  => $size
+                    ]
+                ];
+                $aggs[$k] = $agg;
+            }
+        }
+        // fields aggregates
         $structure = $this->context_factory->getLimitedStructure($options);
         foreach ($structure->getFacetFields() as $name => $field) {
             // 2015-05-26 (mdarse) Removed databox filtering.
             // It was already done by the ACL filter in the query scope, so no
             // document that shouldn't be displayed can go this far.
-            $agg = [];
-            $agg['terms']['field'] = $field->getIndexField(true);
-            $agg['terms']['size'] = $field->getFacetValuesLimit();
+            $agg = [
+                'terms' => [
+                    'field' => $field->getIndexField(true),
+                    'size'  => $field->getFacetValuesLimit()
+                ]
+            ];
             $aggs[$name] = AggregationHelper::wrapPrivateFieldAggregation($field, $agg);
         }
-
         return $aggs;
     }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php

@@ -26,6 +26,10 @@ class ElasticsearchOptions
     /** @var  bool */
     private $highlight;
 
+    /** @var  int[] */
+    private $_customValues;
+    private $activeTab;
+
     /**
      * Factory method to hydrate an instance from serialized options
      *
@@ -34,15 +38,22 @@ class ElasticsearchOptions
      */
     public static function fromArray(array $options)
     {
-        $options = array_replace([
+        $defaultOptions = [
             'host' => '127.0.0.1',
             'port' => 9200,
             'index' => '',
             'shards' => 3,
             'replicas' => 0,
             'minScore' => 4,
-            'highlight' => true
-        ], $options);
+            'highlight' => true,
+            'activeTab' => null,
+        ];
+
+        foreach(self::getAggregableTechnicalFields() as $k => $f) {
+            $defaultOptions[$k.'_limit'] = 0;
+        }
+        $options = array_replace($defaultOptions, $options);
+
 
         $self = new self();
         $self->setHost($options['host']);
@@ -52,6 +63,11 @@ class ElasticsearchOptions
         $self->setReplicas($options['replicas']);
         $self->setMinScore($options['minScore']);
         $self->setHighlight($options['highlight']);
+        $self->setActiveTab($options['activeTab']);
+        foreach(self::getAggregableTechnicalFields() as $k => $f) {
+            $self->setAggregableFieldLimit($k, $options[$k.'_limit']);
+        }
+
 
         return $self;
     }
@@ -61,7 +77,7 @@ class ElasticsearchOptions
      */
     public function toArray()
     {
-        return [
+        $ret = [
             'host' => $this->host,
             'port' => $this->port,
             'index' => $this->indexName,
@@ -69,7 +85,13 @@ class ElasticsearchOptions
             'replicas' => $this->replicas,
             'minScore' => $this->minScore,
             'highlight' => $this->highlight,
+            'activeTab' => $this->activeTab
         ];
+        foreach(self::getAggregableTechnicalFields() as $k => $f) {
+            $ret[$k.'_limit'] = $this->getAggregableFieldLimit($k);
+        }
+
+        return $ret;
     }
 
     /**
@@ -183,4 +205,121 @@ class ElasticsearchOptions
     {
         $this->highlight = $highlight;
     }
+
+    public function setAggregableFieldLimit($key, $value)
+    {
+        $this->_customValues[$key.'_limit'] = $value;
+    }
+
+    public function getAggregableFieldLimit($key)
+    {
+        return $this->_customValues[$key.'_limit'];
+    }
+
+    public function getActiveTab()
+    {
+        return $this->activeTab;
+    }
+    public function setActiveTab($activeTab)
+    {
+        $this->activeTab = $activeTab;
+    }
+
+    public function __get($key)
+    {
+        if(!array_key_exists($key, $this->_customValues)) {
+            $this->_customValues[$key] = 0;
+        }
+        return $this->_customValues[$key];
+    }
+
+    public function __set($key, $value)
+    {
+        $this->_customValues[$key] = $value;
+    }
+
+    public static function getAggregableTechnicalFields()
+    {
+        return [
+            'base_aggregate' => [
+                'label' => 'prod::facet:base_label',
+                'field' => 'databox_name',
+                'query' => 'database:%s',
+            ],
+            'collection_aggregate' => [
+                'label' => 'prod::facet:collection_label',
+                'field' => 'collection_name',
+                'query' => 'collection:%s',
+            ],
+            'doctype_aggregate' => [
+                'label' => 'prod::facet:doctype_label',
+                'field' => 'type',
+                'query' => 'type:%s',
+            ],
+            'camera_model_aggregate' => [
+                'label' => 'Camera Model',
+                'field' => 'metadata_tags.CameraModel.raw',
+                'query' => 'meta.CameraModel:%s',
+            ],
+            'iso_aggregate' => [
+                'label' => 'ISO',
+                'field' => 'metadata_tags.ISO',
+                'query' => 'meta.ISO=%s',
+            ],
+            'aperture_aggregate' => [
+                'label' => 'Aperture',
+                'field' => 'metadata_tags.Aperture',
+                'query' => 'meta.Aperture=%s',
+            ],
+            'shutterspeed_aggregate' => [
+                'label' => 'Shutter speed',
+                'field' => 'metadata_tags.ShutterSpeed',
+                'query' => 'meta.ShutterSpeed=%s',
+            ],
+            'flashfired_aggregate' => [
+                'label' => 'FlashFired',
+                'field' => 'metadata_tags.FlashFired',
+                'query' => 'meta.FlashFired=%s',
+                'choices' => [
+                    "aggregated (2 values: fired = 0 or 1)" => -1,
+                ],
+            ],
+            'framerate_aggregate' => [
+                'label' => 'FrameRate',
+                'field' => 'metadata_tags.FrameRate',
+                'query' => 'meta.FrameRate=%s',
+            ],
+            'audiosamplerate_aggregate' => [
+                'label' => 'Audio Samplerate',
+                'field' => 'metadata_tags.AudioSamplerate',
+                'query' => 'meta.AudioSamplerate=%s',
+            ],
+            'videocodec_aggregate' => [
+                'label' => 'Video codec',
+                'field' => 'metadata_tags.VideoCodec',
+                'query' => 'meta.VideoCodec:%s',
+            ],
+            'audiocodec_aggregate' => [
+                'label' => 'Audio codec',
+                'field' => 'metadata_tags.AudioCodec',
+                'query' => 'meta.AudioCodec:%s',
+            ],
+            'orientation_aggregate' => [
+                'label' => 'Orientation',
+                'field' => 'metadata_tags.Orientation',
+                'query' => 'meta.Orientation=%s',
+            ],
+            'colorspace_aggregate' => [
+                'label' => 'Colorspace',
+                'field' => 'metadata_tags.ColorSpace',
+                'query' => 'meta.ColorSpace:%s',
+            ],
+            'mimetype_aggregate' => [
+                'label' => 'MimeType',
+                'field' => 'metadata_tags.MimeType',
+                'query' => 'meta.MimeType:%s',
+            ],
+        ];
+    }
+
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php

@@ -10,6 +10,7 @@
 namespace Alchemy\Phrasea\SearchEngine\Elastic;
 
 use Symfony\Component\Form\AbstractType;
+use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
 use Symfony\Component\Form\FormBuilderInterface;
 use Symfony\Component\Validator\Constraints\NotBlank;
 use Symfony\Component\Validator\Constraints\Range;
@@ -29,14 +30,20 @@ class ElasticsearchSettingsFormType extends AbstractType
             ->add('indexName', 'text', [
                 'label' => 'ElasticSearch index name',
                 'constraints' => new NotBlank(),
+                'attr' =>['data-class'=>'inline']
             ])
             ->add('esSettingsDropIndexButton', 'button', [
                 'label' => "Drop index",
-                'attr' => ['data-id' => "esSettingsDropIndexButton"]
+                'attr' => [
+                    'data-id' => 'esSettingsDropIndexButton',
+                    'class' => 'btn btn-danger'
+                ]
             ])
             ->add('esSettingsCreateIndexButton', 'button', [
                 'label' => "Create index",
-                'attr' => ['data-id' => "esSettingsCreateIndexButton"]
+                'attr' => ['data-id' => "esSettingsCreateIndexButton",
+                           'class' => 'btn btn-success'
+                ]
             ])
             ->add('shards', 'integer', [
                 'label' => 'Number of shards',
@@ -49,13 +56,59 @@ class ElasticsearchSettingsFormType extends AbstractType
             ->add('minScore', 'integer', [
                 'label' => 'Thesaurus Min score',
                 'constraints' => new Range(['min' => 0]),
-            ])
-            ->add('highlight', 'checkbox', [
-                'label' => 'Activate highlight',
-                'required' => false
-            ])
-            ->add('save', 'submit')
-        ;
+            ]);
+
+            foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
+                if(array_key_exists('choices', $f)) {
+                    // choices[] : choice_key => choice_value
+                    $choices = $f['choices'];
+                }
+                else {
+                    $choices = [
+                        "10 values" => 10,
+                        "20 values" => 20,
+                        "50 values" => 50,
+                        "100 values" => 100,
+                        "all values" => -1
+                    ];
+                }
+                // array_unshift($choices, "not aggregated");  //  always as first choice
+                $choices = array_merge(["not aggregated" => 0], $choices);
+                $builder
+                    ->add($k.'_limit', ChoiceType::class, [
+                        // 'label' => $f['label'],// . ' ' . 'aggregate limit',
+                        'choices_as_values' => true,
+                        'choices'           => $choices,
+                        'attr'              => [
+                            'class' => 'aggregate'
+                        ]
+                    ]);
+            }
+
+            $builder
+                ->add('highlight', 'checkbox', [
+                    'label' => 'Activate highlight',
+                    'required' => false
+                ])
+//                ->add('save', 'submit', [
+//                    'attr' => ['class' => 'btn btn-primary']
+//                ])
+                ->add('esSettingFromIndex', 'button', [
+                    'label' => 'Get setting form index',
+                    'attr' => [
+                        'onClick' => 'esSettingFromIndex()',
+                        'class' => 'btn'
+                    ]
+                ])
+                ->add('dumpField', 'textarea', [
+                    'label' => false,
+                    'required' => false,
+                    'mapped' => false,
+                    'attr' => ['class' => 'dumpfield hide']
+                ])
+                ->add('activeTab', 'hidden');
+
+            ;
     }
 
     public function getName()

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php

@@ -238,4 +238,14 @@ class Indexer
         // Flush just in case, it's a noop when already done
         $bulk->flush();
     }
+
+    public function getSettings(array $params)
+    {
+        try {
+            //Get setting from index
+            return $this->client->indices()->getSettings($params);
+        } catch (\Exception $e) {
+            return $e->getMessage();
+        }
+    }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/FacetsResponse.php

@@ -3,6 +3,7 @@
 namespace Alchemy\Phrasea\SearchEngine\Elastic\Search;
 
 use Alchemy\Phrasea\Exception\RuntimeException;
+use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
 use Alchemy\Phrasea\SearchEngine\SearchEngineSuggestion;
 use Doctrine\Common\Collections\ArrayCollection;
@@ -70,18 +71,14 @@ class FacetsResponse
 
     private function buildQuery($name, $value)
     {
-        switch($name) {
-            case 'Collection_Name':
-                return sprintf('collection:%s', $this->escaper->escapeWord($value));
-            case 'Base_Name':
-                return sprintf('database:%s', $this->escaper->escapeWord($value));
-            case 'Type_Name':
-                return sprintf('type:%s', $this->escaper->escapeWord($value));
-            default:
-                return sprintf('field.%s = %s',
-                    $this->escaper->escapeWord($name),
-                    $this->escaper->escapeWord($value));
+        if(array_key_exists($name, ElasticsearchOptions::getAggregableTechnicalFields())) {
+            $q = ElasticsearchOptions::getAggregableTechnicalFields()[$name]['query'];
+            $ret = sprintf($q, $this->escaper->escapeWord($value));
         }
+        else {
+            $ret = sprintf('field.%s:%s', $this->escaper->escapeWord($name), $this->escaper->escapeWord($value));
+        }
+        return $ret;
     }
 
     private function throwAggregationResponseError()