Merge pull request #1315 from mdarse/cross-fields-multi-word-query

Working cross-fields queries with multiple words (without operators)

bin/console

@@ -18,7 +18,7 @@ use Alchemy\Phrasea\Command\Setup\H264MappingGenerator;
 use Alchemy\Phrasea\Command\SearchEngine\Debug\QueryParseCommand;
 use Alchemy\Phrasea\Command\SearchEngine\IndexCreateCommand;
 use Alchemy\Phrasea\Command\SearchEngine\IndexDropCommand;
-use Alchemy\Phrasea\Command\SearchEngine\IndexFull;
+use Alchemy\Phrasea\Command\SearchEngine\MappingUpdateCommand;
 use Alchemy\Phrasea\Command\SearchEngine\IndexPopulateCommand;
 use Alchemy\Phrasea\Command\Thesaurus\FindConceptsCommand;
 use Alchemy\Phrasea\Command\WebsocketServer;
@@ -126,6 +126,7 @@ $cli->command(new XSendFileMappingGenerator());
 if ($cli['search_engine.type'] === SearchEngineInterface::TYPE_ELASTICSEARCH) {
     $cli->command(new IndexCreateCommand());
     $cli->command(new IndexDropCommand());
+    $cli->command(new MappingUpdateCommand());
     $cli->command(new IndexPopulateCommand());
     $cli->command(new QueryParseCommand());
     $cli->command(new FindConceptsCommand());

lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QuerySampleCommand.php

@@ -0,0 +1,69 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2014 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Command\SearchEngine\Debug;
+
+use Alchemy\Phrasea\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class QuerySampleCommand extends Command
+{
+    protected function configure()
+    {
+        $this
+            ->setName('searchengine:query:sample')
+            ->setDescription('Generate sample queries from grammar')
+        ;
+    }
+
+    protected function doExecute(InputInterface $input, OutputInterface $output)
+    {
+        $grammarPath = $this->container['query_parser.grammar_path'];
+        $output->writeln(sprintf('Generating sample queries from <comment>%s</comment>', $grammarPath));
+        $output->writeln(str_repeat('-', 20));
+
+        $parser = $this->container['query_parser'];
+
+        // UNIFORM
+
+        // $sampler = new \Hoa\Compiler\Llk\Sampler\Uniform(
+        //     $parser,
+        //     new \Hoa\Regex\Visitor\Isotropic(new \Hoa\Math\Sampler\Random()),
+        //     7
+        // );
+
+        // for($i = 0; $i < 10; ++$i) {
+        //     $output->writeln(sprintf('%d => %s', $i, $sampler->uniform()));
+        // }
+
+        // BOUNDED EXAUSTIVE
+
+        $sampler = new \Hoa\Compiler\Llk\Sampler\BoundedExhaustive(
+            $parser,
+            new \Hoa\Regex\Visitor\Isotropic(new \Hoa\Math\Sampler\Random()),
+            6
+        );
+
+        // COVERAGE
+
+        // $sampler = new \Hoa\Compiler\Llk\Sampler\Coverage(
+        //     $parser,
+        //     new \Hoa\Regex\Visitor\Isotropic(new \Hoa\Math\Sampler\Random())
+        // );
+
+        foreach($sampler as $i => $data) {
+            $output->writeln(sprintf('%d => %s', $i, $data));
+        }
+
+        $output->writeln(str_repeat('-', 20));
+    }
+}

lib/Alchemy/Phrasea/Command/SearchEngine/MappingUpdateCommand.php

@@ -0,0 +1,36 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2014 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Command\SearchEngine;
+
+use Alchemy\Phrasea\Command\Command;
+use Alchemy\Phrasea\SearchEngine\Elastic\Indexer;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class MappingUpdateCommand extends Command
+{
+    protected function configure()
+    {
+        $this
+            ->setName('searchengine:mapping:update')
+            ->setDescription('Update index mapping')
+        ;
+    }
+
+    protected function doExecute(InputInterface $input, OutputInterface $output)
+    {
+        $indexer = $this->container['elasticsearch.indexer'];
+
+        $indexer->updateMapping();
+        $output->writeln('Mapping pushed to index');
+    }
+}

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php

@@ -271,9 +271,7 @@ class ElasticSearchEngine implements SearchEngineInterface
     {
         $options = $options ?: new SearchEngineOptions();
 
-        // TODO Pass options to getFields to include/exclude private fields
+        $queryContext = new QueryContext($this->locales, $this->app['locale']);
-        $searchableFields = $this->recordHelper->getFields();
-        $queryContext = new QueryContext($this->locales, $this->app['locale'], $searchableFields);
         $recordQuery = $this->app['query_parser']->compile($string, $queryContext);
 
         $params = $this->createRecordQueryParams($recordQuery, $options, null);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php

@@ -83,8 +83,7 @@ class Indexer
         $params['index'] = $this->options['index'];
         $params['type'] = RecordIndexer::TYPE_NAME;
         $params['body'][RecordIndexer::TYPE_NAME] = $this->recordIndexer->getMapping();
-
+        $params['body'][TermIndexer::TYPE_NAME]   = $this->termIndexer->getMapping();
-        // @todo Add term mapping
 
         // @todo This must throw a new indexation if a mapping is edited
         $this->client->indices()->putMapping($params);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php

@@ -72,6 +72,12 @@ SQL;
                         $record[$type][$key] = array();
                     }
                     $record[$type][$key][] = $value;
+                    // Collect value in the "all" field
+                    $field = sprintf('%s_all', $type);
+                    if (!isset($record[$field])) {
+                        $record[$field] = array();
+                    }
+                    $record[$field][] = $value;
                     break;
 
                 case 'exif':

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php

@@ -206,8 +206,16 @@ class RecordIndexer
         // Caption mapping
         $captionMapping = new Mapping();
         $mapping->add('caption', $captionMapping);
+        $mapping
+            ->add('caption_all', 'string')
+            ->addLocalizedSubfields($this->locales)
+        ;
         $privateCaptionMapping = new Mapping();
         $mapping->add('private_caption', $privateCaptionMapping);
+        $mapping
+            ->add('private_caption_all', 'string')
+            ->addLocalizedSubfields($this->locales)
+        ;
         // Inferred thesaurus concepts
         $conceptPathMapping = new Mapping();
         $mapping->add('concept_path', $conceptPathMapping);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php

@@ -19,24 +19,29 @@ class QueryContext
 
     public function narrowToFields(array $fields)
     {
+        if (is_array($this->fields)) {
             // Ensure we are not escaping from original fields restrictions
             $fields = array_intersect($this->fields, $fields);
             if (!$fields) {
                 throw new QueryException('Query narrowed to non available fields');
             }
+        } else {
+            $fields = null;
+        }
 
         return new static($this->locales, $this->queryLocale, $fields);
     }
 
     public function getLocalizedFields()
     {
+        // TODO Private fields handling
         if ($this->fields === null) {
-            return $this->localizeField('*');
+            return $this->localizeField('caption_all');
         }
 
         $fields = array();
         foreach ($this->fields as $field) {
-            foreach ($this->localizeField($field) as $fields[]);
+            foreach ($this->localizeField(sprintf('caption.%s', $field)) as $fields[]);
         }
 
         return $fields;
@@ -47,10 +52,10 @@ class QueryContext
         $fields = array();
         foreach ($this->locales as $locale) {
             $boost = ($locale === $this->queryLocale) ? '^5' : '';
-            $fields[] = sprintf('caption.%s.%s%s', $field, $locale, $boost);
+            $fields[] = sprintf('%s.%s%s', $field, $locale, $boost);
         }
         // TODO Put generic analyzers on main field instead of "light" sub-field
-        $fields[] = sprintf('caption.%s.%s', $field, 'light^10');
+        $fields[] = sprintf('%s.light^10', $field);
 
         return $fields;
     }