port to 4.1 fix ES index NUL

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php

@@ -155,15 +155,16 @@ class BulkOperation
         // nb: results (items) are returned IN THE SAME ORDER as commands were pushed in the stack
         // so the items[X] match the operationIdentifiers[X]
         foreach ($response['items'] as $key => $item) {
-            foreach($item as $command=>$result) {   // command may be "index" or "delete"
+            foreach ($item as $command=>$result) {   // command may be "index" or "delete"
-                if($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx error
+                if ($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx
-                    throw new Exception(sprintf('%d: %s', $key, var_export($result, true)));
+                    $err = array_key_exists('error', $result) ? var_export($result['error'], true) : ($command . " error " . $result['status']);
+                    throw new Exception(sprintf('%d: %s', $key, $err));
                 }
             }
 
             $operationIdentifier = $this->operationIdentifiers[$key];
 
-            if(is_string($operationIdentifier) || is_int($operationIdentifier)) {   // dont include null keys
+            if (is_string($operationIdentifier) || is_int($operationIdentifier)) {   // dont include null keys
                 $callbackData[$operationIdentifier] = $response['items'][$key];
             }
         }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php

@@ -39,18 +39,13 @@ class MetadataHydrator implements HydratorInterface
 
     public function hydrateRecords(array &$records)
     {
-        $sql = <<<SQL
+        $sql = "(SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private\n"
-            (SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private
+            . " FROM metadatas AS m INNER JOIN metadatas_structure AS ms ON (ms.id = m.meta_struct_id)\n"
-            FROM metadatas AS m
+            . " WHERE record_id IN (?))\n"
-            INNER JOIN metadatas_structure AS ms ON (ms.id = m.meta_struct_id)
+            . "UNION\n"
-            WHERE record_id IN (?))
+            . "(SELECT record_id, t.name AS `key`, t.value AS value, 'exif' AS type, 0 AS private\n"
-
+            . " FROM technical_datas AS t\n"
-            UNION
+            . " WHERE record_id IN (?))\n";
-
-            (SELECT record_id, t.name AS `key`, t.value AS value, 'exif' AS type, 0 AS private
-            FROM technical_datas AS t
-            WHERE record_id IN (?))
-SQL;
 
         $ids = array_keys($records);
         $statement = $this->connection->executeQuery(
@@ -62,7 +57,7 @@ SQL;
         while ($metadata = $statement->fetch()) {
             // Store metadata value
             $key = $metadata['key'];
-            $value = $metadata['value'];
+            $value = trim($metadata['value']);
 
             // Do not keep empty values
             if ($key === '' || $value === '') {
@@ -80,7 +75,7 @@ SQL;
                 case 'caption':
                     // Sanitize fields
                     $value = StringHelper::crlfNormalize($value);
-                    $value = $this->sanitizeValue($value, $this->structure->typeOf($key));
+                    $value = $this->helper->sanitizeValue($value, $this->structure->typeOf($key));
                     // Private caption fields are kept apart
                     $type = $metadata['private'] ? 'private_caption' : 'caption';
                     // Caption are multi-valued
@@ -103,7 +98,7 @@ SQL;
                     }
                     $tag = $this->structure->getMetadataTagByName($key);
                     if ($tag) {
-                        $value = $this->sanitizeValue($value, $tag->getType());
+                        $value = $this->helper->sanitizeValue($value, $tag->getType());
                     }
                     // EXIF data is single-valued
                     $record['metadata_tags'][$key] = $value;
@@ -118,33 +113,6 @@ SQL;
         $this->clearGpsPositionBuffer();
     }
 
-    private function sanitizeValue($value, $type)
-    {
-        switch ($type) {
-            case FieldMapping::TYPE_STRING:
-                return str_replace("\0", "", $value);
-
-            case FieldMapping::TYPE_DATE:
-                return $this->helper->sanitizeDate($value);
-
-            case FieldMapping::TYPE_FLOAT:
-            case FieldMapping::TYPE_DOUBLE:
-                return (float) $value;
-
-            case FieldMapping::TYPE_INTEGER:
-            case FieldMapping::TYPE_LONG:
-            case FieldMapping::TYPE_SHORT:
-            case FieldMapping::TYPE_BYTE:
-                return (int) $value;
-
-            case FieldMapping::TYPE_BOOLEAN:
-                return (bool) $value;
-
-            default:
-                return $value;
-        }
-    }
-
     private function handleGpsPosition(&$records, $id, $tag_name, $value)
     {
         // Get position object

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/TitleHydrator.php

@@ -11,6 +11,8 @@
 
 namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator;
 
+use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
+use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
 use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Driver\Connection as DriverConnection;
 
@@ -18,31 +20,34 @@ class TitleHydrator implements HydratorInterface
 {
     private $connection;
 
-    public function __construct(DriverConnection $connection)
+    /** @var RecordHelper */
+    private $helper;
+
+    public function __construct(DriverConnection $connection, RecordHelper $helper)
     {
         $this->connection = $connection;
+        $this->helper = $helper;
     }
 
     public function hydrateRecords(array &$records)
     {
-        $sql = <<<SQL
+        $sql = "SELECT\n"
-            SELECT
+            . "m.`record_id`,\n"
-                m.`record_id`,
+            . "   CASE ms.`thumbtitle`\n"
-                CASE ms.`thumbtitle`
+            . "     WHEN '1' THEN 'default'\n"
-                  WHEN "1" THEN "default"
+            . "     WHEN '0' THEN 'default'\n"
-                  WHEN "0" THEN "default"
+            . "     ELSE ms.`thumbtitle`\n"
-                  ELSE ms.`thumbtitle`
+            . "   END AS locale,\n"
-                END AS locale,
+            . "   CASE ms.`thumbtitle`\n"
-                CASE ms.`thumbtitle`
+            . "     WHEN '0' THEN r.`originalname`\n"
-                  WHEN "0" THEN r.`originalname`
+            . "     ELSE GROUP_CONCAT(m.`value` ORDER BY ms.`thumbtitle`, ms.`sorter` SEPARATOR ' - ')\n"
-                  ELSE GROUP_CONCAT(m.`value` ORDER BY ms.`thumbtitle`, ms.`sorter` SEPARATOR " - ")
+            . "   END AS title\n"
-                END AS title
+            . "FROM metadatas AS m FORCE INDEX(`record_id`)\n"
-            FROM metadatas AS m FORCE INDEX(`record_id`)
+            . "STRAIGHT_JOIN metadatas_structure AS ms ON (ms.`id` = m.`meta_struct_id`)\n"
-            STRAIGHT_JOIN metadatas_structure AS ms ON (ms.`id` = m.`meta_struct_id`)
+            . "STRAIGHT_JOIN record AS r ON (r.`record_id` = m.`record_id`)\n"
-            STRAIGHT_JOIN record AS r ON (r.`record_id` = m.`record_id`)
+            . "WHERE m.`record_id` IN (?)\n"
-            WHERE m.`record_id` IN (?)
+            . "GROUP BY m.`record_id`, ms.`thumbtitle`\n";
-            GROUP BY m.`record_id`, ms.`thumbtitle`
+
-SQL;
         $statement = $this->connection->executeQuery(
             $sql,
             array(array_keys($records)),
@@ -50,7 +55,7 @@ SQL;
         );
 
         while ($row = $statement->fetch()) {
-            $records[$row['record_id']]['title'][$row['locale']] = $row['title'];
+            $records[$row['record_id']]['title'][$row['locale']] = $this->helper->sanitizeValue($row['title'], FieldMapping::TYPE_STRING);
         }
     }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/RecordHelper.php

@@ -116,4 +116,31 @@ class RecordHelper
             return null;
         }
     }
+
+    public function sanitizeValue($value, $type)
+    {
+        switch ($type) {
+            case FieldMapping::TYPE_DATE:
+                return self::sanitizeDate($value);
+
+            case FieldMapping::TYPE_FLOAT:
+            case FieldMapping::TYPE_DOUBLE:
+                return (float) $value;
+
+            case FieldMapping::TYPE_INTEGER:
+            case FieldMapping::TYPE_LONG:
+            case FieldMapping::TYPE_SHORT:
+            case FieldMapping::TYPE_BYTE:
+                return (int) $value;
+
+            case FieldMapping::TYPE_BOOLEAN:
+                return (bool) $value;
+
+            case FieldMapping::TYPE_STRING:
+                return str_replace("\0", '', $value);
+
+            default:
+                return $value;
+        }
+    }
 }