Merge branch 'master' into PHRAS-2666_slow-notifications_4.1

composer.json

@@ -47,7 +47,7 @@
     "php": ">=5.5.9",
     "ext-intl": "*",
     "alchemy-fr/tcpdf-clone": "~6.0",
-    "alchemy/embed-bundle": "^2.0.6",
+    "alchemy/embed-bundle": "^2.0.7",
     "alchemy/geonames-api-consumer": "~0.1.0",
     "alchemy/mediavorus": "^0.4.4",
     "alchemy/oauth2php": "1.1.0",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "64830cb4d53b32b47e02d4a19df9cef2",
+    "content-hash": "f3b1fc0a30bf14b05e57ce673550d9c0",
     "packages": [
         {
             "name": "alchemy-fr/tcpdf-clone",
@@ -131,16 +131,16 @@
         },
         {
             "name": "alchemy/embed-bundle",
-            "version": "2.0.6",
+            "version": "2.0.7",
             "source": {
                 "type": "git",
                 "url": "https://github.com/alchemy-fr/embed-bundle.git",
-                "reference": "53ba295dfd0554a31c35e93902a5ef6cb8eca31a"
+                "reference": "c585ccf18e53a9a6f2b696ddbbc39521732dfdde"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/alchemy-fr/embed-bundle/zipball/53ba295dfd0554a31c35e93902a5ef6cb8eca31a",
+                "url": "https://api.github.com/repos/alchemy-fr/embed-bundle/zipball/c585ccf18e53a9a6f2b696ddbbc39521732dfdde",
-                "reference": "53ba295dfd0554a31c35e93902a5ef6cb8eca31a",
+                "reference": "c585ccf18e53a9a6f2b696ddbbc39521732dfdde",
                 "shasum": ""
             },
             "require-dev": {
@@ -178,10 +178,10 @@
             ],
             "description": "Embed resources bundle",
             "support": {
-                "source": "https://github.com/alchemy-fr/embed-bundle/tree/2.0.6",
+                "source": "https://github.com/alchemy-fr/embed-bundle/tree/2.0.7",
                 "issues": "https://github.com/alchemy-fr/embed-bundle/issues"
             },
-            "time": "2019-07-11T12:59:49+00:00"
+            "time": "2019-09-02T12:28:19+00:00"
         },
         {
             "name": "alchemy/geonames-api-consumer",
@@ -443,8 +443,8 @@
                 },
                 {
                     "name": "Benoit Burnichon",
-                    "email": "bburnichon@alchemy.fr",
+                    "role": "Lead Developer",
-                    "role": "Lead Developer"
+                    "email": "bburnichon@alchemy.fr"
                 }
             ],
             "description": "Exiftool driver for PHP",

lib/Alchemy/Phrasea/Application.php

@@ -115,6 +115,7 @@ use Symfony\Component\Form\FormBuilderInterface;
 use Symfony\Component\Form\FormInterface;
 use Symfony\Component\Form\FormTypeInterface;
 use Symfony\Component\HttpFoundation\RedirectResponse;
+use Symfony\Component\Process\ExecutableFinder;
 use Unoconv\UnoconvServiceProvider;
 use XPDF\PdfToText;
 use XPDF\XPDFServiceProvider;
@@ -237,8 +238,19 @@ class Application extends SilexApplication
 
         $this->register(new UnicodeServiceProvider());
         $this->register(new ValidatorServiceProvider());
-        $this->register(new XPDFServiceProvider());
+
-        $this->setupXpdf();
+        if ($this['configuration.store']->isSetup()) {
+            $binariesConfig = $this['conf']->get(['main', 'binaries']);
+            $executableFinder = new ExecutableFinder();
+            $this->register(new XPDFServiceProvider(), [
+                'xpdf.configuration' => [
+                    'pdftotext.binaries' => isset($binariesConfig['pdftotext_binary']) ? $binariesConfig['pdftotext_binary'] : $executableFinder->find('pdftotext'),
+                ]
+            ]);
+
+            $this->setupXpdf();
+        }
+
         $this->register(new FileServeServiceProvider());
         $this->register(new ManipulatorServiceProvider());
         $this->register(new PluginServiceProvider());

lib/Alchemy/Phrasea/Controller/Api/V1Controller.php

@@ -1984,7 +1984,7 @@ class V1Controller extends Controller
                 return $this->getBadRequestAction($request);
             }
 
-            $datas = substr($datas, 0, ($n)) . $value . substr($datas, ($n + 2));
+            $datas = substr($datas, 0, ($n)) . $value . substr($datas, ($n + 1));
         }
 
         $record->setStatus(strrev($datas));

lib/Alchemy/Phrasea/Metadata/PhraseanetMetadataSetter.php

@@ -14,6 +14,7 @@ namespace Alchemy\Phrasea\Metadata;
 use Alchemy\Phrasea\Border\File;
 use Alchemy\Phrasea\Databox\DataboxRepository;
 use Alchemy\Phrasea\Metadata\Tag\NoSource;
+use DateTime;
 use PHPExiftool\Driver\Metadata\Metadata;
 
 class PhraseanetMetadataSetter
@@ -66,8 +67,16 @@ class PhraseanetMetadataSetter
                     continue;
                 }
 
-                $data['value'] = $value;
+                if ($field->get_type() == 'date') {
+                    try {
+                        $dateTime = new DateTime($value);
+                        $value = $dateTime->format('Y/m/d H:i:s');
+                    } catch (\Exception $e) {
+                        // $value unchanged
+                    }
+                }
 
+                $data['value'] = $value;
                 $metadataInRecordFormat[] = $data;
             }
         }

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/FieldKey.php

@@ -30,6 +30,11 @@ class FieldKey implements Key, QueryPostProcessor
         return $this->getField($context)->getIndexField($raw);
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return $this->getField($context)->getType();
+    }
+
     public function isValueCompatible($value, QueryContext $context)
     {
         return ValueChecker::isValueCompatible($this->getField($context), $value);

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Key.php

@@ -6,6 +6,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
 
 interface Key
 {
+    public function getFieldType(QueryContext $context);
     public function getIndexField(QueryContext $context, $raw = false);
     public function isValueCompatible($value, QueryContext $context);
     public function __toString();

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MetadataKey.php

@@ -23,6 +23,11 @@ class MetadataKey implements Key
         return $this->getTag($context)->getIndexField($raw);
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return $this->getTag($context)->getType();
+    }
+
     public function isValueCompatible($value, QueryContext $context)
     {
         return ValueChecker::isValueCompatible($this->getTag($context), $value);

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php

@@ -52,6 +52,11 @@ class NativeKey implements Key
         $this->key = $key;
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return $this->type;
+    }
+
     public function getIndexField(QueryContext $context, $raw = false)
     {
         return $this->key;

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/RangeExpression.php

@@ -2,18 +2,20 @@
 
 namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
 
+use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
+use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
 use Assert\Assertion;
-use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\FieldKey;
-use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\Key;
 use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node;
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
-use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryPostProcessor;
 
 class RangeExpression extends Node
 {
+    /** @var FieldKey  */
     private $key;
+
     private $lower_bound;
     private $lower_inclusive;
     private $higher_bound;
@@ -55,20 +57,34 @@ class RangeExpression extends Node
     public function buildQuery(QueryContext $context)
     {
         $params = array();
-        if ($this->lower_bound !== null) {
+        /** @var StructureField $field */
-            $this->assertValueCompatible($this->lower_bound, $context);
+        // $field = $this->key->getField($context);
-            if ($this->lower_inclusive) {
+        $lower_bound = $this->lower_bound;
-                $params['gte'] = $this->lower_bound;
+        $higher_bound = $this->higher_bound;
-            } else {
+
-                $params['gt'] = $this->lower_bound;
+        if($this->key->getFieldType($context) === FieldMapping::TYPE_DATE) {
+            if($lower_bound !== null) {
+                $lower_bound = RecordHelper::sanitizeDate($lower_bound);
+            }
+            if($higher_bound !== null) {
+                $higher_bound = RecordHelper::sanitizeDate($higher_bound);
             }
         }
-        if ($this->higher_bound !== null) {
+
-            $this->assertValueCompatible($this->higher_bound, $context);
+        if ($lower_bound !== null) {
-            if ($this->higher_inclusive) {
+            $this->assertValueCompatible($lower_bound, $context);
-                $params['lte'] = $this->higher_bound;
+            if ($this->lower_inclusive) {
+                $params['gte'] = $lower_bound;
             } else {
-                $params['lt'] = $this->higher_bound;
+                $params['gt'] = $lower_bound;
+            }
+        }
+        if ($higher_bound !== null) {
+            $this->assertValueCompatible($higher_bound, $context);
+            if ($this->higher_inclusive) {
+                $params['lte'] = $higher_bound;
+            } else {
+                $params['lt'] = $higher_bound;
             }
         }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/TimestampKey.php

@@ -34,6 +34,11 @@ class TimestampKey implements Key, Typed
         return FieldMapping::TYPE_DATE;
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return FieldMapping::TYPE_DATE;
+    }
+
     public function getIndexField(QueryContext $context, $raw = false)
     {
         return $this->index_field;

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php

@@ -396,10 +396,10 @@ class ElasticSearchEngine implements SearchEngineInterface
         if ($options->getDateFields() && ($options->getMaxDate() || $options->getMinDate())) {
             $range = [];
             if ($options->getMaxDate()) {
-                $range['lte'] = $options->getMaxDate()->format(FieldMapping::DATE_FORMAT_CAPTION_PHP);
+                $range['lte'] = $options->getMaxDate()->format('Y-m-d');
             }
             if ($options->getMinDate()) {
-                $range['gte'] = $options->getMinDate()->format(FieldMapping::DATE_FORMAT_CAPTION_PHP);
+                $range['gte'] = $options->getMinDate()->format('Y-m-d');
             }
 
             foreach ($options->getDateFields() as $dateField) {

lib/Alchemy/Phrasea/SearchEngine/Elastic/FieldMapping.php

@@ -16,8 +16,7 @@ class FieldMapping
 
     const DATE_FORMAT_MYSQL = 'yyyy-MM-dd HH:mm:ss';
     const DATE_FORMAT_CAPTION = 'yyyy/MM/dd'; // ES format
-    const DATE_FORMAT_MYSQL_OR_CAPTION = 'yyyy-MM-dd HH:mm:ss||yyyy/MM/dd';
+    const DATE_FORMAT_MYSQL_OR_CAPTION = 'yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||yyyy-MM||yyyy';
-    const DATE_FORMAT_CAPTION_PHP = 'Y/m/d';  // PHP format
 
     // Core types
     const TYPE_STRING     = 'string';

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php

@@ -155,15 +155,16 @@ class BulkOperation
         // nb: results (items) are returned IN THE SAME ORDER as commands were pushed in the stack
         // so the items[X] match the operationIdentifiers[X]
         foreach ($response['items'] as $key => $item) {
-            foreach($item as $command=>$result) {   // command may be "index" or "delete"
+            foreach ($item as $command=>$result) {   // command may be "index" or "delete"
-                if($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx error
+                if ($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx
-                    throw new Exception(sprintf('%d: %s', $key, var_export($result, true)));
+                    $err = array_key_exists('error', $result) ? var_export($result['error'], true) : ($command . " error " . $result['status']);
+                    throw new Exception(sprintf('%d: %s', $key, $err));
                 }
             }
 
             $operationIdentifier = $this->operationIdentifiers[$key];
 
-            if(is_string($operationIdentifier) || is_int($operationIdentifier)) {   // dont include null keys
+            if (is_string($operationIdentifier) || is_int($operationIdentifier)) {   // dont include null keys
                 $callbackData[$operationIdentifier] = $response['items'][$key];
             }
         }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php

@@ -39,18 +39,13 @@ class MetadataHydrator implements HydratorInterface
 
     public function hydrateRecords(array &$records)
     {
-        $sql = <<<SQL
+        $sql = "(SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private\n"
-            (SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private
+            . " FROM metadatas AS m INNER JOIN metadatas_structure AS ms ON (ms.id = m.meta_struct_id)\n"
-            FROM metadatas AS m
+            . " WHERE record_id IN (?))\n"
-            INNER JOIN metadatas_structure AS ms ON (ms.id = m.meta_struct_id)
+            . "UNION\n"
-            WHERE record_id IN (?))
+            . "(SELECT record_id, t.name AS `key`, t.value AS value, 'exif' AS type, 0 AS private\n"
-
+            . " FROM technical_datas AS t\n"
-            UNION
+            . " WHERE record_id IN (?))\n";
-
-            (SELECT record_id, t.name AS `key`, t.value AS value, 'exif' AS type, 0 AS private
-            FROM technical_datas AS t
-            WHERE record_id IN (?))
-SQL;
 
         $ids = array_keys($records);
         $statement = $this->connection->executeQuery(
@@ -62,7 +57,7 @@ SQL;
         while ($metadata = $statement->fetch()) {
             // Store metadata value
             $key = $metadata['key'];
-            $value = $metadata['value'];
+            $value = trim($metadata['value']);
 
             // Do not keep empty values
             if ($key === '' || $value === '') {
@@ -80,7 +75,7 @@ SQL;
                 case 'caption':
                     // Sanitize fields
                     $value = StringHelper::crlfNormalize($value);
-                    $value = $this->sanitizeValue($value, $this->structure->typeOf($key));
+                    $value = $this->helper->sanitizeValue($value, $this->structure->typeOf($key));
                     // Private caption fields are kept apart
                     $type = $metadata['private'] ? 'private_caption' : 'caption';
                     // Caption are multi-valued
@@ -103,7 +98,7 @@ SQL;
                     }
                     $tag = $this->structure->getMetadataTagByName($key);
                     if ($tag) {
-                        $value = $this->sanitizeValue($value, $tag->getType());
+                        $value = $this->helper->sanitizeValue($value, $tag->getType());
                     }
                     // EXIF data is single-valued
                     $record['metadata_tags'][$key] = $value;
@@ -118,33 +113,6 @@ SQL;
         $this->clearGpsPositionBuffer();
     }
 
-    private function sanitizeValue($value, $type)
-    {
-        switch ($type) {
-            case FieldMapping::TYPE_STRING:
-                return str_replace("\0", "", $value);
-
-            case FieldMapping::TYPE_DATE:
-                return $this->helper->sanitizeDate($value);
-
-            case FieldMapping::TYPE_FLOAT:
-            case FieldMapping::TYPE_DOUBLE:
-                return (float) $value;
-
-            case FieldMapping::TYPE_INTEGER:
-            case FieldMapping::TYPE_LONG:
-            case FieldMapping::TYPE_SHORT:
-            case FieldMapping::TYPE_BYTE:
-                return (int) $value;
-
-            case FieldMapping::TYPE_BOOLEAN:
-                return (bool) $value;
-
-            default:
-                return $value;
-        }
-    }
-
     private function handleGpsPosition(&$records, $id, $tag_name, $value)
     {
         // Get position object

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/TitleHydrator.php

@@ -11,6 +11,8 @@
 
 namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator;
 
+use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
+use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
 use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Driver\Connection as DriverConnection;
 
@@ -18,31 +20,34 @@ class TitleHydrator implements HydratorInterface
 {
     private $connection;
 
-    public function __construct(DriverConnection $connection)
+    /** @var RecordHelper */
+    private $helper;
+
+    public function __construct(DriverConnection $connection, RecordHelper $helper)
     {
         $this->connection = $connection;
+        $this->helper = $helper;
     }
 
     public function hydrateRecords(array &$records)
     {
-        $sql = <<<SQL
+        $sql = "SELECT\n"
-            SELECT
+            . "m.`record_id`,\n"
-                m.`record_id`,
+            . "   CASE ms.`thumbtitle`\n"
-                CASE ms.`thumbtitle`
+            . "     WHEN '1' THEN 'default'\n"
-                  WHEN "1" THEN "default"
+            . "     WHEN '0' THEN 'default'\n"
-                  WHEN "0" THEN "default"
+            . "     ELSE ms.`thumbtitle`\n"
-                  ELSE ms.`thumbtitle`
+            . "   END AS locale,\n"
-                END AS locale,
+            . "   CASE ms.`thumbtitle`\n"
-                CASE ms.`thumbtitle`
+            . "     WHEN '0' THEN r.`originalname`\n"
-                  WHEN "0" THEN r.`originalname`
+            . "     ELSE GROUP_CONCAT(m.`value` ORDER BY ms.`thumbtitle`, ms.`sorter` SEPARATOR ' - ')\n"
-                  ELSE GROUP_CONCAT(m.`value` ORDER BY ms.`thumbtitle`, ms.`sorter` SEPARATOR " - ")
+            . "   END AS title\n"
-                END AS title
+            . "FROM metadatas AS m FORCE INDEX(`record_id`)\n"
-            FROM metadatas AS m FORCE INDEX(`record_id`)
+            . "STRAIGHT_JOIN metadatas_structure AS ms ON (ms.`id` = m.`meta_struct_id`)\n"
-            STRAIGHT_JOIN metadatas_structure AS ms ON (ms.`id` = m.`meta_struct_id`)
+            . "STRAIGHT_JOIN record AS r ON (r.`record_id` = m.`record_id`)\n"
-            STRAIGHT_JOIN record AS r ON (r.`record_id` = m.`record_id`)
+            . "WHERE m.`record_id` IN (?)\n"
-            WHERE m.`record_id` IN (?)
+            . "GROUP BY m.`record_id`, ms.`thumbtitle`\n";
-            GROUP BY m.`record_id`, ms.`thumbtitle`
+
-SQL;
         $statement = $this->connection->executeQuery(
             $sql,
             array(array_keys($records)),
@@ -50,7 +55,7 @@ SQL;
         );
 
         while ($row = $statement->fetch()) {
-            $records[$row['record_id']]['title'][$row['locale']] = $row['title'];
+            $records[$row['record_id']]['title'][$row['locale']] = $this->helper->sanitizeValue($row['title'], FieldMapping::TYPE_STRING);
         }
     }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/DateFieldMapping.php

@@ -57,6 +57,9 @@ class DateFieldMapping extends ComplexFieldMapping
      */
     protected function getProperties()
     {
-        return array_merge([ 'format' => $this->format ], parent::getProperties());
+        return array_merge([
+            'format' => $this->format,
+            'ignore_malformed' => true
+            ], parent::getProperties());
     }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/RecordHelper.php

@@ -89,31 +89,72 @@ class RecordHelper
         return $this->collectionMap;
     }
 
-    /**
-     * @param string $date
-     * @return bool
-     */
-    public static function validateDate($date)
-    {
-        $d = DateTime::createFromFormat(FieldMapping::DATE_FORMAT_CAPTION_PHP, $date);
-
-        return $d && $d->format(FieldMapping::DATE_FORMAT_CAPTION_PHP) == $date;
-    }
-
     /**
      * @param string $value
      * @return null|string
      */
     public static function sanitizeDate($value)
     {
-        // introduced in https://github.com/alchemy-fr/Phraseanet/commit/775ce804e0257d3a06e4e068bd17330a79eb8370#diff-bee690ed259e0cf73a31dee5295d2edcR286
+        $v_fix = null;
-        // not sure if it's really needed
         try {
-            $date = new \DateTime($value);
+            $a = explode(';', preg_replace('/\D+/', ';', trim($value)));
-
+            switch (count($a)) {
-            return $date->format(FieldMapping::DATE_FORMAT_CAPTION_PHP);
+                case 1:     // yyyy
+                    $date = new \DateTime($a[0] . '-01-01');    // will throw if date is not valid
+                    $v_fix = $date->format('Y');
+                    break;
+                case 2:     // yyyy;mm
+                    $date = new \DateTime( $a[0] . '-' . $a[1] . '-01');
+                    $v_fix = $date->format('Y-m');
+                    break;
+                case 3:     // yyyy;mm;dd
+                    $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2]);
+                    $v_fix = $date->format('Y-m-d');
+                    break;
+                case 4:
+                    $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':00:00');
+                    $v_fix = $date->format('Y-m-d H:i:s');
+                    break;
+                case 5:
+                    $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':00');
+                    $v_fix = $date->format('Y-m-d H:i:s');
+                    break;
+                case 6:
+                    $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':' . $a[5]);
+                    $v_fix = $date->format('Y-m-d H:i:s');
+                    break;
+            }
         } catch (\Exception $e) {
-            return null;
+            // no-op, v_fix = null
+        }
+
+        return $v_fix;
+    }
+
+    public function sanitizeValue($value, $type)
+    {
+        switch ($type) {
+            case FieldMapping::TYPE_DATE:
+                return self::sanitizeDate($value);
+
+            case FieldMapping::TYPE_FLOAT:
+            case FieldMapping::TYPE_DOUBLE:
+                return (float) $value;
+
+            case FieldMapping::TYPE_INTEGER:
+            case FieldMapping::TYPE_LONG:
+            case FieldMapping::TYPE_SHORT:
+            case FieldMapping::TYPE_BYTE:
+                return (int) $value;
+
+            case FieldMapping::TYPE_BOOLEAN:
+                return (bool) $value;
+
+            case FieldMapping::TYPE_STRING:
+                return str_replace("\0", '', $value);
+
+            default:
+                return $value;
         }
     }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryHelper.php

@@ -110,41 +110,50 @@ class QueryHelper
         }
     }
 
-    public static function getRangeFromDateString($string)
+    public static function getRangeFromDateString($value)
     {
-        $formats = ['Y/m/d', 'Y/m', 'Y'];
+        $date_from = null;
-        $deltas = ['+1 day', '+1 month', '+1 year'];
+        $date_to   = null;
-        $to = null;
+        try {
-        while ($format = array_pop($formats)) {
+            $a = explode(';', preg_replace('/\D+/', ';', trim($value)));
-            $delta = array_pop($deltas);
+            switch (count($a)) {
-            $from = date_create_from_format($format, $string);
+                case 1:     // yyyy
-            if ($from !== false) {
+                    $date_to = clone($date_from = new \DateTime($a[0] . '-01-01 00:00:00'));    // will throw if date is not valid
-                // Rewind to start of range
+                    $date_to->add(new \DateInterval('P1Y'));
-                $month = 1;
+                    break;
-                $day = 1;
+                case 2:     // yyyy;mm
-                switch ($format) {
+                    $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-01 00:00:00'));    // will throw if date is not valid
-                    case 'Y/m/d':
+                    $date_to->add(new \DateInterval('P1M'));
-                        $day = (int) $from->format('d');
+                    break;
-                    case 'Y/m':
+                case 3:     // yyyy;mm;dd
-                        $month = (int) $from->format('m');
+                    $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' 00:00:00'));    // will throw if date is not valid
-                    case 'Y':
+                    $date_to->add(new \DateInterval('P1D'));
-                        $year = (int) $from->format('Y');
+                    break;
-                }
+                case 4:
-                date_date_set($from, $year, $month, $day);
+                    $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':00:00'));
-                date_time_set($from, 0, 0, 0);
+                    $date_to->add(new \DateInterval('PT1H'));
-                // Create end of the the range
+                    break;
-                $to = date_modify(clone $from, $delta);
+                case 5:
-                break;
+                    $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':00'));
+                    $date_to->add(new \DateInterval('PT1M'));
+                    break;
+                case 6:
+                    $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':' . $a[5]));
+                    // $date_to->add(new \DateInterval('PT1S'));    // no need since precision is 1 sec, a "equal" will be generated when from==to
+                    break;
             }
         }
+        catch (\Exception $e) {
+            // no-op
+        }
 
-        if (!$from || !$to) {
+        if ($date_from === null || $date_to === null) {
-            throw new \InvalidArgumentException(sprintf('Invalid date "%s".', $string));
+            throw new \InvalidArgumentException(sprintf('Invalid date "%s".', $value));
         }
 
         return [
-            'from' => $from->format(FieldMapping::DATE_FORMAT_CAPTION_PHP),
+            'from' => $date_from->format('Y-m-d H:i:s'),
-            'to'   => $to->format(FieldMapping::DATE_FORMAT_CAPTION_PHP)
+            'to'   => $date_to->format('Y-m-d H:i:s')
         ];
     }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php

@@ -5,7 +5,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Search;
 use Alchemy\Phrasea\SearchEngine\Elastic\AST;
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
 use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
-use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
+use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
 use Hoa\Compiler\Llk\TreeNode;
 use Hoa\Visitor\Element;
@@ -166,6 +166,12 @@ class QueryVisitor implements Visit
         $key = $node->getChild(0)->accept($this);
         $boundary = $node->getChild(1)->accept($this);
 
+        if ($this->isDateKey($key)) {
+            if(($v = RecordHelper::sanitizeDate($boundary)) !== null) {
+                $boundary = $v;
+            }
+        }
+
         switch ($node->getId()) {
             case NodeTypes::LT_EXPR:
                 return AST\KeyValue\RangeExpression::lessThan($key, $boundary);
@@ -195,11 +201,15 @@ class QueryVisitor implements Visit
                 try {
                     // Try to create a range for incomplete dates
                     $range = QueryHelper::getRangeFromDateString($right);
-                    return new AST\KeyValue\RangeExpression(
+                    if ($range['from'] === $range['to']) {
-                        $left,
+                        return new AST\KeyValue\EqualExpression($left, $range['from']);
-                        $range['from'], true,
+                    } else {
-                        $range['to'], false
+                        return new AST\KeyValue\RangeExpression(
-                    );
+                            $left,
+                            $range['from'], true,
+                            $range['to'], false
+                        );
+                    }
                 } catch (\InvalidArgumentException $e) {
                     // Fall back to equal expression
                 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/ValueChecker.php

@@ -3,7 +3,6 @@
 namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
-use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
 use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
 use Assert\Assertion;
 
@@ -20,7 +19,7 @@ class ValueChecker
     {
         Assertion::allIsInstanceOf($list, Typed::class);
         $is_numeric = is_numeric($value);
-        $is_valid_date = RecordHelper::validateDate($value);
+        $is_valid_date = (RecordHelper::sanitizeDate($value) !== null);
         $filtered = [];
         foreach ($list as $item) {
             switch ($item->getType()) {

lib/Alchemy/Phrasea/TaskManager/Job/WriteMetadataJob.php

@@ -127,7 +127,10 @@ class WriteMetadataJob extends AbstractJob
 
                     // check exiftool known tags to skip Phraseanet:tf-*
                     try {
-                        TagFactory::getFromRDFTagname($tagName);
+                        $tag = TagFactory::getFromRDFTagname($tagName);
+                        if(!$tag->isWritable()) {
+                            continue;
+                        }
                     } catch (TagUnknown $e) {
                         continue;
                     }
@@ -147,21 +150,34 @@ class WriteMetadataJob extends AbstractJob
                             $fieldValue = array_pop($fieldValues);
                             $value = $this->removeNulChar($fieldValue->getValue());
 
-                            $value = new Value\Mono($value);
+                            // fix the dates edited into phraseanet
+                            if($fieldStructure->get_type() === $fieldStructure::TYPE_DATE) {
+                                try {
+                                    $value = self::fixDate($value); // will return NULL if the date is not valid
+                                }
+                                catch (\Exception $e) {
+                                    $value = null;    // do NOT write back to iptc
+                                }
+                            }
+
+                            if($value !== null) {   // do not write invalid dates
+                                $value = new Value\Mono($value);
+                            }
                         }
-                    } catch(\Exception $e) {
+                    } catch (\Exception $e) {
                         // the field is not set in the record, erase it
                         if ($fieldStructure->is_multi()) {
                             $value = new Value\Multi(array(''));
-                        }
+                        } else {
-                        else {
                             $value = new Value\Mono('');
                         }
                     }
 
-                    $metadata->add(
+                    if($value !== null) {   // do not write invalid data
-                        new Metadata\Metadata($fieldStructure->get_tag(), $value)
+                        $metadata->add(
-                    );
+                            new Metadata\Metadata($fieldStructure->get_tag(), $value)
+                        );
+                    }
                 }
 
                 $writer = $this->getMetadataWriter($jobData->getApplication());
@@ -220,4 +236,34 @@ class WriteMetadataJob extends AbstractJob
     {
         return str_replace("\0", "", $value);
     }
+
+    /**
+     * re-format a phraseanet date for iptc writing
+     * return NULL if the date is not valid
+     *
+     * @param string $value
+     * @return string|null
+     */
+    private static function fixDate($value)
+    {
+        $date = null;
+        try {
+            $a = explode(';', preg_replace('/\D+/', ';', trim($value)));
+            switch (count($a)) {
+                case 3:     // yyyy;mm;dd
+                    $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2]);
+                    $date = $date->format('Y-m-d H:i:s');
+                    break;
+                case 6:     // yyyy;mm;dd;hh;mm;ss
+                    $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':' . $a[5]);
+                    $date = $date->format('Y-m-d H:i:s');
+                    break;
+            }
+        }
+        catch (\Exception $e) {
+            $date = null;
+        }
+
+        return $date;
+    }
 }

lib/classes/cache/databox.php

@@ -121,9 +121,9 @@ class cache_databox
 
         $conn = $app->getApplicationBox()->get_connection();
 
-        $sql = 'UPDATE sitepreff SET memcached_update = :date';
+        $sql = 'UPDATE sitepreff SET memcached_update = current_timestamp()';
         $stmt = $conn->prepare($sql);
-        $stmt->execute([':date' => $now]);
+        $stmt->execute();
         $stmt->closeCursor();
 
         self::$refreshing = false;

tests/Alchemy/Tests/Phrasea/SearchEngine/AST/RangeExpressionTest.php

@@ -58,6 +58,7 @@ class RangeExpressionTest extends \PHPUnit_Framework_TestCase
     {
         $query_context = $this->prophesize(QueryContext::class)->reveal();
         $key_prophecy = $this->prophesize(Key::class);
+        $key_prophecy->getFieldType($query_context)->willReturn('text');
         $key_prophecy->getIndexField($query_context)->willReturn('foo');
         $key_prophecy->isValueCompatible('bar', $query_context)->willReturn(true);
         $key = $key_prophecy->reveal();
@@ -73,6 +74,7 @@ class RangeExpressionTest extends \PHPUnit_Framework_TestCase
     {
         $query_context = $this->prophesize(QueryContext::class)->reveal();
         $key = $this->prophesize(FieldKey::class);
+        $key->getFieldType($query_context)->willReturn('text');
         $key->getIndexField($query_context)->willReturn('baz');
         $key->isValueCompatible('bar', $query_context)->willReturn(true);
         $key->postProcessQuery(Argument::any(), $query_context)->willReturnArgument(0);

tests/Alchemy/Tests/Phrasea/SearchEngine/resources/queries.csv

@@ -53,6 +53,7 @@ foo < 42|<range:field.foo lt="42">
 foo ≤ 42|<range:field.foo lte="42">
 foo > 42|<range:field.foo gt="42">
 foo ≥ 42|<range:field.foo gte="42">
+foo = 2015/01/01|(<field.foo> == <value:"2015/01/01">)
 foo < 2015/01/01|<range:field.foo lt="2015/01/01">
 foo ≤ 2015/01/01|<range:field.foo lte="2015/01/01">
 foo > 2015/01/01|<range:field.foo gt="2015/01/01">
@@ -93,19 +94,93 @@ id:90 AND foo|(<record_identifier:"90"> AND <text:"foo">)
 id:90 foo|(<record_identifier:"90"> AND <text:"foo">)
 recordid:90|<record_identifier:"90">
 
-# Timestamps
+# Timestamps yyyy
-created_on < "2015/01/01"|<range:creation lt="2015/01/01">
+created_on < "2015"|<range:creation lt="2015">
-created_on ≤ "2015/01/01"|<range:creation lte="2015/01/01">
+created_on ≤ "2015"|<range:creation lte="2015">
-created_on = "2015/01/01"|<range:creation gte="2015/01/01" lt="2015/01/02">
+created_on = "2015"|<range:creation gte="2015-01-01 00:00:00" lt="2016-01-01 00:00:00">
-created_on ≥ "2015/01/01"|<range:creation gte="2015/01/01">
+created_on ≥ "2015"|<range:creation gte="2015">
-created_on > "2015/01/01"|<range:creation gt="2015/01/01">
+created_on > "2015"|<range:creation gt="2015">
-updated_on < "2015/01/01"|<range:update lt="2015/01/01">
+updated_on < "2015"|<range:update lt="2015">
-updated_on ≤ "2015/01/01"|<range:update lte="2015/01/01">
+updated_on ≤ "2015"|<range:update lte="2015">
-updated_on = "2015/01/01"|<range:update gte="2015/01/01" lt="2015/01/02">
+updated_on = "2015"|<range:update gte="2015-01-01 00:00:00" lt="2016-01-01 00:00:00">
-updated_on ≥ "2015/01/01"|<range:update gte="2015/01/01">
+updated_on ≥ "2015"|<range:update gte="2015">
-updated_on > "2015/01/01"|<range:update gt="2015/01/01">
+updated_on > "2015"|<range:update gt="2015">
-created_at > "2015/01/01"|<range:creation gt="2015/01/01">
+created_at > "2015"|<range:creation gt="2015">
-updated_at > "2015/01/01"|<range:update gt="2015/01/01">
+updated_at > "2015"|<range:update gt="2015">
+
+# Timestamps yyyy/mm
+created_on < "2015/01"|<range:creation lt="2015-01">
+created_on ≤ "2015/01"|<range:creation lte="2015-01">
+created_on = "2015/01"|<range:creation gte="2015-01-01 00:00:00" lt="2015-02-01 00:00:00">
+created_on ≥ "2015/01"|<range:creation gte="2015-01">
+created_on > "2015/01"|<range:creation gt="2015-01">
+updated_on < "2015/01"|<range:update lt="2015-01">
+updated_on ≤ "2015/01"|<range:update lte="2015-01">
+updated_on = "2015/01"|<range:update gte="2015-01-01 00:00:00" lt="2015-02-01 00:00:00">
+updated_on ≥ "2015/01"|<range:update gte="2015-01">
+updated_on > "2015/01"|<range:update gt="2015-01">
+created_at > "2015/01"|<range:creation gt="2015-01">
+updated_at > "2015/01"|<range:update gt="2015-01">
+
+# Timestamps yyyy/mm/dd
+created_on < "2015/01/01"|<range:creation lt="2015-01-01">
+created_on ≤ "2015/01/01"|<range:creation lte="2015-01-01">
+created_on = "2015/01/01"|<range:creation gte="2015-01-01 00:00:00" lt="2015-01-02 00:00:00">
+created_on ≥ "2015/01/01"|<range:creation gte="2015-01-01">
+created_on > "2015/01/01"|<range:creation gt="2015-01-01">
+updated_on < "2015/01/01"|<range:update lt="2015-01-01">
+updated_on ≤ "2015/01/01"|<range:update lte="2015-01-01">
+updated_on = "2015/01/01"|<range:update gte="2015-01-01 00:00:00" lt="2015-01-02 00:00:00">
+updated_on ≥ "2015/01/01"|<range:update gte="2015-01-01">
+updated_on > "2015/01/01"|<range:update gt="2015-01-01">
+created_at > "2015/01/01"|<range:creation gt="2015-01-01">
+updated_at > "2015/01/01"|<range:update gt="2015-01-01">
+
+# Timestamps yyyy/mm/dd hh
+created_on < "2015/01/01 12"|<range:creation lt="2015-01-01 12:00:00">
+created_on ≤ "2015/01/01 12"|<range:creation lte="2015-01-01 12:00:00">
+created_on = "2015/01/01 12"|<range:creation gte="2015-01-01 12:00:00" lt="2015-01-01 13:00:00">
+created_on ≥ "2015/01/01 12"|<range:creation gte="2015-01-01 12:00:00">
+created_on > "2015/01/01 12"|<range:creation gt="2015-01-01 12:00:00">
+updated_on < "2015/01/01 12"|<range:update lt="2015-01-01 12:00:00">
+updated_on ≤ "2015/01/01 12"|<range:update lte="2015-01-01 12:00:00">
+updated_on = "2015/01/01 12"|<range:update gte="2015-01-01 12:00:00" lt="2015-01-01 13:00:00">
+updated_on ≥ "2015/01/01 12"|<range:update gte="2015-01-01 12:00:00">
+updated_on > "2015/01/01 12"|<range:update gt="2015-01-01 12:00:00">
+created_at > "2015/01/01 12"|<range:creation gt="2015-01-01 12:00:00">
+updated_at > "2015/01/01 12"|<range:update gt="2015-01-01 12:00:00">
+
+# Timestamps yyyy/mm/dd hh:mm
+created_on < "2015/01/01 12.34"|<range:creation lt="2015-01-01 12:34:00">
+created_on ≤ "2015/01/01 12.34"|<range:creation lte="2015-01-01 12:34:00">
+created_on = "2015/01/01 12.34"|<range:creation gte="2015-01-01 12:34:00" lt="2015-01-01 12:35:00">
+created_on ≥ "2015/01/01 12.34"|<range:creation gte="2015-01-01 12:34:00">
+created_on > "2015/01/01 12.34"|<range:creation gt="2015-01-01 12:34:00">
+updated_on < "2015/01/01 12.34"|<range:update lt="2015-01-01 12:34:00">
+updated_on ≤ "2015/01/01 12.34"|<range:update lte="2015-01-01 12:34:00">
+updated_on = "2015/01/01 12.34"|<range:update gte="2015-01-01 12:34:00" lt="2015-01-01 12:35:00">
+updated_on ≥ "2015/01/01 12.34"|<range:update gte="2015-01-01 12:34:00">
+updated_on > "2015/01/01 12.34"|<range:update gt="2015-01-01 12:34:00">
+created_at > "2015/01/01 12.34"|<range:creation gt="2015-01-01 12:34:00">
+updated_at > "2015/01/01 12.34"|<range:update gt="2015-01-01 12:34:00">
+
+# Timestamps yyyy/mm/dd hh.mm.ss
+created_on < "2015/01/01 12.34.56"|<range:creation lt="2015-01-01 12:34:56">
+created_on ≤ "2015/01/01 12.34.56"|<range:creation lte="2015-01-01 12:34:56">
+created_on = "2015/01/01 12.34.56"|(<creation> == <value:"2015-01-01 12:34:56">)
+created_on ≥ "2015/01/01 12.34.56"|<range:creation gte="2015-01-01 12:34:56">
+created_on > "2015/01/01 12.34.56"|<range:creation gt="2015-01-01 12:34:56">
+updated_on < "2015/01/01 12.34.56"|<range:update lt="2015-01-01 12:34:56">
+updated_on ≤ "2015/01/01 12.34.56"|<range:update lte="2015-01-01 12:34:56">
+updated_on = "2015/01/01 12.34.56"|(<update> == <value:"2015-01-01 12:34:56">)
+updated_on ≥ "2015/01/01 12.34.56"|<range:update gte="2015-01-01 12:34:56">
+updated_on > "2015/01/01 12.34.56"|<range:update gt="2015-01-01 12:34:56">
+created_at > "2015/01/01 12.34.56"|<range:creation gt="2015-01-01 12:34:56">
+updated_at > "2015/01/01 12.34.56"|<range:update gt="2015-01-01 12:34:56">
+
+# timestamps missing zeros
+created_on = "2015/1/2 1.3.5"|(<creation> == <value:"2015-01-02 01:03:05">)
+
 
 # Flag matcher
 flag.foo:true|<flag:foo set>