Merge branch 'master' into PHRAS-3021_add-volumes-custom-cache

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/RecordListFetcherDelegate.php

@@ -24,7 +24,7 @@ class RecordListFetcherDelegate implements FetcherDelegateInterface
 
     public function buildWhereClause()
     {
-        return 'WHERE r.record_id IN (:record_identifiers)';
+        return 'r.record_id IN (:record_identifiers)';
     }
 
     public function getParameters()

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/ScheduledFetcherDelegate.php

@@ -18,7 +18,7 @@ class ScheduledFetcherDelegate implements FetcherDelegateInterface
 {
     public function buildWhereClause()
     {
-        return 'WHERE (r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0';
+        return '(r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0';
     }
 
     public function getParameters()

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php

@@ -30,7 +30,8 @@ class Fetcher
     private $statement;
     private $delegate;
 
-    private $offset = 0;
+    // since we fetch records dy DESC, this will be the HIGHEST record_id fetched during last batch
+    private $upper_rid = PHP_INT_MAX;
     private $batchSize = 1;
     private $buffer = array();
 
@@ -68,12 +69,16 @@ class Fetcher
     {
         // Fetch records rows
         $statement = $this->getExecutedStatement();
-        // printf("Query %d/%d -> %d rows\n", $this->offset, $this->batchSize, $statement->rowCount());
+        // printf("Query %d(%d) -> %d rows\n", $this->upper_rid, $this->batchSize, $statement->rowCount());
 
         $records = [];
+        $this->upper_rid = PHP_INT_MAX;
         while ($record = $statement->fetch()) {
             $records[$record['record_id']] = $record;
-            $this->offset++;
+            $rid = (int)($record['record_id']);
+            if($rid < $this->upper_rid) {
+                $this->upper_rid = (int)($record['record_id']);
+            }
         }
         if (empty($records)) {
             $this->onDrain->__invoke();
@@ -100,7 +105,7 @@ class Fetcher
     public function restart()
     {
         $this->buffer = array();
-        $this->offset = 0;
+        $this->upper_rid = PHP_INT_MAX;
     }
 
     public function setBatchSize($size)
@@ -133,9 +138,9 @@ class Fetcher
                  . "            r.originalname AS original_name, r.mime, r.type, r.parent_record_id,\n"
                  . "            r.credate AS created_on, r.moddate AS updated_on, r.coll_id\n"
                  . "     FROM record r\n"
-                 . "     -- WHERE\n"
+                 . "     WHERE -- WHERE\n"
                  . "     ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "\n"
-                 . "     LIMIT :offset, :limit\n"
+                 . "     LIMIT :limit\n"
                  . "   ) AS r\n"
                  . "   INNER JOIN coll c ON (c.coll_id = r.coll_id)\n"
                  . " )\n"
@@ -143,7 +148,10 @@ class Fetcher
                  . " subdef ON subdef.record_id=r.record_id AND subdef.name='document'\n"
                  . " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "";
 
-            $where = $this->delegate->buildWhereClause();
+            $where = 'record_id < :upper_rid';
+            if( ($w = $this->delegate->buildWhereClause()) != '') {
+                $where = '(' . $where . ') AND (' . $w . ')';
+            }
             $sql = str_replace('-- WHERE', $where, $sql);
 
             // Build parameters list
@@ -167,12 +175,12 @@ class Fetcher
                     }
                 }
                 // Reference bound parameters
-                $statement->bindParam(':offset', $this->offset, PDO::PARAM_INT);
+                $statement->bindParam(':upper_rid', $this->upper_rid, PDO::PARAM_INT);
                 $statement->bindParam(':limit', $this->batchSize, PDO::PARAM_INT);
                 $this->statement = $statement;
             } else {
                 // Inject own query parameters
-                $params[':offset'] = $this->offset;
+                $params[':upper_rid'] = $this->upper_rid;
                 $params[':limit'] = $this->batchSize;
                 $types[':offset'] = $types[':limit'] = PDO::PARAM_INT;
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php

@@ -82,7 +82,14 @@ class RecordIndex implements MappingProvider
         $mapping->add($this->buildMetadataTagMapping('metadata_tags'));
         $mapping->add($this->buildFlagMapping('flags'));
 
-        $mapping->addIntegerField('flags_bitfield')->disableIndexing();
+        // es int type is always int32 (32 bits signed), so on php-64 we may receive overflow values if the last sb (#31) is set.
+        // In mysql we use unsigned type, so the output value (as decimal string) may also overflow on php-32.
+        // Since we use a php lib for es, - with php-int as underlying type -
+        //      we have no way to use a binary-string notation from mysql to es.
+        // The easy way here is to use a long (int64) in es, even if there is 32 status-bits in phraseanet
+        // nb : not fixed on php-32 !
+        $mapping->addLongField('flags_bitfield')->disableIndexing();
+
         $mapping->addObjectField('subdefs')->disableMapping();
         $mapping->addObjectField('title')->disableMapping();