From 662fe95712b0fa2fd3d4ac357f71d9c8c60571de Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Mon, 6 Apr 2020 19:51:52 +0200 Subject: [PATCH 1/3] PHRAS-3015_sb31-crashes-indexer_4.1 fix : set es:flags_bitfield as long (signed 64 bits) --- .../Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php index 9c86c1dc11..869d43cd4a 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php @@ -82,7 +82,14 @@ class RecordIndex implements MappingProvider $mapping->add($this->buildMetadataTagMapping('metadata_tags')); $mapping->add($this->buildFlagMapping('flags')); - $mapping->addIntegerField('flags_bitfield')->disableIndexing(); + // es int type is always int32 (32 bits signed), so on php-64 we may receive overflow values if the last sb (#31) is set. + // In mysql we use unsigned type, so the output value (as decimal string) may also overflow on php-32. + // Since we use a php lib for es, - with php-int as underlying type - + // we have no way to use a binary-string notation from mysql to es. + // The easy way here is to use a long (int64) in es, even if there is 32 status-bits in phraseanet + // nb : not fixed on php-32 ! + $mapping->addLongField('flags_bitfield')->disableIndexing(); + $mapping->addObjectField('subdefs')->disableMapping(); $mapping->addObjectField('title')->disableMapping(); From 2b4accae918996c1519b13eae14b4aacf4748006 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 8 Apr 2020 19:50:57 +0200 Subject: [PATCH 2/3] PHRAS-3023_slow-query-while-indexing_4.1 fix : change fetcher query to paginate with record_id/count (not limit/count) --- .../Elastic/Indexer/Record/Fetcher.php | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php index 8a379b6269..a57339b9fc 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php @@ -30,7 +30,8 @@ class Fetcher private $statement; private $delegate; - private $offset = 0; + // since we fetch records dy DESC, this will be the HIGHEST record_id fetched during last batch + private $upper_rid = PHP_INT_MAX; private $batchSize = 1; private $buffer = array(); @@ -68,12 +69,16 @@ class Fetcher { // Fetch records rows $statement = $this->getExecutedStatement(); - // printf("Query %d/%d -> %d rows\n", $this->offset, $this->batchSize, $statement->rowCount()); + // printf("Query %d(%d) -> %d rows\n", $this->upper_rid, $this->batchSize, $statement->rowCount()); $records = []; + $this->upper_rid = PHP_INT_MAX; while ($record = $statement->fetch()) { $records[$record['record_id']] = $record; - $this->offset++; + $rid = (int)($record['record_id']); + if($rid < $this->upper_rid) { + $this->upper_rid = (int)($record['record_id']); + } } if (empty($records)) { $this->onDrain->__invoke(); @@ -100,7 +105,7 @@ class Fetcher public function restart() { $this->buffer = array(); - $this->offset = 0; + $this->upper_rid = PHP_INT_MAX; } public function setBatchSize($size) @@ -133,9 +138,9 @@ class Fetcher . " r.originalname AS original_name, r.mime, r.type, r.parent_record_id,\n" . " r.credate AS created_on, r.moddate AS updated_on, r.coll_id\n" . " FROM record r\n" - . " -- WHERE\n" + . " WHERE -- WHERE\n" . " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "\n" - . " LIMIT :offset, :limit\n" + . " LIMIT :limit\n" . " ) AS r\n" . " INNER JOIN coll c ON (c.coll_id = r.coll_id)\n" . " )\n" @@ -143,7 +148,10 @@ class Fetcher . " subdef ON subdef.record_id=r.record_id AND subdef.name='document'\n" . " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . ""; - $where = $this->delegate->buildWhereClause(); + $where = 'record_id < :upper_rid'; + if( ($w = $this->delegate->buildWhereClause()) != '') { + $where = '(' . $where . ') AND (' . $w . ')'; + } $sql = str_replace('-- WHERE', $where, $sql); // Build parameters list @@ -167,12 +175,12 @@ class Fetcher } } // Reference bound parameters - $statement->bindParam(':offset', $this->offset, PDO::PARAM_INT); + $statement->bindParam(':upper_rid', $this->upper_rid, PDO::PARAM_INT); $statement->bindParam(':limit', $this->batchSize, PDO::PARAM_INT); $this->statement = $statement; } else { // Inject own query parameters - $params[':offset'] = $this->offset; + $params[':upper_rid'] = $this->upper_rid; $params[':limit'] = $this->batchSize; $types[':offset'] = $types[':limit'] = PDO::PARAM_INT; From 3b99f5bb5090199c7b73be5c24625ba0d19e1ada Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 8 Apr 2020 20:06:19 +0200 Subject: [PATCH 3/3] PHRAS-3023_slow-query-while-indexing_4.1 fix : remove double where --- .../Indexer/Record/Delegate/RecordListFetcherDelegate.php | 2 +- .../Indexer/Record/Delegate/ScheduledFetcherDelegate.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/RecordListFetcherDelegate.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/RecordListFetcherDelegate.php index b0b36222b4..d57595d486 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/RecordListFetcherDelegate.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/RecordListFetcherDelegate.php @@ -24,7 +24,7 @@ class RecordListFetcherDelegate implements FetcherDelegateInterface public function buildWhereClause() { - return 'WHERE r.record_id IN (:record_identifiers)'; + return 'r.record_id IN (:record_identifiers)'; } public function getParameters() diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/ScheduledFetcherDelegate.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/ScheduledFetcherDelegate.php index 79a1088285..aa56883146 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/ScheduledFetcherDelegate.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/ScheduledFetcherDelegate.php @@ -18,7 +18,7 @@ class ScheduledFetcherDelegate implements FetcherDelegateInterface { public function buildWhereClause() { - return 'WHERE (r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0'; + return '(r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0'; } public function getParameters()