hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-18 15:33:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3437 from alchemy-fr/PHRAS-3023_slow-query-while-indexing_4.1

Browse Source 
PHRAS-3023 #comment merge slow query while indexing_4.1
This commit is contained in:
Nicolas Maillat
2020-04-08 23:38:35 +02:00
committed by GitHub
parent 2b97e96b33 3b99f5bb50
commit 9c4c2c11e0
3 changed files with 19 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/RecordListFetcherDelegate.php
View File

@@ -24,7 +24,7 @@ class RecordListFetcherDelegate implements FetcherDelegateInterface
public function buildWhereClause()
{
return 'WHERE r.record_id IN (:record_identifiers)';
return 'r.record_id IN (:record_identifiers)';
}
public function getParameters()

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/ScheduledFetcherDelegate.php
View File

@@ -18,7 +18,7 @@ class ScheduledFetcherDelegate implements FetcherDelegateInterface
{
public function buildWhereClause()
{
return 'WHERE (r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0';
return '(r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0';
}
public function getParameters()

26
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php
View File

@@ -30,7 +30,8 @@ class Fetcher
private $statement;
private $delegate;
private $offset = 0;
// since we fetch records dy DESC, this will be the HIGHEST record_id fetched during last batch
private $upper_rid = PHP_INT_MAX;
private $batchSize = 1;
private $buffer = array();
@@ -68,12 +69,16 @@ class Fetcher
{
// Fetch records rows
$statement = $this->getExecutedStatement();
// printf("Query %d/%d -> %d rows\n", $this->offset, $this->batchSize, $statement->rowCount());
// printf("Query %d(%d) -> %d rows\n", $this->upper_rid, $this->batchSize, $statement->rowCount());
$records = [];
$this->upper_rid = PHP_INT_MAX;
while ($record = $statement->fetch()) {
$records[$record['record_id']] = $record;
$this->offset++;
$rid = (int)($record['record_id']);
if($rid < $this->upper_rid) {
$this->upper_rid = (int)($record['record_id']);
}
}
if (empty($records)) {
$this->onDrain->__invoke();
@@ -100,7 +105,7 @@ class Fetcher
public function restart()
{
$this->buffer = array();
$this->offset = 0;
$this->upper_rid = PHP_INT_MAX;
}
public function setBatchSize($size)
@@ -133,9 +138,9 @@ class Fetcher
. " r.originalname AS original_name, r.mime, r.type, r.parent_record_id,\n"
. " r.credate AS created_on, r.moddate AS updated_on, r.coll_id\n"
. " FROM record r\n"
. " -- WHERE\n"
. " WHERE -- WHERE\n"
. " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "\n"
. " LIMIT :offset, :limit\n"
. " LIMIT :limit\n"
. " ) AS r\n"
. " INNER JOIN coll c ON (c.coll_id = r.coll_id)\n"
. " )\n"
@@ -143,7 +148,10 @@ class Fetcher
. " subdef ON subdef.record_id=r.record_id AND subdef.name='document'\n"
. " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "";
$where = $this->delegate->buildWhereClause();
$where = 'record_id < :upper_rid';
if( ($w = $this->delegate->buildWhereClause()) != '') {
$where = '(' . $where . ') AND (' . $w . ')';
}
$sql = str_replace('-- WHERE', $where, $sql);
// Build parameters list
@@ -167,12 +175,12 @@ class Fetcher
}
}
// Reference bound parameters
$statement->bindParam(':offset', $this->offset, PDO::PARAM_INT);
$statement->bindParam(':upper_rid', $this->upper_rid, PDO::PARAM_INT);
$statement->bindParam(':limit', $this->batchSize, PDO::PARAM_INT);
$this->statement = $statement;
} else {
// Inject own query parameters
$params[':offset'] = $this->offset;
$params[':upper_rid'] = $this->upper_rid;
$params[':limit'] = $this->batchSize;
$types[':offset'] = $types[':limit'] = PDO::PARAM_INT;