hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-18 15:33:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into PHRAS-3021_add-volumes-custom-cache

Browse Source
This commit is contained in:
Nicolas Maillat
2020-04-09 00:36:12 +02:00
committed by GitHub
parent 40471f59e5 457d080b56
commit 958eb864ff
4 changed files with 27 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/RecordListFetcherDelegate.php
View File

@@ -24,7 +24,7 @@ class RecordListFetcherDelegate implements FetcherDelegateInterface
public function buildWhereClause()
{
return 'WHERE r.record_id IN (:record_identifiers)';
return 'r.record_id IN (:record_identifiers)';
}
public function getParameters()

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Delegate/ScheduledFetcherDelegate.php
View File

@@ -18,7 +18,7 @@ class ScheduledFetcherDelegate implements FetcherDelegateInterface
{
public function buildWhereClause()
{
return 'WHERE (r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0';
return '(r.jeton & :to_index) > 0 AND (jeton & :indexing) = 0';
}
public function getParameters()

26
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php
View File

@@ -30,7 +30,8 @@ class Fetcher
private $statement;
private $delegate;
private $offset = 0;
// since we fetch records dy DESC, this will be the HIGHEST record_id fetched during last batch
private $upper_rid = PHP_INT_MAX;
private $batchSize = 1;
private $buffer = array();
@@ -68,12 +69,16 @@ class Fetcher
{
// Fetch records rows
$statement = $this->getExecutedStatement();
// printf("Query %d/%d -> %d rows\n", $this->offset, $this->batchSize, $statement->rowCount());
// printf("Query %d(%d) -> %d rows\n", $this->upper_rid, $this->batchSize, $statement->rowCount());
$records = [];
$this->upper_rid = PHP_INT_MAX;
while ($record = $statement->fetch()) {
$records[$record['record_id']] = $record;
$this->offset++;
$rid = (int)($record['record_id']);
if($rid < $this->upper_rid) {
$this->upper_rid = (int)($record['record_id']);
}
}
if (empty($records)) {
$this->onDrain->__invoke();
@@ -100,7 +105,7 @@ class Fetcher
public function restart()
{
$this->buffer = array();
$this->offset = 0;
$this->upper_rid = PHP_INT_MAX;
}
public function setBatchSize($size)
@@ -133,9 +138,9 @@ class Fetcher
. " r.originalname AS original_name, r.mime, r.type, r.parent_record_id,\n"
. " r.credate AS created_on, r.moddate AS updated_on, r.coll_id\n"
. " FROM record r\n"
. " -- WHERE\n"
. " WHERE -- WHERE\n"
. " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "\n"
. " LIMIT :offset, :limit\n"
. " LIMIT :limit\n"
. " ) AS r\n"
. " INNER JOIN coll c ON (c.coll_id = r.coll_id)\n"
. " )\n"
@@ -143,7 +148,10 @@ class Fetcher
. " subdef ON subdef.record_id=r.record_id AND subdef.name='document'\n"
. " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "";
$where = $this->delegate->buildWhereClause();
$where = 'record_id < :upper_rid';
if( ($w = $this->delegate->buildWhereClause()) != '') {
$where = '(' . $where . ') AND (' . $w . ')';
}
$sql = str_replace('-- WHERE', $where, $sql);
// Build parameters list
@@ -167,12 +175,12 @@ class Fetcher
}
}
// Reference bound parameters
$statement->bindParam(':offset', $this->offset, PDO::PARAM_INT);
$statement->bindParam(':upper_rid', $this->upper_rid, PDO::PARAM_INT);
$statement->bindParam(':limit', $this->batchSize, PDO::PARAM_INT);
$this->statement = $statement;
} else {
// Inject own query parameters
$params[':offset'] = $this->offset;
$params[':upper_rid'] = $this->upper_rid;
$params[':limit'] = $this->batchSize;
$types[':offset'] = $types[':limit'] = PDO::PARAM_INT;

9
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php
View File

@@ -82,7 +82,14 @@ class RecordIndex implements MappingProvider
$mapping->add($this->buildMetadataTagMapping('metadata_tags'));
$mapping->add($this->buildFlagMapping('flags'));
$mapping->addIntegerField('flags_bitfield')->disableIndexing();
// es int type is always int32 (32 bits signed), so on php-64 we may receive overflow values if the last sb (#31) is set.
// In mysql we use unsigned type, so the output value (as decimal string) may also overflow on php-32.
// Since we use a php lib for es, - with php-int as underlying type -
// we have no way to use a binary-string notation from mysql to es.
// The easy way here is to use a long (int64) in es, even if there is 32 status-bits in phraseanet
// nb : not fixed on php-32 !
$mapping->addLongField('flags_bitfield')->disableIndexing();
$mapping->addObjectField('subdefs')->disableMapping();
$mapping->addObjectField('title')->disableMapping();