hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-23 09:53:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

PHRAS-2161_coll-id-in-log-docs_MASTER

Browse Source 
Optimize sql.
Add the "coll_id_from" column in "log_docs" for the "collection" action.
Hydrate a "coll_id" column in "log_view"
This commit is contained in:
Jean-Yves Gaulier
2018-08-08 19:39:54 +02:00
parent 02efc2b9d1
commit e2745cba9e
2 changed files with 289 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

444
lib/Alchemy/Phrasea/Command/Setup/FixLogCollId.php
View File

@@ -24,6 +24,10 @@ class FixLogCollId extends Command
const OPTION_DISTINT_VALUES = 0;
const OPTION_ALL_VALUES = 1;
const PLAYDRY_NONE = 0;
const PLAYDRY_SQL = 1;
const PLAYDRY_CODE = 2;
/** @var InputInterface */
private $input;
/** @var OutputInterface */
@@ -40,13 +44,6 @@ class FixLogCollId extends Command
/** @var bool */
private $keep_tmp_table;
private $parm_k = [':minid', ':maxid'];
private $parm_v = [];
private $again;
const PLAYDRY_NONE = 0;
const PLAYDRY_SQL = 1;
const PLAYDRY_CODE = 2;
public function __construct($name = null)
{
@@ -129,206 +126,34 @@ class FixLogCollId extends Command
$this->input = $input;
$this->output = $output;
$tsql = [
'count' => [
'msg' => "Count work to do",
'sql' => "SELECT\n"
. " SUM(IF(ISNULL(`coll_id`), 0, 1)) AS `n`,\n"
. " COUNT(*) AS `t`,\n"
. " SUM(IF(`coll_id`>0, 1, 0)) AS `p`,\n"
. " SUM(IF(`coll_id`=0, 1, 0)) AS `z`\n"
. " FROM `log_docs`",
'fetch' => false,
'code' => function(ResultStatement $stmt) {
$row = $stmt->fetch();
if(is_null($row['n'])) {
// no coll_id ?
$this->output->writeln(sprintf("<error>The \"log_docs\" table has no \"coll_id\" column ? Please apply patch 410alpha12a</error>"));
$this->again = false;
}
$this->output->writeln("");
$this->output->writeln(sprintf("done: %s / %s (fixed: %s ; can't fix: %s)", $row['n'], $row['t'], $row['p'], $row['z']));
},
'playdry' => self::PLAYDRY_SQL | self::PLAYDRY_CODE,
],
'minmax' => [
'msg' => "Get a batch",
'sql' => "SELECT MIN(`id`) AS `minid`, MAX(`id`) AS `maxid` FROM\n"
. " (SELECT `id` FROM `log_docs` WHERE ISNULL(`coll_id`) ORDER BY `id` DESC LIMIT " . $this->batch_size . ") AS `t",
'fetch' => false,
'code' => function(ResultStatement $stmt) {
$row = $stmt->fetch();
$this->output->writeln("");
$this->output->writeln(sprintf("minid: %s ; maxid : %s\n", is_null($row['minid']) ? 'null' : $row['minid'], is_null($row['maxid']) ? 'null' : $row['maxid']));
if (is_null($row['minid']) || is_null($row['maxid'])) {
$this->again = false;
}
$this->parm_v = [(int)$row['minid'], (int)$row['maxid']];
},
'playdry' => self::PLAYDRY_SQL | self::PLAYDRY_CODE,
],
'trunc' => [
'msg' => "Empty working table",
'sql' => "TRUNCATE TABLE `tmp_colls`",
'fetch' => false,
'code' => null,
'playdry' => self::PLAYDRY_NONE,
],
'offset' => [
'msg' => "Make room for \"collection_from\" actions",
'sql' => "UPDATE `log_docs` SET `id` = `id` * 2 WHERE `id` >= :minid AND `id` <= :maxid ORDER BY `id` DESC",
'fetch' => false,
'code' =>
function(/** @noinspection PhpUnusedParameterInspection */ $stmt) {
// fix new minmax values since id was changed
$this->parm_v = [$this->parm_v[0] << 1, $this->parm_v[1] << 1];
},
'playdry' => self::PLAYDRY_CODE,
],
'compute' => [
'msg' => "Compute coll_id to working table",
'sql' => "INSERT INTO `tmp_colls`\n"
. " SELECT `record_id`, `r1_id` AS `from_id`, `r1_date` AS `from_date`, MIN(`r2_id`) AS `to_id`, MIN(`r2_date`) AS `to_date`, `r1_final` AS `coll_id` FROM\n"
. " (\n"
. " SELECT `r1`.`record_id`, `r1`.`id` AS `r1_id`, `r1`.`date` AS `r1_date`, `r1`.`final` AS `r1_final`, `r2`.`id` AS `r2_id`, `r2`.`date` AS `r2_date` FROM\n"
. " (SELECT `id`, `date`, `record_id`, `action`, `final` FROM `log_docs` WHERE `action` IN('add', 'collection') AND `id` >= :minid AND `id` <= :maxid) AS `r1`\n"
. " LEFT JOIN `log_docs` AS `r2`\n"
. " ON `r2`.`record_id`=`r1`.`record_id` AND `r2`.`action`='collection' AND `r2`.`id`>`r1`.`id`\n"
. " )\n"
. " AS `t` GROUP BY `r1_id` ORDER BY `record_id` ASC, `from_id` ASC",
'fetch' => false,
'code' => null,
'playdry' => self::PLAYDRY_NONE,
],
'copy_result' => [
'msg' => "Copy result back to \"log_docs\"",
'sql' => "UPDATE `tmp_colls` INNER JOIN `log_docs`\n"
. " ON `tmp_colls`.`record_id` = `log_docs`.`record_id`\n"
. " AND `log_docs`.`id` >= `tmp_colls`.`from_id`\n"
. " AND (`log_docs`.`id` < `tmp_colls`.`to_id` OR ISNULL(`tmp_colls`.`to_id`))\n"
. " SET `log_docs`.`coll_id` = `tmp_colls`.`coll_id`",
'fetch' => false,
'code' => null,
'playdry' => self::PLAYDRY_NONE,
],
'collection_from' => [
'msg' => "Insert \"collection_from\" actions",
'sql' => "INSERT INTO `log_docs` (`id`, `log_id`, `date`, `record_id`, `action`, `final`, `coll_id`)\n"
. " SELECT `r1`.`id`-1 AS `id`, `r1`.`log_id`, `r1`.`date`, `r1`.`record_id`, 'collection_from' AS `action`, `r1`.`final`,\n"
. " SUBSTRING_INDEX(GROUP_CONCAT(`r2`.`coll_id` ORDER BY `r1`.`id` DESC), ',', 1) AS `coll_id`\n"
. " FROM `log_docs` AS `r1` LEFT JOIN `log_docs` AS `r2`\n"
. " ON `r2`.`record_id` = `r1`.`record_id` AND `r2`.`id` < `r1`.`id` AND `r2`.`action` IN('collection', 'add')\n"
. " WHERE `r1`.`action` = 'collection' AND `r1`.`id` >= :minid AND `r1`.`id` <= :maxid\n"
. " GROUP BY `r1`.`id`",
'fetch' => false,
'code' => null,
'playdry' => self::PLAYDRY_NONE,
],
'fix_unfound' => [
'msg' => "Set missing coll_id to 0",
'sql' => "UPDATE `log_docs` SET `coll_id` = 0 WHERE `id` >= :minid AND `id` <= :maxid AND ISNULL(`coll_id`)",
'fetch' => false,
'code' => null,
'playdry' => self::PLAYDRY_NONE,
],
'fix_view' => [
'msg' => "Fix \"log_view.coll_id\"",
'sql' => "UPDATE `tmp_colls` AS `c` INNER JOIN `log_view` AS `v`\n"
. " ON `v`.`record_id` = `c`.`record_id` AND `v`.`date` >= `c`.`from_date` AND (`v`.`date` < `c`.`to_date` OR ISNULL(`c`.`to_date`))\n"
. " SET `v`.`coll_id` = `c`.`coll_id`",
'fetch' => false,
'code' => null,
'playdry' => self::PLAYDRY_NONE,
],
];
foreach($this->databoxes as $databox) {
$this->output->writeln("");
$this->output->writeln(sprintf("<info>================================ Working on databox %s (id:%s) ================================</info>", $databox->get_dbname(), $databox->get_sbas_id()));
$sql = "CREATE " . ($this->keep_tmp_table ? "TABLE IF NOT EXISTS" : "TEMPORARY TABLE") . " `tmp_colls` (\n"
. " `record_id` int(11) unsigned NOT NULL,\n"
. " `from_id` int(11) unsigned NOT NULL,\n"
. " `from_date` datetime NOT NULL,\n"
. " `to_id` int(11) unsigned DEFAULT NULL,\n"
. " `to_date` datetime DEFAULT NULL,\n"
. " `coll_id` int(10) unsigned NOT NULL,\n"
. " KEY `record_id` (`record_id`),\n"
. " KEY `from_id` (`from_id`),\n"
. " KEY `from_date` (`from_date`),\n"
. " KEY `to_id` (`to_id`),\n"
. " KEY `to_date` (`to_date`)\n"
. ") ENGINE=InnoDB;";
$this->output->writeln("");
$this->output->writeln(sprintf("<info> ----------------- Creating working %s table -----------------</info>", ($this->keep_tmp_table ? "(temporary)" : "")));
$this->output->writeln("");
if($this->show_sql) {
$this->output->writeln($sql);
if (!$this->showCount($databox)) {
// databox not patched
break;
}
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
$stmt->closeCursor();
for ($this->again=true; $this->again; ) {
$this->createWorkingTable($databox);
foreach($tsql as $work) {
if(!$this->again) {
break;
}
$this->output->writeln("");
$this->output->writeln(sprintf("<info> ----------------- %s ----------------- %s</info>",
$work['msg'],
$this->dry && !($work['playdry'] & self::PLAYDRY_SQL) ? " -- NOT PLAYED IN DRY MODE --" : ""
)
);
$this->output->writeln("");
$sql = str_replace($this->parm_k, $this->parm_v, $work['sql']);
if ($this->show_sql) {
$this->output->writeln($sql);
}
$stmt = null;
if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) {
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
}
if($work['code'] && (!$this->dry || ($work['playdry'] & self::PLAYDRY_CODE))) {
$code = $work['code'];
$code($stmt);
}
if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) {
$stmt->closeCursor();
}
}
if($this->dry) {
// since there was no changes it will loop forever
$this->again = false;
}
// loop to compute coll_id from top to bottom
do {
$n = $this->computeCollId($databox); // in dry mode, n=0
}
while ($n > 0);
// set the "from_coll_id"
$this->computeCollIdFrom($databox);
// copy results back to the log_docs
$this->copyReults($databox);
if (!$this->keep_tmp_table) {
$this->output->writeln(sprintf("<info> ----------------- Drop working table -----------------</info>"));
$sql = "DROP TABLE `tmp_colls`";
if($this->show_sql) {
$this->output->writeln($sql);
}
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
$stmt->closeCursor();
$this->dropWorkingTable($databox);
}
$this->output->writeln("");
@@ -337,4 +162,235 @@ class FixLogCollId extends Command
return 0;
}
private function createWorkingTable(\databox $databox)
{
$this->output->writeln("");
$this->output->writeln(sprintf("<info> ----------------- Creating working %s table -----------------</info>", ($this->keep_tmp_table ? "(temporary)" : "")));
$this->output->writeln("");
$sql = "CREATE " . ($this->keep_tmp_table ? "TABLE IF NOT EXISTS" : "TEMPORARY TABLE") . " `tmp_colls` (\n"
. " `record_id` int(11) unsigned NOT NULL,\n"
. " `from_id` int(11) unsigned NOT NULL,\n"
. " `from_date` datetime NOT NULL,\n"
. " `to_id` int(11) unsigned DEFAULT NULL,\n"
. " `to_date` datetime DEFAULT NULL,\n"
. " `coll_id_from` int(10) unsigned DEFAULT NULL,\n"
. " `coll_id` int(10) unsigned DEFAULT NULL,\n"
. " KEY `record_id` (`record_id`),\n"
. " KEY `from_id` (`from_id`),\n"
. " KEY `from_date` (`from_date`),\n"
. " KEY `to_id` (`to_id`),\n"
. " KEY `to_date` (`to_date`)\n"
. ") ENGINE=InnoDB;";
if($this->show_sql) {
$this->output->writeln($sql);
$this->output->writeln("");
}
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
$stmt->closeCursor();
$sql = "TRUNCATE TABLE `tmp_colls`";
if($this->show_sql) {
$this->output->writeln($sql);
$this->output->writeln("");
}
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
$stmt->closeCursor();
}
private function dropWorkingTable(\databox $databox)
{
$this->output->writeln(sprintf("<info> ----------------- Drop working table -----------------</info>"));
$sql = "DROP TABLE `tmp_colls`";
if($this->show_sql) {
$this->output->writeln($sql);
}
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
$stmt->closeCursor();
}
private function showCount(\databox $databox)
{
$ret = true;
$this->playSQL(
$databox,
[
'msg' => "Count work to do",
'sql' => "SELECT\n"
. " SUM(IF(ISNULL(`coll_id`), 0, 1)) AS `n`,\n"
. " COUNT(*) AS `t`,\n"
. " SUM(IF(`coll_id`>0, 1, 0)) AS `p`,\n"
. " SUM(IF(`coll_id`=0, 1, 0)) AS `z`\n"
. " FROM `log_docs`",
'code' => function(ResultStatement $stmt) use($ret) {
$row = $stmt->fetch();
if(is_null($row['n'])) {
// no coll_id ?
$this->output->writeln(sprintf("<error>The \"log_docs\" table has no \"coll_id\" column ? Please apply patch 410alpha12a</error>"));
$ret = false;
}
$this->output->writeln("");
$this->output->writeln(sprintf("done: %s / %s (fixed: %s ; can't fix: %s)", $row['n'], $row['t'], $row['p'], $row['z']));
},
'playdry' => self::PLAYDRY_SQL | self::PLAYDRY_CODE,
]
);
return $ret;
}
private function computeCollId(\databox $databox)
{
static $sql_lastid = null;
static $stmt_lastid = null;
static $sql_insert = null;
static $stmt_insert = null;
$ret = 0;
if(!$stmt_lastid) {
$sql_lastid = "SELECT @m:=COALESCE(MAX(`from_id`), 0) AS `lastid` FROM `tmp_colls`";
$stmt_lastid = $databox->get_connection()->prepare($sql_lastid);
$sql_insert = "INSERT INTO `tmp_colls`\n"
. " SELECT `r1`.`record_id`, `r1`.`id` AS `from_id`, `r1`.`date` AS `from_date`, MIN(`r2`.`id`) AS `to_id`, MIN(`r2`.`date`) AS `to_date`, NULL AS `coll_id_from`, `r1`.`final` AS `coll_id`\n"
. " FROM (\n"
. " SELECT `id`, `date`, `record_id`, `action`, `final` FROM `log_docs`\n"
. " WHERE `id` > @m AND `action` IN('add', 'collection')\n"
. " ORDER BY `id` ASC\n"
. " LIMIT " . $this->batch_size . "\n"
. " ) AS `r1`\n"
. " LEFT JOIN `log_docs` AS `r2`\n"
. " ON `r2`.`record_id`=`r1`.`record_id` AND `r2`.`action`='collection' AND `r2`.`id`>`r1`.`id`\n"
. " GROUP BY `r1`.`id`\n"
// . " ORDER BY `record_id` ASC, `from_id` ASC"
;
$stmt_insert = $databox->get_connection()->prepare($sql_insert);
$this->output->writeln("");
$this->output->writeln(sprintf("<info> ----------------- Compute coll_id to working table ----------------- %s</info>",
$this->dry ? " -- NOT PLAYED IN DRY MODE --" : ""
)
);
$this->output->writeln("");
if ($this->show_sql) {
$this->output->writeln($sql_lastid);
$this->output->writeln($sql_insert);
}
}
if(!$this->dry) {
$stmt_lastid->execute();
$stmt_lastid->closeCursor();
$stmt_insert->execute();
$ret = $stmt_insert->rowCount();
$stmt_insert->closeCursor();
}
return $ret;
}
private function computeCollIdFrom(\databox $databox)
{
static $sql = null;
static $stmt = null;
$ret = 0;
if(!$stmt) {
$sql = "UPDATE `tmp_colls` AS `t1` INNER JOIN `tmp_colls` AS `t2` ON `t2`.`from_id`=`t1`.`to_id`\n"
. " SET `t2`.`coll_id_from` = `t1`.`coll_id`";
$this->output->writeln("");
$this->output->writeln(sprintf("<info> ----------------- Compute coll_id_from to working table ----------------- %s</info>",
$this->dry ? " -- NOT PLAYED IN DRY MODE --" : ""
)
);
$this->output->writeln("");
if ($this->show_sql) {
$this->output->writeln($sql);
}
$stmt = $databox->get_connection()->prepare($sql);
}
if(!$this->dry) {
$stmt->execute();
$ret = $stmt->rowCount();
$stmt->closeCursor();
}
return $ret;
}
private function copyReults(\databox $databox)
{
$this->playSQL(
$databox,
[
'msg' => "Copy result back to \"log_docs\"",
'sql' => "UPDATE `tmp_colls` AS `t` INNER JOIN `log_docs` AS `d`\n"
. " ON ISNULL(`d`.`coll_id`)\n"
. " AND `t`.`record_id` = `d`.`record_id`\n"
. " AND `d`.`id` >= `t`.`from_id`\n"
. " AND (`d`.`id` < `t`.`to_id` OR ISNULL(`t`.`to_id`))\n"
. " SET `d`.`coll_id_from` = IF(`action`='collection', `t`.`coll_id_from`, NULL),\n"
. " `d`.`coll_id` = `t`.`coll_id`",
'code' => null,
'playdry' => self::PLAYDRY_NONE,
]
);
$this->playSQL(
$databox,
[
'msg' => "Copy result back to \"log_view\"",
'sql' => "UPDATE `log_view` AS `v` INNER JOIN `tmp_colls` AS `t`\n"
. " ON ISNULL(`v`.`coll_id`)\n"
. " AND `t`.`record_id` = `v`.`record_id`\n"
. " AND `v`.`date` >= `t`.`from_date`\n"
. " AND (`v`.`date` < `t`.`to_date` OR ISNULL(`t`.`to_date`))\n"
. " SET `v`.`coll_id` = `t`.`coll_id`",
'code' => null,
'playdry' => self::PLAYDRY_NONE,
]
);
}
private function playSQL(\databox $databox, Array $work)
{
$this->output->writeln("");
$this->output->writeln(sprintf("<info> ----------------- %s ----------------- %s</info>",
$work['msg'],
$this->dry && !($work['playdry'] & self::PLAYDRY_SQL) ? " -- NOT PLAYED IN DRY MODE --" : ""
)
);
$this->output->writeln("");
if ($this->show_sql) {
$this->output->writeln($work['sql']);
}
$stmt = null;
if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) {
$stmt = $databox->get_connection()->prepare($work['sql']);
$stmt->execute();
}
if($work['code'] && (!$this->dry || ($work['playdry'] & self::PLAYDRY_CODE))) {
$code = $work['code'];
$code($stmt);
}
if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) {
$stmt->closeCursor();
}
}
}

56
lib/conf.d/bases_structure.xml
View File

@@ -2509,17 +2509,9 @@
<default></default>
<comment></comment>
</field>
<field>
<name>coll_id</name>
<type>int(11) unsigned</type>
<null></null>
<extra></extra>
<default>0</default>
<comment></comment>
</field>
<field>
<name>action</name>
<type>enum('push','add','validate','edit','collection','status','print','substit','publish','download','mail','ftp','delete','collection_from','')</type>
<type>enum('push','add','validate','edit','collection','status','print','substit','publish','download','mail','ftp','delete','')</type>
<null></null>
<extra></extra>
<default></default>
@@ -2542,6 +2534,22 @@
<default></default>
<comment></comment>
</field>
<field>
<name>coll_id_from</name>
<type>int(11) unsigned</type>
<null>YES</null>
<extra></extra>
<default></default>
<comment></comment>
</field>
<field>
<name>coll_id</name>
<type>int(11) unsigned</type>
<null>YES</null>
<extra></extra>
<default></default>
<comment></comment>
</field>
</fields>
<indexes>
<index>
@@ -2565,6 +2573,13 @@
<field>record_id</field>
</fields>
</index>
<index>
<name>coll_id_from</name>
<type>INDEX</type>
<fields>
<field>coll_id</field>
</fields>
</index>
<index>
<name>coll_id</name>
<type>INDEX</type>
@@ -2650,6 +2665,14 @@
<default></default>
<comment></comment>
</field>
<field>
<name>coll_id</name>
<type>int(11) unsigned</type>
<null>YES</null>
<extra></extra>
<default></default>
<comment></comment>
</field>
</fields>
<indexes>
<index>
@@ -2694,15 +2717,14 @@
<field>site_id</field>
</fields>
</index>
<index>
<name>coll_id</name>
<type>INDEX</type>
<fields>
<field>coll_id</field>
</fields>
</index>
</indexes>
<field>
<name>coll_id</name>
<type>int(11) unsigned</type>
<null>YES</null>
<extra></extra>
<default></default>
<comment></comment>
</field>
<engine>InnoDB</engine>
</table>