From e2745cba9e408d96b810f981721a5c39f09894a8 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 8 Aug 2018 19:39:54 +0200 Subject: [PATCH] PHRAS-2161_coll-id-in-log-docs_MASTER Optimize sql. Add the "coll_id_from" column in "log_docs" for the "collection" action. Hydrate a "coll_id" column in "log_view" --- .../Phrasea/Command/Setup/FixLogCollId.php | 444 ++++++++++-------- lib/conf.d/bases_structure.xml | 56 ++- 2 files changed, 289 insertions(+), 211 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/Setup/FixLogCollId.php b/lib/Alchemy/Phrasea/Command/Setup/FixLogCollId.php index ae0961423c..66d699003d 100644 --- a/lib/Alchemy/Phrasea/Command/Setup/FixLogCollId.php +++ b/lib/Alchemy/Phrasea/Command/Setup/FixLogCollId.php @@ -24,6 +24,10 @@ class FixLogCollId extends Command const OPTION_DISTINT_VALUES = 0; const OPTION_ALL_VALUES = 1; + const PLAYDRY_NONE = 0; + const PLAYDRY_SQL = 1; + const PLAYDRY_CODE = 2; + /** @var InputInterface */ private $input; /** @var OutputInterface */ @@ -40,13 +44,6 @@ class FixLogCollId extends Command /** @var bool */ private $keep_tmp_table; - private $parm_k = [':minid', ':maxid']; - private $parm_v = []; - private $again; - - const PLAYDRY_NONE = 0; - const PLAYDRY_SQL = 1; - const PLAYDRY_CODE = 2; public function __construct($name = null) { @@ -129,206 +126,34 @@ class FixLogCollId extends Command $this->input = $input; $this->output = $output; - $tsql = [ - 'count' => [ - 'msg' => "Count work to do", - 'sql' => "SELECT\n" - . " SUM(IF(ISNULL(`coll_id`), 0, 1)) AS `n`,\n" - . " COUNT(*) AS `t`,\n" - . " SUM(IF(`coll_id`>0, 1, 0)) AS `p`,\n" - . " SUM(IF(`coll_id`=0, 1, 0)) AS `z`\n" - . " FROM `log_docs`", - 'fetch' => false, - 'code' => function(ResultStatement $stmt) { - $row = $stmt->fetch(); - if(is_null($row['n'])) { - // no coll_id ? - $this->output->writeln(sprintf("The \"log_docs\" table has no \"coll_id\" column ? Please apply patch 410alpha12a")); - $this->again = false; - } - $this->output->writeln(""); - $this->output->writeln(sprintf("done: %s / %s (fixed: %s ; can't fix: %s)", $row['n'], $row['t'], $row['p'], $row['z'])); - }, - 'playdry' => self::PLAYDRY_SQL | self::PLAYDRY_CODE, - ], - - 'minmax' => [ - 'msg' => "Get a batch", - 'sql' => "SELECT MIN(`id`) AS `minid`, MAX(`id`) AS `maxid` FROM\n" - . " (SELECT `id` FROM `log_docs` WHERE ISNULL(`coll_id`) ORDER BY `id` DESC LIMIT " . $this->batch_size . ") AS `t", - 'fetch' => false, - 'code' => function(ResultStatement $stmt) { - $row = $stmt->fetch(); - $this->output->writeln(""); - $this->output->writeln(sprintf("minid: %s ; maxid : %s\n", is_null($row['minid']) ? 'null' : $row['minid'], is_null($row['maxid']) ? 'null' : $row['maxid'])); - if (is_null($row['minid']) || is_null($row['maxid'])) { - $this->again = false; - } - $this->parm_v = [(int)$row['minid'], (int)$row['maxid']]; - }, - 'playdry' => self::PLAYDRY_SQL | self::PLAYDRY_CODE, - ], - - 'trunc' => [ - 'msg' => "Empty working table", - 'sql' => "TRUNCATE TABLE `tmp_colls`", - 'fetch' => false, - 'code' => null, - 'playdry' => self::PLAYDRY_NONE, - ], - - 'offset' => [ - 'msg' => "Make room for \"collection_from\" actions", - 'sql' => "UPDATE `log_docs` SET `id` = `id` * 2 WHERE `id` >= :minid AND `id` <= :maxid ORDER BY `id` DESC", - 'fetch' => false, - 'code' => - function(/** @noinspection PhpUnusedParameterInspection */ $stmt) { - // fix new minmax values since id was changed - $this->parm_v = [$this->parm_v[0] << 1, $this->parm_v[1] << 1]; - }, - 'playdry' => self::PLAYDRY_CODE, - ], - - 'compute' => [ - 'msg' => "Compute coll_id to working table", - 'sql' => "INSERT INTO `tmp_colls`\n" - . " SELECT `record_id`, `r1_id` AS `from_id`, `r1_date` AS `from_date`, MIN(`r2_id`) AS `to_id`, MIN(`r2_date`) AS `to_date`, `r1_final` AS `coll_id` FROM\n" - . " (\n" - . " SELECT `r1`.`record_id`, `r1`.`id` AS `r1_id`, `r1`.`date` AS `r1_date`, `r1`.`final` AS `r1_final`, `r2`.`id` AS `r2_id`, `r2`.`date` AS `r2_date` FROM\n" - . " (SELECT `id`, `date`, `record_id`, `action`, `final` FROM `log_docs` WHERE `action` IN('add', 'collection') AND `id` >= :minid AND `id` <= :maxid) AS `r1`\n" - . " LEFT JOIN `log_docs` AS `r2`\n" - . " ON `r2`.`record_id`=`r1`.`record_id` AND `r2`.`action`='collection' AND `r2`.`id`>`r1`.`id`\n" - . " )\n" - . " AS `t` GROUP BY `r1_id` ORDER BY `record_id` ASC, `from_id` ASC", - 'fetch' => false, - 'code' => null, - 'playdry' => self::PLAYDRY_NONE, - ], - - 'copy_result' => [ - 'msg' => "Copy result back to \"log_docs\"", - 'sql' => "UPDATE `tmp_colls` INNER JOIN `log_docs`\n" - . " ON `tmp_colls`.`record_id` = `log_docs`.`record_id`\n" - . " AND `log_docs`.`id` >= `tmp_colls`.`from_id`\n" - . " AND (`log_docs`.`id` < `tmp_colls`.`to_id` OR ISNULL(`tmp_colls`.`to_id`))\n" - . " SET `log_docs`.`coll_id` = `tmp_colls`.`coll_id`", - 'fetch' => false, - 'code' => null, - 'playdry' => self::PLAYDRY_NONE, - ], - - 'collection_from' => [ - 'msg' => "Insert \"collection_from\" actions", - 'sql' => "INSERT INTO `log_docs` (`id`, `log_id`, `date`, `record_id`, `action`, `final`, `coll_id`)\n" - . " SELECT `r1`.`id`-1 AS `id`, `r1`.`log_id`, `r1`.`date`, `r1`.`record_id`, 'collection_from' AS `action`, `r1`.`final`,\n" - . " SUBSTRING_INDEX(GROUP_CONCAT(`r2`.`coll_id` ORDER BY `r1`.`id` DESC), ',', 1) AS `coll_id`\n" - . " FROM `log_docs` AS `r1` LEFT JOIN `log_docs` AS `r2`\n" - . " ON `r2`.`record_id` = `r1`.`record_id` AND `r2`.`id` < `r1`.`id` AND `r2`.`action` IN('collection', 'add')\n" - . " WHERE `r1`.`action` = 'collection' AND `r1`.`id` >= :minid AND `r1`.`id` <= :maxid\n" - . " GROUP BY `r1`.`id`", - 'fetch' => false, - 'code' => null, - 'playdry' => self::PLAYDRY_NONE, - ], - - 'fix_unfound' => [ - 'msg' => "Set missing coll_id to 0", - 'sql' => "UPDATE `log_docs` SET `coll_id` = 0 WHERE `id` >= :minid AND `id` <= :maxid AND ISNULL(`coll_id`)", - 'fetch' => false, - 'code' => null, - 'playdry' => self::PLAYDRY_NONE, - ], - - 'fix_view' => [ - 'msg' => "Fix \"log_view.coll_id\"", - 'sql' => "UPDATE `tmp_colls` AS `c` INNER JOIN `log_view` AS `v`\n" - . " ON `v`.`record_id` = `c`.`record_id` AND `v`.`date` >= `c`.`from_date` AND (`v`.`date` < `c`.`to_date` OR ISNULL(`c`.`to_date`))\n" - . " SET `v`.`coll_id` = `c`.`coll_id`", - 'fetch' => false, - 'code' => null, - 'playdry' => self::PLAYDRY_NONE, - ], - ]; - - foreach($this->databoxes as $databox) { $this->output->writeln(""); $this->output->writeln(sprintf("================================ Working on databox %s (id:%s) ================================", $databox->get_dbname(), $databox->get_sbas_id())); - $sql = "CREATE " . ($this->keep_tmp_table ? "TABLE IF NOT EXISTS" : "TEMPORARY TABLE") . " `tmp_colls` (\n" - . " `record_id` int(11) unsigned NOT NULL,\n" - . " `from_id` int(11) unsigned NOT NULL,\n" - . " `from_date` datetime NOT NULL,\n" - . " `to_id` int(11) unsigned DEFAULT NULL,\n" - . " `to_date` datetime DEFAULT NULL,\n" - . " `coll_id` int(10) unsigned NOT NULL,\n" - . " KEY `record_id` (`record_id`),\n" - . " KEY `from_id` (`from_id`),\n" - . " KEY `from_date` (`from_date`),\n" - . " KEY `to_id` (`to_id`),\n" - . " KEY `to_date` (`to_date`)\n" - . ") ENGINE=InnoDB;"; - - $this->output->writeln(""); - $this->output->writeln(sprintf(" ----------------- Creating working %s table -----------------", ($this->keep_tmp_table ? "(temporary)" : ""))); - $this->output->writeln(""); - if($this->show_sql) { - $this->output->writeln($sql); + if (!$this->showCount($databox)) { + // databox not patched + break; } - $stmt = $databox->get_connection()->prepare($sql); - $stmt->execute(); - $stmt->closeCursor(); - for ($this->again=true; $this->again; ) { + $this->createWorkingTable($databox); - foreach($tsql as $work) { - if(!$this->again) { - break; - } - $this->output->writeln(""); - $this->output->writeln(sprintf(" ----------------- %s ----------------- %s", - $work['msg'], - $this->dry && !($work['playdry'] & self::PLAYDRY_SQL) ? " -- NOT PLAYED IN DRY MODE --" : "" - ) - ); - $this->output->writeln(""); - - $sql = str_replace($this->parm_k, $this->parm_v, $work['sql']); - if ($this->show_sql) { - $this->output->writeln($sql); - } - $stmt = null; - if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) { - $stmt = $databox->get_connection()->prepare($sql); - $stmt->execute(); - } - if($work['code'] && (!$this->dry || ($work['playdry'] & self::PLAYDRY_CODE))) { - $code = $work['code']; - $code($stmt); - } - if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) { - $stmt->closeCursor(); - } - } - - if($this->dry) { - // since there was no changes it will loop forever - $this->again = false; - } + // loop to compute coll_id from top to bottom + do { + $n = $this->computeCollId($databox); // in dry mode, n=0 } + while ($n > 0); + + // set the "from_coll_id" + $this->computeCollIdFrom($databox); + + // copy results back to the log_docs + $this->copyReults($databox); if (!$this->keep_tmp_table) { - $this->output->writeln(sprintf(" ----------------- Drop working table -----------------")); - $sql = "DROP TABLE `tmp_colls`"; - if($this->show_sql) { - $this->output->writeln($sql); - } - $stmt = $databox->get_connection()->prepare($sql); - $stmt->execute(); - $stmt->closeCursor(); + $this->dropWorkingTable($databox); } $this->output->writeln(""); @@ -337,4 +162,235 @@ class FixLogCollId extends Command return 0; } + private function createWorkingTable(\databox $databox) + { + $this->output->writeln(""); + $this->output->writeln(sprintf(" ----------------- Creating working %s table -----------------", ($this->keep_tmp_table ? "(temporary)" : ""))); + $this->output->writeln(""); + + $sql = "CREATE " . ($this->keep_tmp_table ? "TABLE IF NOT EXISTS" : "TEMPORARY TABLE") . " `tmp_colls` (\n" + . " `record_id` int(11) unsigned NOT NULL,\n" + . " `from_id` int(11) unsigned NOT NULL,\n" + . " `from_date` datetime NOT NULL,\n" + . " `to_id` int(11) unsigned DEFAULT NULL,\n" + . " `to_date` datetime DEFAULT NULL,\n" + . " `coll_id_from` int(10) unsigned DEFAULT NULL,\n" + . " `coll_id` int(10) unsigned DEFAULT NULL,\n" + . " KEY `record_id` (`record_id`),\n" + . " KEY `from_id` (`from_id`),\n" + . " KEY `from_date` (`from_date`),\n" + . " KEY `to_id` (`to_id`),\n" + . " KEY `to_date` (`to_date`)\n" + . ") ENGINE=InnoDB;"; + + if($this->show_sql) { + $this->output->writeln($sql); + $this->output->writeln(""); + } + $stmt = $databox->get_connection()->prepare($sql); + $stmt->execute(); + $stmt->closeCursor(); + + $sql = "TRUNCATE TABLE `tmp_colls`"; + + if($this->show_sql) { + $this->output->writeln($sql); + $this->output->writeln(""); + } + $stmt = $databox->get_connection()->prepare($sql); + $stmt->execute(); + $stmt->closeCursor(); + } + + private function dropWorkingTable(\databox $databox) + { + $this->output->writeln(sprintf(" ----------------- Drop working table -----------------")); + $sql = "DROP TABLE `tmp_colls`"; + if($this->show_sql) { + $this->output->writeln($sql); + } + $stmt = $databox->get_connection()->prepare($sql); + $stmt->execute(); + $stmt->closeCursor(); + } + + private function showCount(\databox $databox) + { + $ret = true; + $this->playSQL( + $databox, + [ + 'msg' => "Count work to do", + 'sql' => "SELECT\n" + . " SUM(IF(ISNULL(`coll_id`), 0, 1)) AS `n`,\n" + . " COUNT(*) AS `t`,\n" + . " SUM(IF(`coll_id`>0, 1, 0)) AS `p`,\n" + . " SUM(IF(`coll_id`=0, 1, 0)) AS `z`\n" + . " FROM `log_docs`", + 'code' => function(ResultStatement $stmt) use($ret) { + $row = $stmt->fetch(); + if(is_null($row['n'])) { + // no coll_id ? + $this->output->writeln(sprintf("The \"log_docs\" table has no \"coll_id\" column ? Please apply patch 410alpha12a")); + $ret = false; + } + $this->output->writeln(""); + $this->output->writeln(sprintf("done: %s / %s (fixed: %s ; can't fix: %s)", $row['n'], $row['t'], $row['p'], $row['z'])); + }, + 'playdry' => self::PLAYDRY_SQL | self::PLAYDRY_CODE, + ] + ); + + return $ret; + } + + private function computeCollId(\databox $databox) + { + static $sql_lastid = null; + static $stmt_lastid = null; + + static $sql_insert = null; + static $stmt_insert = null; + + $ret = 0; + if(!$stmt_lastid) { + $sql_lastid = "SELECT @m:=COALESCE(MAX(`from_id`), 0) AS `lastid` FROM `tmp_colls`"; + + $stmt_lastid = $databox->get_connection()->prepare($sql_lastid); + + $sql_insert = "INSERT INTO `tmp_colls`\n" + . " SELECT `r1`.`record_id`, `r1`.`id` AS `from_id`, `r1`.`date` AS `from_date`, MIN(`r2`.`id`) AS `to_id`, MIN(`r2`.`date`) AS `to_date`, NULL AS `coll_id_from`, `r1`.`final` AS `coll_id`\n" + . " FROM (\n" + . " SELECT `id`, `date`, `record_id`, `action`, `final` FROM `log_docs`\n" + . " WHERE `id` > @m AND `action` IN('add', 'collection')\n" + . " ORDER BY `id` ASC\n" + . " LIMIT " . $this->batch_size . "\n" + . " ) AS `r1`\n" + . " LEFT JOIN `log_docs` AS `r2`\n" + . " ON `r2`.`record_id`=`r1`.`record_id` AND `r2`.`action`='collection' AND `r2`.`id`>`r1`.`id`\n" + . " GROUP BY `r1`.`id`\n" + // . " ORDER BY `record_id` ASC, `from_id` ASC" + ; + + $stmt_insert = $databox->get_connection()->prepare($sql_insert); + + $this->output->writeln(""); + $this->output->writeln(sprintf(" ----------------- Compute coll_id to working table ----------------- %s", + $this->dry ? " -- NOT PLAYED IN DRY MODE --" : "" + ) + ); + $this->output->writeln(""); + if ($this->show_sql) { + $this->output->writeln($sql_lastid); + $this->output->writeln($sql_insert); + } + } + + if(!$this->dry) { + $stmt_lastid->execute(); + $stmt_lastid->closeCursor(); + + $stmt_insert->execute(); + $ret = $stmt_insert->rowCount(); + $stmt_insert->closeCursor(); + } + + return $ret; + } + + private function computeCollIdFrom(\databox $databox) + { + static $sql = null; + static $stmt = null; + + $ret = 0; + if(!$stmt) { + $sql = "UPDATE `tmp_colls` AS `t1` INNER JOIN `tmp_colls` AS `t2` ON `t2`.`from_id`=`t1`.`to_id`\n" + . " SET `t2`.`coll_id_from` = `t1`.`coll_id`"; + + $this->output->writeln(""); + $this->output->writeln(sprintf(" ----------------- Compute coll_id_from to working table ----------------- %s", + $this->dry ? " -- NOT PLAYED IN DRY MODE --" : "" + ) + ); + $this->output->writeln(""); + if ($this->show_sql) { + $this->output->writeln($sql); + } + + $stmt = $databox->get_connection()->prepare($sql); + } + + if(!$this->dry) { + $stmt->execute(); + $ret = $stmt->rowCount(); + $stmt->closeCursor(); + } + + return $ret; + } + + private function copyReults(\databox $databox) + { + $this->playSQL( + $databox, + [ + 'msg' => "Copy result back to \"log_docs\"", + 'sql' => "UPDATE `tmp_colls` AS `t` INNER JOIN `log_docs` AS `d`\n" + . " ON ISNULL(`d`.`coll_id`)\n" + . " AND `t`.`record_id` = `d`.`record_id`\n" + . " AND `d`.`id` >= `t`.`from_id`\n" + . " AND (`d`.`id` < `t`.`to_id` OR ISNULL(`t`.`to_id`))\n" + . " SET `d`.`coll_id_from` = IF(`action`='collection', `t`.`coll_id_from`, NULL),\n" + . " `d`.`coll_id` = `t`.`coll_id`", + 'code' => null, + 'playdry' => self::PLAYDRY_NONE, + ] + ); + + $this->playSQL( + $databox, + [ + 'msg' => "Copy result back to \"log_view\"", + 'sql' => "UPDATE `log_view` AS `v` INNER JOIN `tmp_colls` AS `t`\n" + . " ON ISNULL(`v`.`coll_id`)\n" + . " AND `t`.`record_id` = `v`.`record_id`\n" + . " AND `v`.`date` >= `t`.`from_date`\n" + . " AND (`v`.`date` < `t`.`to_date` OR ISNULL(`t`.`to_date`))\n" + . " SET `v`.`coll_id` = `t`.`coll_id`", + 'code' => null, + 'playdry' => self::PLAYDRY_NONE, + ] + ); + + } + + + private function playSQL(\databox $databox, Array $work) + { + $this->output->writeln(""); + $this->output->writeln(sprintf(" ----------------- %s ----------------- %s", + $work['msg'], + $this->dry && !($work['playdry'] & self::PLAYDRY_SQL) ? " -- NOT PLAYED IN DRY MODE --" : "" + ) + ); + $this->output->writeln(""); + + if ($this->show_sql) { + $this->output->writeln($work['sql']); + } + $stmt = null; + if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) { + $stmt = $databox->get_connection()->prepare($work['sql']); + $stmt->execute(); + } + if($work['code'] && (!$this->dry || ($work['playdry'] & self::PLAYDRY_CODE))) { + $code = $work['code']; + $code($stmt); + } + if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) { + $stmt->closeCursor(); + } + } } + diff --git a/lib/conf.d/bases_structure.xml b/lib/conf.d/bases_structure.xml index b0c91e86a9..13ed58fcb9 100644 --- a/lib/conf.d/bases_structure.xml +++ b/lib/conf.d/bases_structure.xml @@ -2509,17 +2509,9 @@ - - coll_id - int(11) unsigned - - - 0 - - action - enum('push','add','validate','edit','collection','status','print','substit','publish','download','mail','ftp','delete','collection_from','') + enum('push','add','validate','edit','collection','status','print','substit','publish','download','mail','ftp','delete','') @@ -2542,6 +2534,22 @@ + + coll_id_from + int(11) unsigned + YES + + + + + + coll_id + int(11) unsigned + YES + + + + @@ -2565,6 +2573,13 @@ record_id + + coll_id_from + INDEX + + coll_id + + coll_id INDEX @@ -2650,6 +2665,14 @@ + + coll_id + int(11) unsigned + YES + + + + @@ -2694,15 +2717,14 @@ site_id + + coll_id + INDEX + + coll_id + + - - coll_id - int(11) unsigned - YES - - - - InnoDB