From a88608636ade2266b514b98439243527c43f1cbe Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 7 Dec 2016 16:29:21 +0100 Subject: [PATCH 1/5] PHRAS-1292_BAD-CHARS-IN-PERMALINKS fix : some invalid chars are now replaced by "_" in permalinks labels --- lib/classes/media/Permalink/Adapter.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/classes/media/Permalink/Adapter.php b/lib/classes/media/Permalink/Adapter.php index d489963066..26901ac1d8 100644 --- a/lib/classes/media/Permalink/Adapter.php +++ b/lib/classes/media/Permalink/Adapter.php @@ -39,6 +39,12 @@ class media_Permalink_Adapter implements cache_cacheableInterface /** @var Application */ protected $app; + private static $bad_chars = [ + "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07", + "\x08", "\x09", "\x0A", "\x0B", "\x0C", "\x0D", "\x0E", "\x0F", + " ", "/", "\\", "%", "+" + ]; + /** * @param Application $app * @param databox $databox @@ -118,7 +124,7 @@ class media_Permalink_Adapter implements cache_cacheableInterface 'record_id' => $this->media_subdef->get_record_id(), 'subdef' => $this->media_subdef->get_name(), /** @Ignore */ - 'label' => str_replace('/', '_', $label), + 'label' => str_replace(self::$bad_chars, '_', $label), 'token' => $this->get_token(), ])); } From a16ce931106827ed22602ff8acf1993c63fb916a Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Thu, 8 Dec 2016 19:34:45 +0100 Subject: [PATCH 2/5] PHRAS-1293_CLI-BUILD-SUBDEFS-RECORDS-IGNORED - fix : no more subdefs ignored when mode=scheduled - change : --min_record is now --min_record_id ; --max_record is now --max_record_id - add : options --maxrecs, --maxduration, --ttl --- lib/Alchemy/Phrasea/Command/BuildSubdefs.php | 182 +++++++++++++------ 1 file changed, 126 insertions(+), 56 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/BuildSubdefs.php b/lib/Alchemy/Phrasea/Command/BuildSubdefs.php index 530ad3aded..811478e9cd 100644 --- a/lib/Alchemy/Phrasea/Command/BuildSubdefs.php +++ b/lib/Alchemy/Phrasea/Command/BuildSubdefs.php @@ -43,9 +43,9 @@ class BuildSubdefs extends Command /** @var string */ private $mode; /** @var int */ - private $recmin; + private $min_record_id; /** @var int */ - private $recmax; + private $max_record_id; /** @var bool */ private $substituted_only; /** @var bool */ @@ -68,6 +68,12 @@ class BuildSubdefs extends Command private $reset_subdef_flag; /** @var bool */ private $set_writemeta_flag; + /** @var integer */ + private $ttl; + /** @var integer */ + private $maxrecs; + /** @var integer */ + private $maxduration; /** @var bool */ private $dry; /** @var bool */ @@ -84,17 +90,31 @@ class BuildSubdefs extends Command parent::__construct($name); $this->presets = [ - 'scheduled' => ['scheduled', 'reset_subdef_flag', 'set_writemeta_flag'], - 'repair' => ['missing_only', 'prune', 'reset_subdef_flag', 'set_writemeta_flag'], - 'all' => ['all', 'reset_subdef_flag', 'set_writemeta_flag'] + 'scheduled' => [ + 'scheduled' => true, + 'reset_subdef_flag' => true, + 'set_writemeta_flag' => true, + 'ttl' => 10 + ], + 'repair' => [ + 'missing_only' => true, + 'prune' => true, + 'reset_subdef_flag' => true, + 'set_writemeta_flag' => true + ], + 'all' => [ + 'all' => true, + 'reset_subdef_flag' => true, + 'set_writemeta_flag' => true + ] ]; $this->setDescription('Build subviews'); $this->addOption('databox', null, InputOption::VALUE_REQUIRED, 'Mandatory : The id (or dbname or viewname) of the databox'); $this->addOption('mode', null, InputOption::VALUE_REQUIRED, 'preset working mode : ' . implode('|', array_keys($this->presets))); $this->addOption('record_type', null, InputOption::VALUE_REQUIRED|InputOption::VALUE_IS_ARRAY, 'Type(s) of records(s) to (re)build ex. "image,video", dafault=ALL'); - $this->addOption('min_record', null, InputOption::VALUE_OPTIONAL, 'Min record id'); - $this->addOption('max_record', null, InputOption::VALUE_OPTIONAL, 'Max record id'); + $this->addOption('min_record_id', null, InputOption::VALUE_OPTIONAL, 'Min record id'); + $this->addOption('max_record_id', null, InputOption::VALUE_OPTIONAL, 'Max record id'); $this->addOption('partition', null, InputOption::VALUE_REQUIRED, 'n/N : work only on records belonging to partition \'n\''); $this->addOption('reverse', null, InputOption::VALUE_NONE, 'Build records from the last to the oldest'); $this->addOption('name', null, InputOption::VALUE_REQUIRED|InputOption::VALUE_IS_ARRAY, 'Name(s) of sub-definition(s) to (re)build, ex. "thumbnail,preview", default=ALL'); @@ -106,32 +126,39 @@ class BuildSubdefs extends Command $this->addOption('prune', null, InputOption::VALUE_NONE, 'Delete subdefs not in structure anymore'); $this->addOption('reset_subdef_flag', null, InputOption::VALUE_NONE, 'Reset "make-subdef" flag (should only be used when working on all subdefs, that is NO --name filter)'); $this->addOption('set_writemeta_flag', null, InputOption::VALUE_NONE, 'Set "write-metadata" flag (should only be used when working on all subdefs, that is NO --name filter)'); + $this->addOption('maxrecs', null, InputOption::VALUE_REQUIRED, 'Maximum count of records to do.'); + $this->addOption('maxduration', null, InputOption::VALUE_REQUIRED, 'Maximum duration (seconds) of job. (job will do at least one record)'); + $this->addOption('ttl', null, InputOption::VALUE_REQUIRED, 'Wait time (seconds) before quit if no records were changed'); $this->addOption('dry', null, InputOption::VALUE_NONE, 'dry run, list but don\'t act'); $this->addOption('show_sql', null, InputOption::VALUE_NONE, 'show sql pre-selecting records'); $this->setHelp("" . "Record filters :\n" - . " --record_type=image,video : Select records of those types ('image','video','audio','document','flash')\n" - . " --min_record=100 : Select records with record_id >= 100\n" - . " --max_record=500 : Select records with record_id <= 500\n" - . " --partition=2/5 : Split databox records in 5 buckets, select records in bucket #2\n" - . " --scheduled : Select records flagged as \"make subdef\"\n" - . " --missing_only : Select only records with a missing and/or unknown subdef name\n" - . " --all : Select all records\n" + . " --record_type=image,video : Select records of those types ('image','video','audio','document','flash').\n" + . " --min_record_id=100 : Select records with record_id >= 100.\n" + . " --max_record_id=500 : Select records with record_id <= 500.\n" + . " --partition=2/5 : Split databox records in 5 buckets, select records in bucket #2.\n" + . " --scheduled : Select records flagged as \"make subdef\".\n" + . " --missing_only : Select only records with a missing and/or unknown subdef name.\n" + . " --all : Select all records.\n" . "Subdef filters :\n" - . " --name=thumbnail,preview : (re)build only thumbnail and preview\n" - . " --with_substituted : (re)build substituted subdefs also (normally ignored)\n" - . " --substituted_only : rebuild substituted subdefs from document (= remove substitution)\n" + . " --name=thumbnail,preview : (re)build only thumbnail and preview.\n" + . " --with_substituted : (re)build substituted subdefs also (normally ignored).\n" + . " --substituted_only : rebuild substituted subdefs from document (= remove substitution).\n" . "Actions :\n" - . " --prune : remove unknown subdefs\n" - . " --reset_subdef_flag : reset the \"make subdef\" scheduling flag (= mark record as done)\n" - . " --set_writemeta_flag : raise the \"write meta\" flag (= mark subdefs as missing metadata)\n" + . " --prune : remove unknown subdefs.\n" + . " --reset_subdef_flag : reset the \"make subdef\" scheduling flag (= mark record as done).\n" + . " --set_writemeta_flag : raise the \"write meta\" flag (= mark subdefs as missing metadata).\n" + . "Job limits :\n" + . " --maxrecs=100 : quit anyway after 100 records are done.\n" + . " --maxduration=3600 : quit anyway after 1 hour (at least one record is done).\n" + . " --ttl=10 : if nothing was done, wait 10 seconds (sleep) before quit.\n" . "Preset modes :\n" - . " --mode=scheduled : Create subdefs for records flagged \"make subdef\", same as \"subview creation\" task\n" - . " (= --scheduled --reset_subdef_flag --set_writemeta_flag)\n" - . " --mode=repair : Create only missing subdefs, prune obsolete subdefs\n" + . " --mode=scheduled : Create subdefs for records flagged \"make subdef\", same as \"subview creation\" task.\n" + . " (= --scheduled --reset_subdef_flag --set_writemeta_flag --ttl=10)\n" + . " --mode=repair : Create only missing subdefs, prune obsolete subdefs.\n" . " (= --missing_only --prune --reset_subdef_flag --set_writemeta_flag)\n" - . " --mode=all : Re-creates all subdefs\n" + . " --mode=all : Re-creates all subdefs.\n" . " (= --all --reset_subdef_flag --set_writemeta_flag\n" ); @@ -206,11 +233,33 @@ class BuildSubdefs extends Command } // get options - $this->mode = $input->getOption('mode'); + $this->mode = $input->getOption('mode'); + if($this->mode) { + if(!in_array($this->mode, array_keys($this->presets))) { + $output->writeln(sprintf("invalid --mode \"%s\"", $this->mode)); + $argsOK = false; + } + else { + $explain = ""; + foreach($this->presets[$this->mode] as $k=>$v) { + if($input->getOption($k) !== false && $input->getOption($k) !== null) { + $output->writeln(sprintf("--mode=%s and --%s are mutually exclusive", $this->mode, $k)); + $argsOK = false; + } + else { + $input->setOption($k, $v); + } + $explain .= ' --' . $k . ($v===true ? '' : ('='.$v)); + } + + $output->writeln(sprintf("mode \"%s\" ==>%s", $this->mode, $explain)); + } + } + $this->show_sql = $input->getOption('show_sql') ? true : false; $this->dry = $input->getOption('dry') ? true : false; - $this->recmin = $input->getOption('min_record'); - $this->recmax = $input->getOption('max_record'); + $this->min_record_id = $input->getOption('min_record_id'); + $this->max_record_id = $input->getOption('max_record_id'); $this->substituted_only = $input->getOption('substituted_only') ? true : false; $this->with_substituted = $input->getOption('with_substituted') ? true : false; $this->missing_only = $input->getOption('missing_only') ? true : false; @@ -220,6 +269,10 @@ class BuildSubdefs extends Command $this->reverse = $input->getOption('reverse') ? true : false; $this->reset_subdef_flag = $input->getOption('reset_subdef_flag') ? true : false; $this->set_writemeta_flag = $input->getOption('set_writemeta_flag') ? true : false; + $this->maxrecs = (int)$input->getOption('maxrecs'); + $this->maxduration = (int)$input->getOption('maxduration'); + $this->ttl = (int)$input->getOption('ttl'); + $types = $this->getOptionAsArray($input, 'record_type', self::OPTION_DISTINT_VALUES); $names = $this->getOptionAsArray($input, 'name', self::OPTION_DISTINT_VALUES); @@ -227,22 +280,6 @@ class BuildSubdefs extends Command $output->writeln("--substituted_only and --with_substituted are mutually exclusive"); $argsOK = false; } - if($this->mode) { - if($this->substituted_only || $this->with_substituted || $this->missing_only || $this->prune || $this->all || $this->reset_subdef_flag || $this->set_writemeta_flag) { - $output->writeln("--mode and (--substituted_only, --with_substituted, --missing_only, --prune, --all, --reset_subdef_flag, --set_writemeta_flag) are mutually exclusive"); - $argsOK = false; - } - if(!in_array($this->mode, array_keys($this->presets))) { - $output->writeln(sprintf("invalid --mode \"%s\"", $this->mode)); - $argsOK = false; - } - else { - $output->writeln(sprintf("mode \"%s\" ==> --%s", $this->mode, implode(' --', $this->presets[$this->mode]))); - foreach($this->presets[$this->mode] as $toset) { - $this->{$toset} = true; - } - } - } if($this->prune && !empty($names)) { $output->writeln("--prune and --name are mutually exclusive"); $argsOK = false; @@ -343,7 +380,10 @@ class BuildSubdefs extends Command $stmt = $this->connection->executeQuery($sql); + $time_start = new \DateTime(); + $nRecordsDone = 0; while( ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) ) { + $recordChanged = false; $type = $row['type']; $msg = []; @@ -367,6 +407,7 @@ class BuildSubdefs extends Command if(!$this->dry) { $subdef->delete(); } + $recordChanged = true; $subdefsDeleted[] = $name; $msg[] = sprintf(" \"%s\" pruned", $name); } @@ -395,6 +436,7 @@ class BuildSubdefs extends Command $subdef->remove_file(); $subdef->set_substituted(false); } + $recordChanged = true; $msg[] = sprintf(" [\"%s\"] deleted", $name); } } @@ -406,7 +448,7 @@ class BuildSubdefs extends Command $subdefGenerator = $this->container['subdef.generator']; $subdefGenerator->generateSubdefs($record, $subdefNamesToDo); } - + $recordChanged = true; $msg[] = sprintf(" [\"%s\"] built", implode('","', $subdefNamesToDo)); } else { @@ -434,19 +476,41 @@ class BuildSubdefs extends Command ':flag_or' => ($this->set_writemeta_flag ? PhraseaTokens::WRITE_META_SUBDEF : 0) ]); } + $recordChanged = true; + } + + if($recordChanged) { + $nRecordsDone++; } } catch(\Exception $e) { - $this->output->write("failed\n"); + $output->write("failed\n"); } if($progress) { $progress->advance(); - $this->output->write(implode(' ', $msg)); + $output->write(implode(' ', $msg)); } else { - $this->output->writeln(implode("\n", $msg)); + $output->writeln(implode("\n", $msg)); } + + if($this->maxrecs > 0 && $nRecordsDone >= $this->maxrecs) { + if($progress) { + $output->writeln(''); + } + $output->writeln(sprintf("Maximum number (%d >= %d) of records done, quit.", $nRecordsDone, $this->maxrecs)); + } + + $time_end = new \DateTime(); + $dt = $time_end->getTimestamp() - $time_start->getTimestamp(); + if($this->maxduration > 0 && $dt >= $this->maxduration && $nRecordsDone > 0) { + if($progress) { + $output->writeln(''); + } + $output->writeln(sprintf("Maximum duration (%d >= %d) done, quit.", $dt, $this->maxduration)); + } + } unset($stmt); @@ -456,6 +520,13 @@ class BuildSubdefs extends Command $output->writeln(''); } + if($nRecordsDone == 0) { + while($this->ttl > 0) { + sleep(1); + $this->ttl--; + } + } + return 0; } @@ -473,8 +544,8 @@ class BuildSubdefs extends Command $sql .= " END\n" . " AS `waited`\n" - . "FROM `record` AS r LEFT JOIN `subdef` AS s USING(`record_id`)\n" - . "WHERE r.`parent_record_id`=0"; + . "FROM `record` AS r LEFT JOIN `subdef` AS s ON((s.`record_id` = r.`record_id`) AND s.`name` != 'document')\n" + . "WHERE r.`parent_record_id`=0\n"; $recordTypes = array_keys($this->subdefsTodoByType); $types = array_map(function($v) {return $this->connection->quote($v);}, $recordTypes); @@ -482,21 +553,20 @@ class BuildSubdefs extends Command if(!empty($types)) { $sql .= " AND r.`type` IN(" . implode(',', $types) . ")\n"; } - if ($this->recmin !== null) { - $sql .= " AND (r.`record_id` >= " . (int)($this->recmin) . ")\n"; + if ($this->min_record_id !== null) { + $sql .= " AND (r.`record_id` >= " . (int)($this->min_record_id) . ")\n"; } - if ($this->recmax) { - $sql .= " AND (r.`record_id` <= " . (int)($this->recmax) . ")\n"; + if ($this->max_record_id) { + $sql .= " AND (r.`record_id` <= " . (int)($this->max_record_id) . ")\n"; } if($this->partitionCount !== null && $this->partitionIndex !== null) { $sql .= " AND MOD(r.`record_id`, " . $this->partitionCount . ")=" . ($this->partitionIndex-1) . "\n"; } if($this->scheduled) { - $sql .= " AND r.`jeton` & " . PhraseaTokens::MAKE_SUBDEF; + $sql .= " AND r.`jeton` & " . PhraseaTokens::MAKE_SUBDEF . "\n"; } - $sql .= " AND s.`name` != 'document'\n" - . "GROUP BY r.`record_id`"; + $sql .= "GROUP BY r.`record_id`"; if(!$this->scheduled && !$this->all) { $sql .= "\nHAVING `exists` != `waited`"; From 1657d96610a80d2792a17fef89838e5dbc5adc8e Mon Sep 17 00:00:00 2001 From: Nicolas Maillat Date: Mon, 12 Dec 2016 11:25:45 +0100 Subject: [PATCH 3/5] add bin/developer orm:generate-proxies deployement add bin/developer orm:generate-proxies for automated deployement --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 6d0bb59ee7..52f1317967 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ clean_assets: config: @php bin/console compile:configuration + @php bin/developer orm:generate-proxies watch: @echo 'config/configuration.yml' | entr make config From f412a031b92f063b66a1c3561b7580292efbf6ac Mon Sep 17 00:00:00 2001 From: jygaulier Date: Mon, 12 Dec 2016 14:46:44 +0100 Subject: [PATCH 4/5] Update BuildSubdefs.php --- lib/Alchemy/Phrasea/Command/BuildSubdefs.php | 29 ++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/BuildSubdefs.php b/lib/Alchemy/Phrasea/Command/BuildSubdefs.php index 811478e9cd..32c43d01eb 100644 --- a/lib/Alchemy/Phrasea/Command/BuildSubdefs.php +++ b/lib/Alchemy/Phrasea/Command/BuildSubdefs.php @@ -350,12 +350,15 @@ class BuildSubdefs extends Command */ protected function doExecute(InputInterface $input, OutputInterface $output) { + $time_start = new \DateTime(); + if(!$this->sanitizeArgs($input, $output)) { return -1; } $this->input = $input; $this->output = $output; + $progress = null; $sql = $this->getSQL(); @@ -367,22 +370,19 @@ class BuildSubdefs extends Command $totalRecords = (int)$this->connection->executeQuery($sqlCount)->fetchColumn(); - if ($totalRecords === 0) { - return 0; - } + $nRecordsDone = 0; - $progress = null; - if($output->getVerbosity() < OutputInterface::VERBOSITY_VERBOSE) { - $progress = new ProgressBar($output, $totalRecords); - $progress->start(); - $progress->display(); - } + $again = true; $stmt = $this->connection->executeQuery($sql); + while($again && ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) ) { + + if($output->getVerbosity() < OutputInterface::VERBOSITY_VERBOSE && $progress === null) { + $progress = new ProgressBar($output, $totalRecords); + $progress->start(); + $progress->display(); + } - $time_start = new \DateTime(); - $nRecordsDone = 0; - while( ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) ) { $recordChanged = false; $type = $row['type']; $msg = []; @@ -489,7 +489,7 @@ class BuildSubdefs extends Command if($progress) { $progress->advance(); - $output->write(implode(' ', $msg)); + //$output->write(implode(' ', $msg)); } else { $output->writeln(implode("\n", $msg)); @@ -500,6 +500,7 @@ class BuildSubdefs extends Command $output->writeln(''); } $output->writeln(sprintf("Maximum number (%d >= %d) of records done, quit.", $nRecordsDone, $this->maxrecs)); + $again = false; } $time_end = new \DateTime(); @@ -509,6 +510,7 @@ class BuildSubdefs extends Command $output->writeln(''); } $output->writeln(sprintf("Maximum duration (%d >= %d) done, quit.", $dt, $this->maxduration)); + $again = false; } } @@ -516,7 +518,6 @@ class BuildSubdefs extends Command unset($stmt); if($progress) { - $progress->finish(); $output->writeln(''); } From b151faaba7594dba139fb399f03ba57cc3412c9e Mon Sep 17 00:00:00 2001 From: Nicolas Maillat Date: Mon, 12 Dec 2016 16:44:02 +0100 Subject: [PATCH 5/5] Change right mask to asset dir PHRAS-1294 Change right mask to asset dir PHRAS-1294 --- lib/Alchemy/Phrasea/Filesystem/FilesystemService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Filesystem/FilesystemService.php b/lib/Alchemy/Phrasea/Filesystem/FilesystemService.php index 9d5e75822d..813aa037de 100644 --- a/lib/Alchemy/Phrasea/Filesystem/FilesystemService.php +++ b/lib/Alchemy/Phrasea/Filesystem/FilesystemService.php @@ -45,7 +45,7 @@ class FilesystemService $pathout = sprintf('%s%s%05d', $repository_path, $comp, $n++); } while (is_dir($pathout) && iterator_count(new \DirectoryIterator($pathout)) > 100); - $this->filesystem->mkdir($pathout, 0750); + $this->filesystem->mkdir($pathout, 0770); return $pathout . DIRECTORY_SEPARATOR; }