From a0521c59fcad1162f2885a3773c9f14e1baa2766 Mon Sep 17 00:00:00 2001 From: aynsix Date: Fri, 22 May 2020 12:24:19 +0300 Subject: [PATCH] port to 4.1 archive task improvement --- .../Phrasea/TaskManager/Job/ArchiveJob.php | 148 ++++++++++++------ lib/classes/databox/status.php | 4 +- 2 files changed, 99 insertions(+), 53 deletions(-) diff --git a/lib/Alchemy/Phrasea/TaskManager/Job/ArchiveJob.php b/lib/Alchemy/Phrasea/TaskManager/Job/ArchiveJob.php index de418a4e71..35bc878008 100644 --- a/lib/Alchemy/Phrasea/TaskManager/Job/ArchiveJob.php +++ b/lib/Alchemy/Phrasea/TaskManager/Job/ArchiveJob.php @@ -181,11 +181,12 @@ class ArchiveJob extends AbstractJob $dom = new \DOMDocument(); $dom->formatOutput = true; + /** @var \DOMElement $root */ $root = $dom->appendChild($dom->createElement('root')); $nnew = $this->listFilesPhase1($app, $dom, $root, $path_in, $server_coll_id, 0, $TColls); if ($app['debug']) { - $this->log('debug', "=========== listFilesPhase1 ========== (returned " . $nnew . ")\n" . $dom->saveXML()); + $this->log('debug', "== listFilesPhase1 returned " . $nnew . ")\n" . $dom->saveXML()); } if (!$this->isStarted()) { @@ -193,15 +194,16 @@ class ArchiveJob extends AbstractJob } // wait for files to be cold - $this->pause($cold); - - if (!$this->isStarted()) { - return; + for($i=0; $i<($cold*2); $i++) { + if (!$this->isStarted()) { + return; + } + $this->pause(0.5); } $this->listFilesPhase2($app, $dom, $root, $path_in, 0); if ($app['debug']) { - $this->log('debug', "=========== listFilesPhase2 ========== : \n" . $dom->saveXML()); + $this->log('debug', "== listFilesPhase2\n" . $dom->saveXML()); } if (!$this->isStarted()) { @@ -210,31 +212,35 @@ class ArchiveJob extends AbstractJob $this->makePairs($dom, $root, $path_in, $path_archived, $path_error, false, 0, $tmask, $tmaskgrp); if ($app['debug']) { - $this->log('debug', "=========== makePairs ========== : \n" . $dom->saveXML()); - } - - $r = $this->removeBadGroups($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError); - if ($app['debug']) { - $this->log('debug', "=========== removeBadGroups ========== (returned " . ((Boolean) $r ? 'true' : 'false') . ") : \n" . $dom->saveXML()); - } - - $this->archive($app, $databox, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError, $moveArchived, $stat0, $stat1); - if ($app['debug']) { - $this->log('debug', "=========== archive ========== : \n" . $dom->saveXML()); + $this->log('debug', "== makePairs\n" . $dom->saveXML()); } if (!$this->isStarted()) { return; } + $this->removeBadGroups($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError); + if ($app['debug']) { + $this->log('debug', "== removeBadGroups\n" . $dom->saveXML()); + } + + if (!$this->isStarted()) { + return; + } + + $this->archive($app, $databox, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError, $moveArchived, $stat0, $stat1); + if ($app['debug']) { + $this->log('debug', "== archive\n" . $dom->saveXML()); + } + $this->bubbleResults($dom, $root, $path_in, 0, \p4field::isyes($settings->copy_spe)); if ($app['debug']) { - $this->log('debug', "=========== bubbleResults ========== : \n" . $dom->saveXML()); + $this->log('debug', "== bubbleResults\n" . $dom->saveXML()); } $moved = $this->moveFiles($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveArchived, $moveError); if ($app['debug']) { - $this->log('debug', "=========== moveFiles ========== (returned " . ($moved ? 'true' : 'false') . ") : \n" . $dom->saveXML()); + $this->log('debug', "== moveFiles returned " . ($moved ? 'true' : 'false') . "\n" . $dom->saveXML()); } } } @@ -243,15 +249,17 @@ class ArchiveJob extends AbstractJob { $nnew = 0; - if (false !== $sxDotPhrasea = @simplexml_load_file($path . '/.phrasea.xml')) { + $magicfile = $magicmethod = null; + + if (($sxDotPhrasea = @simplexml_load_file($path . '/.phrasea.xml')) !== false) { // test for magic file if (($magicfile = trim((string) ($sxDotPhrasea->magicfile))) != '') { $magicmethod = strtoupper($sxDotPhrasea->magicfile['method']); - if ($magicmethod == 'LOCK' && true === $app['filesystem']->exists($path . '/' . $magicfile)) { - return; - } elseif ($magicmethod == 'UNLOCK' && false === $app['filesystem']->exists($path . '/' . $magicfile)) { - return; + if ($magicmethod == 'LOCK' && ($app['filesystem']->exists($path . '/' . $magicfile) === true)) { + return 0; + } elseif ($magicmethod == 'UNLOCK' && ($app['filesystem']->exists($path . '/' . $magicfile) === false)) { + return 0; } } @@ -278,6 +286,7 @@ class ArchiveJob extends AbstractJob continue; } + /** @var \DOMElement $n */ if (is_dir($path . '/' . $file)) { $n = $node->appendChild($dom->createElement('file')); $n->setAttribute('isdir', '1'); @@ -293,6 +302,16 @@ class ArchiveJob extends AbstractJob foreach (["size", "ctime", "mtime"] as $k) { $n->setAttribute($k, $stat[$k]); } + // special file + if($file == '.phrasea.xml') { + $n->setAttribute('match', '*'); + } + // special file + if($file === $magicfile) { + $n->setAttribute('match', '*'); + $node->setAttribute('magicfile', $magicfile); + $node->setAttribute('magicmethod', $magicmethod); + } $nnew++; } $n->setAttribute('cid', $server_coll_id); @@ -335,27 +354,31 @@ class ArchiveJob extends AbstractJob $dnl = @$xp->query('./file[@name="' . $file . '"]', $node); if ($dnl && $dnl->length == 0) { if (is_dir($path . '/' . $file)) { + /** @var \DOMElement $n */ $n = $node->appendChild($dom->createElement('file')); $n->setAttribute('isdir', '1'); $n->setAttribute('name', $file); $nnew += $this->listFilesPhase2($app, $dom, $n, $path . '/' . $file, $depth + 1); } else { + /** @var \DOMElement $n */ $n = $node->appendChild($dom->createElement('file')); $n->setAttribute('name', $file); $nnew++; } - $this->setBranchHot($dom, $n); + $this->setBranchHot($n); } elseif ($dnl && $dnl->length == 1) { - $dnl->item(0)->setAttribute('temperature', 'cold'); + /** @var \DOMElement $n */ + $n = $dnl->item(0); + $n->setAttribute('temperature', 'cold'); if (is_dir($path . '/' . $file)) { - $this->listFilesPhase2($app, $dom, $dnl->item(0), $path . '/' . $file, $depth + 1); + $this->listFilesPhase2($app, $dom, $n, $path . '/' . $file, $depth + 1); } else { $stat = stat($path . '/' . $file); foreach (["size", "ctime", "mtime"] as $k) { - if ($dnl->item(0)->getAttribute($k) != $stat[$k]) { - $this->setBranchHot($dom, $dnl->item(0)); + if ($n->getAttribute($k) != $stat[$k]) { + $this->setBranchHot($n); break; } } @@ -399,13 +422,16 @@ class ArchiveJob extends AbstractJob if ($dnl->length == 1) { // this group is old (don't care about any linked files), just flag it $n->setAttribute('grp', 'tocomplete'); - $dnl->item(0)->setAttribute('match', '*'); + /** @var \DOMElement $_n */ + $_n = $dnl->item(0); + $_n->setAttribute('match', '*'); // recurse only if group is ok $this->makePairs($dom, $n, $path . '/' . $name, $path_archived, $path_error, true, $depth + 1, $tmask, $tmaskgrp); } else { // this group in new (to be created) // do we need one (or both) linked file ? (caption or representation) $err = false; + /** @var \DOMElement[] $flink */ $flink = ['caption' => null, 'representation' => null]; foreach ($flink as $linkName => $v) { @@ -473,12 +499,12 @@ class ArchiveJob extends AbstractJob // this is a file if (!$n->getAttribute('match')) { // because match can be set before - if ($name == '.phrasea.xml') { - // special file(s) always ok - $n->setAttribute('match', '*'); - } else { +// if ($name == '.phrasea.xml') { +// // special file(s) always ok +// $n->setAttribute('match', '*'); +// } else { $this->checkMatch($dom, $n, $tmask); - } +// } } } } @@ -503,7 +529,7 @@ class ArchiveJob extends AbstractJob // if root of hotfolder if hot, die... if ($depth == 0 && $node->getAttribute('temperature') == 'hot') { - return $ret; + return; } $nodesToDel = []; @@ -522,7 +548,7 @@ class ArchiveJob extends AbstractJob $name = $n->getAttribute('name'); if ($n->getAttribute('isdir')) { - $ret |= $this->removeBadGroups($app, $dom, $n, $path . '/' . $name + $this->removeBadGroups($app, $dom, $n, $path . '/' . $name , $path_archived . '/' . $name , $path_error . '/' . $name , $depth + 1, $moveError); @@ -642,7 +668,7 @@ class ArchiveJob extends AbstractJob } if ($node->getAttribute('temperature') == 'hot') { - return; + return 0; } $ret = 0; @@ -758,7 +784,7 @@ class ArchiveJob extends AbstractJob } } - if (!$n->getAttribute('keep')) { + if (!$n->getAttribute('keep') && !$n->getAttribute('match')) { $this->log('debug', sprintf('delete \'%s\'', $path . '/' . $name)); try { @@ -792,7 +818,9 @@ class ArchiveJob extends AbstractJob if ($dnl->length == 1) { // the caption file exists $node->setAttribute('match', $captionFileName); - $dnl->item(0)->setAttribute('match', '*'); + /** @var \DOMElement $n */ + $n = $dnl->item(0); + $n->setAttribute('match', '*'); } else { // the caption file is missing $node->setAttribute('match', '?'); @@ -817,7 +845,7 @@ class ArchiveJob extends AbstractJob return ($f[0] == '.' && $f != '.phrasea.xml' && $f != '.grouping.xml') || $f == 'thumbs.db' || $f == 'par-system'; } - private function setBranchHot(\DOMDocument $dom, \DOMElement $node) + private function setBranchHot(\DOMElement $node) { for ($n = $node; $n; $n = $n->parentNode) { if ($n->nodeType == XML_ELEMENT_NODE) { @@ -847,8 +875,10 @@ class ArchiveJob extends AbstractJob if ($node->getAttribute('grp') == 'tocreate') { $representationFileName = null; + /** @var \DOMElement $representationFileNode */ $representationFileNode = null; $captionFileName = null; + /** @var \DOMElement $captionFileNode */ $captionFileNode = null; $cid = $node->getAttribute('cid'); $genericdoc = null; @@ -903,6 +933,7 @@ class ArchiveJob extends AbstractJob } file_put_contents($groupingFile, ''); + /** @var \DOMElement $n */ $n = $node->appendChild($dom->createElement('file')); $n->setAttribute('name', '.grouping.xml'); $n->setAttribute('temperature', 'cold'); @@ -1025,6 +1056,8 @@ class ArchiveJob extends AbstractJob } $story = \record_adapter::createStory($app, $collection); + + $story->setStatus($status); $app['subdef.substituer']->substituteDocument($story, $media); $story->set_metadatas($metadatas->toMetadataArray($metadatasStructure), true); @@ -1080,12 +1113,14 @@ class ArchiveJob extends AbstractJob $file->addAttribute(new BorderAttribute\Status($app, $status)); - $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfFilepath(), new MonoValue($media->getFile()->getRealPath())))); - $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfDirname(), new MonoValue(dirname($media->getFile()->getRealPath()))))); + /** @var \MediaVorus\File $mediaFile */ + $mediaFile = $media->getFile(); + $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfFilepath(), new MonoValue($mediaFile->getRealPath())))); + $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfDirname(), new MonoValue(dirname($mediaFile->getRealPath()))))); - $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfAtime(), new MonoValue($media->getFile()->getATime())))); - $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfMtime(), new MonoValue($media->getFile()->getMTime())))); - $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfCtime(), new MonoValue($media->getFile()->getCTime())))); + $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfAtime(), new MonoValue($mediaFile->getATime())))); + $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfMtime(), new MonoValue($mediaFile->getMTime())))); + $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfCtime(), new MonoValue($mediaFile->getCTime())))); foreach ($metadatas as $meta) { $file->addAttribute(new BorderAttribute\Metadata($meta)); @@ -1102,8 +1137,13 @@ class ArchiveJob extends AbstractJob $record = null; $postProcess = function ($element, $visa, $code) use (&$record) { - $record = $element; - }; + $r = isset($visa); // one way to avoid "variable not used" with phpstorm 10. ugly. + unset($r); // + $r = isset($code); // one way to avoid "variable not used" with phpstorm 10. ugly. + unset($r); // + + $record = $element; + }; /** @var borderManager $borderManager */ $borderManager = $app['border-manager']; @@ -1206,12 +1246,13 @@ class ArchiveJob extends AbstractJob } } - $this->archiveFileAndCaption($app, $databox, $dom, $node, $captionFileNode, $path, $path_archived, $path_error, $grp_rid, $nodesToDel, $stat0, $stat1, $moveError, $moveArchived); + $this->archiveFileAndCaption($app, $databox, $node, $captionFileNode, $path, $path_archived, $path_error, $grp_rid, $nodesToDel, $stat0, $stat1, $moveError, $moveArchived); } /** * - * @param \DOMDOcument $dom + * @param Application $app + * @param \databox $databox * @param \DOMElement $node * @param \DOMElement $captionFileNode * @param string $path @@ -1219,8 +1260,12 @@ class ArchiveJob extends AbstractJob * @param string $path_error * @param integer $grp_rid * @param array $nodesToDel out, filled with files to delete + * @param $stat0 + * @param $stat1 + * @param $moveError + * @param $moveArchived */ - private function archiveFileAndCaption(Application $app, \databox $databox, \DOMDocument $dom, \DOMElement $node, \DOMElement $captionFileNode = null, $path, $path_archived, $path_error, $grp_rid, array &$nodesToDel, $stat0, $stat1, $moveError, $moveArchived) + private function archiveFileAndCaption(Application $app, \databox $databox, \DOMElement $node, \DOMElement $captionFileNode = null, $path, $path_archived, $path_error, $grp_rid, array &$nodesToDel, $stat0, $stat1, $moveError, $moveArchived) { // quick fix to reconnect if mysql is lost $app->getApplicationBox()->get_connection(); @@ -1422,6 +1467,7 @@ class ArchiveJob extends AbstractJob { $ret = new MetadataBag(); + /** @var \databox_field $databox_field */ foreach ($metadatasStructure as $databox_field) { if ($bag->containsKey($databox_field->get_tag()->getTagname())) { $ret->set($databox_field->get_name(), $bag->get($databox_field->get_tag()->getTagname())); diff --git a/lib/classes/databox/status.php b/lib/classes/databox/status.php index 744041e034..397f1eacea 100644 --- a/lib/classes/databox/status.php +++ b/lib/classes/databox/status.php @@ -157,8 +157,8 @@ class databox_status * compute ((0 M s1) M s2) where M is the "mask" operator * nb : s1,s2 are binary mask strings as "01x0xx1xx0x", no other format (hex) supported * - * @param $stat1 a binary mask "010x1xx0.." STRING - * @param $stat2 a binary mask "x100x1..." STRING + * @param string $stat1 a binary mask "010x1xx0.." + * @param string $stat2 a binary mask "x100x1..." * * @return string */