hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-15 22:13:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

port to 4.1 archive task improvement

Browse Source
This commit is contained in:
aynsix
2020-05-22 12:24:19 +03:00
parent 0b2f630817
commit a0521c59fc
2 changed files with 99 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
lib/Alchemy/Phrasea/TaskManager/Job/ArchiveJob.php
View File

@@ -181,11 +181,12 @@ class ArchiveJob extends AbstractJob
$dom = new \DOMDocument(); $dom = new \DOMDocument();
$dom->formatOutput = true; $dom->formatOutput = true;
/** @var \DOMElement $root */
$root = $dom->appendChild($dom->createElement('root')); $root = $dom->appendChild($dom->createElement('root'));
$nnew = $this->listFilesPhase1($app, $dom, $root, $path_in, $server_coll_id, 0, $TColls); $nnew = $this->listFilesPhase1($app, $dom, $root, $path_in, $server_coll_id, 0, $TColls);
if ($app['debug']) { if ($app['debug']) {
$this->log('debug', "=========== listFilesPhase1 ========== (returned " . $nnew . ")\n" . $dom->saveXML()); $this->log('debug', "== listFilesPhase1 returned " . $nnew . ")\n" . $dom->saveXML());
} }
if (!$this->isStarted()) { if (!$this->isStarted()) {
@@ -193,15 +194,16 @@ class ArchiveJob extends AbstractJob
} }
// wait for files to be cold // wait for files to be cold
$this->pause($cold); for($i=0; $i<($cold*2); $i++) {
if (!$this->isStarted()) {
if (!$this->isStarted()) { return;
return; }
$this->pause(0.5);
} }
$this->listFilesPhase2($app, $dom, $root, $path_in, 0); $this->listFilesPhase2($app, $dom, $root, $path_in, 0);
if ($app['debug']) { if ($app['debug']) {
$this->log('debug', "=========== listFilesPhase2 ========== : \n" . $dom->saveXML()); $this->log('debug', "== listFilesPhase2\n" . $dom->saveXML());
} }
if (!$this->isStarted()) { if (!$this->isStarted()) {
@@ -210,31 +212,35 @@ class ArchiveJob extends AbstractJob
$this->makePairs($dom, $root, $path_in, $path_archived, $path_error, false, 0, $tmask, $tmaskgrp); $this->makePairs($dom, $root, $path_in, $path_archived, $path_error, false, 0, $tmask, $tmaskgrp);
if ($app['debug']) { if ($app['debug']) {
$this->log('debug', "=========== makePairs ========== : \n" . $dom->saveXML()); $this->log('debug', "== makePairs\n" . $dom->saveXML());
}
$r = $this->removeBadGroups($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError);
if ($app['debug']) {
$this->log('debug', "=========== removeBadGroups ========== (returned " . ((Boolean) $r ? 'true' : 'false') . ") : \n" . $dom->saveXML());
}
$this->archive($app, $databox, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError, $moveArchived, $stat0, $stat1);
if ($app['debug']) {
$this->log('debug', "=========== archive ========== : \n" . $dom->saveXML());
} }
if (!$this->isStarted()) { if (!$this->isStarted()) {
return; return;
} }
$this->removeBadGroups($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError);
if ($app['debug']) {
$this->log('debug', "== removeBadGroups\n" . $dom->saveXML());
}
if (!$this->isStarted()) {
return;
}
$this->archive($app, $databox, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError, $moveArchived, $stat0, $stat1);
if ($app['debug']) {
$this->log('debug', "== archive\n" . $dom->saveXML());
}
$this->bubbleResults($dom, $root, $path_in, 0, \p4field::isyes($settings->copy_spe)); $this->bubbleResults($dom, $root, $path_in, 0, \p4field::isyes($settings->copy_spe));
if ($app['debug']) { if ($app['debug']) {
$this->log('debug', "=========== bubbleResults ========== : \n" . $dom->saveXML()); $this->log('debug', "== bubbleResults\n" . $dom->saveXML());
} }
$moved = $this->moveFiles($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveArchived, $moveError); $moved = $this->moveFiles($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveArchived, $moveError);
if ($app['debug']) { if ($app['debug']) {
$this->log('debug', "=========== moveFiles ========== (returned " . ($moved ? 'true' : 'false') . ") : \n" . $dom->saveXML()); $this->log('debug', "== moveFiles returned " . ($moved ? 'true' : 'false') . "\n" . $dom->saveXML());
} }
} }
} }
@@ -243,15 +249,17 @@ class ArchiveJob extends AbstractJob
{ {
$nnew = 0; $nnew = 0;
if (false !== $sxDotPhrasea = @simplexml_load_file($path . '/.phrasea.xml')) { $magicfile = $magicmethod = null;
if (($sxDotPhrasea = @simplexml_load_file($path . '/.phrasea.xml')) !== false) {
// test for magic file // test for magic file
if (($magicfile = trim((string) ($sxDotPhrasea->magicfile))) != '') { if (($magicfile = trim((string) ($sxDotPhrasea->magicfile))) != '') {
$magicmethod = strtoupper($sxDotPhrasea->magicfile['method']); $magicmethod = strtoupper($sxDotPhrasea->magicfile['method']);
if ($magicmethod == 'LOCK' && true === $app['filesystem']->exists($path . '/' . $magicfile)) { if ($magicmethod == 'LOCK' && ($app['filesystem']->exists($path . '/' . $magicfile) === true)) {
return; return 0;
} elseif ($magicmethod == 'UNLOCK' && false === $app['filesystem']->exists($path . '/' . $magicfile)) { } elseif ($magicmethod == 'UNLOCK' && ($app['filesystem']->exists($path . '/' . $magicfile) === false)) {
return; return 0;
} }
} }
@@ -278,6 +286,7 @@ class ArchiveJob extends AbstractJob
continue; continue;
} }
/** @var \DOMElement $n */
if (is_dir($path . '/' . $file)) { if (is_dir($path . '/' . $file)) {
$n = $node->appendChild($dom->createElement('file')); $n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('isdir', '1'); $n->setAttribute('isdir', '1');
@@ -293,6 +302,16 @@ class ArchiveJob extends AbstractJob
foreach (["size", "ctime", "mtime"] as $k) { foreach (["size", "ctime", "mtime"] as $k) {
$n->setAttribute($k, $stat[$k]); $n->setAttribute($k, $stat[$k]);
} }
// special file
if($file == '.phrasea.xml') {
$n->setAttribute('match', '*');
}
// special file
if($file === $magicfile) {
$n->setAttribute('match', '*');
$node->setAttribute('magicfile', $magicfile);
$node->setAttribute('magicmethod', $magicmethod);
}
$nnew++; $nnew++;
} }
$n->setAttribute('cid', $server_coll_id); $n->setAttribute('cid', $server_coll_id);
@@ -335,27 +354,31 @@ class ArchiveJob extends AbstractJob
$dnl = @$xp->query('./file[@name="' . $file . '"]', $node); $dnl = @$xp->query('./file[@name="' . $file . '"]', $node);
if ($dnl && $dnl->length == 0) { if ($dnl && $dnl->length == 0) {
if (is_dir($path . '/' . $file)) { if (is_dir($path . '/' . $file)) {
/** @var \DOMElement $n */
$n = $node->appendChild($dom->createElement('file')); $n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('isdir', '1'); $n->setAttribute('isdir', '1');
$n->setAttribute('name', $file); $n->setAttribute('name', $file);
$nnew += $this->listFilesPhase2($app, $dom, $n, $path . '/' . $file, $depth + 1); $nnew += $this->listFilesPhase2($app, $dom, $n, $path . '/' . $file, $depth + 1);
} else { } else {
/** @var \DOMElement $n */
$n = $node->appendChild($dom->createElement('file')); $n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('name', $file); $n->setAttribute('name', $file);
$nnew++; $nnew++;
} }
$this->setBranchHot($dom, $n); $this->setBranchHot($n);
} elseif ($dnl && $dnl->length == 1) { } elseif ($dnl && $dnl->length == 1) {
$dnl->item(0)->setAttribute('temperature', 'cold'); /** @var \DOMElement $n */
$n = $dnl->item(0);
$n->setAttribute('temperature', 'cold');
if (is_dir($path . '/' . $file)) { if (is_dir($path . '/' . $file)) {
$this->listFilesPhase2($app, $dom, $dnl->item(0), $path . '/' . $file, $depth + 1); $this->listFilesPhase2($app, $dom, $n, $path . '/' . $file, $depth + 1);
} else { } else {
$stat = stat($path . '/' . $file); $stat = stat($path . '/' . $file);
foreach (["size", "ctime", "mtime"] as $k) { foreach (["size", "ctime", "mtime"] as $k) {
if ($dnl->item(0)->getAttribute($k) != $stat[$k]) { if ($n->getAttribute($k) != $stat[$k]) {
$this->setBranchHot($dom, $dnl->item(0)); $this->setBranchHot($n);
break; break;
} }
} }
@@ -399,13 +422,16 @@ class ArchiveJob extends AbstractJob
if ($dnl->length == 1) { if ($dnl->length == 1) {
// this group is old (don't care about any linked files), just flag it // this group is old (don't care about any linked files), just flag it
$n->setAttribute('grp', 'tocomplete'); $n->setAttribute('grp', 'tocomplete');
$dnl->item(0)->setAttribute('match', '*'); /** @var \DOMElement $_n */
$_n = $dnl->item(0);
$_n->setAttribute('match', '*');
// recurse only if group is ok // recurse only if group is ok
$this->makePairs($dom, $n, $path . '/' . $name, $path_archived, $path_error, true, $depth + 1, $tmask, $tmaskgrp); $this->makePairs($dom, $n, $path . '/' . $name, $path_archived, $path_error, true, $depth + 1, $tmask, $tmaskgrp);
} else { } else {
// this group in new (to be created) // this group in new (to be created)
// do we need one (or both) linked file ? (caption or representation) // do we need one (or both) linked file ? (caption or representation)
$err = false; $err = false;
/** @var \DOMElement[] $flink */
$flink = ['caption' => null, 'representation' => null]; $flink = ['caption' => null, 'representation' => null];
foreach ($flink as $linkName => $v) { foreach ($flink as $linkName => $v) {
@@ -473,12 +499,12 @@ class ArchiveJob extends AbstractJob
// this is a file // this is a file
if (!$n->getAttribute('match')) { if (!$n->getAttribute('match')) {
// because match can be set before // because match can be set before
if ($name == '.phrasea.xml') { // if ($name == '.phrasea.xml') {
// special file(s) always ok // // special file(s) always ok
$n->setAttribute('match', '*'); // $n->setAttribute('match', '*');
} else { // } else {
$this->checkMatch($dom, $n, $tmask); $this->checkMatch($dom, $n, $tmask);
} // }
} }
} }
} }
@@ -503,7 +529,7 @@ class ArchiveJob extends AbstractJob
// if root of hotfolder if hot, die... // if root of hotfolder if hot, die...
if ($depth == 0 && $node->getAttribute('temperature') == 'hot') { if ($depth == 0 && $node->getAttribute('temperature') == 'hot') {
return $ret; return;
} }
$nodesToDel = []; $nodesToDel = [];
@@ -522,7 +548,7 @@ class ArchiveJob extends AbstractJob
$name = $n->getAttribute('name'); $name = $n->getAttribute('name');
if ($n->getAttribute('isdir')) { if ($n->getAttribute('isdir')) {
$ret |= $this->removeBadGroups($app, $dom, $n, $path . '/' . $name $this->removeBadGroups($app, $dom, $n, $path . '/' . $name
, $path_archived . '/' . $name , $path_archived . '/' . $name
, $path_error . '/' . $name , $path_error . '/' . $name
, $depth + 1, $moveError); , $depth + 1, $moveError);
@@ -642,7 +668,7 @@ class ArchiveJob extends AbstractJob
} }
if ($node->getAttribute('temperature') == 'hot') { if ($node->getAttribute('temperature') == 'hot') {
return; return 0;
} }
$ret = 0; $ret = 0;
@@ -758,7 +784,7 @@ class ArchiveJob extends AbstractJob
} }
} }
if (!$n->getAttribute('keep')) { if (!$n->getAttribute('keep') && !$n->getAttribute('match')) {
$this->log('debug', sprintf('delete \'%s\'', $path . '/' . $name)); $this->log('debug', sprintf('delete \'%s\'', $path . '/' . $name));
try { try {
@@ -792,7 +818,9 @@ class ArchiveJob extends AbstractJob
if ($dnl->length == 1) { if ($dnl->length == 1) {
// the caption file exists // the caption file exists
$node->setAttribute('match', $captionFileName); $node->setAttribute('match', $captionFileName);
$dnl->item(0)->setAttribute('match', '*'); /** @var \DOMElement $n */
$n = $dnl->item(0);
$n->setAttribute('match', '*');
} else { } else {
// the caption file is missing // the caption file is missing
$node->setAttribute('match', '?'); $node->setAttribute('match', '?');
@@ -817,7 +845,7 @@ class ArchiveJob extends AbstractJob
return ($f[0] == '.' && $f != '.phrasea.xml' && $f != '.grouping.xml') || $f == 'thumbs.db' || $f == 'par-system'; return ($f[0] == '.' && $f != '.phrasea.xml' && $f != '.grouping.xml') || $f == 'thumbs.db' || $f == 'par-system';
} }
private function setBranchHot(\DOMDocument $dom, \DOMElement $node) private function setBranchHot(\DOMElement $node)
{ {
for ($n = $node; $n; $n = $n->parentNode) { for ($n = $node; $n; $n = $n->parentNode) {
if ($n->nodeType == XML_ELEMENT_NODE) { if ($n->nodeType == XML_ELEMENT_NODE) {
@@ -847,8 +875,10 @@ class ArchiveJob extends AbstractJob
if ($node->getAttribute('grp') == 'tocreate') { if ($node->getAttribute('grp') == 'tocreate') {
$representationFileName = null; $representationFileName = null;
/** @var \DOMElement $representationFileNode */
$representationFileNode = null; $representationFileNode = null;
$captionFileName = null; $captionFileName = null;
/** @var \DOMElement $captionFileNode */
$captionFileNode = null; $captionFileNode = null;
$cid = $node->getAttribute('cid'); $cid = $node->getAttribute('cid');
$genericdoc = null; $genericdoc = null;
@@ -903,6 +933,7 @@ class ArchiveJob extends AbstractJob
} }
file_put_contents($groupingFile, '<?xml version="1.0" encoding="ISO-8859-1" ?><record grouping="' . $rid . '" />'); file_put_contents($groupingFile, '<?xml version="1.0" encoding="ISO-8859-1" ?><record grouping="' . $rid . '" />');
/** @var \DOMElement $n */
$n = $node->appendChild($dom->createElement('file')); $n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('name', '.grouping.xml'); $n->setAttribute('name', '.grouping.xml');
$n->setAttribute('temperature', 'cold'); $n->setAttribute('temperature', 'cold');
@@ -1025,6 +1056,8 @@ class ArchiveJob extends AbstractJob
} }
$story = \record_adapter::createStory($app, $collection); $story = \record_adapter::createStory($app, $collection);
$story->setStatus($status);
$app['subdef.substituer']->substituteDocument($story, $media); $app['subdef.substituer']->substituteDocument($story, $media);
$story->set_metadatas($metadatas->toMetadataArray($metadatasStructure), true); $story->set_metadatas($metadatas->toMetadataArray($metadatasStructure), true);
@@ -1080,12 +1113,14 @@ class ArchiveJob extends AbstractJob
$file->addAttribute(new BorderAttribute\Status($app, $status)); $file->addAttribute(new BorderAttribute\Status($app, $status));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfFilepath(), new MonoValue($media->getFile()->getRealPath())))); /** @var \MediaVorus\File $mediaFile */
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfDirname(), new MonoValue(dirname($media->getFile()->getRealPath()))))); $mediaFile = $media->getFile();
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfFilepath(), new MonoValue($mediaFile->getRealPath()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfDirname(), new MonoValue(dirname($mediaFile->getRealPath())))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfAtime(), new MonoValue($media->getFile()->getATime())))); $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfAtime(), new MonoValue($mediaFile->getATime()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfMtime(), new MonoValue($media->getFile()->getMTime())))); $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfMtime(), new MonoValue($mediaFile->getMTime()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfCtime(), new MonoValue($media->getFile()->getCTime())))); $file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfCtime(), new MonoValue($mediaFile->getCTime()))));
foreach ($metadatas as $meta) { foreach ($metadatas as $meta) {
$file->addAttribute(new BorderAttribute\Metadata($meta)); $file->addAttribute(new BorderAttribute\Metadata($meta));
@@ -1102,8 +1137,13 @@ class ArchiveJob extends AbstractJob
$record = null; $record = null;
$postProcess = function ($element, $visa, $code) use (&$record) { $postProcess = function ($element, $visa, $code) use (&$record) {
$record = $element; $r = isset($visa); // one way to avoid "variable not used" with phpstorm 10. ugly.
}; unset($r); //
$r = isset($code); // one way to avoid "variable not used" with phpstorm 10. ugly.
unset($r); //
$record = $element;
};
/** @var borderManager $borderManager */ /** @var borderManager $borderManager */
$borderManager = $app['border-manager']; $borderManager = $app['border-manager'];
@@ -1206,12 +1246,13 @@ class ArchiveJob extends AbstractJob
} }
} }
$this->archiveFileAndCaption($app, $databox, $dom, $node, $captionFileNode, $path, $path_archived, $path_error, $grp_rid, $nodesToDel, $stat0, $stat1, $moveError, $moveArchived); $this->archiveFileAndCaption($app, $databox, $node, $captionFileNode, $path, $path_archived, $path_error, $grp_rid, $nodesToDel, $stat0, $stat1, $moveError, $moveArchived);
} }
/** /**
* *
* @param \DOMDOcument $dom * @param Application $app
* @param \databox $databox
* @param \DOMElement $node * @param \DOMElement $node
* @param \DOMElement $captionFileNode * @param \DOMElement $captionFileNode
* @param string $path * @param string $path
@@ -1219,8 +1260,12 @@ class ArchiveJob extends AbstractJob
* @param string $path_error * @param string $path_error
* @param integer $grp_rid * @param integer $grp_rid
* @param array $nodesToDel out, filled with files to delete * @param array $nodesToDel out, filled with files to delete
* @param $stat0
* @param $stat1
* @param $moveError
* @param $moveArchived
*/ */
private function archiveFileAndCaption(Application $app, \databox $databox, \DOMDocument $dom, \DOMElement $node, \DOMElement $captionFileNode = null, $path, $path_archived, $path_error, $grp_rid, array &$nodesToDel, $stat0, $stat1, $moveError, $moveArchived) private function archiveFileAndCaption(Application $app, \databox $databox, \DOMElement $node, \DOMElement $captionFileNode = null, $path, $path_archived, $path_error, $grp_rid, array &$nodesToDel, $stat0, $stat1, $moveError, $moveArchived)
{ {
// quick fix to reconnect if mysql is lost // quick fix to reconnect if mysql is lost
$app->getApplicationBox()->get_connection(); $app->getApplicationBox()->get_connection();
@@ -1422,6 +1467,7 @@ class ArchiveJob extends AbstractJob
{ {
$ret = new MetadataBag(); $ret = new MetadataBag();
/** @var \databox_field $databox_field */
foreach ($metadatasStructure as $databox_field) { foreach ($metadatasStructure as $databox_field) {
if ($bag->containsKey($databox_field->get_tag()->getTagname())) { if ($bag->containsKey($databox_field->get_tag()->getTagname())) {
$ret->set($databox_field->get_name(), $bag->get($databox_field->get_tag()->getTagname())); $ret->set($databox_field->get_name(), $bag->get($databox_field->get_tag()->getTagname()));

4
lib/classes/databox/status.php
View File

@@ -157,8 +157,8 @@ class databox_status
* compute ((0 M s1) M s2) where M is the "mask" operator * compute ((0 M s1) M s2) where M is the "mask" operator
* nb : s1,s2 are binary mask strings as "01x0xx1xx0x", no other format (hex) supported * nb : s1,s2 are binary mask strings as "01x0xx1xx0x", no other format (hex) supported
* *
* @param $stat1 a binary mask "010x1xx0.." STRING * @param string $stat1 a binary mask "010x1xx0.."
* @param $stat2 a binary mask "x100x1..." STRING * @param string $stat2 a binary mask "x100x1..."
* *
* @return string * @return string
*/ */