hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-15 22:13:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

port to 4.1 archive task improvement

Browse Source
This commit is contained in:
aynsix
2020-05-22 12:24:19 +03:00
parent 0b2f630817
commit a0521c59fc
2 changed files with 99 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
lib/Alchemy/Phrasea/TaskManager/Job/ArchiveJob.php
View File

@@ -181,11 +181,12 @@ class ArchiveJob extends AbstractJob
$dom = new \DOMDocument();
$dom->formatOutput = true;
/** @var \DOMElement $root */
$root = $dom->appendChild($dom->createElement('root'));
$nnew = $this->listFilesPhase1($app, $dom, $root, $path_in, $server_coll_id, 0, $TColls);
if ($app['debug']) {
$this->log('debug', "=========== listFilesPhase1 ========== (returned " . $nnew . ")\n" . $dom->saveXML());
$this->log('debug', "== listFilesPhase1 returned " . $nnew . ")\n" . $dom->saveXML());
}
if (!$this->isStarted()) {
@@ -193,15 +194,16 @@ class ArchiveJob extends AbstractJob
}
// wait for files to be cold
$this->pause($cold);
for($i=0; $i<($cold*2); $i++) {
if (!$this->isStarted()) {
return;
}
$this->pause(0.5);
}
$this->listFilesPhase2($app, $dom, $root, $path_in, 0);
if ($app['debug']) {
$this->log('debug', "=========== listFilesPhase2 ========== : \n" . $dom->saveXML());
$this->log('debug', "== listFilesPhase2\n" . $dom->saveXML());
}
if (!$this->isStarted()) {
@@ -210,31 +212,35 @@ class ArchiveJob extends AbstractJob
$this->makePairs($dom, $root, $path_in, $path_archived, $path_error, false, 0, $tmask, $tmaskgrp);
if ($app['debug']) {
$this->log('debug', "=========== makePairs ========== : \n" . $dom->saveXML());
}
$r = $this->removeBadGroups($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError);
if ($app['debug']) {
$this->log('debug', "=========== removeBadGroups ========== (returned " . ((Boolean) $r ? 'true' : 'false') . ") : \n" . $dom->saveXML());
}
$this->archive($app, $databox, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError, $moveArchived, $stat0, $stat1);
if ($app['debug']) {
$this->log('debug', "=========== archive ========== : \n" . $dom->saveXML());
$this->log('debug', "== makePairs\n" . $dom->saveXML());
}
if (!$this->isStarted()) {
return;
}
$this->removeBadGroups($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError);
if ($app['debug']) {
$this->log('debug', "== removeBadGroups\n" . $dom->saveXML());
}
if (!$this->isStarted()) {
return;
}
$this->archive($app, $databox, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveError, $moveArchived, $stat0, $stat1);
if ($app['debug']) {
$this->log('debug', "== archive\n" . $dom->saveXML());
}
$this->bubbleResults($dom, $root, $path_in, 0, \p4field::isyes($settings->copy_spe));
if ($app['debug']) {
$this->log('debug', "=========== bubbleResults ========== : \n" . $dom->saveXML());
$this->log('debug', "== bubbleResults\n" . $dom->saveXML());
}
$moved = $this->moveFiles($app, $dom, $root, $path_in, $path_archived, $path_error, 0, $moveArchived, $moveError);
if ($app['debug']) {
$this->log('debug', "=========== moveFiles ========== (returned " . ($moved ? 'true' : 'false') . ") : \n" . $dom->saveXML());
$this->log('debug', "== moveFiles returned " . ($moved ? 'true' : 'false') . "\n" . $dom->saveXML());
}
}
}
@@ -243,15 +249,17 @@ class ArchiveJob extends AbstractJob
{
$nnew = 0;
if (false !== $sxDotPhrasea = @simplexml_load_file($path . '/.phrasea.xml')) {
$magicfile = $magicmethod = null;
if (($sxDotPhrasea = @simplexml_load_file($path . '/.phrasea.xml')) !== false) {
// test for magic file
if (($magicfile = trim((string) ($sxDotPhrasea->magicfile))) != '') {
$magicmethod = strtoupper($sxDotPhrasea->magicfile['method']);
if ($magicmethod == 'LOCK' && true === $app['filesystem']->exists($path . '/' . $magicfile)) {
return;
} elseif ($magicmethod == 'UNLOCK' && false === $app['filesystem']->exists($path . '/' . $magicfile)) {
return;
if ($magicmethod == 'LOCK' && ($app['filesystem']->exists($path . '/' . $magicfile) === true)) {
return 0;
} elseif ($magicmethod == 'UNLOCK' && ($app['filesystem']->exists($path . '/' . $magicfile) === false)) {
return 0;
}
}
@@ -278,6 +286,7 @@ class ArchiveJob extends AbstractJob
continue;
}
/** @var \DOMElement $n */
if (is_dir($path . '/' . $file)) {
$n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('isdir', '1');
@@ -293,6 +302,16 @@ class ArchiveJob extends AbstractJob
foreach (["size", "ctime", "mtime"] as $k) {
$n->setAttribute($k, $stat[$k]);
}
// special file
if($file == '.phrasea.xml') {
$n->setAttribute('match', '*');
}
// special file
if($file === $magicfile) {
$n->setAttribute('match', '*');
$node->setAttribute('magicfile', $magicfile);
$node->setAttribute('magicmethod', $magicmethod);
}
$nnew++;
}
$n->setAttribute('cid', $server_coll_id);
@@ -335,27 +354,31 @@ class ArchiveJob extends AbstractJob
$dnl = @$xp->query('./file[@name="' . $file . '"]', $node);
if ($dnl && $dnl->length == 0) {
if (is_dir($path . '/' . $file)) {
/** @var \DOMElement $n */
$n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('isdir', '1');
$n->setAttribute('name', $file);
$nnew += $this->listFilesPhase2($app, $dom, $n, $path . '/' . $file, $depth + 1);
} else {
/** @var \DOMElement $n */
$n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('name', $file);
$nnew++;
}
$this->setBranchHot($dom, $n);
$this->setBranchHot($n);
} elseif ($dnl && $dnl->length == 1) {
$dnl->item(0)->setAttribute('temperature', 'cold');
/** @var \DOMElement $n */
$n = $dnl->item(0);
$n->setAttribute('temperature', 'cold');
if (is_dir($path . '/' . $file)) {
$this->listFilesPhase2($app, $dom, $dnl->item(0), $path . '/' . $file, $depth + 1);
$this->listFilesPhase2($app, $dom, $n, $path . '/' . $file, $depth + 1);
} else {
$stat = stat($path . '/' . $file);
foreach (["size", "ctime", "mtime"] as $k) {
if ($dnl->item(0)->getAttribute($k) != $stat[$k]) {
$this->setBranchHot($dom, $dnl->item(0));
if ($n->getAttribute($k) != $stat[$k]) {
$this->setBranchHot($n);
break;
}
}
@@ -399,13 +422,16 @@ class ArchiveJob extends AbstractJob
if ($dnl->length == 1) {
// this group is old (don't care about any linked files), just flag it
$n->setAttribute('grp', 'tocomplete');
$dnl->item(0)->setAttribute('match', '*');
/** @var \DOMElement $_n */
$_n = $dnl->item(0);
$_n->setAttribute('match', '*');
// recurse only if group is ok
$this->makePairs($dom, $n, $path . '/' . $name, $path_archived, $path_error, true, $depth + 1, $tmask, $tmaskgrp);
} else {
// this group in new (to be created)
// do we need one (or both) linked file ? (caption or representation)
$err = false;
/** @var \DOMElement[] $flink */
$flink = ['caption' => null, 'representation' => null];
foreach ($flink as $linkName => $v) {
@@ -473,12 +499,12 @@ class ArchiveJob extends AbstractJob
// this is a file
if (!$n->getAttribute('match')) {
// because match can be set before
if ($name == '.phrasea.xml') {
// special file(s) always ok
$n->setAttribute('match', '*');
} else {
// if ($name == '.phrasea.xml') {
// // special file(s) always ok
// $n->setAttribute('match', '*');
// } else {
$this->checkMatch($dom, $n, $tmask);
}
// }
}
}
}
@@ -503,7 +529,7 @@ class ArchiveJob extends AbstractJob
// if root of hotfolder if hot, die...
if ($depth == 0 && $node->getAttribute('temperature') == 'hot') {
return $ret;
return;
}
$nodesToDel = [];
@@ -522,7 +548,7 @@ class ArchiveJob extends AbstractJob
$name = $n->getAttribute('name');
if ($n->getAttribute('isdir')) {
$ret |= $this->removeBadGroups($app, $dom, $n, $path . '/' . $name
$this->removeBadGroups($app, $dom, $n, $path . '/' . $name
, $path_archived . '/' . $name
, $path_error . '/' . $name
, $depth + 1, $moveError);
@@ -642,7 +668,7 @@ class ArchiveJob extends AbstractJob
}
if ($node->getAttribute('temperature') == 'hot') {
return;
return 0;
}
$ret = 0;
@@ -758,7 +784,7 @@ class ArchiveJob extends AbstractJob
}
}
if (!$n->getAttribute('keep')) {
if (!$n->getAttribute('keep') && !$n->getAttribute('match')) {
$this->log('debug', sprintf('delete \'%s\'', $path . '/' . $name));
try {
@@ -792,7 +818,9 @@ class ArchiveJob extends AbstractJob
if ($dnl->length == 1) {
// the caption file exists
$node->setAttribute('match', $captionFileName);
$dnl->item(0)->setAttribute('match', '*');
/** @var \DOMElement $n */
$n = $dnl->item(0);
$n->setAttribute('match', '*');
} else {
// the caption file is missing
$node->setAttribute('match', '?');
@@ -817,7 +845,7 @@ class ArchiveJob extends AbstractJob
return ($f[0] == '.' && $f != '.phrasea.xml' && $f != '.grouping.xml') || $f == 'thumbs.db' || $f == 'par-system';
}
private function setBranchHot(\DOMDocument $dom, \DOMElement $node)
private function setBranchHot(\DOMElement $node)
{
for ($n = $node; $n; $n = $n->parentNode) {
if ($n->nodeType == XML_ELEMENT_NODE) {
@@ -847,8 +875,10 @@ class ArchiveJob extends AbstractJob
if ($node->getAttribute('grp') == 'tocreate') {
$representationFileName = null;
/** @var \DOMElement $representationFileNode */
$representationFileNode = null;
$captionFileName = null;
/** @var \DOMElement $captionFileNode */
$captionFileNode = null;
$cid = $node->getAttribute('cid');
$genericdoc = null;
@@ -903,6 +933,7 @@ class ArchiveJob extends AbstractJob
}
file_put_contents($groupingFile, '<?xml version="1.0" encoding="ISO-8859-1" ?><record grouping="' . $rid . '" />');
/** @var \DOMElement $n */
$n = $node->appendChild($dom->createElement('file'));
$n->setAttribute('name', '.grouping.xml');
$n->setAttribute('temperature', 'cold');
@@ -1025,6 +1056,8 @@ class ArchiveJob extends AbstractJob
}
$story = \record_adapter::createStory($app, $collection);
$story->setStatus($status);
$app['subdef.substituer']->substituteDocument($story, $media);
$story->set_metadatas($metadatas->toMetadataArray($metadatasStructure), true);
@@ -1080,12 +1113,14 @@ class ArchiveJob extends AbstractJob
$file->addAttribute(new BorderAttribute\Status($app, $status));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfFilepath(), new MonoValue($media->getFile()->getRealPath()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfDirname(), new MonoValue(dirname($media->getFile()->getRealPath())))));
/** @var \MediaVorus\File $mediaFile */
$mediaFile = $media->getFile();
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfFilepath(), new MonoValue($mediaFile->getRealPath()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfDirname(), new MonoValue(dirname($mediaFile->getRealPath())))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfAtime(), new MonoValue($media->getFile()->getATime()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfMtime(), new MonoValue($media->getFile()->getMTime()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfCtime(), new MonoValue($media->getFile()->getCTime()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfAtime(), new MonoValue($mediaFile->getATime()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfMtime(), new MonoValue($mediaFile->getMTime()))));
$file->addAttribute(new BorderAttribute\Metadata(new Metadata(new PhraseaTag\TfCtime(), new MonoValue($mediaFile->getCTime()))));
foreach ($metadatas as $meta) {
$file->addAttribute(new BorderAttribute\Metadata($meta));
@@ -1102,6 +1137,11 @@ class ArchiveJob extends AbstractJob
$record = null;
$postProcess = function ($element, $visa, $code) use (&$record) {
$r = isset($visa); // one way to avoid "variable not used" with phpstorm 10. ugly.
unset($r); //
$r = isset($code); // one way to avoid "variable not used" with phpstorm 10. ugly.
unset($r); //
$record = $element;
};
@@ -1206,12 +1246,13 @@ class ArchiveJob extends AbstractJob
}
}
$this->archiveFileAndCaption($app, $databox, $dom, $node, $captionFileNode, $path, $path_archived, $path_error, $grp_rid, $nodesToDel, $stat0, $stat1, $moveError, $moveArchived);
$this->archiveFileAndCaption($app, $databox, $node, $captionFileNode, $path, $path_archived, $path_error, $grp_rid, $nodesToDel, $stat0, $stat1, $moveError, $moveArchived);
}
/**
*
* @param \DOMDOcument $dom
* @param Application $app
* @param \databox $databox
* @param \DOMElement $node
* @param \DOMElement $captionFileNode
* @param string $path
@@ -1219,8 +1260,12 @@ class ArchiveJob extends AbstractJob
* @param string $path_error
* @param integer $grp_rid
* @param array $nodesToDel out, filled with files to delete
* @param $stat0
* @param $stat1
* @param $moveError
* @param $moveArchived
*/
private function archiveFileAndCaption(Application $app, \databox $databox, \DOMDocument $dom, \DOMElement $node, \DOMElement $captionFileNode = null, $path, $path_archived, $path_error, $grp_rid, array &$nodesToDel, $stat0, $stat1, $moveError, $moveArchived)
private function archiveFileAndCaption(Application $app, \databox $databox, \DOMElement $node, \DOMElement $captionFileNode = null, $path, $path_archived, $path_error, $grp_rid, array &$nodesToDel, $stat0, $stat1, $moveError, $moveArchived)
{
// quick fix to reconnect if mysql is lost
$app->getApplicationBox()->get_connection();
@@ -1422,6 +1467,7 @@ class ArchiveJob extends AbstractJob
{
$ret = new MetadataBag();
/** @var \databox_field $databox_field */
foreach ($metadatasStructure as $databox_field) {
if ($bag->containsKey($databox_field->get_tag()->getTagname())) {
$ret->set($databox_field->get_name(), $bag->get($databox_field->get_tag()->getTagname()));

4
lib/classes/databox/status.php
View File

@@ -157,8 +157,8 @@ class databox_status
* compute ((0 M s1) M s2) where M is the "mask" operator
* nb : s1,s2 are binary mask strings as "01x0xx1xx0x", no other format (hex) supported
*
* @param $stat1 a binary mask "010x1xx0.." STRING
* @param $stat2 a binary mask "x100x1..." STRING
* @param string $stat1 a binary mask "010x1xx0.."
* @param string $stat2 a binary mask "x100x1..."
*
* @return string
*/