hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-24 10:23:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

manage error on subdef an write meta

Browse Source
This commit is contained in:
aynsix
2020-05-15 21:04:30 +03:00
parent 6a7ca08000
commit 5f9fb0224e
4 changed files with 210 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php
View File

@@ -73,7 +73,6 @@ class AlchemyWorkerServiceProvider implements PluginProviderInterface
$app['elasticsearch.indexer'] $app['elasticsearch.indexer']
)) ))
->setApplicationBox($app['phraseanet.appbox']) ->setApplicationBox($app['phraseanet.appbox'])
->setEntityManagerLocator(new LazyLocator($app, 'orm.em'))
; ;
})); }));

44
lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php
View File

@@ -9,7 +9,10 @@ use Alchemy\Phrasea\Core\Event\Record\MetadataChangedEvent;
use Alchemy\Phrasea\Core\Event\Record\RecordEvent; use Alchemy\Phrasea\Core\Event\Record\RecordEvent;
use Alchemy\Phrasea\Core\Event\Record\RecordEvents; use Alchemy\Phrasea\Core\Event\Record\RecordEvents;
use Alchemy\Phrasea\Core\Event\Record\SubdefinitionCreateEvent; use Alchemy\Phrasea\Core\Event\Record\SubdefinitionCreateEvent;
use Alchemy\Phrasea\Core\PhraseaTokens;
use Alchemy\Phrasea\Databox\Subdef\MediaSubdefRepository; use Alchemy\Phrasea\Databox\Subdef\MediaSubdefRepository;
use Alchemy\Phrasea\Model\Entities\WorkerRunningJob;
use Alchemy\Phrasea\Model\Repositories\WorkerRunningJobRepository;
use Alchemy\Phrasea\WorkerManager\Event\StoryCreateCoverEvent; use Alchemy\Phrasea\WorkerManager\Event\StoryCreateCoverEvent;
use Alchemy\Phrasea\WorkerManager\Event\SubdefinitionCreationFailureEvent; use Alchemy\Phrasea\WorkerManager\Event\SubdefinitionCreationFailureEvent;
use Alchemy\Phrasea\WorkerManager\Event\SubdefinitionWritemetaEvent; use Alchemy\Phrasea\WorkerManager\Event\SubdefinitionWritemetaEvent;
@@ -35,6 +38,10 @@ class RecordSubscriber implements EventSubscriberInterface
* @var callable * @var callable
*/ */
private $appboxLocator; private $appboxLocator;
/**
* @var WorkerRunningJobRepository
*/
private $repoWorker;
public function __construct(Application $app, callable $appboxLocator) public function __construct(Application $app, callable $appboxLocator)
{ {
@@ -42,6 +49,7 @@ class RecordSubscriber implements EventSubscriberInterface
$this->workerResolver = $app['alchemy_worker.type_based_worker_resolver']; $this->workerResolver = $app['alchemy_worker.type_based_worker_resolver'];
$this->app = $app; $this->app = $app;
$this->appboxLocator = $appboxLocator; $this->appboxLocator = $appboxLocator;
$this->repoWorker = $app['repo.worker-running-job'];
} }
public function onSubdefinitionCreate(SubdefinitionCreateEvent $event) public function onSubdefinitionCreate(SubdefinitionCreateEvent $event)
@@ -96,6 +104,23 @@ class RecordSubscriber implements EventSubscriberInterface
] ]
]; ];
$em = $this->repoWorker->getEntityManager();
$workerRunningJob = $this->repoWorker->findOneBy([
'databoxId' => $event->getRecord()->getDataboxId(),
'recordId' => $event->getRecord()->getRecordId(),
'work' => PhraseaTokens::MAKE_SUBDEF,
'workOn' => $event->getSubdefName()
]);
$em->beginTransaction();
try {
$em->remove($workerRunningJob);
$em->flush();
$em->commit();
} catch (\Exception $e) {
$em->rollback();
}
$this->messagePublisher->publishMessage( $this->messagePublisher->publishMessage(
$payload, $payload,
MessagePublisher::RETRY_SUBDEF_QUEUE, MessagePublisher::RETRY_SUBDEF_QUEUE,
@@ -198,6 +223,25 @@ class RecordSubscriber implements EventSubscriberInterface
); );
$this->messagePublisher->pushLog($logMessage); $this->messagePublisher->pushLog($logMessage);
$jeton = ($event->getSubdefName() == "document") ? PhraseaTokens::WRITE_META_DOC : PhraseaTokens::WRITE_META_SUBDEF;
$em = $this->repoWorker->getEntityManager();
$workerRunningJob = $this->repoWorker->findOneBy([
'databoxId' => $event->getRecord()->getDataboxId(),
'recordId' => $event->getRecord()->getRecordId(),
'work' => $jeton,
'workOn' => $event->getSubdefName()
]);
$em->beginTransaction();
try {
$em->remove($workerRunningJob);
$em->flush();
$em->commit();
} catch (\Exception $e) {
$em->rollback();
}
$this->messagePublisher->publishMessage( $this->messagePublisher->publishMessage(
$payload, $payload,
MessagePublisher::RETRY_METADATAS_QUEUE, MessagePublisher::RETRY_METADATAS_QUEUE,

114
lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php
View File

@@ -21,7 +21,6 @@ use Symfony\Component\EventDispatcher\EventDispatcherInterface;
class SubdefCreationWorker implements WorkerInterface class SubdefCreationWorker implements WorkerInterface
{ {
use ApplicationBoxAware; use ApplicationBoxAware;
use EntityManagerAware;
private $subdefGenerator; private $subdefGenerator;
@@ -84,7 +83,7 @@ class SubdefCreationWorker implements WorkerInterface
} }
// tell that a file is in used to create subdef // tell that a file is in used to create subdef
$em = $this->getEntityManager(); $em = $this->repoWorker->getEntityManager();
$em->beginTransaction(); $em->beginTransaction();
try { try {
@@ -107,76 +106,57 @@ class SubdefCreationWorker implements WorkerInterface
$em->rollback(); $em->rollback();
} }
try { $this->subdefGenerator->setLogger($this->logger);
$this->subdefGenerator->setLogger($this->logger);
$this->subdefGenerator->generateSubdefs($record, $wantedSubdef); $this->subdefGenerator->generateSubdefs($record, $wantedSubdef);
// begin to check if the subdef is successfully generated // begin to check if the subdef is successfully generated
$subdef = $record->getDatabox()->get_subdef_structure()->getSubdefGroup($record->getType())->getSubdef($payload['subdefName']); $subdef = $record->getDatabox()->get_subdef_structure()->getSubdefGroup($record->getType())->getSubdef($payload['subdefName']);
$filePathToCheck = null; $filePathToCheck = null;
if ($record->has_subdef($payload['subdefName']) ) { if ($record->has_subdef($payload['subdefName']) ) {
$filePathToCheck = $record->get_subdef($payload['subdefName'])->getRealPath(); $filePathToCheck = $record->get_subdef($payload['subdefName'])->getRealPath();
}
$filePathToCheck = $this->filesystem->generateSubdefPathname($record, $subdef, $filePathToCheck);
if (!$this->filesystem->exists($filePathToCheck)) {
$count = isset($payload['count']) ? $payload['count'] + 1 : 2 ;
$this->dispatcher->dispatch(WorkerEvents::SUBDEFINITION_CREATION_FAILURE, new SubdefinitionCreationFailureEvent(
$record,
$payload['subdefName'],
'Subdef generation failed !',
$count
));
$this->subdefGenerator->setLogger($oldLogger);
return ;
}
// checking ended
// order to write meta for the subdef if needed
$this->dispatcher->dispatch(WorkerEvents::SUBDEFINITION_WRITE_META, new SubdefinitionWritemetaEvent($record, $payload['subdefName']));
$this->subdefGenerator->setLogger($oldLogger);
// update jeton when subdef is created
$this->updateJeton($record);
$parents = $record->get_grouping_parents();
// create a cover for a story
// used when uploaded via uploader-service and grouped as a story
if (!$parents->is_empty() && isset($payload['status']) && $payload['status'] == MessagePublisher::NEW_RECORD_MESSAGE && in_array($payload['subdefName'], array('thumbnail', 'preview'))) {
foreach ($parents->get_elements() as $story) {
if (self::checkIfFirstChild($story, $record)) {
$data = implode('_', [$databoxId, $story->getRecordId(), $recordId, $payload['subdefName']]);
$this->dispatcher->dispatch(WorkerEvents::STORY_CREATE_COVER, new StoryCreateCoverEvent($data));
}
}
}
} catch (\Exception $e) {
$payload = [
'message_type' => MessagePublisher::SUBDEF_CREATION_TYPE,
'payload' => $payload
];
$this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_SUBDEF_QUEUE);
$em->beginTransaction();
try {
$workerRunningJob->setStatus(WorkerRunningJob::FINISHED);
$em->persist($workerRunningJob);
$em->flush();
$em->commit();
} catch (\Exception $e) {
$em->rollback();
}
} }
$filePathToCheck = $this->filesystem->generateSubdefPathname($record, $subdef, $filePathToCheck);
if (!$this->filesystem->exists($filePathToCheck)) {
$count = isset($payload['count']) ? $payload['count'] + 1 : 2 ;
$this->dispatcher->dispatch(WorkerEvents::SUBDEFINITION_CREATION_FAILURE, new SubdefinitionCreationFailureEvent(
$record,
$payload['subdefName'],
'Subdef generation failed !',
$count
));
$this->subdefGenerator->setLogger($oldLogger);
return ;
}
// checking ended
// order to write meta for the subdef if needed
$this->dispatcher->dispatch(WorkerEvents::SUBDEFINITION_WRITE_META, new SubdefinitionWritemetaEvent($record, $payload['subdefName']));
$this->subdefGenerator->setLogger($oldLogger);
// update jeton when subdef is created
$this->updateJeton($record);
$parents = $record->get_grouping_parents();
// create a cover for a story
// used when uploaded via uploader-service and grouped as a story
if (!$parents->is_empty() && isset($payload['status']) && $payload['status'] == MessagePublisher::NEW_RECORD_MESSAGE && in_array($payload['subdefName'], array('thumbnail', 'preview'))) {
foreach ($parents->get_elements() as $story) {
if (self::checkIfFirstChild($story, $record)) {
$data = implode('_', [$databoxId, $story->getRecordId(), $recordId, $payload['subdefName']]);
$this->dispatcher->dispatch(WorkerEvents::STORY_CREATE_COVER, new StoryCreateCoverEvent($data));
}
}
}
// update elastic // update elastic
$this->indexer->flushQueue(); $this->indexer->flushQueue();

257
lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php
View File

@@ -107,165 +107,146 @@ class WriteMetadatasWorker implements WorkerInterface
$em->rollback(); $em->rollback();
} }
try { $record = $databox->get_record($recordId);
$record = $databox->get_record($recordId);
$subdef = $record->get_subdef($payload['subdefName']); $subdef = $record->get_subdef($payload['subdefName']);
if ($subdef->is_physically_present()) { if ($subdef->is_physically_present()) {
$metadata = new MetadataBag(); $metadata = new MetadataBag();
// add Uuid in metadatabag // add Uuid in metadatabag
if ($record->getUuid()) { if ($record->getUuid()) {
$metadata->add( $metadata->add(
new Metadata( new Metadata(
new Tag\XMPExif\ImageUniqueID(), new Tag\XMPExif\ImageUniqueID(),
new Mono($record->getUuid()) new Mono($record->getUuid())
) )
); );
$metadata->add( $metadata->add(
new Metadata( new Metadata(
new Tag\ExifIFD\ImageUniqueID(), new Tag\ExifIFD\ImageUniqueID(),
new Mono($record->getUuid()) new Mono($record->getUuid())
) )
); );
$metadata->add( $metadata->add(
new Metadata( new Metadata(
new Tag\IPTC\UniqueDocumentID(), new Tag\IPTC\UniqueDocumentID(),
new Mono($record->getUuid()) new Mono($record->getUuid())
) )
); );
}
// read document fields and add to metadatabag
$caption = $record->get_caption();
foreach ($databox->get_meta_structure() as $fieldStructure) {
$tagName = $fieldStructure->get_tag()->getTagname();
$fieldName = $fieldStructure->get_name();
// skip fields with no src
if ($tagName == '' || $tagName == 'Phraseanet:no-source') {
continue;
} }
// read document fields and add to metadatabag // check exiftool known tags to skip Phraseanet:tf-*
$caption = $record->get_caption();
foreach ($databox->get_meta_structure() as $fieldStructure) {
$tagName = $fieldStructure->get_tag()->getTagname();
$fieldName = $fieldStructure->get_name();
// skip fields with no src
if ($tagName == '' || $tagName == 'Phraseanet:no-source') {
continue;
}
// check exiftool known tags to skip Phraseanet:tf-*
try {
$tag = TagFactory::getFromRDFTagname($tagName);
if(!$tag->isWritable()) {
continue;
}
} catch (TagUnknown $e) {
continue;
}
try {
$field = $caption->get_field($fieldName);
$fieldValues = $field->get_values();
if ($fieldStructure->is_multi()) {
$values = array();
foreach ($fieldValues as $value) {
$values[] = $this->removeNulChar($value->getValue());
}
$value = new Multi($values);
} else {
$fieldValue = array_pop($fieldValues);
$value = $this->removeNulChar($fieldValue->getValue());
// fix the dates edited into phraseanet
if($fieldStructure->get_type() === $fieldStructure::TYPE_DATE) {
try {
$value = self::fixDate($value); // will return NULL if the date is not valid
}
catch (\Exception $e) {
$value = null; // do NOT write back to iptc
}
}
if($value !== null) { // do not write invalid dates
$value = new Mono($value);
}
}
} catch(\Exception $e) {
// the field is not set in the record, erase it
if ($fieldStructure->is_multi()) {
$value = new Multi(array(''));
}
else {
$value = new Mono('');
}
}
if($value !== null) { // do not write invalid data
$metadata->add(
new Metadata($fieldStructure->get_tag(), $value)
);
}
}
$this->writer->reset();
if ($MWG) {
$this->writer->setModule(Writer::MODULE_MWG, true);
}
$this->writer->erase($subdef->get_name() != 'document' || $clearDoc, true);
// write meta in file
try { try {
$this->writer->write($subdef->getRealPath(), $metadata); $tag = TagFactory::getFromRDFTagname($tagName);
if(!$tag->isWritable()) {
$this->messagePublisher->pushLog(sprintf('meta written for sbasid=%1$d - recordid=%2$d (%3$s)', $databox->get_sbas_id(), $recordId, $subdef->get_name() )); continue;
} catch (PHPExiftoolException $e) { }
$workerMessage = sprintf('meta NOT written for sbasid=%1$d - recordid=%2$d (%3$s) because "%s"', $databox->get_sbas_id(), $recordId, $subdef->get_name() , $e->getMessage()); } catch (TagUnknown $e) {
$this->logger->error($workerMessage); continue;
$count = isset($payload['count']) ? $payload['count'] + 1 : 2 ;
$this->dispatch(WorkerEvents::SUBDEFINITION_WRITE_META, new SubdefinitionWritemetaEvent(
$record,
$payload['subdefName'],
SubdefinitionWritemetaEvent::FAILED,
$workerMessage,
$count
));
} }
// mark write metas finished try {
$this->updateJeton($record); $field = $caption->get_field($fieldName);
} else { $fieldValues = $field->get_values();
if ($fieldStructure->is_multi()) {
$values = array();
foreach ($fieldValues as $value) {
$values[] = $this->removeNulChar($value->getValue());
}
$value = new Multi($values);
} else {
$fieldValue = array_pop($fieldValues);
$value = $this->removeNulChar($fieldValue->getValue());
// fix the dates edited into phraseanet
if($fieldStructure->get_type() === $fieldStructure::TYPE_DATE) {
try {
$value = self::fixDate($value); // will return NULL if the date is not valid
}
catch (\Exception $e) {
$value = null; // do NOT write back to iptc
}
}
if($value !== null) { // do not write invalid dates
$value = new Mono($value);
}
}
} catch(\Exception $e) {
// the field is not set in the record, erase it
if ($fieldStructure->is_multi()) {
$value = new Multi(array(''));
}
else {
$value = new Mono('');
}
}
if($value !== null) { // do not write invalid data
$metadata->add(
new Metadata($fieldStructure->get_tag(), $value)
);
}
}
$this->writer->reset();
if ($MWG) {
$this->writer->setModule(Writer::MODULE_MWG, true);
}
$this->writer->erase($subdef->get_name() != 'document' || $clearDoc, true);
// write meta in file
try {
$this->writer->write($subdef->getRealPath(), $metadata);
$this->messagePublisher->pushLog(sprintf('meta written for sbasid=%1$d - recordid=%2$d (%3$s)', $databox->get_sbas_id(), $recordId, $subdef->get_name() ));
} catch (PHPExiftoolException $e) {
$workerMessage = sprintf('meta NOT written for sbasid=%1$d - recordid=%2$d (%3$s) because "%s"', $databox->get_sbas_id(), $recordId, $subdef->get_name() , $e->getMessage());
$this->logger->error($workerMessage);
$count = isset($payload['count']) ? $payload['count'] + 1 : 2 ; $count = isset($payload['count']) ? $payload['count'] + 1 : 2 ;
$this->dispatch(WorkerEvents::SUBDEFINITION_WRITE_META, new SubdefinitionWritemetaEvent( $this->dispatch(WorkerEvents::SUBDEFINITION_WRITE_META, new SubdefinitionWritemetaEvent(
$record, $record,
$payload['subdefName'], $payload['subdefName'],
SubdefinitionWritemetaEvent::FAILED, SubdefinitionWritemetaEvent::FAILED,
'Subdef is not physically present!', $workerMessage,
$count $count
)); ));
} }
} catch(\Exception $e) { // mark write metas finished
$payload = [ $this->updateJeton($record);
'message_type' => MessagePublisher::WRITE_METADATAS_TYPE, } else {
'payload' => $payload $count = isset($payload['count']) ? $payload['count'] + 1 : 2 ;
];
$this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_METADATAS_QUEUE);
$this->dispatch(WorkerEvents::SUBDEFINITION_WRITE_META, new SubdefinitionWritemetaEvent(
$em->beginTransaction(); $record,
try { $payload['subdefName'],
$workerRunningJob->setStatus(WorkerRunningJob::FINISHED); SubdefinitionWritemetaEvent::FAILED,
$em->persist($workerRunningJob); 'Subdef is not physically present!',
$em->flush(); $count
$em->commit(); ));
} catch (\Exception $e) {
$em->rollback();
}
} }
// tell that we have finished to work on this file // tell that we have finished to work on this file
$em->beginTransaction(); $em->beginTransaction();
try { try {