From 50f016ebb52c6c615a20a30d71acbc5e3d02326a Mon Sep 17 00:00:00 2001 From: aynsix Date: Wed, 27 May 2020 19:55:45 +0300 Subject: [PATCH 1/2] fix mysql server gone away and dead lock on table --- .../WorkerRunningJobRepository.php | 8 ++++ .../Subscriber/RecordSubscriber.php | 38 ++++++++++++------- .../Worker/SubdefCreationWorker.php | 20 ++++++++-- .../Worker/WriteMetadatasWorker.php | 2 + 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php b/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php index a3b584058d..8d8655ec6a 100644 --- a/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php +++ b/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php @@ -102,4 +102,12 @@ class WorkerRunningJobRepository extends EntityRepository { return parent::getEntityManager(); } + + public function reconnect() + { + if($this->_em->getConnection()->ping() === false) { + $this->_em->getConnection()->close(); + $this->_em->getConnection()->connect(); + } + } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php index d225a0c37a..c0df49717d 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php @@ -101,6 +101,9 @@ class RecordSubscriber implements EventSubscriberInterface $repoWorker = $this->getRepoWorker(); $em = $repoWorker->getEntityManager(); + // check connection an re-connect if needed + $repoWorker->reconnect(); + $workerRunningJob = $repoWorker->findOneBy([ 'databoxId' => $event->getRecord()->getDataboxId(), 'recordId' => $event->getRecord()->getRecordId(), @@ -108,13 +111,15 @@ class RecordSubscriber implements EventSubscriberInterface 'workOn' => $event->getSubdefName() ]); - $em->beginTransaction(); - try { - $em->remove($workerRunningJob); - $em->flush(); - $em->commit(); - } catch (\Exception $e) { - $em->rollback(); + if ($workerRunningJob) { + $em->beginTransaction(); + try { + $em->remove($workerRunningJob); + $em->flush(); + $em->commit(); + } catch (\Exception $e) { + $em->rollback(); + } } $this->messagePublisher->publishMessage( @@ -223,6 +228,9 @@ class RecordSubscriber implements EventSubscriberInterface $repoWorker = $this->getRepoWorker(); $em = $repoWorker->getEntityManager(); + // check connection an re-connect if needed + $repoWorker->reconnect(); + $workerRunningJob = $repoWorker->findOneBy([ 'databoxId' => $event->getRecord()->getDataboxId(), 'recordId' => $event->getRecord()->getRecordId(), @@ -230,13 +238,15 @@ class RecordSubscriber implements EventSubscriberInterface 'workOn' => $event->getSubdefName() ]); - $em->beginTransaction(); - try { - $em->remove($workerRunningJob); - $em->flush(); - $em->commit(); - } catch (\Exception $e) { - $em->rollback(); + if ($workerRunningJob) { + $em->beginTransaction(); + try { + $em->remove($workerRunningJob); + $em->flush(); + $em->commit(); + } catch (\Exception $e) { + $em->rollback(); + } } $this->messagePublisher->publishMessage( diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php index a05d8a1aa7..68b08ed020 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php @@ -81,6 +81,7 @@ class SubdefCreationWorker implements WorkerInterface // tell that a file is in used to create subdef $em = $this->repoWorker->getEntityManager(); + $this->repoWorker->reconnect(); $em->beginTransaction(); try { @@ -108,13 +109,13 @@ class SubdefCreationWorker implements WorkerInterface try { $this->subdefGenerator->generateSubdefs($record, $wantedSubdef); } catch (\Exception $e) { - $em->beginTransaction(); try { + $this->repoWorker->reconnect(); + $em->getConnection()->beginTransaction(); $em->remove($workerRunningJob); $em->flush(); $em->commit(); } catch (\Exception $e) { - $em->rollback(); } } @@ -170,7 +171,8 @@ class SubdefCreationWorker implements WorkerInterface $this->indexer->flushQueue(); // tell that we have finished to work on this file - $em->beginTransaction(); + $this->repoWorker->reconnect(); + $em->getConnection()->beginTransaction(); try { $workerRunningJob->setStatus(WorkerRunningJob::FINISHED); $workerRunningJob->setFinished(new \DateTime('now')); @@ -178,7 +180,17 @@ class SubdefCreationWorker implements WorkerInterface $em->flush(); $em->commit(); } catch (\Exception $e) { - $em->rollback(); + try { + $em->getConnection()->beginTransaction(); + $workerRunningJob->setStatus(WorkerRunningJob::FINISHED); + $em->persist($workerRunningJob); + $em->flush(); + $em->commit(); + } catch (\Exception $e) { + $this->messagePublisher->pushLog("rollback on recordID :" . $workerRunningJob->getRecordId()); + $em->rollback(); + } + } } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php index 6c02078ac1..aea5376a73 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php @@ -82,6 +82,7 @@ class WriteMetadatasWorker implements WorkerInterface // tell that a file is in used to create subdef $em = $this->getEntityManager(); + $this->repoWorker->reconnect(); $em->beginTransaction(); try { @@ -245,6 +246,7 @@ class WriteMetadatasWorker implements WorkerInterface // tell that we have finished to work on this file + $this->repoWorker->reconnect(); $em->beginTransaction(); try { $workerRunningJob->setStatus(WorkerRunningJob::FINISHED); From 24261483d74a61972ef57ab98aa264beabef2348 Mon Sep 17 00:00:00 2001 From: aynsix Date: Thu, 28 May 2020 20:29:15 +0300 Subject: [PATCH 2/2] catch exception on the worker --- .../Worker/SubdefCreationWorker.php | 14 ++++++++++++++ .../Worker/WriteMetadatasWorker.php | 19 ++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php index 68b08ed020..1bf324943f 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php @@ -117,6 +117,20 @@ class SubdefCreationWorker implements WorkerInterface $em->commit(); } catch (\Exception $e) { } + } catch (\Throwable $e) { + $count = isset($payload['count']) ? $payload['count'] + 1 : 2 ; + $workerMessage = "Exception throwable catched when create subdef for the recordID: " .$recordId; + + $this->logger->error($workerMessage); + + $this->dispatcher->dispatch(WorkerEvents::SUBDEFINITION_CREATION_FAILURE, new SubdefinitionCreationFailureEvent( + $record, + $payload['subdefName'], + $workerMessage, + $count + )); + + return ; } // begin to check if the subdef is successfully generated diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php index aea5376a73..8ec819cb77 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php @@ -107,7 +107,24 @@ class WriteMetadatasWorker implements WorkerInterface $record = $databox->get_record($recordId); - $subdef = $record->get_subdef($payload['subdefName']); + try { + $subdef = $record->get_subdef($payload['subdefName']); + } catch (\Exception $e) { + $workerMessage = "Exception catched when try to get subdef " .$payload['subdefName']. " from DB for the recordID: " .$recordId; + $this->logger->error($workerMessage); + + $count = isset($payload['count']) ? $payload['count'] + 1 : 2 ; + + $this->dispatch(WorkerEvents::SUBDEFINITION_WRITE_META, new SubdefinitionWritemetaEvent( + $record, + $payload['subdefName'], + SubdefinitionWritemetaEvent::FAILED, + $workerMessage, + $count + )); + + return ; + } if ($subdef->is_physically_present()) { $metadata = new MetadataBag();