hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-15 14:03:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

make delayed queue connfigurable

Browse Source
This commit is contained in:
aynsix
2020-05-13 17:29:45 +03:00
parent 8ace8aeab7
commit daf059d03f
7 changed files with 135 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php
View File

@@ -21,7 +21,12 @@ class AdminConfigurationController extends Controller
{ {
public function indexAction(PhraseaApplication $app, Request $request) public function indexAction(PhraseaApplication $app, Request $request)
{ {
return $this->render('admin/worker-manager/index.html.twig'); /** @var AMQPConnection $serverConnection */
$serverConnection = $this->app['alchemy_worker.amqp.connection'];
return $this->render('admin/worker-manager/index.html.twig', [
'isConnected' => ($serverConnection->getChannel() != null) ? true : false
]);
} }
/** /**
@@ -48,6 +53,7 @@ class AdminConfigurationController extends Controller
$serverConnection = $this->app['alchemy_worker.amqp.connection']; $serverConnection = $this->app['alchemy_worker.amqp.connection'];
// change the queue TTL // change the queue TTL
$serverConnection->reinitializeQueue($retryQueuesToReset); $serverConnection->reinitializeQueue($retryQueuesToReset);
$serverConnection->reinitializeQueue(AMQPConnection::$defaultDelayedQueues);
return $app->redirectPath('worker_admin'); return $app->redirectPath('worker_admin');
} }

6
lib/Alchemy/Phrasea/WorkerManager/Form/WorkerConfigurationType.php
View File

@@ -34,6 +34,12 @@ class WorkerConfigurationType extends AbstractType
->add(MessagePublisher::POPULATE_INDEX_TYPE, 'text', [ ->add(MessagePublisher::POPULATE_INDEX_TYPE, 'text', [
'label' => 'Populate Index retry delay in ms' 'label' => 'Populate Index retry delay in ms'
]) ])
->add('delayedSubdef', 'text', [
'label' => 'Subdef delay in ms'
])
->add('delayedWriteMeta', 'text', [
'label' => 'Write meta delay in ms'
])
; ;
} }

31
lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php
View File

@@ -63,6 +63,9 @@ class AMQPConnection
// default message TTL in retry queue in millisecond // default message TTL in retry queue in millisecond
const RETRY_DELAY = 10000; const RETRY_DELAY = 10000;
// default message TTL in delayed queue in millisecond
const DELAY = 5000;
public function __construct(PropertyAccess $conf) public function __construct(PropertyAccess $conf)
{ {
$defaultConfiguration = [ $defaultConfiguration = [
@@ -150,7 +153,7 @@ class AMQPConnection
$this->channel->queue_declare(self::$defaultRetryQueues[$queueName], false, true, false, false, false, new AMQPTable([ $this->channel->queue_declare(self::$defaultRetryQueues[$queueName], false, true, false, false, false, new AMQPTable([
'x-dead-letter-exchange' => AMQPConnection::ALCHEMY_EXCHANGE, 'x-dead-letter-exchange' => AMQPConnection::ALCHEMY_EXCHANGE,
'x-dead-letter-routing-key' => $queueName, 'x-dead-letter-routing-key' => $queueName,
'x-message-ttl' => $this->getTtlPerRouting($queueName) 'x-message-ttl' => $this->getTtlRetryPerRouting($queueName)
])); ]));
$this->channel->queue_bind(self::$defaultRetryQueues[$queueName], AMQPConnection::RETRY_ALCHEMY_EXCHANGE, self::$defaultRetryQueues[$queueName]); $this->channel->queue_bind(self::$defaultRetryQueues[$queueName], AMQPConnection::RETRY_ALCHEMY_EXCHANGE, self::$defaultRetryQueues[$queueName]);
@@ -161,7 +164,7 @@ class AMQPConnection
$this->channel->queue_declare($queueName, false, true, false, false, false, new AMQPTable([ $this->channel->queue_declare($queueName, false, true, false, false, false, new AMQPTable([
'x-dead-letter-exchange' => AMQPConnection::ALCHEMY_EXCHANGE, 'x-dead-letter-exchange' => AMQPConnection::ALCHEMY_EXCHANGE,
'x-dead-letter-routing-key' => $routing, 'x-dead-letter-routing-key' => $routing,
'x-message-ttl' => $this->getTtlPerRouting($routing) 'x-message-ttl' => $this->getTtlRetryPerRouting($routing)
])); ]));
$this->channel->queue_bind($queueName, AMQPConnection::RETRY_ALCHEMY_EXCHANGE, $queueName); $this->channel->queue_bind($queueName, AMQPConnection::RETRY_ALCHEMY_EXCHANGE, $queueName);
@@ -176,7 +179,7 @@ class AMQPConnection
$this->channel->queue_declare($queueName, false, true, false, false, false, new AMQPTable([ $this->channel->queue_declare($queueName, false, true, false, false, false, new AMQPTable([
'x-dead-letter-exchange' => AMQPConnection::ALCHEMY_EXCHANGE, 'x-dead-letter-exchange' => AMQPConnection::ALCHEMY_EXCHANGE,
'x-dead-letter-routing-key' => $routing, 'x-dead-letter-routing-key' => $routing,
'x-message-ttl' => 5000 'x-message-ttl' => $this->getTtlDelayedPerRouting($routing)
])); ]));
$this->channel->queue_bind($queueName, AMQPConnection::RETRY_ALCHEMY_EXCHANGE, $queueName); $this->channel->queue_bind($queueName, AMQPConnection::RETRY_ALCHEMY_EXCHANGE, $queueName);
@@ -191,6 +194,10 @@ class AMQPConnection
public function reinitializeQueue(array $queuNames) public function reinitializeQueue(array $queuNames)
{ {
if (!isset($this->channel)) {
$this->getChannel();
$this->declareExchange();
}
foreach ($queuNames as $queuName) { foreach ($queuNames as $queuName) {
if (in_array($queuName, self::$defaultQueues)) { if (in_array($queuName, self::$defaultQueues)) {
$this->channel->queue_purge($queuName); $this->channel->queue_purge($queuName);
@@ -216,7 +223,7 @@ class AMQPConnection
* @param $routing * @param $routing
* @return int * @return int
*/ */
private function getTtlPerRouting($routing) private function getTtlRetryPerRouting($routing)
{ {
$config = $this->conf->get(['workers']); $config = $this->conf->get(['workers']);
@@ -233,4 +240,20 @@ class AMQPConnection
return self::RETRY_DELAY; return self::RETRY_DELAY;
} }
private function getTtlDelayedPerRouting($routing)
{
$delayed = [
MessagePublisher::METADATAS_QUEUE => 'delayedWriteMeta',
MessagePublisher::SUBDEF_QUEUE => 'delayedSubdef'
];
$config = $this->conf->get(['workers']);
if (isset($config['retry_queue']) && isset($config['retry_queue'][$delayed[$routing]])) {
return (int)$config['retry_queue'][$delayed[$routing]];
}
return self::DELAY;
}
} }

6
lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php
View File

@@ -72,9 +72,9 @@ class SubdefCreationWorker implements WorkerInterface
'payload' => $payload 'payload' => $payload
]; ];
$this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_SUBDEF_QUEUE); $this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_SUBDEF_QUEUE);
//
$message = MessagePublisher::SUBDEF_CREATION_TYPE.' to be re-published! >> Payload ::'. json_encode($payload); // $message = MessagePublisher::SUBDEF_CREATION_TYPE.' to be re-published! >> Payload ::'. json_encode($payload);
$this->messagePublisher->pushLog($message); // $this->messagePublisher->pushLog($message);
return ; return ;
} }

4
lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php
View File

@@ -77,8 +77,8 @@ class WriteMetadatasWorker implements WorkerInterface
]; ];
$this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_METADATAS_QUEUE); $this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_METADATAS_QUEUE);
$message = MessagePublisher::WRITE_METADATAS_TYPE.' to be re-published! >> Payload ::'. json_encode($payload); // $message = MessagePublisher::WRITE_METADATAS_TYPE.' to be re-published! >> Payload ::'. json_encode($payload);
$this->messagePublisher->pushLog($message); // $this->messagePublisher->pushLog($message);
return ; return ;
} }

136
templates/web/admin/worker-manager/index.html.twig
View File

@@ -1,77 +1,85 @@
<h1>Worker</h1> <h1>Worker</h1>
<div> {% if isConnected %}
<!-- Nav tabs --> <div>
<ul class="nav nav-tabs" id="configurationTabs"> <!-- Nav tabs -->
<ul class="nav nav-tabs" id="configurationTabs">
<li class="worker-configuration" role="presentation"> <li class="worker-configuration" role="presentation">
<a href="#worker-configuration" aria-controls="worker-configuration" role="tab" data-toggle="tab" data-url="/admin/worker-manager/configuration"> <a href="#worker-configuration" aria-controls="worker-configuration" role="tab" data-toggle="tab" data-url="/admin/worker-manager/configuration">
{{ "Configuration" | trans }} {{ "Configuration" | trans }}
</a> </a>
</li> </li>
<li class="worker-searchengine" role="presentation"> <li class="worker-searchengine" role="presentation">
<a href="#worker-searchengine" aria-controls="worker-searchengine" role="tab" data-toggle="tab" data-url="/admin/worker-manager/searchengine"> <a href="#worker-searchengine" aria-controls="worker-searchengine" role="tab" data-toggle="tab" data-url="/admin/worker-manager/searchengine">
{{ "Searchengine" | trans }} {{ "Searchengine" | trans }}
</a> </a>
</li> </li>
<li class="worker-pull-assets" role="presentation"> <li class="worker-pull-assets" role="presentation">
<a href="#worker-pull-assets" aria-controls="worker-pull-assets" role="tab" data-toggle="tab" data-url="/admin/worker-manager/pull-assets"> <a href="#worker-pull-assets" aria-controls="worker-pull-assets" role="tab" data-toggle="tab" data-url="/admin/worker-manager/pull-assets">
{{ "Pull Assets" | trans }} {{ "Pull Assets" | trans }}
</a> </a>
</li> </li>
<li class="worker-subview active" role="presentation"> <li class="worker-subview active" role="presentation">
<a href="#worker-subview" aria-controls="worker-subview" role="tab" data-toggle="tab" data-url="/admin/worker-manager/subview"> <a href="#worker-subview" aria-controls="worker-subview" role="tab" data-toggle="tab" data-url="/admin/worker-manager/subview">
{{ "Subview" | trans }} {{ "Subview" | trans }}
</a> </a>
</li> </li>
<li class="worker-metadata" role="presentation"> <li class="worker-metadata" role="presentation">
<a href="#worker-metadata" aria-controls="worker-metadata" role="tab" data-toggle="tab" data-url="/admin/worker-manager/metadata"> <a href="#worker-metadata" aria-controls="worker-metadata" role="tab" data-toggle="tab" data-url="/admin/worker-manager/metadata">
{{ "Metadata" | trans }} {{ "Metadata" | trans }}
</a> </a>
</li> </li>
</ul> </ul>
<!-- Tab panes --> <!-- Tab panes -->
<div class="tab-content"> <div class="tab-content">
<div role="tabpanel" class="tab-pane fade" id="worker-configuration"></div> <div role="tabpanel" class="tab-pane fade" id="worker-configuration"></div>
<div role="tabpanel" class="tab-pane fade" id="worker-searchengine"></div> <div role="tabpanel" class="tab-pane fade" id="worker-searchengine"></div>
<div role="tabpanel" class="tab-pane fade" id="worker-pull-assets"></div> <div role="tabpanel" class="tab-pane fade" id="worker-pull-assets"></div>
<div role="tabpanel" class="tab-pane fade in active" id="worker-subview"></div> <div role="tabpanel" class="tab-pane fade in active" id="worker-subview"></div>
<div role="tabpanel" class="tab-pane fade" id="worker-metadata"></div> <div role="tabpanel" class="tab-pane fade" id="worker-metadata"></div>
</div>
</div> </div>
</div>
<script type="text/javascript"> <script type="text/javascript">
var contentsDownloaded = {}; var contentsDownloaded = {};
var remoteContent = function(url) { var remoteContent = function(url) {
return $.get(url); return $.get(url);
}; };
var tabs = $('#configurationTabs a[data-toggle="tab"]'); var tabs = $('#configurationTabs a[data-toggle="tab"]');
tabs.on('click', function(){ tabs.on('click', function(){
$(this).tab('show'); $(this).tab('show');
}); });
$('.nav-tabs li').on('show.bs.tab', function (e) { $('.nav-tabs li').on('show.bs.tab', function (e) {
if (contentsDownloaded[e.target.hash] === undefined) { if (contentsDownloaded[e.target.hash] === undefined) {
$(e.target.hash).empty().html('<img src="/assets/common/images/icons/main-loader.gif" alt="loading"/>'); $(e.target.hash).empty().html('<img src="/assets/common/images/icons/main-loader.gif" alt="loading"/>');
} }
}); });
$('.nav-tabs').on('shown.bs.tab', function (e) { $('.nav-tabs').on('shown.bs.tab', function (e) {
if (contentsDownloaded[e.target.hash] === undefined) {
var targetDiv = $(e.target.hash);
remoteContent($(e.target).attr('data-url')).then(function(response) {
targetDiv.empty().html(response);
contentsDownloaded[e.target.hash] = true;
}, function(error) {
console.log(error);
targetDiv.empty().html('<i class="icon-fire">{{ 'admin:worker Retrieve configuration error'|trans }}</i>');
});
}
});
</script>
{% else %}
<h1 class="alert alert-danger">
Error! Check rabbit config to use worker.
</h1>
{% endif %}
if (contentsDownloaded[e.target.hash] === undefined) {
var targetDiv = $(e.target.hash);
remoteContent($(e.target).attr('data-url')).then(function(response) {
targetDiv.empty().html(response);
contentsDownloaded[e.target.hash] = true;
}, function(error) {
console.log(error);
targetDiv.empty().html('<i class="icon-fire">{{ 'admin:worker Retrieve configuration error'|trans }}</i>');
});
}
});
</script>

20
templates/web/admin/worker-manager/worker_configuration.html.twig
View File

@@ -1,6 +1,10 @@
<h3> Config Worker queue retry</h3> <h3> Config Worker queue retry</h3>
<p class="alert alert-danger">
<strong>Warning!</strong>
When applied, it's re-initialize and purge "retry" and "delayed" queue!
</p>
<p>Set up the delay between two attempts per queue!</p> <p>Set up the delay between two attempts per queue! (if not set, default 10000 millisecond)</p>
{{ form_start(form, {'action': path('worker_admin_configuration')}) }} {{ form_start(form, {'action': path('worker_admin_configuration')}) }}
<div class="control-group"> <div class="control-group">
@@ -31,8 +35,20 @@
{{ form_row(form.populateIndex) }} {{ form_row(form.populateIndex) }}
</div> </div>
<h3> Config Worker queue delayed</h3>
<p>if not set ,default 5000 millisecond</p>
<div class="control-group"> <div class="control-group">
<input type="submit" class="btn btn-primary" value={{ "Apply retry delay"|trans }} /> {{ form_row(form.delayedSubdef) }}
</div>
<div class="control-group">
{{ form_row(form.delayedWriteMeta) }}
</div>
<div class="control-group">
<input type="submit" class="btn btn-primary" value={{ "Apply in queue"|trans }} />
</div> </div>