make delayed queue connfigurable

2025-10-15 14:03:27 +00:00 · 2020-05-13 17:29:45 +03:00
parent 8ace8aeab7
commit daf059d03f
7 changed files with 135 additions and 76 deletions
--- a/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php
+++ b/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php
@@ -21,7 +21,12 @@ class AdminConfigurationController extends Controller
 {
    public function indexAction(PhraseaApplication $app, Request $request)
    {
-        return $this->render('admin/worker-manager/index.html.twig');
+        /** @var AMQPConnection $serverConnection */
+        $serverConnection = $this->app['alchemy_worker.amqp.connection'];
+
+        return $this->render('admin/worker-manager/index.html.twig', [
+            'isConnected' => ($serverConnection->getChannel() != null) ? true : false
+        ]);
    }

    /**
@@ -48,6 +53,7 @@ class AdminConfigurationController extends Controller
            $serverConnection = $this->app['alchemy_worker.amqp.connection'];
            // change the queue TTL
            $serverConnection->reinitializeQueue($retryQueuesToReset);
+            $serverConnection->reinitializeQueue(AMQPConnection::$defaultDelayedQueues);

            return $app->redirectPath('worker_admin');
        }
--- a/lib/Alchemy/Phrasea/WorkerManager/Form/WorkerConfigurationType.php
+++ b/lib/Alchemy/Phrasea/WorkerManager/Form/WorkerConfigurationType.php
@@ -34,6 +34,12 @@ class WorkerConfigurationType extends AbstractType
            ->add(MessagePublisher::POPULATE_INDEX_TYPE, 'text', [
                'label' => 'Populate Index retry delay in ms'
            ])
+            ->add('delayedSubdef', 'text', [
+                'label' => 'Subdef delay in ms'
+            ])
+            ->add('delayedWriteMeta', 'text', [
+                'label' => 'Write meta delay in ms'
+            ])
        ;
    }

--- a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php
+++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php
@@ -63,6 +63,9 @@ class AMQPConnection
    // default message TTL in retry queue in millisecond
    const RETRY_DELAY =  10000;

+    // default message TTL in delayed queue in millisecond
+    const DELAY = 5000;
+
    public function __construct(PropertyAccess $conf)
    {
        $defaultConfiguration = [
@@ -150,7 +153,7 @@ class AMQPConnection
            $this->channel->queue_declare(self::$defaultRetryQueues[$queueName], false, true, false, false, false, new AMQPTable([
                'x-dead-letter-exchange'    => AMQPConnection::ALCHEMY_EXCHANGE,
                'x-dead-letter-routing-key' => $queueName,
-                'x-message-ttl'             => $this->getTtlPerRouting($queueName)
+                'x-message-ttl'             => $this->getTtlRetryPerRouting($queueName)
            ]));

            $this->channel->queue_bind(self::$defaultRetryQueues[$queueName], AMQPConnection::RETRY_ALCHEMY_EXCHANGE, self::$defaultRetryQueues[$queueName]);
@@ -161,7 +164,7 @@ class AMQPConnection
            $this->channel->queue_declare($queueName, false, true, false, false, false, new AMQPTable([
                'x-dead-letter-exchange'    => AMQPConnection::ALCHEMY_EXCHANGE,
                'x-dead-letter-routing-key' => $routing,
-                'x-message-ttl'             => $this->getTtlPerRouting($routing)
+                'x-message-ttl'             => $this->getTtlRetryPerRouting($routing)
            ]));

            $this->channel->queue_bind($queueName, AMQPConnection::RETRY_ALCHEMY_EXCHANGE, $queueName);
@@ -176,7 +179,7 @@ class AMQPConnection
            $this->channel->queue_declare($queueName, false, true, false, false, false, new AMQPTable([
                'x-dead-letter-exchange'    => AMQPConnection::ALCHEMY_EXCHANGE,
                'x-dead-letter-routing-key' => $routing,
-                'x-message-ttl'             => 5000
+                'x-message-ttl'             => $this->getTtlDelayedPerRouting($routing)
            ]));

            $this->channel->queue_bind($queueName, AMQPConnection::RETRY_ALCHEMY_EXCHANGE, $queueName);
@@ -191,6 +194,10 @@ class AMQPConnection

    public function reinitializeQueue(array $queuNames)
    {
+        if (!isset($this->channel)) {
+            $this->getChannel();
+            $this->declareExchange();
+        }
        foreach ($queuNames as $queuName) {
            if (in_array($queuName, self::$defaultQueues)) {
                $this->channel->queue_purge($queuName);
@@ -216,7 +223,7 @@ class AMQPConnection
     * @param $routing
     * @return int
     */
-    private function getTtlPerRouting($routing)
+    private function getTtlRetryPerRouting($routing)
    {
        $config = $this->conf->get(['workers']);

@@ -233,4 +240,20 @@ class AMQPConnection

        return self::RETRY_DELAY;
    }
+
+    private function getTtlDelayedPerRouting($routing)
+    {
+        $delayed = [
+            MessagePublisher::METADATAS_QUEUE => 'delayedWriteMeta',
+            MessagePublisher::SUBDEF_QUEUE    => 'delayedSubdef'
+        ];
+
+        $config = $this->conf->get(['workers']);
+
+        if (isset($config['retry_queue']) && isset($config['retry_queue'][$delayed[$routing]])) {
+            return (int)$config['retry_queue'][$delayed[$routing]];
+        }
+
+        return self::DELAY;
+    }
 }
--- a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php
+++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php
@@ -72,9 +72,9 @@ class SubdefCreationWorker implements WorkerInterface
                        'payload'       => $payload
                    ];
                    $this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_SUBDEF_QUEUE);
-
-                    $message = MessagePublisher::SUBDEF_CREATION_TYPE.' to be re-published! >> Payload ::'. json_encode($payload);
-                    $this->messagePublisher->pushLog($message);
+//
+//                    $message = MessagePublisher::SUBDEF_CREATION_TYPE.' to be re-published! >> Payload ::'. json_encode($payload);
+//                    $this->messagePublisher->pushLog($message);

                    return ;
                }
--- a/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php
+++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php
@@ -77,8 +77,8 @@ class WriteMetadatasWorker implements WorkerInterface
                ];
                $this->messagePublisher->publishMessage($payload, MessagePublisher::DELAYED_METADATAS_QUEUE);

-                $message = MessagePublisher::WRITE_METADATAS_TYPE.' to be re-published! >> Payload ::'. json_encode($payload);
-                $this->messagePublisher->pushLog($message);
+//                $message = MessagePublisher::WRITE_METADATAS_TYPE.' to be re-published! >> Payload ::'. json_encode($payload);
+//                $this->messagePublisher->pushLog($message);

                return ;
            }
--- a/templates/web/admin/worker-manager/index.html.twig
+++ b/templates/web/admin/worker-manager/index.html.twig
@@ -1,77 +1,85 @@
 <h1>Worker</h1>

-<div>
-    <!-- Nav tabs -->
-    <ul class="nav nav-tabs" id="configurationTabs">
+{% if isConnected %}
+    <div>
+        <!-- Nav tabs -->
+        <ul class="nav nav-tabs" id="configurationTabs">

-        <li class="worker-configuration" role="presentation">
-            <a href="#worker-configuration" aria-controls="worker-configuration" role="tab" data-toggle="tab" data-url="/admin/worker-manager/configuration">
-                {{ "Configuration" | trans }}
-            </a>
-        </li>
-        <li class="worker-searchengine" role="presentation">
-            <a href="#worker-searchengine" aria-controls="worker-searchengine" role="tab" data-toggle="tab" data-url="/admin/worker-manager/searchengine">
-                {{ "Searchengine" | trans }}
-            </a>
-        </li>
-        <li class="worker-pull-assets" role="presentation">
-            <a href="#worker-pull-assets" aria-controls="worker-pull-assets" role="tab" data-toggle="tab" data-url="/admin/worker-manager/pull-assets">
-                {{ "Pull Assets" | trans }}
-            </a>
-        </li>
-        <li class="worker-subview active" role="presentation">
-            <a href="#worker-subview" aria-controls="worker-subview" role="tab" data-toggle="tab" data-url="/admin/worker-manager/subview">
-                {{ "Subview" | trans }}
-            </a>
-        </li>
-        <li class="worker-metadata" role="presentation">
-            <a href="#worker-metadata" aria-controls="worker-metadata" role="tab" data-toggle="tab" data-url="/admin/worker-manager/metadata">
-                {{ "Metadata" | trans }}
-            </a>
-        </li>
+            <li class="worker-configuration" role="presentation">
+                <a href="#worker-configuration" aria-controls="worker-configuration" role="tab" data-toggle="tab" data-url="/admin/worker-manager/configuration">
+                    {{ "Configuration" | trans }}
+                </a>
+            </li>
+            <li class="worker-searchengine" role="presentation">
+                <a href="#worker-searchengine" aria-controls="worker-searchengine" role="tab" data-toggle="tab" data-url="/admin/worker-manager/searchengine">
+                    {{ "Searchengine" | trans }}
+                </a>
+            </li>
+            <li class="worker-pull-assets" role="presentation">
+                <a href="#worker-pull-assets" aria-controls="worker-pull-assets" role="tab" data-toggle="tab" data-url="/admin/worker-manager/pull-assets">
+                    {{ "Pull Assets" | trans }}
+                </a>
+            </li>
+            <li class="worker-subview active" role="presentation">
+                <a href="#worker-subview" aria-controls="worker-subview" role="tab" data-toggle="tab" data-url="/admin/worker-manager/subview">
+                    {{ "Subview" | trans }}
+                </a>
+            </li>
+            <li class="worker-metadata" role="presentation">
+                <a href="#worker-metadata" aria-controls="worker-metadata" role="tab" data-toggle="tab" data-url="/admin/worker-manager/metadata">
+                    {{ "Metadata" | trans }}
+                </a>
+            </li>

-    </ul>
+        </ul>

-    <!-- Tab panes -->
-    <div class="tab-content">
-        <div role="tabpanel" class="tab-pane fade" id="worker-configuration"></div>
-        <div role="tabpanel" class="tab-pane fade" id="worker-searchengine"></div>
-        <div role="tabpanel" class="tab-pane fade" id="worker-pull-assets"></div>
-        <div role="tabpanel" class="tab-pane fade in active" id="worker-subview"></div>
-        <div role="tabpanel" class="tab-pane fade" id="worker-metadata"></div>
+        <!-- Tab panes -->
+        <div class="tab-content">
+            <div role="tabpanel" class="tab-pane fade" id="worker-configuration"></div>
+            <div role="tabpanel" class="tab-pane fade" id="worker-searchengine"></div>
+            <div role="tabpanel" class="tab-pane fade" id="worker-pull-assets"></div>
+            <div role="tabpanel" class="tab-pane fade in active" id="worker-subview"></div>
+            <div role="tabpanel" class="tab-pane fade" id="worker-metadata"></div>
+        </div>
    </div>
-</div>

-<script type="text/javascript">
-    var contentsDownloaded = {};
-    var remoteContent = function(url) {
-        return $.get(url);
-    };
+    <script type="text/javascript">
+        var contentsDownloaded = {};
+        var remoteContent = function(url) {
+            return $.get(url);
+        };

-    var tabs = $('#configurationTabs a[data-toggle="tab"]');
+        var tabs = $('#configurationTabs a[data-toggle="tab"]');

-    tabs.on('click', function(){
-        $(this).tab('show');
-    });
+        tabs.on('click', function(){
+            $(this).tab('show');
+        });

-    $('.nav-tabs li').on('show.bs.tab', function (e) {
-        if (contentsDownloaded[e.target.hash] === undefined) {
-            $(e.target.hash).empty().html('<img src="/assets/common/images/icons/main-loader.gif" alt="loading"/>');
-        }
-    });
+        $('.nav-tabs li').on('show.bs.tab', function (e) {
+            if (contentsDownloaded[e.target.hash] === undefined) {
+                $(e.target.hash).empty().html('<img src="/assets/common/images/icons/main-loader.gif" alt="loading"/>');
+            }
+        });

-    $('.nav-tabs').on('shown.bs.tab', function (e) {
+        $('.nav-tabs').on('shown.bs.tab', function (e) {
+
+            if (contentsDownloaded[e.target.hash] === undefined) {
+                var targetDiv = $(e.target.hash);
+
+                remoteContent($(e.target).attr('data-url')).then(function(response) {
+                    targetDiv.empty().html(response);
+                    contentsDownloaded[e.target.hash] = true;
+                }, function(error) {
+                    console.log(error);
+                    targetDiv.empty().html('<i class="icon-fire">{{ 'admin:worker Retrieve configuration error'|trans }}</i>');
+                });
+            }
+        });
+    </script>
+{% else %}
+    <h1 class="alert alert-danger">
+        Error! Check rabbit config to use worker.
+    </h1>
+{% endif %}

-        if (contentsDownloaded[e.target.hash] === undefined) {
-            var targetDiv = $(e.target.hash);

-            remoteContent($(e.target).attr('data-url')).then(function(response) {
-                targetDiv.empty().html(response);
-                contentsDownloaded[e.target.hash] = true;
-            }, function(error) {
-                console.log(error);
-                targetDiv.empty().html('<i class="icon-fire">{{ 'admin:worker Retrieve configuration error'|trans }}</i>');
-            });
-        }
-    });
-</script>
--- a/templates/web/admin/worker-manager/worker_configuration.html.twig
+++ b/templates/web/admin/worker-manager/worker_configuration.html.twig
@@ -1,6 +1,10 @@
 <h3> Config Worker queue retry</h3>
+<p class="alert alert-danger">
+    <strong>Warning!</strong>
+    When applied, it's re-initialize and purge "retry" and "delayed" queue!
+</p>

-<p>Set up the delay between two attempts per queue!</p>
+<p>Set up the delay between two attempts per queue! (if not set, default 10000 millisecond)</p>
 {{ form_start(form, {'action': path('worker_admin_configuration')}) }}

 <div class="control-group">
@@ -31,8 +35,20 @@
    {{ form_row(form.populateIndex) }}
 </div>

+<h3> Config Worker queue delayed</h3>
+<p>if not set ,default 5000 millisecond</p>
+
 <div class="control-group">
-    <input type="submit" class="btn btn-primary" value={{ "Apply retry delay"|trans }} />
+    {{ form_row(form.delayedSubdef) }}
+</div>
+
+<div class="control-group">
+    {{ form_row(form.delayedWriteMeta) }}
+</div>
+
+
+<div class="control-group">
+    <input type="submit" class="btn btn-primary" value={{ "Apply in queue"|trans }} />
 </div>