Merge branch 'master' into PHRAS-3519_filesystem-atomic-move

.circleci/config.yml

@@ -14,31 +14,26 @@ jobs:
           CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
           CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
         docker:
-        - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
+        - image: cimg/php:7.0.33-node
         - image: circleci/rabbitmq:3.7.7
         steps:
         - checkout
-        - run: phpenv versions
-        - run: phpenv global 7.0.7
         - run: php -v
         - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
         - run:
             working_directory: ~/alchemy-fr/Phraseanet
-            command: nvm install v10.12.0 && nvm alias default v10.12.0
-        - run:
-            working_directory: ~/alchemy-fr/Phraseanet
-            command: 'sudo service mysql status || sudo service mysql start;'
+            command: 'sudo apt update && sudo apt install mysql-server && sudo service mysql status || sudo service mysql start;'
         # Dependencies
         #   This would typically go in either a build or a build-and-test job when using workflows
         # Restore the dependency cache
         - restore_cache:
             keys:
             # This branch if available
-            - v1-dep-{{ .Branch }}-
+            - v2-dep-{{ .Branch }}-
             # Default branch if not
-            - v1-dep-master-
+            - v2-dep-master-
             # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-            - v1-dep-
+            - v2-dep-
         # This is based on your 1.0 configuration file or project settings
         - run: echo 127.0.0.1 redis elasticsearch db rabbitmq | sudo tee -a /etc/hosts
         - run: git clone https://github.com/alanxz/rabbitmq-c
@@ -46,23 +41,20 @@ jobs:
         - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install
         # disabled because pear.php.net is down cause of security failure
         #- run: pecl channel-update pear.php.net
-        - run: yes '' | pecl install amqp-1.9.3
-        - run: yes '' | pecl install imagick
-        - run: sudo apt-get install libzmq-dev
-        - run: yes '' | pecl install zmq-beta
-        - run: echo "extension = amqp.so" > /opt/circleci/php/$(phpenv global)/etc/conf.d/amqp.ini
-        - run: echo "extension = zmq.so" > /opt/circleci/php/$(phpenv global)/etc/conf.d/zmq.ini
-        - run: echo "date.timezone = UTC" > /opt/circleci/php/$(phpenv global)/etc/conf.d/timezone.ini
-        - run: sed -i 's/^\(session.cache_limiter = \).*/\1""/' /opt/circleci/php/$(phpenv global)/etc/php.ini
+        - run: sudo apt -y install  openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-zmq php7.0-intl php7.0-sqlite3 libmagickwand-dev libmagickcore-dev
+        - run: printf "\n" | sudo pecl install amqp-1.11.0beta
+        - run: printf "\n" | sudo pecl install imagick
+        - run: sudo -- bash -c  "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini"
+        - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini
+        - run: sudo -- bash -c  'echo -e "[mysqld]\ndefault-authentication-plugin=mysql_native_password" > /etc/mysql/conf.d/mysql_native_password.cnf' && sudo service mysql restart
         - run: npm rebuild node-sass
         # This is based on your 1.0 configuration file or project settings
         - run: composer install --no-progress --no-interaction --optimize-autoloader
+        - run: curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
+        - run: source ~/.profile && nvm install v10.12.0 && nvm alias default v10.12.0 
         # This is based on your 1.0 configuration file or project settings
         - run: node -v
         - run: npm -v
-        - run: npm install -g yarn
-        - run: rm ~/.yarn/bin/yarn
-        - run: ln -s /opt/circleci/nodejs/v10.12.0/bin/yarn ~/.yarn/bin/yarn
         - run: yarn install
         - run: if [[ ! -e elasticsearch-2.3.3 ]]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.3.3.tar.gz && tar -xvf elasticsearch-2.3.3.tar.gz && elasticsearch-2.3.3/bin/plugin install analysis-icu; fi
         - run:
@@ -70,7 +62,7 @@ jobs:
             background: true
         # Save dependency cache
         - save_cache:
-            key: v1-dep-{{ .Branch }}-{{ epoch }}
+            key: v2-dep-{{ .Branch }}-{{ epoch }}
             paths:
             # This is a broad list of cache paths to include many possible development environments
             # You can probably delete some of these entries
@@ -87,10 +79,10 @@ jobs:
             - node_modules
             - ~/.composer
         # This is based on your 1.0 configuration file or project settings
-        - run: mysql -u ubuntu -e 'CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;';
+        - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'localhost' IDENTIFIED WITH mysql_native_password BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;"
         # This is based on your 1.0 configuration file or project settings
         - run: ./bin/developer system:uninstall -v
-        - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=ubuntu --db-template=fr-simple --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y;
+        - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=localhost --db-user=phraseanet --db-template=fr-simple --db-password=phraseanet --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y;
         - run: ./bin/developer ini:setup-tests-dbs -v
         - run: ./bin/console searchengine:index:create -v
         - run: ./bin/developer phraseanet:regenerate-sqlite -v

.env

@@ -1,9 +1,9 @@
 
-# For dev purpose COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.mailhog.yml:docker-compose.pma.yml:docker-compose.override.yml
-# Dev tools and utilities: docker-compose.elk.yml:docker-compose.ftp.yml:docker-compose.squid.yml
+# For dev purpose COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.rabbitmq.yml:docker-compose.mailhog.yml:docker-compose.pma.yml:docker-compose.override.yml
+# Dev tools and utilities add : docker-compose.elk.yml:docker-compose.ftp.yml:docker-compose.squid.yml
 # For old Phraseanet scheduler launching use : docker-compose.scheduler.yml
 
-COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.mailhog.yml
+COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.rabbitmq.yml:docker-compose.mailhog.yml
 
 # Registry from where you pull Docker images
 PHRASEANET_DOCKER_REGISTRY=local
@@ -29,11 +29,13 @@ PHRASEANET_UPGRADE=0
 
 # --------------- RabbitMQ SETTING ----------------------
 
+
 RABBITMQ_DEFAULT_USER=alchemy
 RABBITMQ_DEFAULT_PASS=vdh4dpe5Wy3R
 RABBITMQ_MANAGEMENT_PORT=10811
 
 
+
 # --------------- GATEWAY SETTING (nginx) -----------------------
 
 GATEWAY_SEND_TIMEOUT=120
@@ -121,6 +123,15 @@ PHRASEANET_SWFTOOLS_TIMEOUT=120
 PHRASEANET_UNOCON_TIMEOUT=120
 PHRASEANET_EXIFTOOL_TIMEOUT=120
 
+# RabbitMQ parameters
+
+PHRASEANET_RABBITMQ_HOST=rabbitmq
+PHRASEANET_RABBITMQ_PORT=5672
+PHRASEANET_RABBITMQ_SSL=false
+PHRASEANET_RABBITMQ_VHOST=/
+PHRASEANET_RABBITMQ_HEARTBEAT=30
+
+
 # network : comma separated list of IP ou SUBNETS
 PHRASEANET_TRUSTED_PROXIES=
 PHRASEANET_DEBUG_ALLOWED_IP=
@@ -235,6 +246,12 @@ IMAGEMAGICK_POLICY_TEMPORARY_PATH=/tmp
 # PhpMyAdmin port
 PHRASEANET_PHPMYADMIN_PORT=8089
 
+
+# mailhog web GUI port mapping
+
+MAILHOG_GUI_PORT=8025
+
+
 # Xdebug
 XDEBUG_ENABLED=0
 XDEBUG_PROFILER_ENABLED=0

Dockerfile

@@ -13,7 +13,10 @@ RUN echo "deb http://deb.debian.org/debian stretch main non-free" > /etc/apt/sou
         apt-transport-https \
         ca-certificates \
         gnupg2 \
-    && sed -i 's/mozilla\/DST_Root_CA_X3.crt/!mozilla\/DST_Root_CA_X3.crt/g' /etc/ca-certificates.conf \
+        wget \
+    && wget -O certs.deb http://ftp.fr.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20210119_all.deb \
+    && dpkg --fsys-tarfile certs.deb | tar -xOf - ./usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt > /usr/local/share/ca-certificates/ISRG_Root_X1.crt \
+    && rm -rf /usr/share/ca-certificates/mozilla/DST_Root_CA_X3.crt \
     && update-ca-certificates --fresh \
     && apt-get update \
     && apt-get install -y --no-install-recommends \

bin/console

@@ -64,6 +64,7 @@ use Alchemy\Phrasea\Command\User\UserListCommand;
 use Alchemy\Phrasea\Command\User\UserPasswordCommand;
 use Alchemy\Phrasea\Core\Version;
 use Alchemy\Phrasea\WorkerManager\Command\WorkerExecuteCommand;
+use Alchemy\Phrasea\WorkerManager\Command\WorkerHeartbeatCommand;
 use Alchemy\Phrasea\WorkerManager\Command\WorkerRunServiceCommand;
 use Alchemy\Phrasea\WorkerManager\Command\WorkerShowConfigCommand;
 
@@ -171,6 +172,7 @@ $cli->command(new QuerySampleCommand());
 $cli->command(new FindConceptsCommand());
 
 $cli->command(new WorkerExecuteCommand());
+$cli->command(new WorkerHeartbeatCommand());
 $cli->command(new WorkerRunServiceCommand());
 $cli->command(new WorkerShowConfigCommand());
 

config/configuration.sample.yml

@@ -333,6 +333,7 @@ workers:
           user: guest
           password: guest
           vhost: /
+          heartbeat: 60
     queues:
         writeMetadatas:           # this Q is "delayable" in case of record is locked
             ttl_retry: 1500         # overwrite 1000 ms default delay

docker-compose.mailhog.yml

@@ -4,7 +4,8 @@ services:
    mailhog:
     image: mailhog/mailhog
     ports:
-    - 1025:1025
-    - 8025:8025
+    - ${MAILHOG_GUI_PORT}:8025
+    environment:
+    - MAILHOG_GUI_PORT
     networks:
       - internal

docker-compose.rabbitmq.yml

@@ -0,0 +1,12 @@
+version: "3.4"
+
+services: 
+
+  rabbitmq:
+    image: rabbitmq:3.6.16-management
+    restart: on-failure
+    environment:
+    - RABBITMQ_DEFAULT_USER
+    - RABBITMQ_DEFAULT_PASS
+    networks:
+      - internal

docker-compose.yml

@@ -74,6 +74,11 @@ services:
     - PHRASEANET_SERVER_NAME
     - PHRASEANET_AVAILABLE_LANGUAGE
     - PHRASEANET_DEFAULT_LANGUAGE
+    - PHRASEANET_RABBITMQ_HOST
+    - PHRASEANET_RABBITMQ_PORT
+    - PHRASEANET_RABBITMQ_SSL
+    - PHRASEANET_RABBITMQ_VHOST
+    - PHRASEANET_RABBITMQ_HEARTBEAT
     - PHRASEANET_RABBITMQ_USER=$RABBITMQ_DEFAULT_USER
     - PHRASEANET_RABBITMQ_PASSWORD=$RABBITMQ_DEFAULT_PASS
     - PHRASEANET_EMITTER_EMAIL
@@ -195,15 +200,6 @@ services:
     networks:
       - internal
 
-  rabbitmq:
-    image: rabbitmq:3.6.16-management
-    restart: on-failure
-    environment:
-    - RABBITMQ_DEFAULT_USER
-    - RABBITMQ_DEFAULT_PASS
-    networks:
-      - internal
-
   redis:
     image: redis
     restart: on-failure

docker/phraseanet/entrypoint.sh

@@ -101,6 +101,19 @@ if [[ -f "$FILE" && $PHRASEANET_SETUP = 1 ]]; then
     bin/setup system:config set registry.api-clients.api-require-ssl $PHRASEANET_API_SSL
     bin/setup system:config set registry.api-clients.api-auth-token-header-only $PHRASEANET_API_AUTH_TOKEN_HEADER_ONLY
 
+
+    echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet setting RABBITMQ"
+    bin/setup system:config set workers.queue.worker-queue.host $PHRASEANET_RABBITMQ_HOST
+    bin/setup system:config set workers.queue.worker-queue.port $PHRASEANET_RABBITMQ_PORT
+    bin/setup system:config set workers.queue.worker-queue.vhost $PHRASEANET_RABBITMQ_VHOST
+    bin/setup system:config set workers.queue.worker-queue.ssl $PHRASEANET_RABBITMQ_SSL
+    bin/setup system:config set workers.queue.worker-queue.heartbeat $PHRASEANET_RABBITMQ_HEARTBEAT
+    bin/setup system:config set workers.queue.worker-queue.user $PHRASEANET_RABBITMQ_USER
+    bin/setup system:config set workers.queue.worker-queue.password $PHRASEANET_RABBITMQ_PASSWORD
+    
+
+
+
     echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet setting SMTP "
     if [[ $PHRASEANET_SMTP_ENABLED && $PHRASEANET_SMTP_ENABLED = true ]]; then
         bin/setup system:config set registry.email.smtp-enabled $PHRASEANET_SMTP_ENABLED

docker/phraseanet/worker/entrypoint.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+HEARTBEAT_INTERVAL=20
+
 DOCKER_DIR="./docker/phraseanet"
 PHR_USER=app
 
@@ -54,76 +56,88 @@ if [[ $BLACKFIRE_ENABLED = "true" ]]; then
   service blackfire-agent start
   echo "Blackfire setup done"
 else
-    echo `date +"%Y-%m-%d %H:%M:%S"` " - blackfire extension deactivation."
-    rm -f /usr/local/etc/php/conf.d/zz-blackfire.ini
+  echo `date +"%Y-%m-%d %H:%M:%S"` " - blackfire extension deactivation."
+  rm -f /usr/local/etc/php/conf.d/zz-blackfire.ini
 fi
 
-
 rm -rf bin/run-worker.sh
 
 if [ ! -z "$PHRASEANET_SCHEDULER" ] ; then
-
   command="bin/console task-manager:scheduler:run"
- echo $command >> bin/run-worker.sh 
- echo "Phraseanet workers container will be launched in Scheduler mode with bin/console task-manager:scheduler:run"
-
+  echo $command >> bin/run-worker.sh
+  echo "Phraseanet workers container will be launched in Scheduler mode with bin/console task-manager:scheduler:run"
 else
- if [ ! -z "$PHRASEANET_EXPLODE_WORKER" ] && [ ${PHRASEANET_EXPLODE_WORKER} == "1" ]; then
+  if [ ! -z "$PHRASEANET_EXPLODE_WORKER" ] && [ ${PHRASEANET_EXPLODE_WORKER} == "1" ]; then
     if [ ! -z "$PHRASEANET_WORKERS_LAUNCH_METHOD" ] && [ ${PHRASEANET_WORKERS_LAUNCH_METHOD} == "supervisor" ]; then
-     echo "Multiples Phraseanet workers will be launched by supervisor"
-     for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1`
-     do
+      echo "Multiples Phraseanet workers will be launched by supervisor"
+      for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1`
+      do
         worker_job_file="$(echo $i | cut -d'_' -f3).conf"
         if [ ${!i} -gt "0" ] ; then
           envsubst < "/var/alchemy/Phraseanet/docker/phraseanet/worker/supervisor_conf.d/$worker_job_file" > /etc/supervisor/conf.d/$worker_job_file
-          echo "Add Worker : " $worker_job_file " -- with parallelism: (-m) " ${!i} 
+          echo "Add worker: " $worker_job_file " -- with parallelism: (-m) " ${!i}
         else
-          echo "NO Worker define for : " $worker_job_file " -- because parallelism (-m) is set to : " ${!i} 
+          echo "No worker defined for: " $worker_job_file " -- because parallelism (-m) is set to : " ${!i}
         fi
       done
-     command="/usr/bin/supervisord -n -c /etc/supervisor/supervisor.conf"
-     echo $command >> bin/run-worker.sh
-     PHR_USER=root
+      command="/usr/bin/supervisord -n -c /etc/supervisor/supervisor.conf"
+      echo $command >> bin/run-worker.sh
+      PHR_USER=root
     else
-     echo "Multiples Phraseanet workers will be launched with bin/console worker:execute"
-     NBR_WORKERS=0
-     for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1`
-     do
+      echo "Multiples Phraseanet workers will be launched with bin/console worker:execute"
+      NBR_WORKERS=0
+
+      echo "bin/console worker:heartbeat --heartbeat ${HEARTBEAT_INTERVAL} &" >> bin/run-worker.sh
+
+      for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1`
+      do
         queue_name="$(echo $i | cut -d'_' -f3)"
         m=$i
         if [ ${!m} -gt "0" ] ; then
-        command="bin/console worker:execute --queue-name=$queue_name -m ${!m} &"
-        echo $command >> bin/run-worker.sh
-        echo "Worker " $queue_name " define with parallelism " ${!m}
-        NBR_WORKERS=$(expr $NBR_WORKERS + 1)
-      else
-        echo " NO Worker defined for : " $m " -- because parallelism (-m) is set to : " ${!m}
-      fi
-     done
-    
-    echo $NBR_WORKERS " workers define"
-    echo $NBR_WORKERS > bin/workers_count.txt
-    chown root:app bin/workers_count.txt
-    chmod 760 bin/workers_count.txt
-    echo '
-    NBR_WORKERS=$(< bin/workers_count.txt)
-    WORKER_LOOP_VALUE=20s
-      while true;
-      do
-        sleep $WORKER_LOOP_VALUE
-        nb_process=`ps faux | grep "worker:execute" | grep php | wc -l`
-        date_time_process=`date +"%Y-%m-%d %H:%M:%S"`
-        echo $date_time_process "-" $nb_process "running workers"
-        if [ $nb_process -lt $NBR_WORKERS ]
-          then
-            exit 1
-            break
+          command="bin/console worker:execute --queue-name=$queue_name -m ${!m} &"
+          echo $command >> bin/run-worker.sh
+          echo "Worker " $queue_name " defined with parallelism " ${!m}
+          NBR_WORKERS=$(expr $NBR_WORKERS + 1)
+        else
+          echo "No worker defined for: " $m " -- because parallelism (-m) is set to : " ${!m}
         fi
-      done  ' >> bin/run-worker.sh
-      fi  
+      done
+
+      echo $NBR_WORKERS " workers defined"
+      echo $NBR_WORKERS > bin/workers_count.txt
+      chown root:app bin/workers_count.txt
+      chmod 760 bin/workers_count.txt
+      echo "HEARTBEAT_INTERVAL=${HEARTBEAT_INTERVAL}" >> bin/run-worker.sh
+      echo '
+NBR_WORKERS=$(< bin/workers_count.txt)
+sleep 1 # let worker:heartbeat fail before process check
+
+function check() {
+  nb_process=`ps faux | grep "worker:execute" | grep php | wc -l`
+  nb_heartbeat=`ps faux | grep "worker:heartbeat" | grep php | wc -l`
+  date_time_process=`date +"%Y-%m-%d %H:%M:%S"`
+  echo $date_time_process "-" $nb_process "running workers"
+  if [ $nb_process -lt $NBR_WORKERS ]; then
+    echo "One or more worker:execute is not running, exiting..."
+    exit 1
+  elif [ $nb_heartbeat -lt 1 ]; then
+    echo "worker:heartbeat is not running, exiting..."
+    exit 1
+  fi
+}
+
+# early check
+check
+
+while true; do
+  sleep ${HEARTBEAT_INTERVAL}s
+  check
+done' >> bin/run-worker.sh
+    fi
   else
     command="bin/console worker:execute"
     echo $command >> bin/run-worker.sh
   fi
 fi
+
 runuser -u $PHR_USER -- $@

lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php

@@ -62,15 +62,19 @@ class SetupTestsDbs extends Command
 
         $ab_name = StringHelper::SqlQuote($settings['database']['ab_name'], StringHelper::SQL_IDENTIFIER);
         $db_name = StringHelper::SqlQuote($settings['database']['db_name'], StringHelper::SQL_IDENTIFIER);
-
+/*
         $this->container['orm.em']->getConnection()->executeUpdate(
-            'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION'
+            'CREATE USER '.$user.'@'.$host.' IDENTIFIED WITH mysql_native_password BY '.$pass
         );
 
         $this->container['orm.em']->getConnection()->executeUpdate(
-            'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION'
+            'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host
         );
 
+        $this->container['orm.em']->getConnection()->executeUpdate(
+            'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host
+        );
+*/
         $this->container['orm.em']->getConnection()->executeUpdate('SET @@global.sql_mode= ""');
 
         return 0;

lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php

@@ -37,7 +37,8 @@ class WorkerConfigurationServiceProvider implements ServiceProviderInterface
                     'port'      => 5672,
                     'user'      => 'guest',
                     'password'  => 'guest',
-                    'vhost'     => '/'
+                    'vhost'     => '/',
+                    'heartbeat' => 60,
                 ]
             ];
 

lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php

@@ -94,6 +94,7 @@ class WorkerExecuteCommand extends Command
         }
 
         $serverConnection->connectionClose();
-    }
 
+        return 0;
+    }
 }

lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php

@@ -0,0 +1,45 @@
+<?php
+
+namespace Alchemy\Phrasea\WorkerManager\Command;
+
+use Alchemy\Phrasea\Command\Command;
+use Alchemy\Phrasea\WorkerManager\Queue\AMQPConnection;
+use Alchemy\Phrasea\WorkerManager\Queue\HeartbeatHandler;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class WorkerHeartbeatCommand extends Command
+{
+    const DEFAULT_INTERVAL = 30;
+
+    public function __construct()
+    {
+        parent::__construct('worker:heartbeat');
+
+        $this
+            ->setDescription('Heartbeat connection to track drops or broken pipes')
+            ->addOption('heartbeat', null, InputOption::VALUE_REQUIRED, sprintf('in seconds (default: %d)', self::DEFAULT_INTERVAL))
+        ;
+
+        return $this;
+    }
+
+    protected function doExecute(InputInterface $input, OutputInterface $output)
+    {
+        /** @var AMQPConnection $serverConnection */
+        $serverConnection = $this->container['alchemy_worker.amqp.connection'];
+
+        $connection = $serverConnection->getConnection();
+
+        $interval = $input->getOption('heartbeat');
+        if (empty($interval)) {
+            $interval = self::DEFAULT_INTERVAL;
+        }
+
+        $heartbeatHandler = new HeartbeatHandler($connection);
+        $heartbeatHandler->run($interval);
+
+        return 0;
+    }
+}

lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php

@@ -141,7 +141,8 @@ class AMQPConnection
             'ssl'       => false,
             'user'      => 'guest',
             'password'  => 'guest',
-            'vhost'     => '/'
+            'vhost'     => '/',
+            'heartbeat' => 60,
         ];
 
         $this->hostConfig = $conf->get(['workers', 'queue', 'worker-queue'], $defaultConfiguration);
@@ -311,10 +312,12 @@ class AMQPConnection
     {
         if (!isset($this->connection)) {
             try {
+                $heartbeat = $this->hostConfig['heartbeat'] ?? 60;
+
                 // if we are in ssl connection type
                 if (isset($this->hostConfig['ssl']) && $this->hostConfig['ssl'] === true) {
                     $sslOptions = [
-                        'verify_peer' => true
+                        'verify_peer' => true,
                     ];
 
                     $this->connection =  new AMQPSSLConnection(
@@ -323,7 +326,10 @@ class AMQPConnection
                         $this->hostConfig['user'],
                         $this->hostConfig['password'],
                         $this->hostConfig['vhost'],
-                        $sslOptions
+                        $sslOptions,
+                        [
+                            'heartbeat' => $heartbeat,
+                        ]
                     );
                 } else {
                     $this->connection =  new AMQPStreamConnection(
@@ -331,7 +337,16 @@ class AMQPConnection
                         $this->hostConfig['port'],
                         $this->hostConfig['user'],
                         $this->hostConfig['password'],
-                        $this->hostConfig['vhost']
+                        $this->hostConfig['vhost'],
+                        false,
+                        'AMQPLAIN',
+                        null,
+                        'en_US',
+                        3.0,
+                        3.0,
+                        null,
+                        false,
+                        $heartbeat
                     );
                 }
             }
@@ -425,7 +440,6 @@ class AMQPConnection
                 break;
             default:
                 throw new Exception(sprintf('undefined q type "%s', $queueName));
-                break;
         }
 
         return $this->channel;

lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Alchemy\Phrasea\WorkerManager\Queue;
+
+use PhpAmqpLib\Connection\AbstractConnection;
+
+class HeartbeatHandler
+{
+    /**
+     * @var AbstractConnection
+     */
+    private $connection;
+
+    public function __construct(AbstractConnection $connection)
+    {
+        $this->connection = $connection;
+    }
+
+    /**
+     * @param int $interval
+     */
+    public function run($interval)
+    {
+        while (true) {
+            if (!$this->connection->isConnected()) {
+                return;
+            }
+
+            sleep((int) $interval / 2);
+
+            $this->connection->checkHeartBeat();
+        }
+    }
+}

lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php

@@ -58,6 +58,7 @@ class RecordSubscriber implements EventSubscriberInterface
 
             if ($subdefs !== null) {
                 foreach ($subdefs as $subdef) {
+
                     $payload = [
                         'message_type' => MessagePublisher::SUBDEF_CREATION_TYPE,
                         'payload' => [

lib/classes/ACL.php

@@ -1666,7 +1666,7 @@ class ACL implements cache_cacheableInterface
 
         $lim_max = $this->_limited[$base_id]['dmax'] && $this->_limited[$base_id]['dmax'] < $datetime;
 
-        return $lim_max || $lim_min;
+        return ($lim_max || $lim_min);
     }
 
     /**
@@ -1697,8 +1697,8 @@ class ACL implements cache_cacheableInterface
             ':time_limited' => $limit ? 1 : 0,
             ':usr_id' => $this->user->getId(),
             ':base_id' => $base_id,
-            ':limited_from' => NullableDateTime::format($limit_from, DATE_ISO8601),
-            ':limited_to' => NullableDateTime::format($limit_to, DATE_ISO8601),
+            ':limited_from' => NullableDateTime::format($limit_from),
+            ':limited_to' => NullableDateTime::format($limit_to),
         ]);
         $stmt->closeCursor();
 

lib/classes/media/subdef.php

@@ -261,6 +261,8 @@ class media_subdef extends media_abstract implements cache_cacheableInterface
     public function remove_file()
     {
         if ($this->is_physically_present() && is_writable($this->getRealPath())) {
+            // @unlink($this->getWatermarkRealPath());
+            @unlink($this->getStampRealPath());
             unlink($this->getRealPath());
 
             $this->delete_data_from_cache();
@@ -372,11 +374,10 @@ class media_subdef extends media_abstract implements cache_cacheableInterface
 
     public function getEtag()
     {
-        if (!$this->etag && $this->is_physically_present()) {
+        if ((!$this->etag && $this->is_physically_present())) {
             $file = new SplFileInfo($this->getRealPath());
-
             if ($file->isFile()) {
-                $this->setEtag(md5($file->getRealPath() . $file->getMTime()));
+                $this->generateEtag($file);
             }
         }
 
@@ -565,6 +566,9 @@ class media_subdef extends media_abstract implements cache_cacheableInterface
      */
     public function rotate($angle, Alchemyst $alchemyst, MediaVorus $mediavorus)
     {
+        // @unlink($this->getWatermarkRealPath());
+        @unlink($this->getStampRealPath());
+
         if (!$this->is_physically_present()) {
             throw new \Alchemy\Phrasea\Exception\RuntimeException('You can not rotate a substitution');
         }
@@ -583,7 +587,11 @@ class media_subdef extends media_abstract implements cache_cacheableInterface
         $this->width = $media->getWidth();
         $this->height = $media->getHeight();
 
-        return $this->save();
+        // generate a new etag after rotation
+        $file = new SplFileInfo($this->getRealPath());
+        $this->generateEtag($file);  // with repository save
+
+        return $this;
     }
 
     /**
@@ -809,6 +817,14 @@ class media_subdef extends media_abstract implements cache_cacheableInterface
         return $this->app['phraseanet.h264']->getUrl($this->getRealPath());
     }
 
+    /**
+     * @param SplFileInfo $file
+     */
+    private function generateEtag(SplFileInfo $file)
+    {
+        $this->setEtag(md5($file->getRealPath() . $file->getMTime()));
+    }
+
     /**
      * @return $this
      */

lib/conf.d/configuration.yml

@@ -334,6 +334,7 @@ workers:
             user: ''
             password: ''
             vhost: /
+            heartbeat: 60
 
 externalservice:
     ginger:

resources/hudson/InstallDBs.yml

@@ -1,7 +1,7 @@
 database:
-  host: 127.0.0.1
+  host: localhost
   port: 3306
-  user: phr_user
-  password: iWvGxPE8
+  user: phraseanet
+  password: phraseanet
   ab_name: ab_setup_test
   db_name: db_setup_test

templates/web/common/menubar.html.twig

@@ -416,10 +416,13 @@
         });
     });
 
-    /**manage session and redirect to login page**/
-    var usr_id = '{{ app.getAuthenticator().user.getId }}';
-    var module = '{{ module }}';
+    // no need to launch pollNotifications if not connected on load page
+    {% if app.getAuthenticator().isAuthenticated() %}
+        /**manage session and redirect to login page**/
+        var usr_id = '{{ app.getAuthenticator().user.getId }}';
+        var module = '{{ module }}';
 
-    // start pooling recursively
-    window.setTimeout( function() { commonModule.pollNotifications(usr_id, module === 'prod', true); }, 2000);
+        // start pooling recursively
+        window.setTimeout( function() { commonModule.pollNotifications(usr_id, module === 'prod', true); }, 2000);
+    {% endif %}
 </script>

tests/classes/ACLTest.php

@@ -777,8 +777,8 @@ class ACLTest extends \PhraseanetTestCase
                 $plusone = new DateTime('+1 day');
                 $this->object->set_limits($base_id, true, $minusone, $plusone);
                 $limits = $this->object->get_limits($base_id);
-                $this->assertEquals($limits['dmin'], $minusone);
                 $this->assertEquals($limits['dmax'], $plusone);
+                $this->assertEquals($limits['dmin'], $minusone);
                 $minustwo = new DateTime('-2 day');
                 $plustwo = new DateTime('-2 day');
                 $this->object->set_limits($base_id, true, $minustwo, $plustwo);

tests/classes/record/adapterTest.php

@@ -407,7 +407,7 @@ class record_adapterTest extends \PhraseanetAuthenticatedTestCase
                 $separator = '';
             }
 
-            $multi_imploded = implode(' ' . $separator . ' ', ['un', 'jeu', 'de', 'test']);
+            $multi_imploded = implode(' ' . $separator . ' ', ['test', 'de', 'jeu', 'un']);
 
             if ($meta_el->is_multi()) {
                 $initial_values = [];