diff --git a/.circleci/config.yml b/.circleci/config.yml index 42d3743cce..b42ab416b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,31 +14,26 @@ jobs: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results docker: - - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 + - image: cimg/php:7.0.33-node - image: circleci/rabbitmq:3.7.7 steps: - checkout - - run: phpenv versions - - run: phpenv global 7.0.7 - run: php -v - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - run: working_directory: ~/alchemy-fr/Phraseanet - command: nvm install v10.12.0 && nvm alias default v10.12.0 - - run: - working_directory: ~/alchemy-fr/Phraseanet - command: 'sudo service mysql status || sudo service mysql start;' + command: 'sudo apt update && sudo apt install mysql-server && sudo service mysql status || sudo service mysql start;' # Dependencies # This would typically go in either a build or a build-and-test job when using workflows # Restore the dependency cache - restore_cache: keys: # This branch if available - - v1-dep-{{ .Branch }}- + - v2-dep-{{ .Branch }}- # Default branch if not - - v1-dep-master- + - v2-dep-master- # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly - - v1-dep- + - v2-dep- # This is based on your 1.0 configuration file or project settings - run: echo 127.0.0.1 redis elasticsearch db rabbitmq | sudo tee -a /etc/hosts - run: git clone https://github.com/alanxz/rabbitmq-c @@ -46,23 +41,20 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: yes '' | pecl install amqp-1.9.3 - - run: yes '' | pecl install imagick - - run: sudo apt-get install libzmq-dev - - run: yes '' | pecl install zmq-beta - - run: echo "extension = amqp.so" > /opt/circleci/php/$(phpenv global)/etc/conf.d/amqp.ini - - run: echo "extension = zmq.so" > /opt/circleci/php/$(phpenv global)/etc/conf.d/zmq.ini - - run: echo "date.timezone = UTC" > /opt/circleci/php/$(phpenv global)/etc/conf.d/timezone.ini - - run: sed -i 's/^\(session.cache_limiter = \).*/\1""/' /opt/circleci/php/$(phpenv global)/etc/php.ini + - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-zmq php7.0-intl php7.0-sqlite3 libmagickwand-dev libmagickcore-dev + - run: printf "\n" | sudo pecl install amqp-1.11.0beta + - run: printf "\n" | sudo pecl install imagick + - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" + - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini + - run: sudo -- bash -c 'echo -e "[mysqld]\ndefault-authentication-plugin=mysql_native_password" > /etc/mysql/conf.d/mysql_native_password.cnf' && sudo service mysql restart - run: npm rebuild node-sass # This is based on your 1.0 configuration file or project settings - run: composer install --no-progress --no-interaction --optimize-autoloader + - run: curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash + - run: source ~/.profile && nvm install v10.12.0 && nvm alias default v10.12.0 # This is based on your 1.0 configuration file or project settings - run: node -v - run: npm -v - - run: npm install -g yarn - - run: rm ~/.yarn/bin/yarn - - run: ln -s /opt/circleci/nodejs/v10.12.0/bin/yarn ~/.yarn/bin/yarn - run: yarn install - run: if [[ ! -e elasticsearch-2.3.3 ]]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.3.3.tar.gz && tar -xvf elasticsearch-2.3.3.tar.gz && elasticsearch-2.3.3/bin/plugin install analysis-icu; fi - run: @@ -70,7 +62,7 @@ jobs: background: true # Save dependency cache - save_cache: - key: v1-dep-{{ .Branch }}-{{ epoch }} + key: v2-dep-{{ .Branch }}-{{ epoch }} paths: # This is a broad list of cache paths to include many possible development environments # You can probably delete some of these entries @@ -87,10 +79,10 @@ jobs: - node_modules - ~/.composer # This is based on your 1.0 configuration file or project settings - - run: mysql -u ubuntu -e 'CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;'; + - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'localhost' IDENTIFIED WITH mysql_native_password BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" # This is based on your 1.0 configuration file or project settings - run: ./bin/developer system:uninstall -v - - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=ubuntu --db-template=fr-simple --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; + - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=localhost --db-user=phraseanet --db-template=fr-simple --db-password=phraseanet --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; - run: ./bin/developer ini:setup-tests-dbs -v - run: ./bin/console searchengine:index:create -v - run: ./bin/developer phraseanet:regenerate-sqlite -v diff --git a/.env b/.env index f6c87571a3..017cfce4ea 100644 --- a/.env +++ b/.env @@ -1,9 +1,9 @@ -# For dev purpose COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.mailhog.yml:docker-compose.pma.yml:docker-compose.override.yml -# Dev tools and utilities: docker-compose.elk.yml:docker-compose.ftp.yml:docker-compose.squid.yml +# For dev purpose COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.rabbitmq.yml:docker-compose.mailhog.yml:docker-compose.pma.yml:docker-compose.override.yml +# Dev tools and utilities add : docker-compose.elk.yml:docker-compose.ftp.yml:docker-compose.squid.yml # For old Phraseanet scheduler launching use : docker-compose.scheduler.yml -COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.mailhog.yml +COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.rabbitmq.yml:docker-compose.mailhog.yml # Registry from where you pull Docker images PHRASEANET_DOCKER_REGISTRY=local @@ -29,11 +29,13 @@ PHRASEANET_UPGRADE=0 # --------------- RabbitMQ SETTING ---------------------- + RABBITMQ_DEFAULT_USER=alchemy RABBITMQ_DEFAULT_PASS=vdh4dpe5Wy3R RABBITMQ_MANAGEMENT_PORT=10811 + # --------------- GATEWAY SETTING (nginx) ----------------------- GATEWAY_SEND_TIMEOUT=120 @@ -121,6 +123,15 @@ PHRASEANET_SWFTOOLS_TIMEOUT=120 PHRASEANET_UNOCON_TIMEOUT=120 PHRASEANET_EXIFTOOL_TIMEOUT=120 +# RabbitMQ parameters + +PHRASEANET_RABBITMQ_HOST=rabbitmq +PHRASEANET_RABBITMQ_PORT=5672 +PHRASEANET_RABBITMQ_SSL=false +PHRASEANET_RABBITMQ_VHOST=/ +PHRASEANET_RABBITMQ_HEARTBEAT=30 + + # network : comma separated list of IP ou SUBNETS PHRASEANET_TRUSTED_PROXIES= PHRASEANET_DEBUG_ALLOWED_IP= @@ -235,6 +246,12 @@ IMAGEMAGICK_POLICY_TEMPORARY_PATH=/tmp # PhpMyAdmin port PHRASEANET_PHPMYADMIN_PORT=8089 + +# mailhog web GUI port mapping + +MAILHOG_GUI_PORT=8025 + + # Xdebug XDEBUG_ENABLED=0 XDEBUG_PROFILER_ENABLED=0 diff --git a/Dockerfile b/Dockerfile index c365ced64e..a38ddb6048 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,10 @@ RUN echo "deb http://deb.debian.org/debian stretch main non-free" > /etc/apt/sou apt-transport-https \ ca-certificates \ gnupg2 \ - && sed -i 's/mozilla\/DST_Root_CA_X3.crt/!mozilla\/DST_Root_CA_X3.crt/g' /etc/ca-certificates.conf \ + wget \ + && wget -O certs.deb http://ftp.fr.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20210119_all.deb \ + && dpkg --fsys-tarfile certs.deb | tar -xOf - ./usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt > /usr/local/share/ca-certificates/ISRG_Root_X1.crt \ + && rm -rf /usr/share/ca-certificates/mozilla/DST_Root_CA_X3.crt \ && update-ca-certificates --fresh \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/bin/console b/bin/console index a3e0ff1ea2..9b0c8c23d8 100755 --- a/bin/console +++ b/bin/console @@ -64,6 +64,7 @@ use Alchemy\Phrasea\Command\User\UserListCommand; use Alchemy\Phrasea\Command\User\UserPasswordCommand; use Alchemy\Phrasea\Core\Version; use Alchemy\Phrasea\WorkerManager\Command\WorkerExecuteCommand; +use Alchemy\Phrasea\WorkerManager\Command\WorkerHeartbeatCommand; use Alchemy\Phrasea\WorkerManager\Command\WorkerRunServiceCommand; use Alchemy\Phrasea\WorkerManager\Command\WorkerShowConfigCommand; @@ -171,6 +172,7 @@ $cli->command(new QuerySampleCommand()); $cli->command(new FindConceptsCommand()); $cli->command(new WorkerExecuteCommand()); +$cli->command(new WorkerHeartbeatCommand()); $cli->command(new WorkerRunServiceCommand()); $cli->command(new WorkerShowConfigCommand()); diff --git a/config/configuration.sample.yml b/config/configuration.sample.yml index 853284738b..3c1d98be83 100644 --- a/config/configuration.sample.yml +++ b/config/configuration.sample.yml @@ -333,6 +333,7 @@ workers: user: guest password: guest vhost: / + heartbeat: 60 queues: writeMetadatas: # this Q is "delayable" in case of record is locked ttl_retry: 1500 # overwrite 1000 ms default delay diff --git a/docker-compose.mailhog.yml b/docker-compose.mailhog.yml index 0f02183fe2..d01cc95055 100644 --- a/docker-compose.mailhog.yml +++ b/docker-compose.mailhog.yml @@ -4,7 +4,8 @@ services: mailhog: image: mailhog/mailhog ports: - - 1025:1025 - - 8025:8025 + - ${MAILHOG_GUI_PORT}:8025 + environment: + - MAILHOG_GUI_PORT networks: - - internal + - internal \ No newline at end of file diff --git a/docker-compose.rabbitmq.yml b/docker-compose.rabbitmq.yml new file mode 100644 index 0000000000..71c894e962 --- /dev/null +++ b/docker-compose.rabbitmq.yml @@ -0,0 +1,12 @@ +version: "3.4" + +services: + + rabbitmq: + image: rabbitmq:3.6.16-management + restart: on-failure + environment: + - RABBITMQ_DEFAULT_USER + - RABBITMQ_DEFAULT_PASS + networks: + - internal diff --git a/docker-compose.yml b/docker-compose.yml index a6d1295309..18e99eac6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,11 @@ services: - PHRASEANET_SERVER_NAME - PHRASEANET_AVAILABLE_LANGUAGE - PHRASEANET_DEFAULT_LANGUAGE + - PHRASEANET_RABBITMQ_HOST + - PHRASEANET_RABBITMQ_PORT + - PHRASEANET_RABBITMQ_SSL + - PHRASEANET_RABBITMQ_VHOST + - PHRASEANET_RABBITMQ_HEARTBEAT - PHRASEANET_RABBITMQ_USER=$RABBITMQ_DEFAULT_USER - PHRASEANET_RABBITMQ_PASSWORD=$RABBITMQ_DEFAULT_PASS - PHRASEANET_EMITTER_EMAIL @@ -195,15 +200,6 @@ services: networks: - internal - rabbitmq: - image: rabbitmq:3.6.16-management - restart: on-failure - environment: - - RABBITMQ_DEFAULT_USER - - RABBITMQ_DEFAULT_PASS - networks: - - internal - redis: image: redis restart: on-failure diff --git a/docker/phraseanet/entrypoint.sh b/docker/phraseanet/entrypoint.sh index ae734deaa5..ebaef86c24 100755 --- a/docker/phraseanet/entrypoint.sh +++ b/docker/phraseanet/entrypoint.sh @@ -101,6 +101,19 @@ if [[ -f "$FILE" && $PHRASEANET_SETUP = 1 ]]; then bin/setup system:config set registry.api-clients.api-require-ssl $PHRASEANET_API_SSL bin/setup system:config set registry.api-clients.api-auth-token-header-only $PHRASEANET_API_AUTH_TOKEN_HEADER_ONLY + + echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet setting RABBITMQ" + bin/setup system:config set workers.queue.worker-queue.host $PHRASEANET_RABBITMQ_HOST + bin/setup system:config set workers.queue.worker-queue.port $PHRASEANET_RABBITMQ_PORT + bin/setup system:config set workers.queue.worker-queue.vhost $PHRASEANET_RABBITMQ_VHOST + bin/setup system:config set workers.queue.worker-queue.ssl $PHRASEANET_RABBITMQ_SSL + bin/setup system:config set workers.queue.worker-queue.heartbeat $PHRASEANET_RABBITMQ_HEARTBEAT + bin/setup system:config set workers.queue.worker-queue.user $PHRASEANET_RABBITMQ_USER + bin/setup system:config set workers.queue.worker-queue.password $PHRASEANET_RABBITMQ_PASSWORD + + + + echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet setting SMTP " if [[ $PHRASEANET_SMTP_ENABLED && $PHRASEANET_SMTP_ENABLED = true ]]; then bin/setup system:config set registry.email.smtp-enabled $PHRASEANET_SMTP_ENABLED diff --git a/docker/phraseanet/worker/entrypoint.sh b/docker/phraseanet/worker/entrypoint.sh index 9501730e84..8e3b098141 100755 --- a/docker/phraseanet/worker/entrypoint.sh +++ b/docker/phraseanet/worker/entrypoint.sh @@ -2,6 +2,8 @@ set -e +HEARTBEAT_INTERVAL=20 + DOCKER_DIR="./docker/phraseanet" PHR_USER=app @@ -54,76 +56,88 @@ if [[ $BLACKFIRE_ENABLED = "true" ]]; then service blackfire-agent start echo "Blackfire setup done" else - echo `date +"%Y-%m-%d %H:%M:%S"` " - blackfire extension deactivation." - rm -f /usr/local/etc/php/conf.d/zz-blackfire.ini + echo `date +"%Y-%m-%d %H:%M:%S"` " - blackfire extension deactivation." + rm -f /usr/local/etc/php/conf.d/zz-blackfire.ini fi - rm -rf bin/run-worker.sh if [ ! -z "$PHRASEANET_SCHEDULER" ] ; then - command="bin/console task-manager:scheduler:run" - echo $command >> bin/run-worker.sh - echo "Phraseanet workers container will be launched in Scheduler mode with bin/console task-manager:scheduler:run" - + echo $command >> bin/run-worker.sh + echo "Phraseanet workers container will be launched in Scheduler mode with bin/console task-manager:scheduler:run" else - if [ ! -z "$PHRASEANET_EXPLODE_WORKER" ] && [ ${PHRASEANET_EXPLODE_WORKER} == "1" ]; then + if [ ! -z "$PHRASEANET_EXPLODE_WORKER" ] && [ ${PHRASEANET_EXPLODE_WORKER} == "1" ]; then if [ ! -z "$PHRASEANET_WORKERS_LAUNCH_METHOD" ] && [ ${PHRASEANET_WORKERS_LAUNCH_METHOD} == "supervisor" ]; then - echo "Multiples Phraseanet workers will be launched by supervisor" - for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` - do + echo "Multiples Phraseanet workers will be launched by supervisor" + for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` + do worker_job_file="$(echo $i | cut -d'_' -f3).conf" if [ ${!i} -gt "0" ] ; then envsubst < "/var/alchemy/Phraseanet/docker/phraseanet/worker/supervisor_conf.d/$worker_job_file" > /etc/supervisor/conf.d/$worker_job_file - echo "Add Worker : " $worker_job_file " -- with parallelism: (-m) " ${!i} + echo "Add worker: " $worker_job_file " -- with parallelism: (-m) " ${!i} else - echo "NO Worker define for : " $worker_job_file " -- because parallelism (-m) is set to : " ${!i} + echo "No worker defined for: " $worker_job_file " -- because parallelism (-m) is set to : " ${!i} fi - done - command="/usr/bin/supervisord -n -c /etc/supervisor/supervisor.conf" - echo $command >> bin/run-worker.sh - PHR_USER=root + done + command="/usr/bin/supervisord -n -c /etc/supervisor/supervisor.conf" + echo $command >> bin/run-worker.sh + PHR_USER=root else - echo "Multiples Phraseanet workers will be launched with bin/console worker:execute" - NBR_WORKERS=0 - for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` - do + echo "Multiples Phraseanet workers will be launched with bin/console worker:execute" + NBR_WORKERS=0 + + echo "bin/console worker:heartbeat --heartbeat ${HEARTBEAT_INTERVAL} &" >> bin/run-worker.sh + + for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` + do queue_name="$(echo $i | cut -d'_' -f3)" m=$i if [ ${!m} -gt "0" ] ; then - command="bin/console worker:execute --queue-name=$queue_name -m ${!m} &" - echo $command >> bin/run-worker.sh - echo "Worker " $queue_name " define with parallelism " ${!m} - NBR_WORKERS=$(expr $NBR_WORKERS + 1) - else - echo " NO Worker defined for : " $m " -- because parallelism (-m) is set to : " ${!m} - fi - done - - echo $NBR_WORKERS " workers define" - echo $NBR_WORKERS > bin/workers_count.txt - chown root:app bin/workers_count.txt - chmod 760 bin/workers_count.txt - echo ' - NBR_WORKERS=$(< bin/workers_count.txt) - WORKER_LOOP_VALUE=20s - while true; - do - sleep $WORKER_LOOP_VALUE - nb_process=`ps faux | grep "worker:execute" | grep php | wc -l` - date_time_process=`date +"%Y-%m-%d %H:%M:%S"` - echo $date_time_process "-" $nb_process "running workers" - if [ $nb_process -lt $NBR_WORKERS ] - then - exit 1 - break + command="bin/console worker:execute --queue-name=$queue_name -m ${!m} &" + echo $command >> bin/run-worker.sh + echo "Worker " $queue_name " defined with parallelism " ${!m} + NBR_WORKERS=$(expr $NBR_WORKERS + 1) + else + echo "No worker defined for: " $m " -- because parallelism (-m) is set to : " ${!m} fi - done ' >> bin/run-worker.sh - fi + done + + echo $NBR_WORKERS " workers defined" + echo $NBR_WORKERS > bin/workers_count.txt + chown root:app bin/workers_count.txt + chmod 760 bin/workers_count.txt + echo "HEARTBEAT_INTERVAL=${HEARTBEAT_INTERVAL}" >> bin/run-worker.sh + echo ' +NBR_WORKERS=$(< bin/workers_count.txt) +sleep 1 # let worker:heartbeat fail before process check + +function check() { + nb_process=`ps faux | grep "worker:execute" | grep php | wc -l` + nb_heartbeat=`ps faux | grep "worker:heartbeat" | grep php | wc -l` + date_time_process=`date +"%Y-%m-%d %H:%M:%S"` + echo $date_time_process "-" $nb_process "running workers" + if [ $nb_process -lt $NBR_WORKERS ]; then + echo "One or more worker:execute is not running, exiting..." + exit 1 + elif [ $nb_heartbeat -lt 1 ]; then + echo "worker:heartbeat is not running, exiting..." + exit 1 + fi +} + +# early check +check + +while true; do + sleep ${HEARTBEAT_INTERVAL}s + check +done' >> bin/run-worker.sh + fi else command="bin/console worker:execute" echo $command >> bin/run-worker.sh fi fi + runuser -u $PHR_USER -- $@ diff --git a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php index f79a4c34ac..5f2d68a94a 100644 --- a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php +++ b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php @@ -62,15 +62,19 @@ class SetupTestsDbs extends Command $ab_name = StringHelper::SqlQuote($settings['database']['ab_name'], StringHelper::SQL_IDENTIFIER); $db_name = StringHelper::SqlQuote($settings['database']['db_name'], StringHelper::SQL_IDENTIFIER); - +/* $this->container['orm.em']->getConnection()->executeUpdate( - 'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION' + 'CREATE USER '.$user.'@'.$host.' IDENTIFIED WITH mysql_native_password BY '.$pass ); $this->container['orm.em']->getConnection()->executeUpdate( - 'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION' + 'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host ); + $this->container['orm.em']->getConnection()->executeUpdate( + 'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host + ); +*/ $this->container['orm.em']->getConnection()->executeUpdate('SET @@global.sql_mode= ""'); return 0; diff --git a/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php index 9931d4649f..37c137cfc1 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php @@ -37,7 +37,8 @@ class WorkerConfigurationServiceProvider implements ServiceProviderInterface 'port' => 5672, 'user' => 'guest', 'password' => 'guest', - 'vhost' => '/' + 'vhost' => '/', + 'heartbeat' => 60, ] ]; diff --git a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php index b03e716fc0..fad30a381c 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php @@ -94,6 +94,7 @@ class WorkerExecuteCommand extends Command } $serverConnection->connectionClose(); - } + return 0; + } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php new file mode 100644 index 0000000000..ef6b923839 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php @@ -0,0 +1,45 @@ +setDescription('Heartbeat connection to track drops or broken pipes') + ->addOption('heartbeat', null, InputOption::VALUE_REQUIRED, sprintf('in seconds (default: %d)', self::DEFAULT_INTERVAL)) + ; + + return $this; + } + + protected function doExecute(InputInterface $input, OutputInterface $output) + { + /** @var AMQPConnection $serverConnection */ + $serverConnection = $this->container['alchemy_worker.amqp.connection']; + + $connection = $serverConnection->getConnection(); + + $interval = $input->getOption('heartbeat'); + if (empty($interval)) { + $interval = self::DEFAULT_INTERVAL; + } + + $heartbeatHandler = new HeartbeatHandler($connection); + $heartbeatHandler->run($interval); + + return 0; + } +} diff --git a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php index d6a092095c..5489098d42 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php @@ -141,7 +141,8 @@ class AMQPConnection 'ssl' => false, 'user' => 'guest', 'password' => 'guest', - 'vhost' => '/' + 'vhost' => '/', + 'heartbeat' => 60, ]; $this->hostConfig = $conf->get(['workers', 'queue', 'worker-queue'], $defaultConfiguration); @@ -311,10 +312,12 @@ class AMQPConnection { if (!isset($this->connection)) { try { + $heartbeat = $this->hostConfig['heartbeat'] ?? 60; + // if we are in ssl connection type if (isset($this->hostConfig['ssl']) && $this->hostConfig['ssl'] === true) { $sslOptions = [ - 'verify_peer' => true + 'verify_peer' => true, ]; $this->connection = new AMQPSSLConnection( @@ -323,7 +326,10 @@ class AMQPConnection $this->hostConfig['user'], $this->hostConfig['password'], $this->hostConfig['vhost'], - $sslOptions + $sslOptions, + [ + 'heartbeat' => $heartbeat, + ] ); } else { $this->connection = new AMQPStreamConnection( @@ -331,7 +337,16 @@ class AMQPConnection $this->hostConfig['port'], $this->hostConfig['user'], $this->hostConfig['password'], - $this->hostConfig['vhost'] + $this->hostConfig['vhost'], + false, + 'AMQPLAIN', + null, + 'en_US', + 3.0, + 3.0, + null, + false, + $heartbeat ); } } @@ -425,7 +440,6 @@ class AMQPConnection break; default: throw new Exception(sprintf('undefined q type "%s', $queueName)); - break; } return $this->channel; diff --git a/lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php b/lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php new file mode 100644 index 0000000000..aa07b47767 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php @@ -0,0 +1,36 @@ +connection = $connection; + } + + /** + * @param int $interval + */ + public function run($interval) + { + while (true) { + if (!$this->connection->isConnected()) { + return; + } + + sleep((int) $interval / 2); + + $this->connection->checkHeartBeat(); + } + } +} diff --git a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php index 8f8eb2972a..fdfe723ccb 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php @@ -58,6 +58,7 @@ class RecordSubscriber implements EventSubscriberInterface if ($subdefs !== null) { foreach ($subdefs as $subdef) { + $payload = [ 'message_type' => MessagePublisher::SUBDEF_CREATION_TYPE, 'payload' => [ diff --git a/lib/classes/ACL.php b/lib/classes/ACL.php index 500f8457b8..8a42182467 100644 --- a/lib/classes/ACL.php +++ b/lib/classes/ACL.php @@ -1666,7 +1666,7 @@ class ACL implements cache_cacheableInterface $lim_max = $this->_limited[$base_id]['dmax'] && $this->_limited[$base_id]['dmax'] < $datetime; - return $lim_max || $lim_min; + return ($lim_max || $lim_min); } /** @@ -1697,8 +1697,8 @@ class ACL implements cache_cacheableInterface ':time_limited' => $limit ? 1 : 0, ':usr_id' => $this->user->getId(), ':base_id' => $base_id, - ':limited_from' => NullableDateTime::format($limit_from, DATE_ISO8601), - ':limited_to' => NullableDateTime::format($limit_to, DATE_ISO8601), + ':limited_from' => NullableDateTime::format($limit_from), + ':limited_to' => NullableDateTime::format($limit_to), ]); $stmt->closeCursor(); diff --git a/lib/classes/media/subdef.php b/lib/classes/media/subdef.php index 26192dc4d0..fe1367d045 100644 --- a/lib/classes/media/subdef.php +++ b/lib/classes/media/subdef.php @@ -261,6 +261,8 @@ class media_subdef extends media_abstract implements cache_cacheableInterface public function remove_file() { if ($this->is_physically_present() && is_writable($this->getRealPath())) { + // @unlink($this->getWatermarkRealPath()); + @unlink($this->getStampRealPath()); unlink($this->getRealPath()); $this->delete_data_from_cache(); @@ -372,11 +374,10 @@ class media_subdef extends media_abstract implements cache_cacheableInterface public function getEtag() { - if (!$this->etag && $this->is_physically_present()) { + if ((!$this->etag && $this->is_physically_present())) { $file = new SplFileInfo($this->getRealPath()); - if ($file->isFile()) { - $this->setEtag(md5($file->getRealPath() . $file->getMTime())); + $this->generateEtag($file); } } @@ -565,6 +566,9 @@ class media_subdef extends media_abstract implements cache_cacheableInterface */ public function rotate($angle, Alchemyst $alchemyst, MediaVorus $mediavorus) { + // @unlink($this->getWatermarkRealPath()); + @unlink($this->getStampRealPath()); + if (!$this->is_physically_present()) { throw new \Alchemy\Phrasea\Exception\RuntimeException('You can not rotate a substitution'); } @@ -583,7 +587,11 @@ class media_subdef extends media_abstract implements cache_cacheableInterface $this->width = $media->getWidth(); $this->height = $media->getHeight(); - return $this->save(); + // generate a new etag after rotation + $file = new SplFileInfo($this->getRealPath()); + $this->generateEtag($file); // with repository save + + return $this; } /** @@ -809,6 +817,14 @@ class media_subdef extends media_abstract implements cache_cacheableInterface return $this->app['phraseanet.h264']->getUrl($this->getRealPath()); } + /** + * @param SplFileInfo $file + */ + private function generateEtag(SplFileInfo $file) + { + $this->setEtag(md5($file->getRealPath() . $file->getMTime())); + } + /** * @return $this */ diff --git a/lib/conf.d/configuration.yml b/lib/conf.d/configuration.yml index e7f680e581..7cf86d71d0 100644 --- a/lib/conf.d/configuration.yml +++ b/lib/conf.d/configuration.yml @@ -334,6 +334,7 @@ workers: user: '' password: '' vhost: / + heartbeat: 60 externalservice: ginger: diff --git a/resources/hudson/InstallDBs.yml b/resources/hudson/InstallDBs.yml index 589c2c6724..266f5e47bb 100644 --- a/resources/hudson/InstallDBs.yml +++ b/resources/hudson/InstallDBs.yml @@ -1,7 +1,7 @@ database: - host: 127.0.0.1 + host: localhost port: 3306 - user: phr_user - password: iWvGxPE8 + user: phraseanet + password: phraseanet ab_name: ab_setup_test db_name: db_setup_test diff --git a/templates/web/common/menubar.html.twig b/templates/web/common/menubar.html.twig index d8021de270..cc528743eb 100644 --- a/templates/web/common/menubar.html.twig +++ b/templates/web/common/menubar.html.twig @@ -416,10 +416,13 @@ }); }); - /**manage session and redirect to login page**/ - var usr_id = '{{ app.getAuthenticator().user.getId }}'; - var module = '{{ module }}'; + // no need to launch pollNotifications if not connected on load page + {% if app.getAuthenticator().isAuthenticated() %} + /**manage session and redirect to login page**/ + var usr_id = '{{ app.getAuthenticator().user.getId }}'; + var module = '{{ module }}'; - // start pooling recursively - window.setTimeout( function() { commonModule.pollNotifications(usr_id, module === 'prod', true); }, 2000); + // start pooling recursively + window.setTimeout( function() { commonModule.pollNotifications(usr_id, module === 'prod', true); }, 2000); + {% endif %} diff --git a/tests/classes/ACLTest.php b/tests/classes/ACLTest.php index 816a9bc171..3eb01ef94e 100644 --- a/tests/classes/ACLTest.php +++ b/tests/classes/ACLTest.php @@ -777,8 +777,8 @@ class ACLTest extends \PhraseanetTestCase $plusone = new DateTime('+1 day'); $this->object->set_limits($base_id, true, $minusone, $plusone); $limits = $this->object->get_limits($base_id); - $this->assertEquals($limits['dmin'], $minusone); $this->assertEquals($limits['dmax'], $plusone); + $this->assertEquals($limits['dmin'], $minusone); $minustwo = new DateTime('-2 day'); $plustwo = new DateTime('-2 day'); $this->object->set_limits($base_id, true, $minustwo, $plustwo); diff --git a/tests/classes/record/adapterTest.php b/tests/classes/record/adapterTest.php index 4405b055e3..83e52146c7 100644 --- a/tests/classes/record/adapterTest.php +++ b/tests/classes/record/adapterTest.php @@ -407,7 +407,7 @@ class record_adapterTest extends \PhraseanetAuthenticatedTestCase $separator = ''; } - $multi_imploded = implode(' ' . $separator . ' ', ['un', 'jeu', 'de', 'test']); + $multi_imploded = implode(' ' . $separator . ' ', ['test', 'de', 'jeu', 'un']); if ($meta_el->is_multi()) { $initial_values = [];