From c15fe0590fc055be13b92ffbdff1273d897fedb0 Mon Sep 17 00:00:00 2001 From: Arthur de Moulins Date: Thu, 30 Sep 2021 12:23:46 +0200 Subject: [PATCH 01/48] PHRAS-3435 set heartbeat to 60 on AMQP connection --- .../WorkerManager/Queue/AMQPConnection.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php index d6a092095c..2eded6bf6f 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php @@ -323,7 +323,10 @@ class AMQPConnection $this->hostConfig['user'], $this->hostConfig['password'], $this->hostConfig['vhost'], - $sslOptions + $sslOptions, + [ + 'heartbeat' => 60, + ] ); } else { $this->connection = new AMQPStreamConnection( @@ -331,7 +334,16 @@ class AMQPConnection $this->hostConfig['port'], $this->hostConfig['user'], $this->hostConfig['password'], - $this->hostConfig['vhost'] + $this->hostConfig['vhost'], + false, + 'AMQPLAIN', + null, + 'en_US', + 3.0, + 3.0, + null, + false, + 60 ); } } From cec6dd117132beb1e1efe1570b5a66867f7c2151 Mon Sep 17 00:00:00 2001 From: Gaspard Jacobson <52928254+gjacobjn@users.noreply.github.com> Date: Sun, 3 Oct 2021 23:00:01 +0200 Subject: [PATCH 02/48] Update Dockerfile --- Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c365ced64e..e150833075 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,18 @@ FROM php:7.0-fpm-stretch as phraseanet-system ENV FFMPEG_VERSION=4.2.2 RUN echo "deb http://deb.debian.org/debian stretch main non-free" > /etc/apt/sources.list \ - && apt-get update \ + && apt-get update \&& dpkg --fsys-tarfile certs.deb | tar -xOf - ./usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt > /usr/local/share/ca-certificates/ISRG_Root_X1.crt \ + && rm -rf /usr/share/ca-certificates/mozilla/DST_Root_CA_X3.crt \ + && update-ca-certificates \ && apt-get install -y \ apt-transport-https \ ca-certificates \ gnupg2 \ - && sed -i 's/mozilla\/DST_Root_CA_X3.crt/!mozilla\/DST_Root_CA_X3.crt/g' /etc/ca-certificates.conf \ - && update-ca-certificates --fresh \ + wget \ + && wget -O certs.deb http://ftp.fr.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20210119_all.deb \ + && dpkg --fsys-tarfile certs.deb | tar -xOf - ./usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt > /usr/local/share/ca-certificates/ISRG_Root_X1.crt \ + && rm -rf /usr/share/ca-certificates/mozilla/DST_Root_CA_X3.crt \ + && update-ca-certificates \ && apt-get update \ && apt-get install -y --no-install-recommends \ zlib1g-dev \ From ce6fadbb0606a7b46cb84f40da3df8a34c95317a Mon Sep 17 00:00:00 2001 From: Gaspard Jacobson <52928254+gjacobjn@users.noreply.github.com> Date: Mon, 4 Oct 2021 11:57:03 +0200 Subject: [PATCH 03/48] Update Dockerfile --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e150833075..1c1a565ff8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,14 +8,12 @@ FROM php:7.0-fpm-stretch as phraseanet-system ENV FFMPEG_VERSION=4.2.2 RUN echo "deb http://deb.debian.org/debian stretch main non-free" > /etc/apt/sources.list \ - && apt-get update \&& dpkg --fsys-tarfile certs.deb | tar -xOf - ./usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt > /usr/local/share/ca-certificates/ISRG_Root_X1.crt \ - && rm -rf /usr/share/ca-certificates/mozilla/DST_Root_CA_X3.crt \ - && update-ca-certificates \ + && apt-get update \ && apt-get install -y \ apt-transport-https \ ca-certificates \ gnupg2 \ - wget \ + wget \ && wget -O certs.deb http://ftp.fr.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20210119_all.deb \ && dpkg --fsys-tarfile certs.deb | tar -xOf - ./usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt > /usr/local/share/ca-certificates/ISRG_Root_X1.crt \ && rm -rf /usr/share/ca-certificates/mozilla/DST_Root_CA_X3.crt \ From a852e151f9c0e95865d254728364784619014660 Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 4 Oct 2021 14:40:42 +0100 Subject: [PATCH 04/48] update circle ubuntu image 14.04 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 42d3743cce..a6b66aa9fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results docker: - - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 + - image: circleci/build-image:ubuntu-14.04-XXL-1327-3401d20 - image: circleci/rabbitmq:3.7.7 steps: - checkout From 8c20b2a1e456e4cbb12f335e75e4efd53677259a Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 4 Oct 2021 16:08:58 +0100 Subject: [PATCH 05/48] migrate to cim/php --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6b66aa9fb..111e4a3128 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,12 +14,10 @@ jobs: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results docker: - - image: circleci/build-image:ubuntu-14.04-XXL-1327-3401d20 + - image: cimg/php:7.0.33-node - image: circleci/rabbitmq:3.7.7 steps: - checkout - - run: phpenv versions - - run: phpenv global 7.0.7 - run: php -v - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - run: From 2f4db78e17e89f4306307e8460f1b81adfeb668a Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 4 Oct 2021 16:18:45 +0100 Subject: [PATCH 06/48] fix nvm error/use npm --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 111e4a3128..1be1d6d32e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,9 +20,6 @@ jobs: - checkout - run: php -v - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - - run: - working_directory: ~/alchemy-fr/Phraseanet - command: nvm install v10.12.0 && nvm alias default v10.12.0 - run: working_directory: ~/alchemy-fr/Phraseanet command: 'sudo service mysql status || sudo service mysql start;' From ab9a88602910a436b65f98bf2fbac5c43d44f467 Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 4 Oct 2021 16:25:47 +0100 Subject: [PATCH 07/48] fix missing mysql server on image --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1be1d6d32e..c3303111b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - run: working_directory: ~/alchemy-fr/Phraseanet - command: 'sudo service mysql status || sudo service mysql start;' + command: 'sudo apt-get install mysql-server && sudo service mysql status || sudo service mysql start;' # Dependencies # This would typically go in either a build or a build-and-test job when using workflows # Restore the dependency cache @@ -82,10 +82,10 @@ jobs: - node_modules - ~/.composer # This is based on your 1.0 configuration file or project settings - - run: mysql -u ubuntu -e 'CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;'; + - run: mysql -u root -e 'CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;'; # This is based on your 1.0 configuration file or project settings - run: ./bin/developer system:uninstall -v - - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=ubuntu --db-template=fr-simple --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; + - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=root --db-template=fr-simple --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; - run: ./bin/developer ini:setup-tests-dbs -v - run: ./bin/console searchengine:index:create -v - run: ./bin/developer phraseanet:regenerate-sqlite -v From 3790b8713888f32e5256edd7bdcf48194956121f Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 4 Oct 2021 16:51:24 +0100 Subject: [PATCH 08/48] fix missing mysql server on image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3303111b9..dafa4339a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - run: working_directory: ~/alchemy-fr/Phraseanet - command: 'sudo apt-get install mysql-server && sudo service mysql status || sudo service mysql start;' + command: 'sudo apt install mariadb-server-10.3 && sudo service mysql status || sudo service mysql start;' # Dependencies # This would typically go in either a build or a build-and-test job when using workflows # Restore the dependency cache From 17fbb48f1ed6409133bacc01b3dc73a65b970246 Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 4 Oct 2021 16:57:23 +0100 Subject: [PATCH 09/48] fix missing mysql server on image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dafa4339a1..4c4821f19c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - run: working_directory: ~/alchemy-fr/Phraseanet - command: 'sudo apt install mariadb-server-10.3 && sudo service mysql status || sudo service mysql start;' + command: 'sudo apt install mysql-server && sudo service mysql status || sudo service mysql start;' # Dependencies # This would typically go in either a build or a build-and-test job when using workflows # Restore the dependency cache From 343f70cc67f5c215cb451ed2777a5d315ac62e23 Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 4 Oct 2021 17:07:48 +0100 Subject: [PATCH 10/48] fix mysql install error --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c4821f19c..358dbaf975 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - run: working_directory: ~/alchemy-fr/Phraseanet - command: 'sudo apt install mysql-server && sudo service mysql status || sudo service mysql start;' + command: 'sudo apt update && sudo apt install mysql-server && sudo service mysql status || sudo service mysql start;' # Dependencies # This would typically go in either a build or a build-and-test job when using workflows # Restore the dependency cache From 3c8376a41c24b62cd28c052bae015461f14a2271 Mon Sep 17 00:00:00 2001 From: aynsix Date: Tue, 5 Oct 2021 15:06:32 +0300 Subject: [PATCH 11/48] fix image preview --- lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php | 2 +- templates/web/common/thumbnail.html.twig | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php index 5a3a9e145c..fa3c53bc6b 100644 --- a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php +++ b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php @@ -343,7 +343,7 @@ class PhraseanetExtension extends \Twig_Extension } elseif ($record instanceof \record_adapter) { if (null !== $thumbnail = $record->get_subdef($subdefName)) { if ('' !== $path = $thumbnail->getRealPath()) { - $etag = $thumbnail->getEtag(); + $etag = dechex(crc32(dechex($record->get_subdef($subdefName)->get_modification_date()->getTimestamp() ^ 0x5A5A5A5A))); return $staticMode->getUrl($path, $etag); } } diff --git a/templates/web/common/thumbnail.html.twig b/templates/web/common/thumbnail.html.twig index 45a31dfb05..1a3b1353ba 100644 --- a/templates/web/common/thumbnail.html.twig +++ b/templates/web/common/thumbnail.html.twig @@ -40,6 +40,10 @@ frameborder="0" allowfullscreen> {% else %} + {% if thumbnail.get_name() == 'preview' %} + {% set url = app.getAuthenticator().isAuthenticated() ? record_subdef_url(thumbnail.get_record(), "preview") : thumbnail.get_permalink().get_url() %} + {% endif %} + {% set fit_size = fitIn( {"width":original_w, "height":original_h}, {"width":box_w, "height":box_h} From d887b3b137f6955492aa5c0fb2a9dbd22a700271 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Tue, 5 Oct 2021 20:03:57 +0200 Subject: [PATCH 12/48] PHRAS-3544_invalidate-stamp delete stamp cache file after image rotation or build subdef --- .../Phrasea/WorkerManager/Subscriber/RecordSubscriber.php | 6 ++++++ lib/classes/media/subdef.php | 3 +++ 2 files changed, 9 insertions(+) diff --git a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php index 8f8eb2972a..5075353d04 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php @@ -23,6 +23,7 @@ use Alchemy\Phrasea\WorkerManager\Worker\Factory\WorkerFactoryInterface; use Alchemy\Phrasea\WorkerManager\Worker\Resolver\TypeBasedWorkerResolver; use databox; use Exception; +use media_subdef; use Symfony\Component\EventDispatcher\EventSubscriberInterface; class RecordSubscriber implements EventSubscriberInterface @@ -57,7 +58,12 @@ class RecordSubscriber implements EventSubscriberInterface $subdefs = $record->getDatabox()->get_subdef_structure()->getSubdefGroup($record->getType()); if ($subdefs !== null) { + /** @var media_subdef $subdef */ foreach ($subdefs as $subdef) { + + // @unlink($subdef->getWatermarkRealPath()); + @unlink($subdef->getStampRealPath()); + $payload = [ 'message_type' => MessagePublisher::SUBDEF_CREATION_TYPE, 'payload' => [ diff --git a/lib/classes/media/subdef.php b/lib/classes/media/subdef.php index 53b8b290bc..3953ccc37e 100644 --- a/lib/classes/media/subdef.php +++ b/lib/classes/media/subdef.php @@ -555,6 +555,9 @@ class media_subdef extends media_abstract implements cache_cacheableInterface */ public function rotate($angle, Alchemyst $alchemyst, MediaVorus $mediavorus) { + // @unlink($this->getWatermarkRealPath()); + @unlink($this->getStampRealPath()); + if (!$this->is_physically_present()) { throw new \Alchemy\Phrasea\Exception\RuntimeException('You can not rotate a substitution'); } From 28913644584f1bdccedf5ba5bd0ea48e77d9d426 Mon Sep 17 00:00:00 2001 From: aynsix Date: Wed, 6 Oct 2021 10:40:24 +0300 Subject: [PATCH 13/48] change etag after rotation --- .../Phrasea/Twig/PhraseanetExtension.php | 2 +- lib/classes/media/subdef.php | 21 +++++++++++++++---- templates/web/common/thumbnail.html.twig | 4 ---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php index fa3c53bc6b..5a3a9e145c 100644 --- a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php +++ b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php @@ -343,7 +343,7 @@ class PhraseanetExtension extends \Twig_Extension } elseif ($record instanceof \record_adapter) { if (null !== $thumbnail = $record->get_subdef($subdefName)) { if ('' !== $path = $thumbnail->getRealPath()) { - $etag = dechex(crc32(dechex($record->get_subdef($subdefName)->get_modification_date()->getTimestamp() ^ 0x5A5A5A5A))); + $etag = $thumbnail->getEtag(); return $staticMode->getUrl($path, $etag); } } diff --git a/lib/classes/media/subdef.php b/lib/classes/media/subdef.php index 53b8b290bc..d151b71c94 100644 --- a/lib/classes/media/subdef.php +++ b/lib/classes/media/subdef.php @@ -362,11 +362,12 @@ class media_subdef extends media_abstract implements cache_cacheableInterface public function getEtag() { - if (!$this->etag && $this->is_physically_present()) { - $file = new SplFileInfo($this->getRealPath()); - + $file = new SplFileInfo($this->getRealPath()); + // etag is generated from md5 of path and file last modified time + // so check if etag is to be changed + if ((!$this->etag && $this->is_physically_present()) || $this->etag != md5($file->getRealPath() . $file->getMTime())) { if ($file->isFile()) { - $this->setEtag(md5($file->getRealPath() . $file->getMTime())); + $this->generateEtag($file); } } @@ -573,6 +574,10 @@ class media_subdef extends media_abstract implements cache_cacheableInterface $this->width = $media->getWidth(); $this->height = $media->getHeight(); + // generate a new etag after rotation + $file = new SplFileInfo($this->getRealPath()); + $this->generateEtag($file); + return $this->save(); } @@ -799,6 +804,14 @@ class media_subdef extends media_abstract implements cache_cacheableInterface return $this->app['phraseanet.h264']->getUrl($this->getRealPath()); } + /** + * @param SplFileInfo $file + */ + private function generateEtag(SplFileInfo $file) + { + $this->setEtag(md5($file->getRealPath() . $file->getMTime())); + } + /** * @return $this */ diff --git a/templates/web/common/thumbnail.html.twig b/templates/web/common/thumbnail.html.twig index 1a3b1353ba..45a31dfb05 100644 --- a/templates/web/common/thumbnail.html.twig +++ b/templates/web/common/thumbnail.html.twig @@ -40,10 +40,6 @@ frameborder="0" allowfullscreen> {% else %} - {% if thumbnail.get_name() == 'preview' %} - {% set url = app.getAuthenticator().isAuthenticated() ? record_subdef_url(thumbnail.get_record(), "preview") : thumbnail.get_permalink().get_url() %} - {% endif %} - {% set fit_size = fitIn( {"width":original_w, "height":original_h}, {"width":box_w, "height":box_h} From 8a29dc8c0d516c296e84db69ee7180e192b260f2 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Wed, 6 Oct 2021 11:44:09 +0200 Subject: [PATCH 14/48] PHRAS-3544_invalidate-stamp move delete to subdef class --- .../Phrasea/WorkerManager/Subscriber/RecordSubscriber.php | 5 ----- lib/classes/media/subdef.php | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php index 5075353d04..fdfe723ccb 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/RecordSubscriber.php @@ -23,7 +23,6 @@ use Alchemy\Phrasea\WorkerManager\Worker\Factory\WorkerFactoryInterface; use Alchemy\Phrasea\WorkerManager\Worker\Resolver\TypeBasedWorkerResolver; use databox; use Exception; -use media_subdef; use Symfony\Component\EventDispatcher\EventSubscriberInterface; class RecordSubscriber implements EventSubscriberInterface @@ -58,12 +57,8 @@ class RecordSubscriber implements EventSubscriberInterface $subdefs = $record->getDatabox()->get_subdef_structure()->getSubdefGroup($record->getType()); if ($subdefs !== null) { - /** @var media_subdef $subdef */ foreach ($subdefs as $subdef) { - // @unlink($subdef->getWatermarkRealPath()); - @unlink($subdef->getStampRealPath()); - $payload = [ 'message_type' => MessagePublisher::SUBDEF_CREATION_TYPE, 'payload' => [ diff --git a/lib/classes/media/subdef.php b/lib/classes/media/subdef.php index 3953ccc37e..c6028acfaa 100644 --- a/lib/classes/media/subdef.php +++ b/lib/classes/media/subdef.php @@ -251,6 +251,8 @@ class media_subdef extends media_abstract implements cache_cacheableInterface public function remove_file() { if ($this->is_physically_present() && is_writable($this->getRealPath())) { + // @unlink($this->getWatermarkRealPath()); + @unlink($this->getStampRealPath()); unlink($this->getRealPath()); $this->delete_data_from_cache(); From 569a1b4dcfe8bb433c67f1b335bf325bc89bb381 Mon Sep 17 00:00:00 2001 From: aynsix Date: Wed, 6 Oct 2021 14:53:26 +0300 Subject: [PATCH 15/48] fix --- lib/classes/media/subdef.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/classes/media/subdef.php b/lib/classes/media/subdef.php index d151b71c94..7f4f2739bd 100644 --- a/lib/classes/media/subdef.php +++ b/lib/classes/media/subdef.php @@ -362,10 +362,8 @@ class media_subdef extends media_abstract implements cache_cacheableInterface public function getEtag() { - $file = new SplFileInfo($this->getRealPath()); - // etag is generated from md5 of path and file last modified time - // so check if etag is to be changed - if ((!$this->etag && $this->is_physically_present()) || $this->etag != md5($file->getRealPath() . $file->getMTime())) { + if ((!$this->etag && $this->is_physically_present())) { + $file = new SplFileInfo($this->getRealPath()); if ($file->isFile()) { $this->generateEtag($file); } @@ -576,9 +574,9 @@ class media_subdef extends media_abstract implements cache_cacheableInterface // generate a new etag after rotation $file = new SplFileInfo($this->getRealPath()); - $this->generateEtag($file); + $this->generateEtag($file); // with repository save - return $this->save(); + return $this; } /** From 8bcfe89ba38d128f243d6f03c82a40d495ac7d8a Mon Sep 17 00:00:00 2001 From: Arthur de Moulins Date: Wed, 6 Oct 2021 15:10:32 +0200 Subject: [PATCH 16/48] PHRAS-3435 add worker:heartbeat command to detect broken connections --- bin/console | 2 + config/configuration.sample.yml | 1 + docker/phraseanet/worker/entrypoint.sh | 112 ++++++++++-------- .../WorkerConfigurationServiceProvider.php | 3 +- .../Command/WorkerExecuteCommand.php | 3 +- .../Command/WorkerHeartbeatCommand.php | 45 +++++++ .../WorkerManager/Queue/AMQPConnection.php | 12 +- .../WorkerManager/Queue/HeartbeatHandler.php | 36 ++++++ lib/conf.d/configuration.yml | 1 + 9 files changed, 159 insertions(+), 56 deletions(-) create mode 100644 lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php create mode 100644 lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php diff --git a/bin/console b/bin/console index a3e0ff1ea2..9b0c8c23d8 100755 --- a/bin/console +++ b/bin/console @@ -64,6 +64,7 @@ use Alchemy\Phrasea\Command\User\UserListCommand; use Alchemy\Phrasea\Command\User\UserPasswordCommand; use Alchemy\Phrasea\Core\Version; use Alchemy\Phrasea\WorkerManager\Command\WorkerExecuteCommand; +use Alchemy\Phrasea\WorkerManager\Command\WorkerHeartbeatCommand; use Alchemy\Phrasea\WorkerManager\Command\WorkerRunServiceCommand; use Alchemy\Phrasea\WorkerManager\Command\WorkerShowConfigCommand; @@ -171,6 +172,7 @@ $cli->command(new QuerySampleCommand()); $cli->command(new FindConceptsCommand()); $cli->command(new WorkerExecuteCommand()); +$cli->command(new WorkerHeartbeatCommand()); $cli->command(new WorkerRunServiceCommand()); $cli->command(new WorkerShowConfigCommand()); diff --git a/config/configuration.sample.yml b/config/configuration.sample.yml index 853284738b..3c1d98be83 100644 --- a/config/configuration.sample.yml +++ b/config/configuration.sample.yml @@ -333,6 +333,7 @@ workers: user: guest password: guest vhost: / + heartbeat: 60 queues: writeMetadatas: # this Q is "delayable" in case of record is locked ttl_retry: 1500 # overwrite 1000 ms default delay diff --git a/docker/phraseanet/worker/entrypoint.sh b/docker/phraseanet/worker/entrypoint.sh index 9501730e84..8e3b098141 100755 --- a/docker/phraseanet/worker/entrypoint.sh +++ b/docker/phraseanet/worker/entrypoint.sh @@ -2,6 +2,8 @@ set -e +HEARTBEAT_INTERVAL=20 + DOCKER_DIR="./docker/phraseanet" PHR_USER=app @@ -54,76 +56,88 @@ if [[ $BLACKFIRE_ENABLED = "true" ]]; then service blackfire-agent start echo "Blackfire setup done" else - echo `date +"%Y-%m-%d %H:%M:%S"` " - blackfire extension deactivation." - rm -f /usr/local/etc/php/conf.d/zz-blackfire.ini + echo `date +"%Y-%m-%d %H:%M:%S"` " - blackfire extension deactivation." + rm -f /usr/local/etc/php/conf.d/zz-blackfire.ini fi - rm -rf bin/run-worker.sh if [ ! -z "$PHRASEANET_SCHEDULER" ] ; then - command="bin/console task-manager:scheduler:run" - echo $command >> bin/run-worker.sh - echo "Phraseanet workers container will be launched in Scheduler mode with bin/console task-manager:scheduler:run" - + echo $command >> bin/run-worker.sh + echo "Phraseanet workers container will be launched in Scheduler mode with bin/console task-manager:scheduler:run" else - if [ ! -z "$PHRASEANET_EXPLODE_WORKER" ] && [ ${PHRASEANET_EXPLODE_WORKER} == "1" ]; then + if [ ! -z "$PHRASEANET_EXPLODE_WORKER" ] && [ ${PHRASEANET_EXPLODE_WORKER} == "1" ]; then if [ ! -z "$PHRASEANET_WORKERS_LAUNCH_METHOD" ] && [ ${PHRASEANET_WORKERS_LAUNCH_METHOD} == "supervisor" ]; then - echo "Multiples Phraseanet workers will be launched by supervisor" - for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` - do + echo "Multiples Phraseanet workers will be launched by supervisor" + for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` + do worker_job_file="$(echo $i | cut -d'_' -f3).conf" if [ ${!i} -gt "0" ] ; then envsubst < "/var/alchemy/Phraseanet/docker/phraseanet/worker/supervisor_conf.d/$worker_job_file" > /etc/supervisor/conf.d/$worker_job_file - echo "Add Worker : " $worker_job_file " -- with parallelism: (-m) " ${!i} + echo "Add worker: " $worker_job_file " -- with parallelism: (-m) " ${!i} else - echo "NO Worker define for : " $worker_job_file " -- because parallelism (-m) is set to : " ${!i} + echo "No worker defined for: " $worker_job_file " -- because parallelism (-m) is set to : " ${!i} fi - done - command="/usr/bin/supervisord -n -c /etc/supervisor/supervisor.conf" - echo $command >> bin/run-worker.sh - PHR_USER=root + done + command="/usr/bin/supervisord -n -c /etc/supervisor/supervisor.conf" + echo $command >> bin/run-worker.sh + PHR_USER=root else - echo "Multiples Phraseanet workers will be launched with bin/console worker:execute" - NBR_WORKERS=0 - for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` - do + echo "Multiples Phraseanet workers will be launched with bin/console worker:execute" + NBR_WORKERS=0 + + echo "bin/console worker:heartbeat --heartbeat ${HEARTBEAT_INTERVAL} &" >> bin/run-worker.sh + + for i in `env | grep PHRASEANET_WORKER_ | cut -d'=' -f1` + do queue_name="$(echo $i | cut -d'_' -f3)" m=$i if [ ${!m} -gt "0" ] ; then - command="bin/console worker:execute --queue-name=$queue_name -m ${!m} &" - echo $command >> bin/run-worker.sh - echo "Worker " $queue_name " define with parallelism " ${!m} - NBR_WORKERS=$(expr $NBR_WORKERS + 1) - else - echo " NO Worker defined for : " $m " -- because parallelism (-m) is set to : " ${!m} - fi - done - - echo $NBR_WORKERS " workers define" - echo $NBR_WORKERS > bin/workers_count.txt - chown root:app bin/workers_count.txt - chmod 760 bin/workers_count.txt - echo ' - NBR_WORKERS=$(< bin/workers_count.txt) - WORKER_LOOP_VALUE=20s - while true; - do - sleep $WORKER_LOOP_VALUE - nb_process=`ps faux | grep "worker:execute" | grep php | wc -l` - date_time_process=`date +"%Y-%m-%d %H:%M:%S"` - echo $date_time_process "-" $nb_process "running workers" - if [ $nb_process -lt $NBR_WORKERS ] - then - exit 1 - break + command="bin/console worker:execute --queue-name=$queue_name -m ${!m} &" + echo $command >> bin/run-worker.sh + echo "Worker " $queue_name " defined with parallelism " ${!m} + NBR_WORKERS=$(expr $NBR_WORKERS + 1) + else + echo "No worker defined for: " $m " -- because parallelism (-m) is set to : " ${!m} fi - done ' >> bin/run-worker.sh - fi + done + + echo $NBR_WORKERS " workers defined" + echo $NBR_WORKERS > bin/workers_count.txt + chown root:app bin/workers_count.txt + chmod 760 bin/workers_count.txt + echo "HEARTBEAT_INTERVAL=${HEARTBEAT_INTERVAL}" >> bin/run-worker.sh + echo ' +NBR_WORKERS=$(< bin/workers_count.txt) +sleep 1 # let worker:heartbeat fail before process check + +function check() { + nb_process=`ps faux | grep "worker:execute" | grep php | wc -l` + nb_heartbeat=`ps faux | grep "worker:heartbeat" | grep php | wc -l` + date_time_process=`date +"%Y-%m-%d %H:%M:%S"` + echo $date_time_process "-" $nb_process "running workers" + if [ $nb_process -lt $NBR_WORKERS ]; then + echo "One or more worker:execute is not running, exiting..." + exit 1 + elif [ $nb_heartbeat -lt 1 ]; then + echo "worker:heartbeat is not running, exiting..." + exit 1 + fi +} + +# early check +check + +while true; do + sleep ${HEARTBEAT_INTERVAL}s + check +done' >> bin/run-worker.sh + fi else command="bin/console worker:execute" echo $command >> bin/run-worker.sh fi fi + runuser -u $PHR_USER -- $@ diff --git a/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php index 9931d4649f..37c137cfc1 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/WorkerConfigurationServiceProvider.php @@ -37,7 +37,8 @@ class WorkerConfigurationServiceProvider implements ServiceProviderInterface 'port' => 5672, 'user' => 'guest', 'password' => 'guest', - 'vhost' => '/' + 'vhost' => '/', + 'heartbeat' => 60, ] ]; diff --git a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php index b03e716fc0..fad30a381c 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerExecuteCommand.php @@ -94,6 +94,7 @@ class WorkerExecuteCommand extends Command } $serverConnection->connectionClose(); - } + return 0; + } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php new file mode 100644 index 0000000000..ef6b923839 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerHeartbeatCommand.php @@ -0,0 +1,45 @@ +setDescription('Heartbeat connection to track drops or broken pipes') + ->addOption('heartbeat', null, InputOption::VALUE_REQUIRED, sprintf('in seconds (default: %d)', self::DEFAULT_INTERVAL)) + ; + + return $this; + } + + protected function doExecute(InputInterface $input, OutputInterface $output) + { + /** @var AMQPConnection $serverConnection */ + $serverConnection = $this->container['alchemy_worker.amqp.connection']; + + $connection = $serverConnection->getConnection(); + + $interval = $input->getOption('heartbeat'); + if (empty($interval)) { + $interval = self::DEFAULT_INTERVAL; + } + + $heartbeatHandler = new HeartbeatHandler($connection); + $heartbeatHandler->run($interval); + + return 0; + } +} diff --git a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php index 2eded6bf6f..5489098d42 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php @@ -141,7 +141,8 @@ class AMQPConnection 'ssl' => false, 'user' => 'guest', 'password' => 'guest', - 'vhost' => '/' + 'vhost' => '/', + 'heartbeat' => 60, ]; $this->hostConfig = $conf->get(['workers', 'queue', 'worker-queue'], $defaultConfiguration); @@ -311,10 +312,12 @@ class AMQPConnection { if (!isset($this->connection)) { try { + $heartbeat = $this->hostConfig['heartbeat'] ?? 60; + // if we are in ssl connection type if (isset($this->hostConfig['ssl']) && $this->hostConfig['ssl'] === true) { $sslOptions = [ - 'verify_peer' => true + 'verify_peer' => true, ]; $this->connection = new AMQPSSLConnection( @@ -325,7 +328,7 @@ class AMQPConnection $this->hostConfig['vhost'], $sslOptions, [ - 'heartbeat' => 60, + 'heartbeat' => $heartbeat, ] ); } else { @@ -343,7 +346,7 @@ class AMQPConnection 3.0, null, false, - 60 + $heartbeat ); } } @@ -437,7 +440,6 @@ class AMQPConnection break; default: throw new Exception(sprintf('undefined q type "%s', $queueName)); - break; } return $this->channel; diff --git a/lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php b/lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php new file mode 100644 index 0000000000..aa07b47767 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/HeartbeatHandler.php @@ -0,0 +1,36 @@ +connection = $connection; + } + + /** + * @param int $interval + */ + public function run($interval) + { + while (true) { + if (!$this->connection->isConnected()) { + return; + } + + sleep((int) $interval / 2); + + $this->connection->checkHeartBeat(); + } + } +} diff --git a/lib/conf.d/configuration.yml b/lib/conf.d/configuration.yml index e7f680e581..7cf86d71d0 100644 --- a/lib/conf.d/configuration.yml +++ b/lib/conf.d/configuration.yml @@ -334,6 +334,7 @@ workers: user: '' password: '' vhost: / + heartbeat: 60 externalservice: ginger: From bdbd5aedf41b25694da323e1180bf2c4c88fa9a0 Mon Sep 17 00:00:00 2001 From: Nicolas Maillat Date: Fri, 8 Oct 2021 10:24:37 +0200 Subject: [PATCH 17/48] ca-certificates update --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1c1a565ff8..a38ddb6048 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN echo "deb http://deb.debian.org/debian stretch main non-free" > /etc/apt/sou && wget -O certs.deb http://ftp.fr.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20210119_all.deb \ && dpkg --fsys-tarfile certs.deb | tar -xOf - ./usr/share/ca-certificates/mozilla/ISRG_Root_X1.crt > /usr/local/share/ca-certificates/ISRG_Root_X1.crt \ && rm -rf /usr/share/ca-certificates/mozilla/DST_Root_CA_X3.crt \ - && update-ca-certificates \ + && update-ca-certificates --fresh \ && apt-get update \ && apt-get install -y --no-install-recommends \ zlib1g-dev \ From 47d7c6e8d78a3617a57e1f081b6411f5b17b47a0 Mon Sep 17 00:00:00 2001 From: aynsix Date: Mon, 11 Oct 2021 15:58:44 +0300 Subject: [PATCH 18/48] PHRAS-3528 fix download zip loop --- templates/web/common/menubar.html.twig | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/templates/web/common/menubar.html.twig b/templates/web/common/menubar.html.twig index d8021de270..cc528743eb 100644 --- a/templates/web/common/menubar.html.twig +++ b/templates/web/common/menubar.html.twig @@ -416,10 +416,13 @@ }); }); - /**manage session and redirect to login page**/ - var usr_id = '{{ app.getAuthenticator().user.getId }}'; - var module = '{{ module }}'; + // no need to launch pollNotifications if not connected on load page + {% if app.getAuthenticator().isAuthenticated() %} + /**manage session and redirect to login page**/ + var usr_id = '{{ app.getAuthenticator().user.getId }}'; + var module = '{{ module }}'; - // start pooling recursively - window.setTimeout( function() { commonModule.pollNotifications(usr_id, module === 'prod', true); }, 2000); + // start pooling recursively + window.setTimeout( function() { commonModule.pollNotifications(usr_id, module === 'prod', true); }, 2000); + {% endif %} From 961ccb91d5c6e7afced12f08dbc3c4aca233c88f Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 14:59:59 +0200 Subject: [PATCH 19/48] fix mysql install error --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 358dbaf975..36889cb3aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,7 +31,7 @@ jobs: # This branch if available - v1-dep-{{ .Branch }}- # Default branch if not - - v1-dep-master- + - v2-dep-master- # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly - v1-dep- # This is based on your 1.0 configuration file or project settings @@ -65,7 +65,7 @@ jobs: background: true # Save dependency cache - save_cache: - key: v1-dep-{{ .Branch }}-{{ epoch }} + key: v2-dep-{{ .Branch }}-{{ epoch }} paths: # This is a broad list of cache paths to include many possible development environments # You can probably delete some of these entries From 89fcc0f63bd110a65c339709a273b65839da8a26 Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 15:03:31 +0200 Subject: [PATCH 20/48] fix mysql install error --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 36889cb3aa..090dbf6f75 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,11 +29,11 @@ jobs: - restore_cache: keys: # This branch if available - - v1-dep-{{ .Branch }}- + - v2-dep-{{ .Branch }}- # Default branch if not - v2-dep-master- # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly - - v1-dep- + - v2-dep- # This is based on your 1.0 configuration file or project settings - run: echo 127.0.0.1 redis elasticsearch db rabbitmq | sudo tee -a /etc/hosts - run: git clone https://github.com/alanxz/rabbitmq-c From 3d84a984e0c8c928248f6899ede39f1c0dcb0824 Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 15:18:35 +0200 Subject: [PATCH 21/48] fix missing mysql server on image --- .circleci/config.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 090dbf6f75..f39b68f9bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,12 +41,10 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: yes '' | pecl install amqp-1.9.3 - - run: yes '' | pecl install imagick - - run: sudo apt-get install libzmq-dev - - run: yes '' | pecl install zmq-beta - - run: echo "extension = amqp.so" > /opt/circleci/php/$(phpenv global)/etc/conf.d/amqp.ini - - run: echo "extension = zmq.so" > /opt/circleci/php/$(phpenv global)/etc/conf.d/zmq.ini + - run: sudo apt -y install php7.0-amqp + - run: sudo apt -y install php7.0-imagick + - run: sudo apt -y install libzmq-dev + - run: sudo apt -y install php7.0-zmq - run: echo "date.timezone = UTC" > /opt/circleci/php/$(phpenv global)/etc/conf.d/timezone.ini - run: sed -i 's/^\(session.cache_limiter = \).*/\1""/' /opt/circleci/php/$(phpenv global)/etc/php.ini - run: npm rebuild node-sass @@ -55,9 +53,6 @@ jobs: # This is based on your 1.0 configuration file or project settings - run: node -v - run: npm -v - - run: npm install -g yarn - - run: rm ~/.yarn/bin/yarn - - run: ln -s /opt/circleci/nodejs/v10.12.0/bin/yarn ~/.yarn/bin/yarn - run: yarn install - run: if [[ ! -e elasticsearch-2.3.3 ]]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.3.3.tar.gz && tar -xvf elasticsearch-2.3.3.tar.gz && elasticsearch-2.3.3/bin/plugin install analysis-icu; fi - run: From 527200639c48db217116a8a36a8f8b53258564dd Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 15:21:14 +0200 Subject: [PATCH 22/48] fix missing mysql server on image --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f39b68f9bd..38a92021d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,6 @@ jobs: #- run: pecl channel-update pear.php.net - run: sudo apt -y install php7.0-amqp - run: sudo apt -y install php7.0-imagick - - run: sudo apt -y install libzmq-dev - run: sudo apt -y install php7.0-zmq - run: echo "date.timezone = UTC" > /opt/circleci/php/$(phpenv global)/etc/conf.d/timezone.ini - run: sed -i 's/^\(session.cache_limiter = \).*/\1""/' /opt/circleci/php/$(phpenv global)/etc/php.ini From f0dabad541c047b771a85569e7ace0155413027f Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 15:34:55 +0200 Subject: [PATCH 23/48] fix missing mysql server on image --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 38a92021d2..33a07579c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,8 +44,8 @@ jobs: - run: sudo apt -y install php7.0-amqp - run: sudo apt -y install php7.0-imagick - run: sudo apt -y install php7.0-zmq - - run: echo "date.timezone = UTC" > /opt/circleci/php/$(phpenv global)/etc/conf.d/timezone.ini - - run: sed -i 's/^\(session.cache_limiter = \).*/\1""/' /opt/circleci/php/$(phpenv global)/etc/php.ini + - run: echo "date.timezone = UTC" > /etc/php/7.0/mods-available/timezone.ini + - run: sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini - run: npm rebuild node-sass # This is based on your 1.0 configuration file or project settings - run: composer install --no-progress --no-interaction --optimize-autoloader From c5217efec116c30c0b365c79d67ed4031f401c85 Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 15:37:28 +0200 Subject: [PATCH 24/48] fix missing mysql server on image --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 33a07579c0..e496a829a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,8 +44,8 @@ jobs: - run: sudo apt -y install php7.0-amqp - run: sudo apt -y install php7.0-imagick - run: sudo apt -y install php7.0-zmq - - run: echo "date.timezone = UTC" > /etc/php/7.0/mods-available/timezone.ini - - run: sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini + - run: sudo echo "date.timezone = UTC" > /etc/php/7.0/mods-available/timezone.ini + - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini - run: npm rebuild node-sass # This is based on your 1.0 configuration file or project settings - run: composer install --no-progress --no-interaction --optimize-autoloader From cb02ee45e75d4dea9061932f5f99bdc928f9938b Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 15:42:37 +0200 Subject: [PATCH 25/48] fix missing mysql server on image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e496a829a2..c7bd49bb2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ jobs: - run: sudo apt -y install php7.0-amqp - run: sudo apt -y install php7.0-imagick - run: sudo apt -y install php7.0-zmq - - run: sudo echo "date.timezone = UTC" > /etc/php/7.0/mods-available/timezone.ini + - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini - run: npm rebuild node-sass # This is based on your 1.0 configuration file or project settings From 498e63e6fdbf7147d7e7c4383ffc2f968713a97c Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 16:04:38 +0200 Subject: [PATCH 26/48] fix missing mysql server on image --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c7bd49bb2f..64e12832b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,9 +41,8 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: sudo apt -y install php7.0-amqp - - run: sudo apt -y install php7.0-imagick - - run: sudo apt -y install php7.0-zmq + - run: sudo apt -y install php7.0-amqp php7.0-imagick php7.0-zmq + - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini - run: npm rebuild node-sass From fb2f6009ca5a607653c6070cf670b381cd5b9aab Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 16:28:33 +0200 Subject: [PATCH 27/48] fix missing mysql server on image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 64e12832b4..d9315717f5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: sudo apt -y install php7.0-amqp php7.0-imagick php7.0-zmq + - run: sudo apt -y install php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini From 987d1a44b5a37e9b7a3f0eef3aa317fc268fe85c Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 16:44:43 +0200 Subject: [PATCH 28/48] fix missing mysql server on image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9315717f5..323192a86b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: sudo apt -y install php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl + - run: sudo apt -y install python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini From 28fe445a2a1d89a6fc5bb1621d1ff0c752321beb Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 18:28:27 +0200 Subject: [PATCH 29/48] add nvm to fix node version --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 323192a86b..52b4678d94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,6 +48,8 @@ jobs: - run: npm rebuild node-sass # This is based on your 1.0 configuration file or project settings - run: composer install --no-progress --no-interaction --optimize-autoloader + - run: curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash + - run: source ~/.profile && nvm install v10.12.0 && nvm alias default v10.12.0 # This is based on your 1.0 configuration file or project settings - run: node -v - run: npm -v From a900c9d13e40ff6f0866f4257a34d74ceb1418db Mon Sep 17 00:00:00 2001 From: Moctar Date: Mon, 11 Oct 2021 18:33:58 +0200 Subject: [PATCH 30/48] add openjdk for elasticsearch --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 52b4678d94..ef0ec3d921 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: sudo apt -y install python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl + - run: sudo apt -y install openjdk-11-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini From 062fff88c6b45a734981ce06aa24a53789a7aba1 Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 11:05:09 +0200 Subject: [PATCH 31/48] add mysql phraseanet user fix openjdk to 8 --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef0ec3d921..7e151f9b57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: sudo apt -y install openjdk-11-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl + - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl php7.0-sqlite3 - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini @@ -77,10 +77,10 @@ jobs: - node_modules - ~/.composer # This is based on your 1.0 configuration file or project settings - - run: mysql -u root -e 'CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;'; + - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'127.0.0.1' IDENTIFIED BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" # This is based on your 1.0 configuration file or project settings - run: ./bin/developer system:uninstall -v - - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=root --db-template=fr-simple --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; + - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=phraseanet --db-template=fr-simple --db-password=phraseanet --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; - run: ./bin/developer ini:setup-tests-dbs -v - run: ./bin/console searchengine:index:create -v - run: ./bin/developer phraseanet:regenerate-sqlite -v From 52aa4fa70672bd24a63ac48706c71f41d4668f65 Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 11:43:03 +0200 Subject: [PATCH 32/48] add mysql phraseanet user fix openjdk to 8 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e151f9b57..a95afb6480 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,10 +77,10 @@ jobs: - node_modules - ~/.composer # This is based on your 1.0 configuration file or project settings - - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'127.0.0.1' IDENTIFIED BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" + - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'localhost' IDENTIFIED BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" # This is based on your 1.0 configuration file or project settings - run: ./bin/developer system:uninstall -v - - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=phraseanet --db-template=fr-simple --db-password=phraseanet --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; + - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=localhost --db-user=phraseanet --db-template=fr-simple --db-password=phraseanet --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; - run: ./bin/developer ini:setup-tests-dbs -v - run: ./bin/console searchengine:index:create -v - run: ./bin/developer phraseanet:regenerate-sqlite -v From aa30995ffe68e3b13a3f97a8b64f937ed10ae068 Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 11:51:19 +0200 Subject: [PATCH 33/48] mysql fix mysql_native_password matter --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a95afb6480..8f561d683d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,7 +77,7 @@ jobs: - node_modules - ~/.composer # This is based on your 1.0 configuration file or project settings - - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'localhost' IDENTIFIED BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" + - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'localhost' IDENTIFIED WITH mysql_native_password BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" # This is based on your 1.0 configuration file or project settings - run: ./bin/developer system:uninstall -v - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=localhost --db-user=phraseanet --db-template=fr-simple --db-password=phraseanet --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; From 4a5cd7bd5e930ed2597013b123af62e1f23058e6 Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 13:54:28 +0200 Subject: [PATCH 34/48] mysql fix struct modes --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f561d683d..b48849858b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,6 +45,7 @@ jobs: - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini + - run: sudo -- bash -c 'echo -e "[mysqld]\ndefault-authentication-plugin=mysql_native_password" > /etc/mysql/conf.d/mysql_native_password.cnf' && sudo service mysql restart - run: npm rebuild node-sass # This is based on your 1.0 configuration file or project settings - run: composer install --no-progress --no-interaction --optimize-autoloader @@ -77,7 +78,7 @@ jobs: - node_modules - ~/.composer # This is based on your 1.0 configuration file or project settings - - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'localhost' IDENTIFIED WITH mysql_native_password BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" + - run: sudo mysql -u root -e "CREATE USER 'phraseanet'@'localhost' IDENTIFIED WITH mysql_native_password BY 'phraseanet';GRANT ALL PRIVILEGES ON * . * TO 'phraseanet'@'localhost';CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;" # This is based on your 1.0 configuration file or project settings - run: ./bin/developer system:uninstall -v - run: ./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=localhost --db-user=phraseanet --db-template=fr-simple --db-password=phraseanet --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y; From 479337673041454ce75082008af680269ece9c0a Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 14:06:43 +0200 Subject: [PATCH 35/48] mysql set localhost using InstallDBs.yml --- resources/hudson/InstallDBs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/hudson/InstallDBs.yml b/resources/hudson/InstallDBs.yml index 589c2c6724..82f7e09578 100644 --- a/resources/hudson/InstallDBs.yml +++ b/resources/hudson/InstallDBs.yml @@ -1,5 +1,5 @@ database: - host: 127.0.0.1 + host: localhost port: 3306 user: phr_user password: iWvGxPE8 From f1938ca00fecbd7bc0640531f00f458a96a714fc Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 14:20:40 +0200 Subject: [PATCH 36/48] mysql set localhost and update grant on setup tests --- lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php index f79a4c34ac..eadee88f12 100644 --- a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php +++ b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php @@ -64,11 +64,15 @@ class SetupTestsDbs extends Command $db_name = StringHelper::SqlQuote($settings['database']['db_name'], StringHelper::SQL_IDENTIFIER); $this->container['orm.em']->getConnection()->executeUpdate( - 'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION' + 'CREATE USER '.$user.'@'.$host.' IDENTIFIED WITH mysql_native_password BY '.$pass.' WITH GRANT OPTION' ); $this->container['orm.em']->getConnection()->executeUpdate( - 'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION' + 'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host.' WITH GRANT OPTION' + ); + + $this->container['orm.em']->getConnection()->executeUpdate( + 'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host.' WITH GRANT OPTION' ); $this->container['orm.em']->getConnection()->executeUpdate('SET @@global.sql_mode= ""'); From 9c37484cfd9458b4ddf56cfea376c9c04179ba12 Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 14:28:18 +0200 Subject: [PATCH 37/48] mysql set localhost and update grant on setup tests --- lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php index eadee88f12..4c05d386f3 100644 --- a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php +++ b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php @@ -68,11 +68,11 @@ class SetupTestsDbs extends Command ); $this->container['orm.em']->getConnection()->executeUpdate( - 'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host.' WITH GRANT OPTION' + 'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host ); $this->container['orm.em']->getConnection()->executeUpdate( - 'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host.' WITH GRANT OPTION' + 'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host ); $this->container['orm.em']->getConnection()->executeUpdate('SET @@global.sql_mode= ""'); From d454e3bf701ca0d1c405914531ce38e657c98698 Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 14:33:34 +0200 Subject: [PATCH 38/48] mysql set localhost and update grant on setup tests --- lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php index 4c05d386f3..057de85223 100644 --- a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php +++ b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php @@ -64,7 +64,7 @@ class SetupTestsDbs extends Command $db_name = StringHelper::SqlQuote($settings['database']['db_name'], StringHelper::SQL_IDENTIFIER); $this->container['orm.em']->getConnection()->executeUpdate( - 'CREATE USER '.$user.'@'.$host.' IDENTIFIED WITH mysql_native_password BY '.$pass.' WITH GRANT OPTION' + 'CREATE USER '.$user.'@'.$host.' IDENTIFIED WITH mysql_native_password BY '.$pass ); $this->container['orm.em']->getConnection()->executeUpdate( From 8c17f3199632437453410642396efe5498eb94fc Mon Sep 17 00:00:00 2001 From: Moctar Date: Tue, 12 Oct 2021 14:56:59 +0200 Subject: [PATCH 39/48] tests - unifiy mysql test user --- lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php | 4 ++-- resources/hudson/InstallDBs.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php index 057de85223..5f2d68a94a 100644 --- a/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php +++ b/lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php @@ -62,7 +62,7 @@ class SetupTestsDbs extends Command $ab_name = StringHelper::SqlQuote($settings['database']['ab_name'], StringHelper::SQL_IDENTIFIER); $db_name = StringHelper::SqlQuote($settings['database']['db_name'], StringHelper::SQL_IDENTIFIER); - +/* $this->container['orm.em']->getConnection()->executeUpdate( 'CREATE USER '.$user.'@'.$host.' IDENTIFIED WITH mysql_native_password BY '.$pass ); @@ -74,7 +74,7 @@ class SetupTestsDbs extends Command $this->container['orm.em']->getConnection()->executeUpdate( 'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host ); - +*/ $this->container['orm.em']->getConnection()->executeUpdate('SET @@global.sql_mode= ""'); return 0; diff --git a/resources/hudson/InstallDBs.yml b/resources/hudson/InstallDBs.yml index 82f7e09578..266f5e47bb 100644 --- a/resources/hudson/InstallDBs.yml +++ b/resources/hudson/InstallDBs.yml @@ -1,7 +1,7 @@ database: host: localhost port: 3306 - user: phr_user - password: iWvGxPE8 + user: phraseanet + password: phraseanet ab_name: ab_setup_test db_name: db_setup_test From 966dbb88d32eb4be3eaf9ea18c51e41dacf56fde Mon Sep 17 00:00:00 2001 From: Moctar Date: Wed, 13 Oct 2021 15:27:14 +0200 Subject: [PATCH 40/48] tests - update imagick version --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b48849858b..5cedf54afe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,6 +43,7 @@ jobs: #- run: pecl channel-update pear.php.net - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl php7.0-sqlite3 - run: printf "\n" | sudo pecl install amqp-1.11.0beta + - run: yes '' | pecl install imagick - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini - run: sudo -- bash -c 'echo -e "[mysqld]\ndefault-authentication-plugin=mysql_native_password" > /etc/mysql/conf.d/mysql_native_password.cnf' && sudo service mysql restart From 246ca5d30eabd52d7fa5afc51b2cb93dbfc98d59 Mon Sep 17 00:00:00 2001 From: Moctar Date: Wed, 13 Oct 2021 15:30:43 +0200 Subject: [PATCH 41/48] tests - update imagick version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5cedf54afe..07ab110210 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: #- run: pecl channel-update pear.php.net - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl php7.0-sqlite3 - run: printf "\n" | sudo pecl install amqp-1.11.0beta - - run: yes '' | pecl install imagick + - run: yes '' | sudo pecl install imagick - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini - run: sudo -- bash -c 'echo -e "[mysqld]\ndefault-authentication-plugin=mysql_native_password" > /etc/mysql/conf.d/mysql_native_password.cnf' && sudo service mysql restart From 900e87d88b6e3544448b29dd560f83d680b50584 Mon Sep 17 00:00:00 2001 From: Moctar Date: Wed, 13 Oct 2021 15:46:26 +0200 Subject: [PATCH 42/48] tests - update imagick version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07ab110210..e58dcaa81a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl php7.0-sqlite3 + - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl php7.0-sqlite3 libmagickwand-dev libmagickcore-dev - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: yes '' | sudo pecl install imagick - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" From fcd3122167995ad3909218679e1d29a910e4c137 Mon Sep 17 00:00:00 2001 From: Moctar Date: Wed, 13 Oct 2021 15:50:54 +0200 Subject: [PATCH 43/48] tests - update imagick version --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e58dcaa81a..57acf0b7f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,8 +42,8 @@ jobs: # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl php7.0-sqlite3 libmagickwand-dev libmagickcore-dev - - run: printf "\n" | sudo pecl install amqp-1.11.0beta - - run: yes '' | sudo pecl install imagick + - run: printf "\n" | sudo pecl install amqp-1.11.0beta + - run: printf "\n" | sudo pecl install imagick - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" - run: sudo sed -i 's/^\(session.cache_limiter = \).*/\1""/' /etc/php/7.0/cli/php.ini - run: sudo -- bash -c 'echo -e "[mysqld]\ndefault-authentication-plugin=mysql_native_password" > /etc/mysql/conf.d/mysql_native_password.cnf' && sudo service mysql restart From c9aa1c918b75a271ed2d51e66541c8dc8e300dcd Mon Sep 17 00:00:00 2001 From: Moctar Date: Wed, 13 Oct 2021 16:34:09 +0200 Subject: [PATCH 44/48] tests - update imagick version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 57acf0b7f3..b42ab416b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ jobs: - run: cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install # disabled because pear.php.net is down cause of security failure #- run: pecl channel-update pear.php.net - - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-imagick php7.0-zmq php7.0-intl php7.0-sqlite3 libmagickwand-dev libmagickcore-dev + - run: sudo apt -y install openjdk-8-jre-headless python-dev-is-python2 php7.0-amqp php7.0-zmq php7.0-intl php7.0-sqlite3 libmagickwand-dev libmagickcore-dev - run: printf "\n" | sudo pecl install amqp-1.11.0beta - run: printf "\n" | sudo pecl install imagick - run: sudo -- bash -c "echo 'date.timezone = UTC' > /etc/php/7.0/mods-available/timezone.ini" From 8025fa027ce75ba9fcd949aa311c1cd13849ff69 Mon Sep 17 00:00:00 2001 From: aynsix Date: Fri, 15 Oct 2021 10:51:29 +0300 Subject: [PATCH 45/48] test --- lib/classes/ACL.php | 6 +++--- tests/classes/ACLTest.php | 2 +- tests/classes/record/adapterTest.php | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/classes/ACL.php b/lib/classes/ACL.php index 500f8457b8..8a42182467 100644 --- a/lib/classes/ACL.php +++ b/lib/classes/ACL.php @@ -1666,7 +1666,7 @@ class ACL implements cache_cacheableInterface $lim_max = $this->_limited[$base_id]['dmax'] && $this->_limited[$base_id]['dmax'] < $datetime; - return $lim_max || $lim_min; + return ($lim_max || $lim_min); } /** @@ -1697,8 +1697,8 @@ class ACL implements cache_cacheableInterface ':time_limited' => $limit ? 1 : 0, ':usr_id' => $this->user->getId(), ':base_id' => $base_id, - ':limited_from' => NullableDateTime::format($limit_from, DATE_ISO8601), - ':limited_to' => NullableDateTime::format($limit_to, DATE_ISO8601), + ':limited_from' => NullableDateTime::format($limit_from), + ':limited_to' => NullableDateTime::format($limit_to), ]); $stmt->closeCursor(); diff --git a/tests/classes/ACLTest.php b/tests/classes/ACLTest.php index 816a9bc171..3eb01ef94e 100644 --- a/tests/classes/ACLTest.php +++ b/tests/classes/ACLTest.php @@ -777,8 +777,8 @@ class ACLTest extends \PhraseanetTestCase $plusone = new DateTime('+1 day'); $this->object->set_limits($base_id, true, $minusone, $plusone); $limits = $this->object->get_limits($base_id); - $this->assertEquals($limits['dmin'], $minusone); $this->assertEquals($limits['dmax'], $plusone); + $this->assertEquals($limits['dmin'], $minusone); $minustwo = new DateTime('-2 day'); $plustwo = new DateTime('-2 day'); $this->object->set_limits($base_id, true, $minustwo, $plustwo); diff --git a/tests/classes/record/adapterTest.php b/tests/classes/record/adapterTest.php index 4405b055e3..cb5163ffa8 100644 --- a/tests/classes/record/adapterTest.php +++ b/tests/classes/record/adapterTest.php @@ -407,7 +407,7 @@ class record_adapterTest extends \PhraseanetAuthenticatedTestCase $separator = ''; } - $multi_imploded = implode(' ' . $separator . ' ', ['un', 'jeu', 'de', 'test']); + $multi_imploded = implode(' ' . $separator . ' ', ['test', 'de', 'je', 'un']); if ($meta_el->is_multi()) { $initial_values = []; From 38306ae460bea18c25fcb42709aed1b31e733692 Mon Sep 17 00:00:00 2001 From: aynsix Date: Fri, 15 Oct 2021 11:04:09 +0300 Subject: [PATCH 46/48] test --- tests/classes/record/adapterTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/classes/record/adapterTest.php b/tests/classes/record/adapterTest.php index cb5163ffa8..83e52146c7 100644 --- a/tests/classes/record/adapterTest.php +++ b/tests/classes/record/adapterTest.php @@ -407,7 +407,7 @@ class record_adapterTest extends \PhraseanetAuthenticatedTestCase $separator = ''; } - $multi_imploded = implode(' ' . $separator . ' ', ['test', 'de', 'je', 'un']); + $multi_imploded = implode(' ' . $separator . ' ', ['test', 'de', 'jeu', 'un']); if ($meta_el->is_multi()) { $initial_values = []; From 0d3cfe43044b991f7f1f112813a9d2cf62e453b7 Mon Sep 17 00:00:00 2001 From: nmaillat Date: Fri, 15 Oct 2021 19:18:38 +0200 Subject: [PATCH 47/48] PHRAS-3551 split dc for RabbitMQ and env for mailhog mapping --- .env | 23 ++++++++++++++++++++--- docker-compose.mailhog.yml | 7 ++++--- docker-compose.rabbitmq.yml | 12 ++++++++++++ docker-compose.yml | 5 +++++ docker/phraseanet/entrypoint.sh | 13 +++++++++++++ 5 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 docker-compose.rabbitmq.yml diff --git a/.env b/.env index f6c87571a3..017cfce4ea 100644 --- a/.env +++ b/.env @@ -1,9 +1,9 @@ -# For dev purpose COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.mailhog.yml:docker-compose.pma.yml:docker-compose.override.yml -# Dev tools and utilities: docker-compose.elk.yml:docker-compose.ftp.yml:docker-compose.squid.yml +# For dev purpose COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.rabbitmq.yml:docker-compose.mailhog.yml:docker-compose.pma.yml:docker-compose.override.yml +# Dev tools and utilities add : docker-compose.elk.yml:docker-compose.ftp.yml:docker-compose.squid.yml # For old Phraseanet scheduler launching use : docker-compose.scheduler.yml -COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.mailhog.yml +COMPOSE_FILE=docker-compose.yml:docker-compose.db.yml:docker-compose.rabbitmq.yml:docker-compose.mailhog.yml # Registry from where you pull Docker images PHRASEANET_DOCKER_REGISTRY=local @@ -29,11 +29,13 @@ PHRASEANET_UPGRADE=0 # --------------- RabbitMQ SETTING ---------------------- + RABBITMQ_DEFAULT_USER=alchemy RABBITMQ_DEFAULT_PASS=vdh4dpe5Wy3R RABBITMQ_MANAGEMENT_PORT=10811 + # --------------- GATEWAY SETTING (nginx) ----------------------- GATEWAY_SEND_TIMEOUT=120 @@ -121,6 +123,15 @@ PHRASEANET_SWFTOOLS_TIMEOUT=120 PHRASEANET_UNOCON_TIMEOUT=120 PHRASEANET_EXIFTOOL_TIMEOUT=120 +# RabbitMQ parameters + +PHRASEANET_RABBITMQ_HOST=rabbitmq +PHRASEANET_RABBITMQ_PORT=5672 +PHRASEANET_RABBITMQ_SSL=false +PHRASEANET_RABBITMQ_VHOST=/ +PHRASEANET_RABBITMQ_HEARTBEAT=30 + + # network : comma separated list of IP ou SUBNETS PHRASEANET_TRUSTED_PROXIES= PHRASEANET_DEBUG_ALLOWED_IP= @@ -235,6 +246,12 @@ IMAGEMAGICK_POLICY_TEMPORARY_PATH=/tmp # PhpMyAdmin port PHRASEANET_PHPMYADMIN_PORT=8089 + +# mailhog web GUI port mapping + +MAILHOG_GUI_PORT=8025 + + # Xdebug XDEBUG_ENABLED=0 XDEBUG_PROFILER_ENABLED=0 diff --git a/docker-compose.mailhog.yml b/docker-compose.mailhog.yml index 0f02183fe2..d01cc95055 100644 --- a/docker-compose.mailhog.yml +++ b/docker-compose.mailhog.yml @@ -4,7 +4,8 @@ services: mailhog: image: mailhog/mailhog ports: - - 1025:1025 - - 8025:8025 + - ${MAILHOG_GUI_PORT}:8025 + environment: + - MAILHOG_GUI_PORT networks: - - internal + - internal \ No newline at end of file diff --git a/docker-compose.rabbitmq.yml b/docker-compose.rabbitmq.yml new file mode 100644 index 0000000000..71c894e962 --- /dev/null +++ b/docker-compose.rabbitmq.yml @@ -0,0 +1,12 @@ +version: "3.4" + +services: + + rabbitmq: + image: rabbitmq:3.6.16-management + restart: on-failure + environment: + - RABBITMQ_DEFAULT_USER + - RABBITMQ_DEFAULT_PASS + networks: + - internal diff --git a/docker-compose.yml b/docker-compose.yml index a6d1295309..645d174542 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,11 @@ services: - PHRASEANET_SERVER_NAME - PHRASEANET_AVAILABLE_LANGUAGE - PHRASEANET_DEFAULT_LANGUAGE + - PHRASEANET_RABBITMQ_HOST + - PHRASEANET_RABBITMQ_PORT + - PHRASEANET_RABBIT_SSL + - PHRASEANET_RABBITMQ_VHOST + - PHRASEANET_RABBITMQ_HEARTBEAT - PHRASEANET_RABBITMQ_USER=$RABBITMQ_DEFAULT_USER - PHRASEANET_RABBITMQ_PASSWORD=$RABBITMQ_DEFAULT_PASS - PHRASEANET_EMITTER_EMAIL diff --git a/docker/phraseanet/entrypoint.sh b/docker/phraseanet/entrypoint.sh index ae734deaa5..ebaef86c24 100755 --- a/docker/phraseanet/entrypoint.sh +++ b/docker/phraseanet/entrypoint.sh @@ -101,6 +101,19 @@ if [[ -f "$FILE" && $PHRASEANET_SETUP = 1 ]]; then bin/setup system:config set registry.api-clients.api-require-ssl $PHRASEANET_API_SSL bin/setup system:config set registry.api-clients.api-auth-token-header-only $PHRASEANET_API_AUTH_TOKEN_HEADER_ONLY + + echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet setting RABBITMQ" + bin/setup system:config set workers.queue.worker-queue.host $PHRASEANET_RABBITMQ_HOST + bin/setup system:config set workers.queue.worker-queue.port $PHRASEANET_RABBITMQ_PORT + bin/setup system:config set workers.queue.worker-queue.vhost $PHRASEANET_RABBITMQ_VHOST + bin/setup system:config set workers.queue.worker-queue.ssl $PHRASEANET_RABBITMQ_SSL + bin/setup system:config set workers.queue.worker-queue.heartbeat $PHRASEANET_RABBITMQ_HEARTBEAT + bin/setup system:config set workers.queue.worker-queue.user $PHRASEANET_RABBITMQ_USER + bin/setup system:config set workers.queue.worker-queue.password $PHRASEANET_RABBITMQ_PASSWORD + + + + echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet setting SMTP " if [[ $PHRASEANET_SMTP_ENABLED && $PHRASEANET_SMTP_ENABLED = true ]]; then bin/setup system:config set registry.email.smtp-enabled $PHRASEANET_SMTP_ENABLED From 0230306206bfdd40d59bb42d363206c97f0851c7 Mon Sep 17 00:00:00 2001 From: nmaillat Date: Sat, 16 Oct 2021 00:04:31 +0200 Subject: [PATCH 48/48] PHRAS-3551 fix env [SKIP CI] --- docker-compose.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 645d174542..18e99eac6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,7 +76,7 @@ services: - PHRASEANET_DEFAULT_LANGUAGE - PHRASEANET_RABBITMQ_HOST - PHRASEANET_RABBITMQ_PORT - - PHRASEANET_RABBIT_SSL + - PHRASEANET_RABBITMQ_SSL - PHRASEANET_RABBITMQ_VHOST - PHRASEANET_RABBITMQ_HEARTBEAT - PHRASEANET_RABBITMQ_USER=$RABBITMQ_DEFAULT_USER @@ -200,15 +200,6 @@ services: networks: - internal - rabbitmq: - image: rabbitmq:3.6.16-management - restart: on-failure - environment: - - RABBITMQ_DEFAULT_USER - - RABBITMQ_DEFAULT_PASS - networks: - - internal - redis: image: redis restart: on-failure