diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ce9464ae5..ce2f8c9a5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: CIRCLE_TEST_REPORTS: /tmp/circleci-test-results docker: - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 - command: /sbin/init + - image: circleci/rabbitmq:3.7.7 steps: - checkout - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS @@ -24,10 +24,7 @@ jobs: command: nvm install v10.12.0 && nvm alias default v10.12.0 - run: working_directory: ~/alchemy-fr/Phraseanet - command: 'sudo service memcached status || sudo service memcached start; sudo - redis-cli ping >/dev/null 2>&1 || sudo service redis-server start; sudo - service mysql status || sudo service mysql start; sudo service rabbitmq-server - status || sudo service rabbitmq-server start; ' + command: 'sudo service mysql status || sudo service mysql start;' # Dependencies # This would typically go in either a build or a build-and-test job when using workflows # Restore the dependency cache @@ -124,7 +121,7 @@ workflows: context: "AWS London" create-repo: true dockerfile: Dockerfile - extra-build-args: "--target phraseanet" + extra-build-args: "--target phraseanet-fpm" region: AWS_DEFAULT_REGION repo: "${AWS_RESOURCE_NAME_PREFIX}/phraseanet" tag: "alpha-0.1" @@ -139,3 +136,14 @@ workflows: region: AWS_DEFAULT_REGION repo: "${AWS_RESOURCE_NAME_PREFIX}/phraseanet-nginx" tag: "alpha-0.1" + - aws-ecr/build_and_push_image: + account-url: AWS_ACCOUNT_URL + aws-access-key-id: AWS_ACCESS_KEY_ID + aws-secret-access-key: AWS_SECRET_ACCESS_KEY + context: "AWS London" + create-repo: true + dockerfile: Dockerfile + extra-build-args: "--target phraseanet-worker" + region: AWS_DEFAULT_REGION + repo: "${AWS_RESOURCE_NAME_PREFIX}/phraseanet" + tag: "alpha-0.1" diff --git a/Dockerfile b/Dockerfile index 9f1821fb6d..7ff718ab2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" \ - && php -r "if (hash_file('sha384', 'composer-setup.php') === '48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" \ + && php -r "if (hash_file('sha384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" \ && php composer-setup.php --install-dir=/usr/local/bin --filename=composer \ && php -r "unlink('composer-setup.php');" diff --git a/Vagrantfile b/Vagrantfile index a899a77ca5..f4d4ea0049 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,16 +1,5 @@ Vagrant.require_version ">= 1.5" - -class MyCustomError < StandardError - attr_reader :code - - def initialize(code) - @code = code - end - - def to_s - "[#{code} #{super}]" - end -end +require 'json' # Check to determine whether we're on a windows or linux/os-x host, # later on we use this to launch ansible in the supported way @@ -39,7 +28,16 @@ unless Vagrant.has_plugin?('vagrant-hostmanager') raise "vagrant-hostmanager is not installed! Please run\n vagrant plugin install vagrant-hostmanager\n\n" end -if ARGV[1] == '--provision' +# Check to determine if box_meta JSON is present +# if provisionned : pick name of box +if File.file?(".vagrant/machines/default/virtualbox/box_meta") + data = File.read(".vagrant/machines/default/virtualbox/box_meta") + parsed_json = JSON.parse(data) + $box = parsed_json["name"] +end + +# if not : run prompt to configure provisioning +if !File.file?(".vagrant/machines/default/virtualbox/box_meta") && ARGV[0] == 'up' print "\033[34m \nChoose a Build type :\n\n(1) Use prebuilt Phraseanet Box\n(2) Build Phraseanet from scratch (xenial)\n\033[00m" type = STDIN.gets.chomp print "\n" @@ -47,9 +45,10 @@ if ARGV[1] == '--provision' case (type) when '1' $box = "alchemy/Phraseanet-vagrant-dev_php" + $playbook = "resources/ansible/playbook-boxes.yml" when '2' $box = "ubuntu/xenial64" - print("\033[91mComplete build selected, don't forget to uncomment all roles on playbook.yml\n\n\033[00m") + $playbook = "resources/ansible/playbook.yml" else raise "\033[31mYou should specify Build type before running vagrant\n\n (Available : 1, 2)\n\n\033[00m" end @@ -133,8 +132,6 @@ else if $env == "linux" $hostIps = `ifconfig | sed -nE 's/[[:space:]]*inet ([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})(.*)$/\\1/p'`.split("\n"); else $hostIps = `resources/ansible/inventories/GetIpAdresses.cmd`; - # raise MyCustomError.new($hostIps), "HOST IP" - end end @@ -157,31 +154,28 @@ Vagrant.configure("2") do |config| ] end - config.vm.box = ($box) ? $box : "ubuntu/xenial64" - - # In case, Phraseanet box, choose the php version - # For php 7.0 use box 0.0.1 - # For php 7.1 use box 0.0.2 - #config.vm.box_version = "0.0.1" - + config.vm.box = $box config.ssh.forward_agent = true config_net(config) # If ansible is in your path it will provision from your HOST machine # If ansible is not found in the path it will be instaled in the VM and provisioned from there if which('ansible-playbook') - config.vm.provision "ansible_local" do |ansible| - ansible.playbook = "resources/ansible/playbook.yml" - ansible.limit = 'all' - ansible.verbose = 'vvv' - ansible.extra_vars = { - hostname: $hostname, - host_addresses: $hostIps, - phpversion: phpversion, - postfix: { - postfix_domain: $hostname + ".vb" + + if $playbook + config.vm.provision "ansible_local" do |ansible| + ansible.playbook = $playbook + ansible.limit = 'all' + ansible.verbose = 'vvv' + ansible.extra_vars = { + hostname: $hostname, + host_addresses: $hostIps, + phpversion: phpversion, + postfix: { + postfix_domain: $hostname + ".vb" + } } - } + end end config.vm.provision "ansible_local", run: "always" do |ansible| @@ -194,10 +188,6 @@ Vagrant.configure("2") do |config| } end else - # raise MyCustomError.new([$hostname, $phpVersion, $hostIps]), "HOST IP" - # raise MyCustomError.new($hostIps), "HOST IP" - # raise MyCustomError.new($hostIps), "HOST IP" - config.vm.provision :shell, path: "resources/ansible/windows.sh", args: [$hostname, $phpVersion, $hostIps] # config.vm.provision :shell, run: "always", path: "resources/ansible/windows-always.sh", args: ["default"] end diff --git a/cache/.gitkeep b/cache/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/composer.json b/composer.json index ca14bb88f0..47635b0e47 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ "php": ">=5.5.9", "ext-intl": "*", "alchemy-fr/tcpdf-clone": "~6.0", - "alchemy/embed-bundle": "^2.0.6", + "alchemy/embed-bundle": "^2.0.7", "alchemy/geonames-api-consumer": "~0.1.0", "alchemy/mediavorus": "^0.4.4", "alchemy/oauth2php": "1.1.0", diff --git a/composer.lock b/composer.lock index 194fd91634..eb4523a2a5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6341722edcd5bfa22e933f5411225144", + "content-hash": "f3b1fc0a30bf14b05e57ce673550d9c0", "packages": [ { "name": "alchemy-fr/tcpdf-clone", @@ -131,16 +131,16 @@ }, { "name": "alchemy/embed-bundle", - "version": "2.0.6", + "version": "2.0.7", "source": { "type": "git", "url": "https://github.com/alchemy-fr/embed-bundle.git", - "reference": "53ba295dfd0554a31c35e93902a5ef6cb8eca31a" + "reference": "c585ccf18e53a9a6f2b696ddbbc39521732dfdde" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/alchemy-fr/embed-bundle/zipball/53ba295dfd0554a31c35e93902a5ef6cb8eca31a", - "reference": "53ba295dfd0554a31c35e93902a5ef6cb8eca31a", + "url": "https://api.github.com/repos/alchemy-fr/embed-bundle/zipball/c585ccf18e53a9a6f2b696ddbbc39521732dfdde", + "reference": "c585ccf18e53a9a6f2b696ddbbc39521732dfdde", "shasum": "" }, "require-dev": { @@ -178,10 +178,10 @@ ], "description": "Embed resources bundle", "support": { - "source": "https://github.com/alchemy-fr/embed-bundle/tree/2.0.6", + "source": "https://github.com/alchemy-fr/embed-bundle/tree/2.0.7", "issues": "https://github.com/alchemy-fr/embed-bundle/issues" }, - "time": "2019-07-11T12:59:49+00:00" + "time": "2019-09-02T12:28:19+00:00" }, { "name": "alchemy/geonames-api-consumer", diff --git a/lib/Alchemy/Phrasea/Application.php b/lib/Alchemy/Phrasea/Application.php index f586a4f93a..13c7728f86 100644 --- a/lib/Alchemy/Phrasea/Application.php +++ b/lib/Alchemy/Phrasea/Application.php @@ -115,6 +115,7 @@ use Symfony\Component\Form\FormBuilderInterface; use Symfony\Component\Form\FormInterface; use Symfony\Component\Form\FormTypeInterface; use Symfony\Component\HttpFoundation\RedirectResponse; +use Symfony\Component\Process\ExecutableFinder; use Unoconv\UnoconvServiceProvider; use XPDF\PdfToText; use XPDF\XPDFServiceProvider; @@ -237,8 +238,19 @@ class Application extends SilexApplication $this->register(new UnicodeServiceProvider()); $this->register(new ValidatorServiceProvider()); - $this->register(new XPDFServiceProvider()); - $this->setupXpdf(); + + if ($this['configuration.store']->isSetup()) { + $binariesConfig = $this['conf']->get(['main', 'binaries']); + $executableFinder = new ExecutableFinder(); + $this->register(new XPDFServiceProvider(), [ + 'xpdf.configuration' => [ + 'pdftotext.binaries' => isset($binariesConfig['pdftotext_binary']) ? $binariesConfig['pdftotext_binary'] : $executableFinder->find('pdftotext'), + ] + ]); + + $this->setupXpdf(); + } + $this->register(new FileServeServiceProvider()); $this->register(new ManipulatorServiceProvider()); $this->register(new PluginServiceProvider()); @@ -653,7 +665,7 @@ class Application extends SilexApplication private function setupGeonames() { $this['geonames.server-uri'] = $this->share(function (Application $app) { - return $app['conf']->get(['registry', 'webservices', 'geonames-server'], 'http://geonames.alchemyasp.com/'); + return $app['conf']->get(['registry', 'webservices', 'geonames-server'], 'https://geonames.alchemyasp.com/'); }); } diff --git a/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php b/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php index 54c246a897..39361d46da 100644 --- a/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php +++ b/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php @@ -944,7 +944,7 @@ class V1Controller extends Controller } $originalName = $pi['filename'] . '.' . $pi['extension']; - $newPathname = $tempfile; + $uploadedFilename = $newPathname = $tempfile; } } else { @@ -956,8 +956,11 @@ class V1Controller extends Controller if (!$file->isValid()) { return $this->getBadRequestAction($request, 'Data corrupted, please try again'); } + + $uploadedFilename = $file->getPathname(); $originalName = $file->getClientOriginalName(); $newPathname = $file->getPathname() . '.' . $file->getClientOriginalExtension(); + if (false === rename($file->getPathname(), $newPathname)) { return Result::createError($request, 403, 'Error while renaming file')->createResponse(); } @@ -1010,6 +1013,11 @@ class V1Controller extends Controller $nosubdef = $request->get('nosubdefs') === '' || \p4field::isyes($request->get('nosubdefs')); $this->getBorderManager()->process($session, $Package, $callback, $behavior, $nosubdef); + // remove $newPathname on temporary directory + if ($newPathname !== $uploadedFilename) { + @rename($newPathname, $uploadedFilename); + } + $ret = ['entity' => null]; if ($output instanceof \record_adapter) { @@ -1081,6 +1089,11 @@ class V1Controller extends Controller } } + // remove $newPathname on temporary directory + if ($renamedFilename !== $uploadedFilename) { + @rename($renamedFilename, $uploadedFilename); + } + return Result::create($request, $ret)->createResponse(); } @@ -1984,7 +1997,7 @@ class V1Controller extends Controller return $this->getBadRequestAction($request); } - $datas = substr($datas, 0, ($n)) . $value . substr($datas, ($n + 2)); + $datas = substr($datas, 0, ($n)) . $value . substr($datas, ($n + 1)); } $record->setStatus(strrev($datas)); diff --git a/lib/Alchemy/Phrasea/Controller/Prod/PushController.php b/lib/Alchemy/Phrasea/Controller/Prod/PushController.php index 569537fed3..f9ce528e41 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/PushController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/PushController.php @@ -463,6 +463,8 @@ class PushController extends Controller } try { + $manager = $this->getEntityManager(); + $password = $this->getRandomGenerator()->generateString(128); $user = $this->getUserManipulator()->createUser($email, $password, $email); @@ -476,12 +478,15 @@ class PushController extends Controller $user->setCompany($request->request->get('company')); } if ($request->request->get('job')) { - $user->setCompany($request->request->get('job')); + $user->setJob($request->request->get('job')); } - if ($request->request->get('form_geonameid')) { - $this->getUserManipulator()->setGeonameId($user, $request->request->get('form_geonameid')); + if ($request->request->get('city')) { + $this->getUserManipulator()->setGeonameId($user, $request->request->get('city')); } + $manager->persist($user); + $manager->flush(); + $result['message'] = $this->app->trans('User successfully created'); $result['success'] = true; $result['user'] = $this->formatUser($user); diff --git a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php index a362540935..1e8724c0ee 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php @@ -179,7 +179,6 @@ class QueryController extends Controller }; $userManipulator->setUserSetting($user, 'last_jsonquery', (string)$request->request->get('jsQuery')); - $jsQuery = @json_decode((string)$request->request->get('jsQuery'), true); if(($ft = $findFulltext($jsQuery['query'])) !== null) { $userManipulator->setUserSetting($user, 'start_page_query', $ft); @@ -317,7 +316,7 @@ class QueryController extends Controller ' . '' - . $this->app->trans('%total% reponses', ['%total%' => ''.$result->getTotal().'']) . ''; + . $this->app->trans('%total% reponses', ['%total%' => ''.number_format($result->getTotal(),null, null, ' ').'']) . ''; $json['infos'] = $infoResult; $json['navigationTpl'] = $string; diff --git a/lib/Alchemy/Phrasea/Controller/Root/LoginController.php b/lib/Alchemy/Phrasea/Controller/Root/LoginController.php index ecb65abc47..c0a05be25f 100644 --- a/lib/Alchemy/Phrasea/Controller/Root/LoginController.php +++ b/lib/Alchemy/Phrasea/Controller/Root/LoginController.php @@ -265,7 +265,7 @@ class LoginController extends Controller return $this->render('login/register-classic.html.twig', array_merge( $this->getDefaultTemplateVariables($request), [ - 'geonames_server_uri' => str_replace(sprintf('%s:', parse_url($url, PHP_URL_SCHEME)), '', $url), + 'geonames_server_uri' => $url, 'form' => $form->createView() ])); } diff --git a/lib/Alchemy/Phrasea/Core/Configuration/RegistryFormManipulator.php b/lib/Alchemy/Phrasea/Core/Configuration/RegistryFormManipulator.php index fbf4526d05..fbce72ccc7 100644 --- a/lib/Alchemy/Phrasea/Core/Configuration/RegistryFormManipulator.php +++ b/lib/Alchemy/Phrasea/Core/Configuration/RegistryFormManipulator.php @@ -146,7 +146,7 @@ class RegistryFormManipulator ], 'webservices' => [ 'google-charts-enabled' => true, - 'geonames-server' => 'http://geonames.alchemyasp.com/', + 'geonames-server' => 'https://geonames.alchemyasp.com/', 'captchas-enabled' => false, 'recaptcha-public-key' => '', 'recaptcha-private-key' => '', diff --git a/lib/Alchemy/Phrasea/Core/Version.php b/lib/Alchemy/Phrasea/Core/Version.php index b2bf620798..0e15108724 100644 --- a/lib/Alchemy/Phrasea/Core/Version.php +++ b/lib/Alchemy/Phrasea/Core/Version.php @@ -16,7 +16,7 @@ class Version /** * @var string */ - private $number = '4.1.0-alpha.14a'; + private $number = '4.1.0-alpha.15a'; /** * @var string diff --git a/lib/Alchemy/Phrasea/Metadata/PhraseanetMetadataSetter.php b/lib/Alchemy/Phrasea/Metadata/PhraseanetMetadataSetter.php index 45d098bd11..c72c884203 100644 --- a/lib/Alchemy/Phrasea/Metadata/PhraseanetMetadataSetter.php +++ b/lib/Alchemy/Phrasea/Metadata/PhraseanetMetadataSetter.php @@ -14,6 +14,7 @@ namespace Alchemy\Phrasea\Metadata; use Alchemy\Phrasea\Border\File; use Alchemy\Phrasea\Databox\DataboxRepository; use Alchemy\Phrasea\Metadata\Tag\NoSource; +use DateTime; use PHPExiftool\Driver\Metadata\Metadata; class PhraseanetMetadataSetter @@ -66,8 +67,16 @@ class PhraseanetMetadataSetter continue; } - $data['value'] = $value; + if ($field->get_type() == 'date') { + try { + $dateTime = new DateTime($value); + $value = $dateTime->format('Y/m/d H:i:s'); + } catch (\Exception $e) { + // $value unchanged + } + } + $data['value'] = $value; $metadataInRecordFormat[] = $data; } } diff --git a/lib/Alchemy/Phrasea/Model/Repositories/BasketRepository.php b/lib/Alchemy/Phrasea/Model/Repositories/BasketRepository.php index 6304f3eaa6..ee65e0bd1e 100644 --- a/lib/Alchemy/Phrasea/Model/Repositories/BasketRepository.php +++ b/lib/Alchemy/Phrasea/Model/Repositories/BasketRepository.php @@ -212,8 +212,7 @@ class BasketRepository extends EntityRepository case self::RECEIVED: $dql = 'SELECT b FROM Phraseanet:Basket b - JOIN b.elements e - WHERE b.user = :usr_id AND b.pusher_id IS NOT NULL'; + JOIN b.elements e'; $params = [ 'usr_id' => $user->getId() ]; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/FieldKey.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/FieldKey.php index f4f8553738..293e8c69dd 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/FieldKey.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/FieldKey.php @@ -30,6 +30,11 @@ class FieldKey implements Key, QueryPostProcessor return $this->getField($context)->getIndexField($raw); } + public function getFieldType(QueryContext $context) + { + return $this->getField($context)->getType(); + } + public function isValueCompatible($value, QueryContext $context) { return ValueChecker::isValueCompatible($this->getField($context), $value); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Key.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Key.php index 20ee045a59..d664fccb94 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Key.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Key.php @@ -6,6 +6,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext; interface Key { + public function getFieldType(QueryContext $context); public function getIndexField(QueryContext $context, $raw = false); public function isValueCompatible($value, QueryContext $context); public function __toString(); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MetadataKey.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MetadataKey.php index 292b6e7237..61f9303fe0 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MetadataKey.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MetadataKey.php @@ -23,6 +23,11 @@ class MetadataKey implements Key return $this->getTag($context)->getIndexField($raw); } + public function getFieldType(QueryContext $context) + { + return $this->getTag($context)->getType(); + } + public function isValueCompatible($value, QueryContext $context) { return ValueChecker::isValueCompatible($this->getTag($context), $value); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php index e486483b71..3c2d334f48 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php @@ -52,6 +52,11 @@ class NativeKey implements Key $this->key = $key; } + public function getFieldType(QueryContext $context) + { + return $this->type; + } + public function getIndexField(QueryContext $context, $raw = false) { return $this->key; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/RangeExpression.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/RangeExpression.php index ec785354ae..17f901a77c 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/RangeExpression.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/RangeExpression.php @@ -2,18 +2,20 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue; +use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; +use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper; +use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField; use Assert\Assertion; -use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\FieldKey; -use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\Key; use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext; -use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryPostProcessor; class RangeExpression extends Node { + /** @var FieldKey */ private $key; + private $lower_bound; private $lower_inclusive; private $higher_bound; @@ -55,20 +57,34 @@ class RangeExpression extends Node public function buildQuery(QueryContext $context) { $params = array(); - if ($this->lower_bound !== null) { - $this->assertValueCompatible($this->lower_bound, $context); - if ($this->lower_inclusive) { - $params['gte'] = $this->lower_bound; - } else { - $params['gt'] = $this->lower_bound; + /** @var StructureField $field */ + // $field = $this->key->getField($context); + $lower_bound = $this->lower_bound; + $higher_bound = $this->higher_bound; + + if($this->key->getFieldType($context) === FieldMapping::TYPE_DATE) { + if($lower_bound !== null) { + $lower_bound = RecordHelper::sanitizeDate($lower_bound); + } + if($higher_bound !== null) { + $higher_bound = RecordHelper::sanitizeDate($higher_bound); } } - if ($this->higher_bound !== null) { - $this->assertValueCompatible($this->higher_bound, $context); - if ($this->higher_inclusive) { - $params['lte'] = $this->higher_bound; + + if ($lower_bound !== null) { + $this->assertValueCompatible($lower_bound, $context); + if ($this->lower_inclusive) { + $params['gte'] = $lower_bound; } else { - $params['lt'] = $this->higher_bound; + $params['gt'] = $lower_bound; + } + } + if ($higher_bound !== null) { + $this->assertValueCompatible($higher_bound, $context); + if ($this->higher_inclusive) { + $params['lte'] = $higher_bound; + } else { + $params['lt'] = $higher_bound; } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/TimestampKey.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/TimestampKey.php index b879945982..a5d34909d7 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/TimestampKey.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/TimestampKey.php @@ -34,6 +34,11 @@ class TimestampKey implements Key, Typed return FieldMapping::TYPE_DATE; } + public function getFieldType(QueryContext $context) + { + return FieldMapping::TYPE_DATE; + } + public function getIndexField(QueryContext $context, $raw = false) { return $this->index_field; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index ed9d3ceeed..63d9ec108e 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -396,10 +396,10 @@ class ElasticSearchEngine implements SearchEngineInterface if ($options->getDateFields() && ($options->getMaxDate() || $options->getMinDate())) { $range = []; if ($options->getMaxDate()) { - $range['lte'] = $options->getMaxDate()->format(FieldMapping::DATE_FORMAT_CAPTION_PHP); + $range['lte'] = $options->getMaxDate()->format('Y-m-d'); } if ($options->getMinDate()) { - $range['gte'] = $options->getMinDate()->format(FieldMapping::DATE_FORMAT_CAPTION_PHP); + $range['gte'] = $options->getMinDate()->format('Y-m-d'); } foreach ($options->getDateFields() as $dateField) { diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/FieldMapping.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/FieldMapping.php index 076034b7b1..7a60b066e6 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/FieldMapping.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/FieldMapping.php @@ -16,8 +16,7 @@ class FieldMapping const DATE_FORMAT_MYSQL = 'yyyy-MM-dd HH:mm:ss'; const DATE_FORMAT_CAPTION = 'yyyy/MM/dd'; // ES format - const DATE_FORMAT_MYSQL_OR_CAPTION = 'yyyy-MM-dd HH:mm:ss||yyyy/MM/dd'; - const DATE_FORMAT_CAPTION_PHP = 'Y/m/d'; // PHP format + const DATE_FORMAT_MYSQL_OR_CAPTION = 'yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||yyyy-MM||yyyy'; // Core types const TYPE_STRING = 'string'; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php index 6ea641492d..b8c834d9f0 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php @@ -155,15 +155,16 @@ class BulkOperation // nb: results (items) are returned IN THE SAME ORDER as commands were pushed in the stack // so the items[X] match the operationIdentifiers[X] foreach ($response['items'] as $key => $item) { - foreach($item as $command=>$result) { // command may be "index" or "delete" - if($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx error - throw new Exception(sprintf('%d: %s', $key, var_export($result, true))); + foreach ($item as $command=>$result) { // command may be "index" or "delete" + if ($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx + $err = array_key_exists('error', $result) ? var_export($result['error'], true) : ($command . " error " . $result['status']); + throw new Exception(sprintf('%d: %s', $key, $err)); } } $operationIdentifier = $this->operationIdentifiers[$key]; - if(is_string($operationIdentifier) || is_int($operationIdentifier)) { // dont include null keys + if (is_string($operationIdentifier) || is_int($operationIdentifier)) { // dont include null keys $callbackData[$operationIdentifier] = $response['items'][$key]; } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php index 32b93502d4..5dc2471cb9 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php @@ -39,18 +39,13 @@ class MetadataHydrator implements HydratorInterface public function hydrateRecords(array &$records) { - $sql = <<connection->executeQuery( @@ -62,7 +57,7 @@ SQL; while ($metadata = $statement->fetch()) { // Store metadata value $key = $metadata['key']; - $value = $metadata['value']; + $value = trim($metadata['value']); // Do not keep empty values if ($key === '' || $value === '') { @@ -80,7 +75,7 @@ SQL; case 'caption': // Sanitize fields $value = StringHelper::crlfNormalize($value); - $value = $this->sanitizeValue($value, $this->structure->typeOf($key)); + $value = $this->helper->sanitizeValue($value, $this->structure->typeOf($key)); // Private caption fields are kept apart $type = $metadata['private'] ? 'private_caption' : 'caption'; // Caption are multi-valued @@ -103,7 +98,7 @@ SQL; } $tag = $this->structure->getMetadataTagByName($key); if ($tag) { - $value = $this->sanitizeValue($value, $tag->getType()); + $value = $this->helper->sanitizeValue($value, $tag->getType()); } // EXIF data is single-valued $record['metadata_tags'][$key] = $value; @@ -118,33 +113,6 @@ SQL; $this->clearGpsPositionBuffer(); } - private function sanitizeValue($value, $type) - { - switch ($type) { - case FieldMapping::TYPE_STRING: - return str_replace("\0", "", $value); - - case FieldMapping::TYPE_DATE: - return $this->helper->sanitizeDate($value); - - case FieldMapping::TYPE_FLOAT: - case FieldMapping::TYPE_DOUBLE: - return (float) $value; - - case FieldMapping::TYPE_INTEGER: - case FieldMapping::TYPE_LONG: - case FieldMapping::TYPE_SHORT: - case FieldMapping::TYPE_BYTE: - return (int) $value; - - case FieldMapping::TYPE_BOOLEAN: - return (bool) $value; - - default: - return $value; - } - } - private function handleGpsPosition(&$records, $id, $tag_name, $value) { // Get position object diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/TitleHydrator.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/TitleHydrator.php index 88382aa7a7..76dc4704b4 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/TitleHydrator.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/TitleHydrator.php @@ -11,6 +11,8 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator; +use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; +use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Driver\Connection as DriverConnection; @@ -18,31 +20,34 @@ class TitleHydrator implements HydratorInterface { private $connection; - public function __construct(DriverConnection $connection) + /** @var RecordHelper */ + private $helper; + + public function __construct(DriverConnection $connection, RecordHelper $helper) { $this->connection = $connection; + $this->helper = $helper; } public function hydrateRecords(array &$records) { - $sql = <<connection->executeQuery( $sql, array(array_keys($records)), @@ -50,7 +55,7 @@ SQL; ); while ($row = $statement->fetch()) { - $records[$row['record_id']]['title'][$row['locale']] = $row['title']; + $records[$row['record_id']]['title'][$row['locale']] = $this->helper->sanitizeValue($row['title'], FieldMapping::TYPE_STRING); } } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/DateFieldMapping.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/DateFieldMapping.php index a96ef45f4a..128a16467f 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/DateFieldMapping.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/DateFieldMapping.php @@ -57,6 +57,9 @@ class DateFieldMapping extends ComplexFieldMapping */ protected function getProperties() { - return array_merge([ 'format' => $this->format ], parent::getProperties()); + return array_merge([ + 'format' => $this->format, + 'ignore_malformed' => true + ], parent::getProperties()); } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/RecordHelper.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/RecordHelper.php index ffb0c71adf..74f3a297a8 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/RecordHelper.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/RecordHelper.php @@ -89,31 +89,72 @@ class RecordHelper return $this->collectionMap; } - /** - * @param string $date - * @return bool - */ - public static function validateDate($date) - { - $d = DateTime::createFromFormat(FieldMapping::DATE_FORMAT_CAPTION_PHP, $date); - - return $d && $d->format(FieldMapping::DATE_FORMAT_CAPTION_PHP) == $date; - } - /** * @param string $value * @return null|string */ public static function sanitizeDate($value) { - // introduced in https://github.com/alchemy-fr/Phraseanet/commit/775ce804e0257d3a06e4e068bd17330a79eb8370#diff-bee690ed259e0cf73a31dee5295d2edcR286 - // not sure if it's really needed + $v_fix = null; try { - $date = new \DateTime($value); - - return $date->format(FieldMapping::DATE_FORMAT_CAPTION_PHP); + $a = explode(';', preg_replace('/\D+/', ';', trim($value))); + switch (count($a)) { + case 1: // yyyy + $date = new \DateTime($a[0] . '-01-01'); // will throw if date is not valid + $v_fix = $date->format('Y'); + break; + case 2: // yyyy;mm + $date = new \DateTime( $a[0] . '-' . $a[1] . '-01'); + $v_fix = $date->format('Y-m'); + break; + case 3: // yyyy;mm;dd + $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2]); + $v_fix = $date->format('Y-m-d'); + break; + case 4: + $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':00:00'); + $v_fix = $date->format('Y-m-d H:i:s'); + break; + case 5: + $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':00'); + $v_fix = $date->format('Y-m-d H:i:s'); + break; + case 6: + $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':' . $a[5]); + $v_fix = $date->format('Y-m-d H:i:s'); + break; + } } catch (\Exception $e) { - return null; + // no-op, v_fix = null + } + + return $v_fix; + } + + public function sanitizeValue($value, $type) + { + switch ($type) { + case FieldMapping::TYPE_DATE: + return self::sanitizeDate($value); + + case FieldMapping::TYPE_FLOAT: + case FieldMapping::TYPE_DOUBLE: + return (float) $value; + + case FieldMapping::TYPE_INTEGER: + case FieldMapping::TYPE_LONG: + case FieldMapping::TYPE_SHORT: + case FieldMapping::TYPE_BYTE: + return (int) $value; + + case FieldMapping::TYPE_BOOLEAN: + return (bool) $value; + + case FieldMapping::TYPE_STRING: + return str_replace("\0", '', $value); + + default: + return $value; } } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryHelper.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryHelper.php index 1acdd5935c..071ac94b1c 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryHelper.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryHelper.php @@ -110,41 +110,50 @@ class QueryHelper } } - public static function getRangeFromDateString($string) + public static function getRangeFromDateString($value) { - $formats = ['Y/m/d', 'Y/m', 'Y']; - $deltas = ['+1 day', '+1 month', '+1 year']; - $to = null; - while ($format = array_pop($formats)) { - $delta = array_pop($deltas); - $from = date_create_from_format($format, $string); - if ($from !== false) { - // Rewind to start of range - $month = 1; - $day = 1; - switch ($format) { - case 'Y/m/d': - $day = (int) $from->format('d'); - case 'Y/m': - $month = (int) $from->format('m'); - case 'Y': - $year = (int) $from->format('Y'); - } - date_date_set($from, $year, $month, $day); - date_time_set($from, 0, 0, 0); - // Create end of the the range - $to = date_modify(clone $from, $delta); - break; + $date_from = null; + $date_to = null; + try { + $a = explode(';', preg_replace('/\D+/', ';', trim($value))); + switch (count($a)) { + case 1: // yyyy + $date_to = clone($date_from = new \DateTime($a[0] . '-01-01 00:00:00')); // will throw if date is not valid + $date_to->add(new \DateInterval('P1Y')); + break; + case 2: // yyyy;mm + $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-01 00:00:00')); // will throw if date is not valid + $date_to->add(new \DateInterval('P1M')); + break; + case 3: // yyyy;mm;dd + $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' 00:00:00')); // will throw if date is not valid + $date_to->add(new \DateInterval('P1D')); + break; + case 4: + $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':00:00')); + $date_to->add(new \DateInterval('PT1H')); + break; + case 5: + $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':00')); + $date_to->add(new \DateInterval('PT1M')); + break; + case 6: + $date_to = clone($date_from = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':' . $a[5])); + // $date_to->add(new \DateInterval('PT1S')); // no need since precision is 1 sec, a "equal" will be generated when from==to + break; } } + catch (\Exception $e) { + // no-op + } - if (!$from || !$to) { - throw new \InvalidArgumentException(sprintf('Invalid date "%s".', $string)); + if ($date_from === null || $date_to === null) { + throw new \InvalidArgumentException(sprintf('Invalid date "%s".', $value)); } return [ - 'from' => $from->format(FieldMapping::DATE_FORMAT_CAPTION_PHP), - 'to' => $to->format(FieldMapping::DATE_FORMAT_CAPTION_PHP) + 'from' => $date_from->format('Y-m-d H:i:s'), + 'to' => $date_to->format('Y-m-d H:i:s') ]; } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php index 522666bcaa..52c0fa54a6 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php @@ -5,7 +5,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Search; use Alchemy\Phrasea\SearchEngine\Elastic\AST; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; +use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure; use Hoa\Compiler\Llk\TreeNode; use Hoa\Visitor\Element; @@ -166,6 +166,12 @@ class QueryVisitor implements Visit $key = $node->getChild(0)->accept($this); $boundary = $node->getChild(1)->accept($this); + if ($this->isDateKey($key)) { + if(($v = RecordHelper::sanitizeDate($boundary)) !== null) { + $boundary = $v; + } + } + switch ($node->getId()) { case NodeTypes::LT_EXPR: return AST\KeyValue\RangeExpression::lessThan($key, $boundary); @@ -195,11 +201,15 @@ class QueryVisitor implements Visit try { // Try to create a range for incomplete dates $range = QueryHelper::getRangeFromDateString($right); - return new AST\KeyValue\RangeExpression( - $left, - $range['from'], true, - $range['to'], false - ); + if ($range['from'] === $range['to']) { + return new AST\KeyValue\EqualExpression($left, $range['from']); + } else { + return new AST\KeyValue\RangeExpression( + $left, + $range['from'], true, + $range['to'], false + ); + } } catch (\InvalidArgumentException $e) { // Fall back to equal expression } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/ValueChecker.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/ValueChecker.php index 31f65c580e..f27defa2bb 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/ValueChecker.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/ValueChecker.php @@ -3,7 +3,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; -use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper; use Assert\Assertion; @@ -20,7 +19,7 @@ class ValueChecker { Assertion::allIsInstanceOf($list, Typed::class); $is_numeric = is_numeric($value); - $is_valid_date = RecordHelper::validateDate($value); + $is_valid_date = (RecordHelper::sanitizeDate($value) !== null); $filtered = []; foreach ($list as $item) { switch ($item->getType()) { diff --git a/lib/Alchemy/Phrasea/TaskManager/Job/WriteMetadataJob.php b/lib/Alchemy/Phrasea/TaskManager/Job/WriteMetadataJob.php index f35b8e806e..795b0b2a83 100644 --- a/lib/Alchemy/Phrasea/TaskManager/Job/WriteMetadataJob.php +++ b/lib/Alchemy/Phrasea/TaskManager/Job/WriteMetadataJob.php @@ -127,7 +127,10 @@ class WriteMetadataJob extends AbstractJob // check exiftool known tags to skip Phraseanet:tf-* try { - TagFactory::getFromRDFTagname($tagName); + $tag = TagFactory::getFromRDFTagname($tagName); + if(!$tag->isWritable()) { + continue; + } } catch (TagUnknown $e) { continue; } @@ -147,21 +150,34 @@ class WriteMetadataJob extends AbstractJob $fieldValue = array_pop($fieldValues); $value = $this->removeNulChar($fieldValue->getValue()); - $value = new Value\Mono($value); + // fix the dates edited into phraseanet + if($fieldStructure->get_type() === $fieldStructure::TYPE_DATE) { + try { + $value = self::fixDate($value); // will return NULL if the date is not valid + } + catch (\Exception $e) { + $value = null; // do NOT write back to iptc + } + } + + if($value !== null) { // do not write invalid dates + $value = new Value\Mono($value); + } } - } catch(\Exception $e) { + } catch (\Exception $e) { // the field is not set in the record, erase it if ($fieldStructure->is_multi()) { $value = new Value\Multi(array('')); - } - else { + } else { $value = new Value\Mono(''); } } - $metadata->add( - new Metadata\Metadata($fieldStructure->get_tag(), $value) - ); + if($value !== null) { // do not write invalid data + $metadata->add( + new Metadata\Metadata($fieldStructure->get_tag(), $value) + ); + } } $writer = $this->getMetadataWriter($jobData->getApplication()); @@ -220,4 +236,34 @@ class WriteMetadataJob extends AbstractJob { return str_replace("\0", "", $value); } + + /** + * re-format a phraseanet date for iptc writing + * return NULL if the date is not valid + * + * @param string $value + * @return string|null + */ + private static function fixDate($value) + { + $date = null; + try { + $a = explode(';', preg_replace('/\D+/', ';', trim($value))); + switch (count($a)) { + case 3: // yyyy;mm;dd + $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2]); + $date = $date->format('Y-m-d H:i:s'); + break; + case 6: // yyyy;mm;dd;hh;mm;ss + $date = new \DateTime($a[0] . '-' . $a[1] . '-' . $a[2] . ' ' . $a[3] . ':' . $a[4] . ':' . $a[5]); + $date = $date->format('Y-m-d H:i:s'); + break; + } + } + catch (\Exception $e) { + $date = null; + } + + return $date; + } } diff --git a/lib/classes/cache/databox.php b/lib/classes/cache/databox.php index 75460fa962..3687deb211 100644 --- a/lib/classes/cache/databox.php +++ b/lib/classes/cache/databox.php @@ -121,9 +121,9 @@ class cache_databox $conn = $app->getApplicationBox()->get_connection(); - $sql = 'UPDATE sitepreff SET memcached_update = :date'; + $sql = 'UPDATE sitepreff SET memcached_update = current_timestamp()'; $stmt = $conn->prepare($sql); - $stmt->execute([':date' => $now]); + $stmt->execute(); $stmt->closeCursor(); self::$refreshing = false; diff --git a/lib/conf.d/bases_structure.xml b/lib/conf.d/bases_structure.xml index a4d0d2f9c5..a666eaeb5c 100644 --- a/lib/conf.d/bases_structure.xml +++ b/lib/conf.d/bases_structure.xml @@ -1278,6 +1278,27 @@ id + + usr_id + INDEX + + usr_id + + + + unread + INDEX + + unread + + + + created_on + INDEX + + created_on + + InnoDB diff --git a/package.json b/package.json index d93ac787a6..84823d0938 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "normalize-css": "^2.1.0", "npm": "^6.0.0", "npm-modernizr": "^2.8.3", - "phraseanet-production-client": "^0.34.53-d", + "phraseanet-production-client": "0.34.72-d", "requirejs": "^2.3.5", "tinymce": "^4.0.28", "underscore": "^1.8.3", diff --git a/resources/ansible/playbook-boxes.yml b/resources/ansible/playbook-boxes.yml new file mode 100644 index 0000000000..3418297c6c --- /dev/null +++ b/resources/ansible/playbook-boxes.yml @@ -0,0 +1,21 @@ +--- +- hosts: all + sudo: true + vars_files: + - vars/all.yml + roles: +# - server +# - repositories +# - vagrant_local + - nginx +# - mariadb +# - elasticsearch +# - rabbitmq +# - php + - xdebug +# - composer + - mailcatcher +# - node +# - yarn +# - ffmpeg + - app diff --git a/resources/ansible/playbook.yml b/resources/ansible/playbook.yml index 370b427481..c4f76aaa61 100644 --- a/resources/ansible/playbook.yml +++ b/resources/ansible/playbook.yml @@ -4,18 +4,18 @@ vars_files: - vars/all.yml roles: -# - server -# - repositories -# - vagrant_local + - server + - repositories + - vagrant_local - nginx -# - mariadb -# - elasticsearch -# - rabbitmq -# - php + - mariadb + - elasticsearch + - rabbitmq + - php - xdebug -# - composer -# - mailcatcher -# - node -# - yarn -# - ffmpeg + - composer + - mailcatcher + - node + - yarn + - ffmpeg - app diff --git a/resources/ansible/roles/mailcatcher/tasks/main.yml b/resources/ansible/roles/mailcatcher/tasks/main.yml index 7d1ca213ea..45a377ebe8 100644 --- a/resources/ansible/roles/mailcatcher/tasks/main.yml +++ b/resources/ansible/roles/mailcatcher/tasks/main.yml @@ -13,7 +13,7 @@ - name: Install mailcatcher gem # gem module is flaky, this is consistent - command: gem install mailcatcher --conservative + command: gem install mailcatcher -v 0.6.4 --conservative ignore_errors: yes - name: Install mailcatcher supervisord conf diff --git a/resources/www/common/js/components/common.js b/resources/www/common/js/components/common.js index d19965e696..39c6ddf3c8 100644 --- a/resources/www/common/js/components/common.js +++ b/resources/www/common/js/components/common.js @@ -31,17 +31,13 @@ var commonModule = (function ($, p4) { $(this).removeClass('context-menu-item-hover'); }); - // $('#help-trigger').contextMenu('#mainMenu .helpcontextmenu', {openEvt: 'click', dropDown: true, theme: 'vista', dropDown: true, - // showTransition: 'slideDown', - // hideTransition: 'hide', - // shadow: false - // }); $('body').on('click', '.infoDialog', function (event) { infoDialog($(this)); }); }); + function showOverlay(n, appendto, callback, zIndex) { var div = "OVERLAY"; diff --git a/resources/www/common/styles/main.scss b/resources/www/common/styles/main.scss index 6a99013e9b..805e018608 100644 --- a/resources/www/common/styles/main.scss +++ b/resources/www/common/styles/main.scss @@ -88,4 +88,20 @@ $mainMenuLinkBackgroundHoverColor: transparent; box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.39); } } +/*Help menu*/ +.contextMenu.helpcontextmenu { + display: none; + &.shown { + display: block; + position: absolute; + z-index: 101; + } +} +.help-trigger { + display:inline-block; + cursor:pointer; + padding: 0 4px; + border:none; + margin-left: -12px; +} diff --git a/resources/www/prod/images/Basket/Browser/archive.png b/resources/www/prod/images/Basket/Browser/archive.png index f7373f9614..ab89643a35 100644 Binary files a/resources/www/prod/images/Basket/Browser/archive.png and b/resources/www/prod/images/Basket/Browser/archive.png differ diff --git a/resources/www/prod/images/Basket/Browser/archived.png b/resources/www/prod/images/Basket/Browser/archived.png index 3e34e45736..6015eafc48 100644 Binary files a/resources/www/prod/images/Basket/Browser/archived.png and b/resources/www/prod/images/Basket/Browser/archived.png differ diff --git a/resources/www/report/styles/main.scss b/resources/www/report/styles/main.scss index 75030126a9..218fce1f28 100644 --- a/resources/www/report/styles/main.scss +++ b/resources/www/report/styles/main.scss @@ -110,7 +110,7 @@ $select-height: 26px; } .input-prepend { margin: 0; - + .dmin, .dmax { border: 0 none; height: 16px; @@ -203,7 +203,7 @@ $select-height: 26px; height: $select-height; } .select-styled { - position: absolute; + position: absolute; top: 0; right: 0; bottom: 0; @@ -232,9 +232,9 @@ $select-height: 26px; background-color: darken($select-background, 5); } } - + .select-options { - display: none; + display: none; position: absolute; top: 12px; right: -12px; @@ -292,7 +292,7 @@ $select-height: 26px; -webkit-transform: translate(-50%, -50%); -ms-transform: translate(-50%, -50%); transform: translate(-50%, -50%); - + &:hover { background-color: #2c2c2c; } @@ -355,4 +355,8 @@ $select-height: 26px; &.ui-widget-content{ background-color: #212121; } -} \ No newline at end of file +} + +#mainMenu li .context-menu-item-inner a:hover { + color: #000; +} diff --git a/templates/web/admin/editusers.html.twig b/templates/web/admin/editusers.html.twig index 4b07dea1d3..2421976775 100644 --- a/templates/web/admin/editusers.html.twig +++ b/templates/web/admin/editusers.html.twig @@ -512,7 +512,7 @@ {{ 'admin::compte-utilisateur poste' | trans }} - + @@ -528,7 +528,7 @@ {{ 'admin::compte-utilisateur activite' | trans }} - + diff --git a/templates/web/common/index_bootstrap.html.twig b/templates/web/common/index_bootstrap.html.twig index 6249ef8e66..9b9f27a228 100644 --- a/templates/web/common/index_bootstrap.html.twig +++ b/templates/web/common/index_bootstrap.html.twig @@ -32,7 +32,7 @@ {% endblock %} - +
{% include 'common/menubar.html.twig' %}
diff --git a/templates/web/common/menubar.html.twig b/templates/web/common/menubar.html.twig index 84ab930e7d..3821f42e79 100644 --- a/templates/web/common/menubar.html.twig +++ b/templates/web/common/menubar.html.twig @@ -25,103 +25,103 @@ @@ -235,11 +235,10 @@ {{ 'phraseanet:: aide' | trans }} - - + +
@@ -301,6 +300,10 @@ + diff --git a/templates/web/prod/WorkZone/Browser/Results.html.twig b/templates/web/prod/WorkZone/Browser/Results.html.twig index 47ba7802dd..50455d69f1 100644 --- a/templates/web/prod/WorkZone/Browser/Results.html.twig +++ b/templates/web/prod/WorkZone/Browser/Results.html.twig @@ -4,28 +4,9 @@ -
- {% if Total == 0 %} - {{ 'No results' | trans }} - {% elseif Total == 1 %} - {{ '1 result' | trans }} - {% else %} - {% trans with {'%Total%' : Total} %}%Total% results{% endtrans %} - {% endif %} - - {% if Page - 1 > 0 %} - - - - {% endif %} - - Page {{ Page }} / {{ MaxPage }} - - {% if Page + 1 <= MaxPage %} - - - - {% endif %} + {% transchoice Total %} + {0} No results|{1} Result|]1,Inf[ Results + {% endtranschoice %}
@@ -33,43 +14,39 @@
{% for Basket in Baskets %} -
- × -
- - - - + - -
+
+   +
+
+ +
{% set BasketElement = Basket.getElements().first() %} {% if BasketElement %} - {{thumbnail.format(BasketElement.getRecord(app).get_thumbnail(), 80, 80, '', true, false)}} + {{thumbnail.format(BasketElement.getRecord(app).get_thumbnail(), 150, 106, '', true, false)}} {% endif %} -
- {{ Basket.getElements().count() }} -
-
+ +

{% if Basket.getValidation() is empty or Basket.getValidation().isInitiator(app.getAuthenticatedUser()) %} - - - +   - - - +   {% endif %} - {{ Basket.getName() }} - + {{ Basket.getName() }} +
+ {{ Basket.getElements().count() }} {{ ' records' }} +

+ {% if Basket.getPusher() %}

- {% set user_name = '' ~ Basket.getPusher(app).get_display_name() ~ '' %} + {% set user_name = '' ~ Basket.getPusher(app).getDisplayName() ~ '' %} {% trans with {'%user_name%' : user_name} %}Received from %user_name%{% endtrans %}

{% endif %} @@ -92,12 +69,11 @@

{% endif %} -
+
{% endfor %} @@ -117,19 +93,16 @@
{% if Page - 1 > 0 %} - - - +   {% endif %} Page {{ Page }} / {{ MaxPage }} {% if Page + 1 <= MaxPage %} - - - +   {% endif %}
+ diff --git a/templates/web/prod/actions/Tools/videoEditor.html.twig b/templates/web/prod/actions/Tools/videoEditor.html.twig index df8f61332c..6d10080cce 100644 --- a/templates/web/prod/actions/Tools/videoEditor.html.twig +++ b/templates/web/prod/actions/Tools/videoEditor.html.twig @@ -45,23 +45,36 @@ {% set dataH = constant('media_subdef::TC_DATA_HEIGHT') %} {% set technical_info = record.get_technical_infos %} + + {#Set the preview Ratio #} {% set width = technical_info[dataW].value %} {% set height = technical_info[dataH].value %} + {% set prevRatio = '' %} {% if width and height %} - {% set rawRatio = (width / height)|number_format(3, '.') %} - {% set rawRatioLength = rawRatio|length %} - {% set ratio = rawRatio|slice(0,rawRatioLength-1) %} - {% else %} - {% set ratio = '' %} + {% set ratio = (width / height)|number_format(2, '.') %} {% endif %} -
@@ -109,7 +122,7 @@ + class="alt_canvas" id="{{ subdef.get_name() }}" > {% endfor %} diff --git a/templates/web/prod/actions/edit_default.html.twig b/templates/web/prod/actions/edit_default.html.twig index afe8bc12c6..bc4a8a3de9 100644 --- a/templates/web/prod/actions/edit_default.html.twig +++ b/templates/web/prod/actions/edit_default.html.twig @@ -196,8 +196,10 @@
-
- +
+
diff --git a/templates/web/prod/index.html.twig b/templates/web/prod/index.html.twig index d2b34b8ff1..c5cde9d778 100644 --- a/templates/web/prod/index.html.twig +++ b/templates/web/prod/index.html.twig @@ -146,7 +146,7 @@ $(document).ready(function(){ prodApp.appEvents.emit('workzone.doRemoveWarning', "{% if app['settings'].getUserSetting(app.getAuthenticatedUser(), "warning_on_delete_story") %}true{% else %}false{% endif %}"); prodApp.appEvents.emit('search.updateFacetData'); - + $('body').addClass('{{ cssfile }}'); }); diff --git a/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiJsonTest.php b/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiJsonTest.php index d78194c4d7..c8f2cdb254 100644 --- a/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiJsonTest.php +++ b/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiJsonTest.php @@ -14,6 +14,7 @@ use Alchemy\Phrasea\Model\Entities\LazaretSession; use Alchemy\Phrasea\Model\Entities\Task; use Alchemy\Phrasea\Model\Entities\User; use Alchemy\Phrasea\SearchEngine\SearchEngineOptions; +use Alchemy\Phrasea\Status\StatusStructureProviderInterface; use Doctrine\Common\Collections\ArrayCollection; use Guzzle\Common\Exception\GuzzleException; use Ramsey\Uuid\Uuid; @@ -1256,13 +1257,33 @@ class ApiJsonTest extends ApiTestCase $record1 = $this->getRecord1(); $route = '/api/v1/records/' . $record1->getDataboxId() . '/' . $record1->getRecordId() . '/setstatus/'; - $record_status = strrev($record1->getStatus()); + $initialRecordStatus = $record_status = strrev($record1->getStatus()); + + /** @var StatusStructureProviderInterface $statusProvider */ + $statusProvider = $app['status.provider']; + + // initialize status structure for test eg: 4 to 15 bit + foreach (range(4, 15) as $n) { + $properties = [ + 'searchable' => '0', + 'printable' => '0', + 'name' => 'status_test_' . $n, + 'labelon' => '', + 'labeloff' => '', + 'labels_on' => [], + 'labels_off' => [], + ]; + + $statusProvider->updateStatus($record1->getStatusStructure(), $n, $properties); + } + $statusStructure = $record1->getStatusStructure(); $tochange = []; foreach ($statusStructure as $n => $datas) { $tochange[$n] = substr($record_status, ($n - 1), 1) == '0' ? '1' : '0'; } + $this->evaluateMethodNotAllowedRoute($route, ['GET', 'PUT', 'DELETE']); $response = $this->request('POST', $route, $this->getParameters(['status' => $tochange]), ['HTTP_Accept' => $this->getAcceptMimeType()]); @@ -1281,6 +1302,27 @@ class ApiJsonTest extends ApiTestCase $this->assertEquals(substr($record_status, ($n), 1), $tochange[$n]); } + // test record_status in string + $record_status_expected = $record_status; + + $pos = strpos($record_status, '1'); + $bitToChange[$pos] = '1'; + + $response = $this->request('POST', $route, $this->getParameters(['status' => $bitToChange]), ['HTTP_Accept' => $this->getAcceptMimeType()]); + $content = $this->unserialize($response->getContent()); + + // Get fresh record_1 + $testRecord = new \record_adapter($app, $testRecord->getDataboxId(), $testRecord->getRecordId()); + + $this->evaluateResponse200($response); + $this->evaluateMeta200($content); + + $this->evaluateRecordsStatusResponse($testRecord, $content); + + $record_new_status = strrev($testRecord->getStatus()); + $this->assertEquals($record_status_expected, $record_new_status); + + foreach ($tochange as $n => $value) { $tochange[$n] = $value == '0' ? '1' : '0'; } @@ -1301,6 +1343,8 @@ class ApiJsonTest extends ApiTestCase $this->assertEquals(substr($record_status, ($n), 1), $tochange[$n]); } + $this->assertEquals($initialRecordStatus, $record_status); + $record1->setStatus(str_repeat('0', 32)); } diff --git a/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/RangeExpressionTest.php b/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/RangeExpressionTest.php index 88a850fe03..d1d376e37b 100644 --- a/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/RangeExpressionTest.php +++ b/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/RangeExpressionTest.php @@ -58,6 +58,7 @@ class RangeExpressionTest extends \PHPUnit_Framework_TestCase { $query_context = $this->prophesize(QueryContext::class)->reveal(); $key_prophecy = $this->prophesize(Key::class); + $key_prophecy->getFieldType($query_context)->willReturn('text'); $key_prophecy->getIndexField($query_context)->willReturn('foo'); $key_prophecy->isValueCompatible('bar', $query_context)->willReturn(true); $key = $key_prophecy->reveal(); @@ -73,6 +74,7 @@ class RangeExpressionTest extends \PHPUnit_Framework_TestCase { $query_context = $this->prophesize(QueryContext::class)->reveal(); $key = $this->prophesize(FieldKey::class); + $key->getFieldType($query_context)->willReturn('text'); $key->getIndexField($query_context)->willReturn('baz'); $key->isValueCompatible('bar', $query_context)->willReturn(true); $key->postProcessQuery(Argument::any(), $query_context)->willReturnArgument(0); diff --git a/tests/Alchemy/Tests/Phrasea/SearchEngine/resources/queries.csv b/tests/Alchemy/Tests/Phrasea/SearchEngine/resources/queries.csv index 9f7d079880..5252788eaa 100644 --- a/tests/Alchemy/Tests/Phrasea/SearchEngine/resources/queries.csv +++ b/tests/Alchemy/Tests/Phrasea/SearchEngine/resources/queries.csv @@ -53,6 +53,7 @@ foo < 42| foo ≤ 42| foo > 42| foo ≥ 42| +foo = 2015/01/01|( == ) foo < 2015/01/01| foo ≤ 2015/01/01| foo > 2015/01/01| @@ -93,19 +94,93 @@ id:90 AND foo|( AND ) id:90 foo|( AND ) recordid:90| -# Timestamps -created_on < "2015/01/01"| -created_on ≤ "2015/01/01"| -created_on = "2015/01/01"| -created_on ≥ "2015/01/01"| -created_on > "2015/01/01"| -updated_on < "2015/01/01"| -updated_on ≤ "2015/01/01"| -updated_on = "2015/01/01"| -updated_on ≥ "2015/01/01"| -updated_on > "2015/01/01"| -created_at > "2015/01/01"| -updated_at > "2015/01/01"| +# Timestamps yyyy +created_on < "2015"| +created_on ≤ "2015"| +created_on = "2015"| +created_on ≥ "2015"| +created_on > "2015"| +updated_on < "2015"| +updated_on ≤ "2015"| +updated_on = "2015"| +updated_on ≥ "2015"| +updated_on > "2015"| +created_at > "2015"| +updated_at > "2015"| + +# Timestamps yyyy/mm +created_on < "2015/01"| +created_on ≤ "2015/01"| +created_on = "2015/01"| +created_on ≥ "2015/01"| +created_on > "2015/01"| +updated_on < "2015/01"| +updated_on ≤ "2015/01"| +updated_on = "2015/01"| +updated_on ≥ "2015/01"| +updated_on > "2015/01"| +created_at > "2015/01"| +updated_at > "2015/01"| + +# Timestamps yyyy/mm/dd +created_on < "2015/01/01"| +created_on ≤ "2015/01/01"| +created_on = "2015/01/01"| +created_on ≥ "2015/01/01"| +created_on > "2015/01/01"| +updated_on < "2015/01/01"| +updated_on ≤ "2015/01/01"| +updated_on = "2015/01/01"| +updated_on ≥ "2015/01/01"| +updated_on > "2015/01/01"| +created_at > "2015/01/01"| +updated_at > "2015/01/01"| + +# Timestamps yyyy/mm/dd hh +created_on < "2015/01/01 12"| +created_on ≤ "2015/01/01 12"| +created_on = "2015/01/01 12"| +created_on ≥ "2015/01/01 12"| +created_on > "2015/01/01 12"| +updated_on < "2015/01/01 12"| +updated_on ≤ "2015/01/01 12"| +updated_on = "2015/01/01 12"| +updated_on ≥ "2015/01/01 12"| +updated_on > "2015/01/01 12"| +created_at > "2015/01/01 12"| +updated_at > "2015/01/01 12"| + +# Timestamps yyyy/mm/dd hh:mm +created_on < "2015/01/01 12.34"| +created_on ≤ "2015/01/01 12.34"| +created_on = "2015/01/01 12.34"| +created_on ≥ "2015/01/01 12.34"| +created_on > "2015/01/01 12.34"| +updated_on < "2015/01/01 12.34"| +updated_on ≤ "2015/01/01 12.34"| +updated_on = "2015/01/01 12.34"| +updated_on ≥ "2015/01/01 12.34"| +updated_on > "2015/01/01 12.34"| +created_at > "2015/01/01 12.34"| +updated_at > "2015/01/01 12.34"| + +# Timestamps yyyy/mm/dd hh.mm.ss +created_on < "2015/01/01 12.34.56"| +created_on ≤ "2015/01/01 12.34.56"| +created_on = "2015/01/01 12.34.56"|( == ) +created_on ≥ "2015/01/01 12.34.56"| +created_on > "2015/01/01 12.34.56"| +updated_on < "2015/01/01 12.34.56"| +updated_on ≤ "2015/01/01 12.34.56"| +updated_on = "2015/01/01 12.34.56"|( == ) +updated_on ≥ "2015/01/01 12.34.56"| +updated_on > "2015/01/01 12.34.56"| +created_at > "2015/01/01 12.34.56"| +updated_at > "2015/01/01 12.34.56"| + +# timestamps missing zeros +created_on = "2015/1/2 1.3.5"|( == ) + # Flag matcher flag.foo:true| diff --git a/yarn.lock b/yarn.lock index c469bd7d52..4cfedda66c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7555,10 +7555,10 @@ phraseanet-common@^0.4.1: js-cookie "^2.1.0" pym.js "^1.3.1" -phraseanet-production-client@^0.34.53-d: - version "0.34.53-d" - resolved "https://registry.yarnpkg.com/phraseanet-production-client/-/phraseanet-production-client-0.34.53-d.tgz#517e5d9ccae2f1f0f3cdb7ac194e70579cbcc65f" - integrity sha512-/Vd5kd/YRapDpUWzBZZvm5hHwieXdU2cBEtC3D/FPbeSEI3dmYEDnuznDxBafPZvep65KdAtufPw+B6ulWNgBQ== +phraseanet-production-client@0.34.72-d: + version "0.34.72-d" + resolved "https://registry.yarnpkg.com/phraseanet-production-client/-/phraseanet-production-client-0.34.72-d.tgz#028a5ccd589e696b5433eea9d53d9367966613c8" + integrity sha512-IPaDRqXwyJegoKmzr56bggxTzN4TnmuAqU4O7rDEhh0aqdCiuC8rlH/yzKoLeEIMSrESCw5mBhrI//ccntvv9w== dependencies: "@mapbox/mapbox-gl-language" "^0.9.2" "@turf/turf" "^5.1.6"