Merge branch 'master' into PHRAS-1798_update_version_name

.bowerrc

@@ -1,3 +0,0 @@
-{
-  "directory": "www/bower_components"
-}

.travis.yml

@@ -5,14 +5,11 @@ branches:
   except:
     - /^3\..*$/
     - /^scrutinizer-.*$/
+    - master
+    - 4.0
 
 matrix:
   fast_finish: true
-  allow_failures:
-    - php: 7.0
-      env: TEST_SUITE=1
-    - php: 7.0
-      env: TEST_SUITE=2
   include:
     - php: 5.5
       env: TEST_SUITE=1
@@ -41,22 +38,31 @@ before_install:
   - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.6.0.deb && sudo dpkg -i --force-confnew elasticsearch-1.6.0.deb
   - sudo /usr/share/elasticsearch/bin/plugin install elasticsearch/elasticsearch-analysis-icu/2.6.0
   - sudo service elasticsearch start
+  - sudo apt-get install libzmq-dev
+  - yes '' | pecl install zmq-beta
   - echo 'session.cache_limiter = ""' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
   - echo 'extension="redis.so"' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
   - if [[ $TRAVIS_PHP_VERSION = 5.* ]];then echo 'extension="memcache.so"' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini; fi;
   - echo 'extension="memcached.so"' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
-  - echo "extension=zmq.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
   - yes | pecl install imagick-beta
-  - sudo apt-get install librabbitmq-dev
-  - pecl install amqp-1.4.0
+  # ok php 5
+  #- sudo apt-get install librabbitmq-dev
+  #- pecl install amqp-1.4.0
+  # ok php 5 and php 7
+  - git clone https://github.com/alanxz/rabbitmq-c
+  - cd rabbitmq-c
+  - git checkout 2ca1774489328cde71195f5fa95e17cf3a80cb8a
+  - git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install
+  - cd ..
+  - yes '' | pecl install amqp-1.9.3
+
 install:
-  - if [[ $TRAVIS_PHP_VERSION = 5.* ]];then travis_retry composer install --no-progress --no-interaction --optimize-autoloader; fi;
-  - if [[ $TRAVIS_PHP_VERSION = 7.* ]];then travis_retry composer update --no-progress --no-interaction --optimize-autoloader; fi;
+  - travis_retry composer install --no-progress --no-interaction --optimize-autoloader
   - travis_retry npm install
 before_script:
   - mysql -e 'CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;'
   - "./bin/developer system:uninstall"
-  - "./bin/setup system:install --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=root --db-template=fr --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y;"
+  - "./bin/setup system:install --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=root --db-template=fr-simple --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y;"
   - |
      ./bin/developer ini:reset --email=test@phraseanet.com --password=test --run-patches --no-setup-dbs;
       php resources/hudson/cleanupSubdefs.php;

Makefile

@@ -8,7 +8,7 @@ install_composer:
 	composer install
 
 install_asset_dependencies:
-	npm install
+	yarn
 	./node_modules/.bin/gulp build
 
 install_assets:
@@ -17,7 +17,6 @@ install_assets:
 clean_assets:
 	rm -rf ./node_modules
 	rm -rf ./www/assets
-	rm -rf ./www/bower_components
 	mkdir ./node_modules
 	touch ./node_modules/.gitkeep
 

Vagrantfile

@@ -1,8 +1,15 @@
 Vagrant.require_version ">= 1.5"
+$php = [ "5.6", "7.0", "7.1", "7.2" ]
+$phpVersion = ENV['phpversion'] ? ENV['phpversion'] : "5.6";
 
 unless Vagrant.has_plugin?('vagrant-hostmanager')
     raise "vagrant-hostmanager is not installed! Please run\n  vagrant plugin install vagrant-hostmanager\n\n"
 end
+
+unless $php.include?($phpVersion)
+    raise "You should specify php version before running vagrant\n\n (Available : 5.6, 7.0, 7.1, 7.2 | default => 5.6)\n\n Exemple: phpversion='7.0' vagrant up \n\n"
+end
+
 $root = File.dirname(File.expand_path(__FILE__))
 
 # Check to determine whether we're on a windows or linux/os-x host,
@@ -54,7 +61,6 @@ else
 end
 
 Vagrant.configure("2") do |config|
-
     # Configure hostmanager
     config.hostmanager.enabled = true
     config.hostmanager.manage_host = true
@@ -88,6 +94,7 @@ Vagrant.configure("2") do |config|
             ansible.extra_vars = {
                 hostname: $hostname,
                 host_addresses: $hostIps,
+                phpversion: $phpVersion,
                 postfix: {
                     postfix_domain: $hostname + ".vb"
                 }
@@ -104,7 +111,7 @@ Vagrant.configure("2") do |config|
             }
         end
     else
-        config.vm.provision :shell, path: "resources/ansible/windows.sh", args: ["default"]
+        config.vm.provision :shell, path: "resources/ansible/windows.sh", args: ["default", $phpVersion]
        # config.vm.provision :shell, run: "always", path: "resources/ansible/windows-always.sh", args: ["default"]
     end
 

bin/console

@@ -18,6 +18,7 @@ use Alchemy\Phrasea\Command\SearchEngine\Debug\QueryParseCommand;
 use Alchemy\Phrasea\Command\SearchEngine\Debug\QuerySampleCommand;
 use Alchemy\Phrasea\Command\SearchEngine\IndexCreateCommand;
 use Alchemy\Phrasea\Command\SearchEngine\IndexDropCommand;
+use Alchemy\Phrasea\Command\SearchEngine\IndexManipulateCommand;
 use Alchemy\Phrasea\Command\SearchEngine\IndexPopulateCommand;
 use Alchemy\Phrasea\Command\Thesaurus\FindConceptsCommand;
 use Alchemy\Phrasea\Core\Version;
@@ -123,6 +124,7 @@ $cli->command(new H264MappingGenerator());
 $cli->command(new XSendFileConfigurationDumper());
 $cli->command(new XSendFileMappingGenerator());
 
+$cli->command(new IndexManipulateCommand());
 $cli->command(new IndexCreateCommand());
 $cli->command(new IndexDropCommand());
 $cli->command(new MappingUpdateCommand());

bin/setup

@@ -12,6 +12,7 @@
 namespace KonsoleKommander;
 
 use Alchemy\Phrasea\Command\Setup\ConfigurationEditor;
+use Alchemy\Phrasea\Command\Setup\FixAutoincrements;
 use Alchemy\Phrasea\Core\Version;
 use Alchemy\Phrasea\Command\UpgradeDBDatas;
 use Alchemy\Phrasea\Command\Setup\Install;
@@ -74,8 +75,9 @@ $app->command(new PluginsReset());
 $app->command(new EnablePlugin());
 $app->command(new DisablePlugin());
 $app->command(new CheckEnvironment('check:system'));
-$app->command(new Install('system:install'));
+$app->command(new Install('system:install', $app['phraseanet.structure-template']));
 $app->command(new CrossDomainGenerator());
+$app->command(new FixAutoincrements('system:fix-autoincrements'));
 
 $app['phraseanet.setup_mode'] = true;
 

bower.json

@@ -1,46 +0,0 @@
-{
-  "name": "Phraseanet",
-  "version": "4.0.0",
-  "dependencies": {
-    "jquery": "~1.11.3",
-    "jquery-ui": "~1.10.4",
-    "jquery-mobile-bower": "~1.3.0",
-    "underscore": "~1.4.4",
-    "font-awesome": "~3.2.1",
-    "modernizr": "~2.8.3",
-    "normalize-css": "~2.1.3",
-    "json2": "latest",
-    "humane-js": "~3.0.6",
-    "jquery-file-upload": "~8.3.2",
-    "blueimp-load-image": "latest",
-    "requirejs": "~2.1",
-    "backbone-amd": "~1.0",
-    "underscore-amd": "~1.4",
-    "i18next": "~1.6",
-    "bootstrap-multiselect": "v0.9",
-    "zxcvbn": "https://github.com/lowe/zxcvbn.git",
-    "geonames-server-jquery-plugin": "~0.2",
-    "tinymce": "~4.0",
-    "jquery-galleria": "1.2.9",
-    "jquery.cookie": "~1.4",
-    "fancytree": "~2.7",
-    "bootstrap-sass": "v2.3.2.2",
-    "jquery.lazyload": "~1.9.7",
-    "jquery-treeview": "~1.4.2",
-    "alchemy-embed-medias": "~0.3.4",
-    "html5shiv": "^3.7.3",
-    "jquery-simplecolorpicker": "^0.3.1"
-  },
-  "devDependencies": {
-    "mocha": "latest",
-    "sinonjs": "~1.7.0",
-    "chai": "~1.6",
-    "squire": "~0",
-    "sinon-chai": "~2.5",
-    "qunit": "https://github.com/jquery/qunit.git#1.11.0",
-    "js-fixtures": "https://github.com/badunk/js-fixtures/archive/master.zip"
-  },
-  "resolutions": {
-    "jquery": "~1.11.3"
-  }
-}

circle.yml

@@ -7,7 +7,7 @@ general:
 
 machine:
   php:
-    version: 5.5.31
+    version: 7.0.24 #5.6.22
   node:
     version: stable
   services:
@@ -18,16 +18,16 @@ machine:
 
 dependencies:
   cache_directories:
-    - elasticsearch-1.6.0 # relative to the build directory
+    - elasticsearch-2.3.3 # relative to the build directory
     - node_modules
     - ~/.composer
   pre:
     - git clone https://github.com/alanxz/rabbitmq-c
     - cd rabbitmq-c && git checkout 2ca1774489328cde71195f5fa95e17cf3a80cb8a
     - cd rabbitmq-c && git submodule init && git submodule update && autoreconf -i && ./configure && make && sudo make install
-    - pecl install amqp-1.6.0
+    - pecl channel-update pear.php.net
+    - yes '' | pecl install amqp-1.9.3
     - yes '' | pecl install imagick
-    - pecl install json
     - sudo apt-get install libzmq-dev
     - yes '' | pecl install zmq-beta
     - echo "extension = amqp.so" > /opt/circleci/php/$(phpenv global)/etc/conf.d/amqp.ini
@@ -38,21 +38,28 @@ dependencies:
   override:
     - composer install --no-progress --no-interaction --optimize-autoloader
   post:
+    - node -v
+    - npm -v
     - npm install
-    - if [[ ! -e elasticsearch-1.6.0 ]]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.6.0.tar.gz && tar -xvf elasticsearch-1.6.0.tar.gz && elasticsearch-1.6.0/bin/plugin install elasticsearch/elasticsearch-analysis-icu/2.6.0; fi
-    - elasticsearch-1.6.0/bin/elasticsearch: {background: true}
+    - if [[ ! -e elasticsearch-2.3.3 ]]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.3.3.tar.gz && tar -xvf elasticsearch-2.3.3.tar.gz && elasticsearch-2.3.3/bin/plugin install analysis-icu; fi
+    - elasticsearch-2.3.3/bin/elasticsearch: {background: true}
 
 database:
   override:
     - mysql -u ubuntu -e 'CREATE DATABASE update39_test;CREATE DATABASE ab_test;CREATE DATABASE db_test;SET @@global.sql_mode=STRICT_ALL_TABLES;SET @@global.max_allowed_packet=33554432;SET @@global.wait_timeout=999999;';
   post:
     - "./bin/developer system:uninstall -v"
-    - "./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=ubuntu --db-template=fr --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y;"
+    - "./bin/setup system:install -v --email=test@phraseanet.com --password=test --db-host=127.0.0.1 --db-user=ubuntu --db-template=fr-simple --db-password= --databox=db_test --appbox=ab_test --server-name=http://127.0.0.1 -y;"
     - "./bin/developer ini:setup-tests-dbs -v"
     - "./bin/console searchengine:index:create -v"
     - "./bin/developer phraseanet:regenerate-sqlite -v"
     - "./bin/developer phraseanet:generate-js-fixtures -v"
 
+#test:
+#  override:
+#    - case $CIRCLE_NODE_INDEX in 0) EXIT=0; php -d memory_limit=-1 bin/phpunit --log-junit $CIRCLE_TEST_REPORTS/phpunit/junit-unit.xml --exclude-group legacy || EXIT=$?; php -d memory_limit=-1 bin/phpunit --log-junit $CIRCLE_TEST_REPORTS/phpunit/junit-legacy-no-web.xml --group legacy --exclude-group web || EXIT=$?; exit $EXIT;; 1) php -d memory_limit=-1 bin/phpunit --log-junit $CIRCLE_TEST_REPORTS/phpunit/junit-legacy-web.xml --group web ;; esac:
+#    #- php -d memory_limit=-1 bin/phpunit -c phpunit.xml.dist;
+
 test:
   override:
     - case $CIRCLE_NODE_INDEX in 0) EXIT=0; php -d memory_limit=-1 bin/phpunit --log-junit $CIRCLE_TEST_REPORTS/phpunit/junit-unit.xml --exclude-group legacy || EXIT=$?; php -d memory_limit=-1 bin/phpunit --log-junit $CIRCLE_TEST_REPORTS/phpunit/junit-legacy-no-web.xml --group legacy --exclude-group web || EXIT=$?; exit $EXIT;; 1) php -d memory_limit=-1 bin/phpunit --log-junit $CIRCLE_TEST_REPORTS/phpunit/junit-legacy-web.xml --group web ;; esac:

composer.json

@@ -67,7 +67,6 @@
     "doctrine/migrations": "^1.0.0",
     "doctrine/orm": "^2.4.0",
     "elasticsearch/elasticsearch": "~2.0",
-    "facebook/php-sdk": "~3.2.3",
     "firebase/php-jwt": "^3.0.0",
     "gedmo/doctrine-extensions": "~2.3.0",
     "goodby/csv": "^1.3.0",
@@ -119,7 +118,8 @@
     "zend/gdata": "~1.12.1",
     "alchemy/worker-bundle": "^0.1.6",
     "alchemy/queue-bundle": "^0.1.5",
-    "google/recaptcha": "^1.1"
+    "google/recaptcha": "^1.1",
+    "facebook/graph-sdk": "^5.6"
   },
   "require-dev": {
     "mikey179/vfsStream": "~1.5",

config/configuration.sample.yml

@@ -102,6 +102,7 @@ border-manager:
             enabled: false
             options:
                 colorspaces: [cmyk, grayscale, rgb]
+                media_types: [Image]
         -
             type: Checker\Dimension
             enabled: false
@@ -135,6 +136,7 @@ authentication:
             options:
                 app-id: ''
                 secret: ''
+                default-graph-version: 'v2.10'
         twitter:
             enabled: false
             options:
@@ -237,9 +239,40 @@ embed_bundle:
         player: flexpaper
         enable-pdfjs: true
 geocoding-providers:
-  -
-    name: 'mapBox'
-    public-key: ''
+    -
+        map-provider: mapboxWebGL
+        enabled: false
+        public-key: ''
+        map-layers:
+            -
+                name: Light
+                value: 'mapbox://styles/mapbox/light-v9'
+            -
+                name: Streets
+                value: 'mapbox://styles/mapbox/streets-v9'
+            -
+                name: Basic
+                value: 'mapbox://styles/mapbox/basic-v9'
+            -
+                name: Satellite
+                value: 'mapbox://styles/mapbox/satellite-v9'
+            -
+                name: Dark
+                value: 'mapbox://styles/mapbox/dark-v9'
+        transition-mapboxgl:
+            -
+                animate: true
+                speed: '2.2'
+                curve: '1.42'
+        default-position:
+            - '48.879162'
+            - '2.335062'
+        default-zoom: 5
+        marker-default-zoom: 9
+        geonames-field-mapping: true
+        cityfields: City, Ville
+        provincefields: Province
+        countryfields: Country, Pays
 workers:
   queue:
     worker-queue:

grammar/query.pp

@@ -28,6 +28,9 @@
 // Rest
 %token  database        database
 %token  collection      collection
+%token  sha256          sha256
+%token  geolocation     geolocation
+%token  uuid            uuid
 %token  type            type
 %token  id              id|recordid
 %token  created_on      created_(on|at)
@@ -86,12 +89,15 @@ match_key:
 
 #native_key:
     <database>
+  | <sha256>
+  | <uuid>
   | <collection>
   | <type>
   | <id>
 
 key:
-    timestamp_key()
+    geolocation_key()
+  | timestamp_key()
   | ::meta_prefix::  meta_key()
   | ::field_prefix:: field_key()
   |                  field_key()
@@ -100,6 +106,9 @@ key:
     <created_on>
   | <updated_on>
 
+#geolocation_key:
+    <geolocation>
+
 #meta_key:
   word_or_keyword()+
 
@@ -169,6 +178,9 @@ keyword:
   | <or>
   | <database>
   | <collection>
+  | <sha256>
+  | <geolocation>
+  | <uuid>
   | <type>
   | <id>
   | <created_on>

lib/Alchemy/Phrasea/Application/RouteLoader.php

@@ -55,6 +55,7 @@ class RouteLoader
         '/prod/records/property'       => Providers\Prod\Property::class,
         '/prod/share/'                 => Providers\Prod\Share::class,
         '/prod/story'                  => Providers\Prod\Story::class,
+        '/prod/subdefs'                => Providers\Prod\Subdefs::class,
         '/prod/tools/'                 => Providers\Prod\Tools::class,
         '/prod/tooltip'                => Providers\Prod\Tooltip::class,
         '/prod/TOU/'                   => Providers\Prod\TOU::class,

lib/Alchemy/Phrasea/Authentication/Provider/Facebook.php

@@ -21,13 +21,14 @@ use Symfony\Component\HttpFoundation\Session\SessionInterface;
 
 class Facebook extends AbstractProvider
 {
-    /** @var \Facebook */
+    /** @var \Facebook\Facebook */
     private $facebook;
 
-    public function __construct(\Facebook $facebook, UrlGenerator $generator)
+    public function __construct(\Facebook\Facebook $facebook, UrlGenerator $generator, SessionInterface $session)
     {
         $this->facebook = $facebook;
         $this->generator = $generator;
+        $this->session = $session;
     }
 
     /**
@@ -49,16 +50,20 @@ class Facebook extends AbstractProvider
     /**
      * {@inheritdoc}
      */
-    public function authenticate()
+    public function authenticate(array $params = array())
     {
-        return new RedirectResponse($this->facebook->getLoginUrl([
-            'scope'        => 'email',
-            'redirect_uri' => $this->generator->generate(
-                'login_authentication_provider_callback',
-                ['providerId' => $this->getId()],
-                UrlGenerator::ABSOLUTE_URL
+        $params = array_merge(['providerId' => $this->getId()], $params);
+
+        return new RedirectResponse(
+            $this->facebook->getRedirectLoginHelper()->getLoginUrl(
+                $this->generator->generate(
+                    'login_authentication_provider_callback',
+                    $params,
+                    UrlGenerator::ABSOLUTE_URL
+                ),
+                ['email']
             )
-        ]));
+        );
     }
 
     /**
@@ -66,15 +71,15 @@ class Facebook extends AbstractProvider
      */
     public function logout()
     {
-        $this->facebook->destroySession();
+        $this->session->remove('fb_access_token');
     }
 
     /**
-     * @param \Facebook $facebook
+     * @param \Facebook\Facebook $facebook
      *
      * @return Facebook
      */
-    public function setFacebook(\Facebook $facebook)
+    public function setFacebook(\Facebook\Facebook $facebook)
     {
         $this->facebook = $facebook;
 
@@ -82,35 +87,55 @@ class Facebook extends AbstractProvider
     }
 
     /**
-     * @return \Facebook
+     * @return \Facebook\Facebook
      */
     public function getFacebook()
     {
         return $this->facebook;
     }
 
+    /**
+     * @param $dataToRetrieve
+     * @return \Facebook\GraphNodes\GraphUser
+     */
+    protected function getGraphUser($dataToRetrieve)
+    {
+        try {
+            $response = $this->facebook->get(
+                "/me?fields=" . implode(',', $dataToRetrieve),
+                $this->session->get('fb_access_token')
+            );
+        } catch(\Facebook\Exceptions\FacebookResponseException $e) {
+            throw new NotAuthenticatedException('Graph returned an error: ' . $e->getMessage());
+            exit;
+        } catch(\Facebook\Exceptions\FacebookSDKException $e) {
+            throw new NotAuthenticatedException('Facebook SDK returned an error: ' . $e->getMessage());
+            exit;
+        }
+
+        if (!$response)
+        {
+            throw new NotAuthenticatedException('Not authenticated');
+        }
+
+        return $response->getGraphUser();
+    }
+
     /**
      * {@inheritdoc}
      */
     public function getIdentity()
     {
-        try {
-            $data = $this->facebook->api('/me');
-            $identity = new Identity();
+        $user = $this->getGraphUser(['id', 'name', 'email', 'picture', 'last_name', 'first_name']);
+        
+        $identity = new Identity();
 
-            $identity->set(Identity::PROPERTY_ID, $data['id']);
-            $identity->set(Identity::PROPERTY_IMAGEURL, sprintf(
-                'https://graph.facebook.com/%s/picture?return_ssl_resources=1',
-                $data['username']
-            ));
-            $identity->set(Identity::PROPERTY_EMAIL, $data['email']);
-            $identity->set(Identity::PROPERTY_FIRSTNAME, $data['first_name']);
-            $identity->set(Identity::PROPERTY_LASTNAME, $data['last_name']);
-            $identity->set(Identity::PROPERTY_USERNAME, $data['username']);
-
-        } catch (\FacebookApiException $e) {
-            throw new NotAuthenticatedException('Unable to get profile informations', $e->getCode(), $e);
-        }
+        $identity->set(Identity::PROPERTY_ID, $user['id']);
+        $identity->set(Identity::PROPERTY_IMAGEURL, $user['picture']);
+        $identity->set(Identity::PROPERTY_EMAIL, $user['email']);
+        $identity->set(Identity::PROPERTY_FIRSTNAME, $user['first_name']);
+        $identity->set(Identity::PROPERTY_LASTNAME, $user['last_name']);
+        $identity->set(Identity::PROPERTY_USERNAME, $user['first_name']);
 
         return $identity;
     }
@@ -120,9 +145,44 @@ class Facebook extends AbstractProvider
      */
     public function onCallback(Request $request)
     {
-        if (!$this->facebook->getUser()) {
-            throw new NotAuthenticatedException('Facebook authentication failed');
+        $helper = $this->facebook->getRedirectLoginHelper();
+
+        try {
+            $accessToken = $helper->getAccessToken();
+        } catch(\Facebook\Exceptions\FacebookResponseException $e) {
+            throw new NotAuthenticatedException('Graph returned an error: ' . $e->getMessage());
+            exit;
+        } catch(\Facebook\Exceptions\FacebookSDKException $e) {
+            throw new NotAuthenticatedException('Facebook SDK returned an error: ' . $e->getMessage());
+            exit;
         }
+
+        if (! isset($accessToken)) {
+            if ($helper->getError()) {
+                $error  = "Error: " . $helper->getError() . "\n";
+                $error .= "Error Code: " . $helper->getErrorCode() . "\n";
+                $error .= "Error Reason: " . $helper->getErrorReason() . "\n";
+                $error .= "Error Description: " . $helper->getErrorDescription() . "\n";
+
+                throw new NotAuthenticatedException($error);
+
+            } else {
+                throw new NotAuthenticatedException('Facebook authentication failed');
+            }
+            exit;
+        }
+
+        $oAuth2Client = $this->facebook->getOAuth2Client();
+
+        if (! $accessToken->isLongLived()) {
+            try {
+                $accessToken = $oAuth2Client->getLongLivedAccessToken($accessToken);
+            } catch (\Facebook\Exceptions\FacebookSDKException $e) {
+                throw new NotAuthenticatedException('Error getting long-lived access token: ' . $e->getMessage() );
+                exit;
+            }
+        }
+        $this->session->set('fb_access_token', (string) $accessToken);
     }
 
     /**
@@ -130,11 +190,9 @@ class Facebook extends AbstractProvider
      */
     public function getToken()
     {
-        if (0 >= $this->facebook->getUser()) {
-            throw new NotAuthenticatedException('Provider has not authenticated');
-        }
+        $user = $this->getGraphUser(['id']);
 
-        return new Token($this, $this->facebook->getUser());
+        return new Token($this, $user['id']);
     }
 
     /**
@@ -194,11 +252,12 @@ class Facebook extends AbstractProvider
      */
     public static function create(UrlGenerator $generator, SessionInterface $session, array $options)
     {
-        $config['appId'] = $options['app-id'];
-        $config['secret'] = $options['secret'];
+        $config['app_id'] = $options['app-id'];
+        $config['app_secret'] = $options['secret'];
+        $config['default_graph_version'] = $options['default-graph-version'];
 
-        $facebook = new \Facebook($config);
+        $facebook = new \Facebook\Facebook($config);
 
-        return new static($facebook, $generator);
+        return new static($facebook, $generator, $session);
     }
 }

lib/Alchemy/Phrasea/Authentication/Provider/Github.php

@@ -78,8 +78,10 @@ class Github extends AbstractProvider
     /**
      * {@inheritdoc}
      */
-    public function authenticate()
+    public function authenticate(array $params = array())
     {
+        $params = array_merge(['providerId' => $this->getId()], $params);
+
         $state = $this->createState();
 
         $this->session->set('github.provider.state', $state);
@@ -90,7 +92,7 @@ class Github extends AbstractProvider
             'state' => $state,
             'redirect_uri' => $this->generator->generate(
                 'login_authentication_provider_callback',
-                ['providerId' => $this->getId()],
+                $params,
                 UrlGenerator::ABSOLUTE_URL
             ),
         ], '', '&'));

lib/Alchemy/Phrasea/Authentication/Provider/GooglePlus.php

@@ -41,14 +41,6 @@ class GooglePlus extends AbstractProvider
             'https://www.googleapis.com/auth/userinfo.profile',
         ]);
 
-        $this->client->setRedirectUri(
-            $this->generator->generate(
-                'login_authentication_provider_callback', [
-                'providerId' => $this->getId(),
-            ], UrlGenerator::ABSOLUTE_URL
-            )
-        );
-
         $this->client->setApprovalPrompt("auto");
 
         if ($this->session->has('google-plus.provider.token')) {
@@ -115,8 +107,18 @@ class GooglePlus extends AbstractProvider
     /**
      * {@inheritdoc}
      */
-    public function authenticate()
+    public function authenticate(array $params = array())
     {
+        $params = array_merge(['providerId' => $this->getId()], $params);
+
+        $this->client->setRedirectUri(
+            $this->generator->generate(
+                'login_authentication_provider_callback',
+                $params,
+                UrlGenerator::ABSOLUTE_URL
+            )
+        );
+
         $state = $this->createState();
 
         $this->session->set('google-plus.provider.state', $state);

lib/Alchemy/Phrasea/Authentication/Provider/Linkedin.php

@@ -78,8 +78,10 @@ class Linkedin extends AbstractProvider
     /**
      * {@inheritdoc}
      */
-    public function authenticate()
+    public function authenticate(array $params = array())
     {
+        $params = array_merge(['providerId' => $this->getId()], $params);
+
         $state = $this->createState();
 
         $this->session->set('linkedin.provider.state', $state);
@@ -91,7 +93,7 @@ class Linkedin extends AbstractProvider
              'state' => $state,
              'redirect_uri' => $this->generator->generate(
                  'login_authentication_provider_callback',
-                 ['providerId' => $this->getId()],
+                 $params,
                  UrlGenerator::ABSOLUTE_URL
              ),
         ], '', '&'));

lib/Alchemy/Phrasea/Authentication/Provider/ProviderInterface.php

@@ -43,9 +43,11 @@ interface ProviderInterface
     /**
      * Redirects to the actual authentication provider
      *
+     * @param array $params
+     *
      * @return RedirectResponse
      */
-    public function authenticate();
+    public function authenticate(array $params);
 
     /**
      * Logout from the provider, removes the token if possible

lib/Alchemy/Phrasea/Authentication/Provider/Twitter.php

@@ -69,14 +69,16 @@ class Twitter extends AbstractProvider
     /**
      * {@inheritdoc}
      */
-    public function authenticate()
+    public function authenticate(array $params = array())
     {
+        $params = array_merge(['providerId' => $this->getId()], $params);
+
         $code = $this->twitter->request(
             'POST',
             $this->twitter->url('oauth/request_token', ''),
             ['oauth_callback' => $this->generator->generate(
                 'login_authentication_provider_callback',
-                ['providerId' => $this->getId()],
+                $params,
                 UrlGenerator::ABSOLUTE_URL
             )]
         );

lib/Alchemy/Phrasea/Authentication/Provider/Viadeo.php

@@ -79,8 +79,10 @@ class Viadeo extends AbstractProvider
     /**
      * {@inheritdoc}
      */
-    public function authenticate()
+    public function authenticate(array $params = array())
     {
+        $params = array_merge(['providerId' => $this->getId()], $params);
+
         $state = $this->createState();
 
         $this->session->set('viadeo.provider.state', $state);
@@ -91,7 +93,7 @@ class Viadeo extends AbstractProvider
             'response_type' => 'code',
             'redirect_uri' => $this->generator->generate(
                 'login_authentication_provider_callback',
-                ['providerId' => $this->getId()],
+                $params,
                 UrlGenerator::ABSOLUTE_URL
             ),
         ]));

lib/Alchemy/Phrasea/Authentication/RegistrationService.php

@@ -10,9 +10,9 @@ use Alchemy\Phrasea\Core\Configuration\RegistrationManager;
 use Alchemy\Phrasea\Core\Event\RegistrationEvent;
 use Alchemy\Phrasea\Core\PhraseaEvents;
 use Alchemy\Phrasea\Exception\RuntimeException;
-use Alchemy\Phrasea\Model\Entities\Registration;
 use Alchemy\Phrasea\Model\Entities\User;
 use Alchemy\Phrasea\Model\Entities\UsrAuthProvider;
+use Alchemy\Phrasea\Model\Entities\WebhookEvent;
 use Alchemy\Phrasea\Model\Manipulator\RegistrationManipulator;
 use Alchemy\Phrasea\Model\Manipulator\TokenManipulator;
 use Alchemy\Phrasea\Model\Manipulator\UserManipulator;
@@ -176,7 +176,7 @@ class RegistrationService
         );
 
         if ($userAuthenticationProvider) {
-            return $userAuthenticationProvider->getUser($this->app);
+            return $userAuthenticationProvider->getUser();
         }
 
         return null;
@@ -190,8 +190,7 @@ class RegistrationService
             $provider = $this->oauthProviderCollection->get($providerId);
         }
 
-        $inscriptions = $this->registrationManager->getRegistrationSummary();
-        $authorizedCollections = $this->getAuthorizedCollections($selectedCollections, $inscriptions);
+        $authorizedCollections = $this->getAuthorizedCollections($selectedCollections);
 
         if (!isset($data['login'])) {
             $data['login'] = $data['email'];
@@ -205,7 +204,7 @@ class RegistrationService
 
         foreach (self::$userPropertySetterMap as $property => $method) {
             if (isset($data[$property])) {
-                call_user_func(array($user, $method), $data[$property]);
+                $user->$method($data[$property]);
             }
         }
 
@@ -217,7 +216,17 @@ class RegistrationService
             $this->entityManager->flush();
         }
 
-        $this->applyAclsToUser($authorizedCollections, $user);
+        if ($this->configuration->get(['registry', 'registration', 'auto-register-enabled'])) {
+            $acl = $this->aclProvider->get($user);
+            foreach ($authorizedCollections as $baseId => $collection) {
+                if( ($model = $collection->getAutoregisterModel($data['email'])) !== null) {
+                    if( ($template_user = $this->userRepository->findByLogin($model)) !== null) {
+                        $acl->apply_model($template_user, [$baseId]);
+                    }
+                }
+            }
+        }
+
         $this->createCollectionAccessDemands($user, $authorizedCollections);
         $user->setMailLocked(true);
 
@@ -226,8 +235,7 @@ class RegistrationService
 
     public function createCollectionRequests(User $user, array $collections)
     {
-        $inscriptions = $this->registrationManager->getRegistrationSummary($user);
-        $authorizedCollections = $this->getAuthorizedCollections($collections, $inscriptions);
+        $authorizedCollections = $this->getAuthorizedCollections($collections);
 
         $this->createCollectionAccessDemands($user, $authorizedCollections);
     }
@@ -279,7 +287,7 @@ class RegistrationService
 
     /**
      * @param array $selectedCollections
-     * @return array
+     * @return \collection[]
      */
     private function getAuthorizedCollections(array $selectedCollections = null)
     {
@@ -292,8 +300,8 @@ class RegistrationService
                     continue;
                 }
 
-                if ($canRegister = \igorw\get_in($inscriptions, [$databox->get_sbas_id(), 'config', 'collections', $collection->get_base_id(), 'can-register'])) {
-                    $authorizedCollections[$collection->get_base_id()] = $canRegister;
+                if (\igorw\get_in($inscriptions, [$databox->get_sbas_id(), 'config', 'collections', $collection->get_base_id(), 'can-register'])) {
+                    $authorizedCollections[$collection->get_base_id()] = $collection;
                 }
             }
         }
@@ -301,42 +309,41 @@ class RegistrationService
         return $authorizedCollections;
     }
 
-    /**
-     * @param $authorizedCollections
-     * @param $user
-     * @return mixed
-     * @throws \Exception
-     */
-    private function applyAclsToUser(array $authorizedCollections, User $user)
-    {
-        $acl = $this->aclProvider->get($user);
-
-        if ($this->configuration->get(['registry', 'registration', 'auto-register-enabled'])) {
-            $template_user = $this->userRepository->findByLogin(User::USER_AUTOREGISTER);
-            $acl->apply_model($template_user, array_keys($authorizedCollections));
-        }
-    }
-
     /**
      * @param User $user
      * @param array $authorizedCollections
      */
     private function createCollectionAccessDemands(User $user, $authorizedCollections)
     {
-        $successfulRegistrations = [];
         $acl = $this->aclProvider->get($user);
-        $autoReg = $acl->get_granted_base();
+
         $registrationManipulator = $this->registrationManipulator;
-
-        array_walk($authorizedCollections, function ($authorization, $baseId) use ($registrationManipulator, $user, &$successfulRegistrations, $acl) {
-            if (false === $authorization || $acl->has_access_to_base($baseId)) {
-                return;
+        $successfulRegistrations = [];
+        foreach($authorizedCollections as $baseId => $collection) {
+            if(!$acl->has_access_to_base($baseId)) {
+                $registrationManipulator->createRegistration($user, $collection);
+                $successfulRegistrations[$baseId] = $collection;
             }
+        }
+
+
+        $autoReg = $acl->get_granted_base();
+        $granted = [];
+        foreach ($autoReg as $baseId => $collection) {
+            $granted[$baseId] = $collection->get_label($this->app['locale']);
+        }
+        if(count($granted) > 0) {
+            $this->app['manipulator.webhook-event']->create(
+                WebhookEvent::USER_REGISTRATION_GRANTED,
+                WebhookEvent::USER_REGISTRATION_TYPE,
+                [
+                    'user_id'  => $user->getId(),
+                    'granted'  => $granted,
+                    'rejected' => []
+                ]
+            );
+        }
 
-            $collection = \collection::getByBaseId($this->app, $baseId);
-            $registrationManipulator->createRegistration($user, $collection);
-            $successfulRegistrations[$baseId] = $collection;
-        });
 
         $this->eventDispatcher->dispatch(PhraseaEvents::REGISTRATION_AUTOREGISTER, new RegistrationEvent($user, $autoReg));
         $this->eventDispatcher->dispatch(PhraseaEvents::REGISTRATION_CREATE, new RegistrationEvent($user, $successfulRegistrations));

lib/Alchemy/Phrasea/Border/Checker/Colorspace.php

@@ -14,15 +14,18 @@ namespace Alchemy\Phrasea\Border\Checker;
 use Alchemy\Phrasea\Application;
 use Alchemy\Phrasea\Border\File;
 use Doctrine\ORM\EntityManager;
+use MediaVorus\Media\Document;
 use Symfony\Component\Translation\TranslatorInterface;
 
 class Colorspace extends AbstractChecker
 {
     protected $colorspaces;
+    protected $mediatypes;
 
     const COLORSPACE_RGB = 'rgb';
     const COLORSPACE_CMYK = 'cmyk';
     const COLORSPACE_GRAYSCALE = 'grayscale';
+    const COLORSPACE_RGBA = 'rgba';
 
     public function __construct(Application $app, array $options)
     {
@@ -30,7 +33,12 @@ class Colorspace extends AbstractChecker
             throw new \InvalidArgumentException('Missing "colorspaces" options');
         }
 
+        if (!isset($options['media_types'])) {
+            throw new \InvalidArgumentException('Missing "media_types" options');
+        }
+
         $this->colorspaces = array_map('strtolower', (array) $options['colorspaces']);
+        $this->mediatypes = $options['media_types'];
         parent::__construct($app);
     }
 
@@ -40,6 +48,8 @@ class Colorspace extends AbstractChecker
 
         if (0 === count($this->colorspaces)) {
             $boolean = true; //bypass color if empty array
+        } elseif (0 !== count($this->mediatypes) && $file->getMedia()->getType() !== NULL && !in_array($file->getMedia()->getType(), $this->mediatypes)) {
+            $boolean = true; //bypass color checker if media type is not in the config
         } elseif (method_exists($file->getMedia(), 'getColorSpace')) {
             $colorspace = null;
             switch ($file->getMedia()->getColorSpace())
@@ -54,6 +64,9 @@ class Colorspace extends AbstractChecker
                 case \MediaVorus\Media\Image::COLORSPACE_GRAYSCALE:
                     $colorspace = self::COLORSPACE_GRAYSCALE;
                     break;
+                case \MediaVorus\Media\Image::COLORSPACE_RGBA:
+                    $colorspace = self::COLORSPACE_RGBA;
+                    break;
             }
 
             $boolean = $colorspace !== null && in_array(strtolower($colorspace), $this->colorspaces);

lib/Alchemy/Phrasea/Border/File.php

@@ -49,6 +49,7 @@ class File
     protected $originalName;
     protected $md5;
     protected $attributes;
+    public static $xmpTag = ['XMP-xmpMM:DocumentID'];
 
     /**
      * Constructor
@@ -102,6 +103,7 @@ class File
             'IPTC:UniqueDocumentID',
             'ExifIFD:ImageUniqueID',
             'Canon:ImageUniqueID',
+            'XMP-xmpMM:DocumentID',
         ];
 
         if (!$this->uuid) {
@@ -112,6 +114,9 @@ class File
             foreach ($availableUUIDs as $meta) {
                 if ($metadatas->containsKey($meta)) {
                     $candidate = $metadatas->get($meta)->getValue()->asString();
+                    if(in_array($meta, self::$xmpTag)){
+                        $candidate = self::sanitizeXmpUuid($candidate);
+                    }
                     if (Uuid::isValid($candidate)) {
                         $uuid = $candidate;
                         break;
@@ -287,4 +292,13 @@ class File
 
         return new File($app, $media, $collection, $originalName);
     }
+
+    /**
+     * Sanitize XMP UUID
+     * @param $uuid
+     * @return mixed
+     */
+    public static function sanitizeXmpUuid($uuid){
+        return str_replace('xmp.did:', '', $uuid);
+    }
 }

lib/Alchemy/Phrasea/Command/Databox/CreateDataboxCommand.php

@@ -3,6 +3,7 @@
 namespace Alchemy\Phrasea\Command\Databox;
 
 use Alchemy\Phrasea\Command\Command;
+use Alchemy\Phrasea\Core\Configuration\StructureTemplate;
 use Alchemy\Phrasea\Databox\DataboxConnectionSettings;
 use Alchemy\Phrasea\Databox\DataboxService;
 use Alchemy\Phrasea\Model\Repositories\UserRepository;
@@ -10,6 +11,7 @@ use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Helper\DialogHelper;
 
 class CreateDataboxCommand extends Command
 {
@@ -18,7 +20,7 @@ class CreateDataboxCommand extends Command
     {
         $this->setName('databox:create')
             ->addArgument('databox', InputArgument::REQUIRED, 'Database name for the databox', null)
-            ->addArgument('owner', InputArgument::REQUIRED, 'Email of the databox admin user', null)
+            ->addArgument('owner', InputArgument::REQUIRED, 'Login of the databox admin user', null)
             ->addOption('connection', 'c', InputOption::VALUE_NONE, 'Flag to set new database settings')
             ->addOption('db-host', null, InputOption::VALUE_OPTIONAL, 'MySQL server host', 'localhost')
             ->addOption('db-port', null, InputOption::VALUE_OPTIONAL, 'MySQL server port', 3306)
@@ -28,13 +30,16 @@ class CreateDataboxCommand extends Command
                 'db-template',
                 null,
                 InputOption::VALUE_OPTIONAL,
-                'Metadata structure language template (available are fr (french) and en (english))',
-                'fr'
+                'Databox template',
+                null
             );
     }
 
     protected function doExecute(InputInterface $input, OutputInterface $output)
     {
+        /** @var DialogHelper $dialog */
+        $dialog = $this->getHelperSet()->get('dialog');
+
         $databoxName = $input->getArgument('databox');
         $connectionSettings = $input->getOption('connection') == false ? null : new DataboxConnectionSettings(
             $input->getOption('db-host'),
@@ -45,18 +50,63 @@ class CreateDataboxCommand extends Command
 
         /** @var UserRepository $userRepository */
         $userRepository = $this->container['repo.users'];
+
+        $owner = $userRepository->findByLogin($input->getArgument('owner'));
+        if(!$owner) {
+            $output->writeln(sprintf("<error>Unknown user \"%s\"</error>", $input->getArgument('owner')));
+
+            return 1;
+        }
+
         /** @var DataboxService $databoxService */
         $databoxService = $this->container['databox.service'];
 
-        $owner = $userRepository->findByEmail($input->getArgument('owner'));
+        if($databoxService->exists($databoxName, $connectionSettings)) {
+            $output->writeln(sprintf("<error>Database \"%s\" already exists</error>", $databoxName));
 
-        $databoxService->createDatabox(
-            $databoxName,
-            $input->getOption('db-template') . '-simple',
-            $owner,
-            $connectionSettings
-        );
+            return 1;
+        }
+
+        /** @var StructureTemplate $templates */
+        $templates = $this->container['phraseanet.structure-template'];
+
+        // if a template name is provided, check that this template exists
+        $templateName = $input->getOption('db-template');
+        if($templateName && !$templates->getByName($templateName)) {
+            throw new \Exception_InvalidArgument(sprintf("Databox template \"%s\" not found.", $templateName));
+        }
+        if(!$templateName) {
+            // propose a default template : the first available if "en-simple" does not exists.
+            $defaultDBoxTemplate = $templates->getDefault();
+
+            do {
+                $templateName = $dialog->ask($output, 'Choose a template from ('.$templates->toString().') for metadata structure <comment>[default: "'.$defaultDBoxTemplate.'"]</comment> : ', $defaultDBoxTemplate);
+                if(!$templates->getByName($templateName)){
+                    $output->writeln("    <error>Data-Box template : Template not found, try again.</error>");
+                }
+            }
+            while (!$templates->getByName($templateName));
+        }
+
+        try {
+            $databoxService->createDatabox(
+                $databoxName,
+                $templateName,
+                $owner,
+                $connectionSettings
+            );
+        }
+        catch(\Exception $e) {
+            $output->writeln(sprintf("<error>Failed to create database \"%s\", error=\"%s\"</error>"
+                , $databoxName
+                , $e->getMessage()
+            ));
+
+            return 1;
+        }
 
         $output->writeln('Databox created');
+
+        return 0;
     }
 }

lib/Alchemy/Phrasea/Command/Developer/IniReset.php

@@ -13,6 +13,8 @@ namespace Alchemy\Phrasea\Command\Developer;
 use Alchemy\Phrasea\Command\Command;
 use Alchemy\Phrasea\Core\Version;
 use Alchemy\Phrasea\Exception\RuntimeException;
+use Alchemy\Phrasea\Utilities\StringHelper;
+use Doctrine\DBAL\Connection;
 use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -112,11 +114,15 @@ class IniReset extends Command
         // get data paths
         $dataPath = $this->container['conf']->get(['main', 'storage', 'subdefs'], $this->container['root.path'].'/datas');
 
-        $schema = $this->container['orm.em']->getConnection()->getSchemaManager();
+        /** @var Connection $connection */
+        $connection = $this->container['orm.em']->getConnection();
+        $schema = $connection->getSchemaManager();
+
         $output->writeln('Creating database "'.$dbs['ab'].'"...<info>OK</info>');
-        $schema->dropAndCreateDatabase($dbs['ab']);
+        $schema->dropAndCreateDatabase(StringHelper::SqlQuote($dbs['ab'], StringHelper::SQL_IDENTIFIER));
+
         $output->writeln('Creating database "'.$dbName.'"...<info>OK</info>');
-        $schema->dropAndCreateDatabase($dbName);
+        $schema->dropAndCreateDatabase(StringHelper::SqlQuote($dbName, StringHelper::SQL_IDENTIFIER));
 
         // inject v3.1 fixtures
         if ($input->getOption('run-patches')) {
@@ -212,7 +218,7 @@ class IniReset extends Command
             } else {
                 $output->write(sprintf('Upgrading... from version <info>%s</info> to <info>%s</info>', $this->app->getApplicationBox()->get_version(), $version->getNumber()), true);
             }
-            
+
             $cmd = 'php ' . __DIR__ . '/../../../../../bin/setup system:upgrade -y -f -v';
             $process = new Process($cmd);
             $process->setTimeout(600);

lib/Alchemy/Phrasea/Command/Developer/JsFixtures.php

@@ -31,7 +31,7 @@ class JsFixtures extends Command
     protected function doExecute(InputInterface $input, OutputInterface $output)
     {
         if (!file_exists($this->container['db.fixture.info']['path'])) {
-            throw new RuntimeException('You must generate sqlite db first, run "bin/console phraseanet:regenerate-sqlite" command.');
+            throw new RuntimeException('You must generate sqlite db first, run "bin/developer phraseanet:regenerate-sqlite" command.');
         }
 
         $this->container['orm.em'] = $this->container->extend('orm.em', function($em, $app) {

lib/Alchemy/Phrasea/Command/Developer/RegenerateSqliteDb.php

@@ -432,8 +432,8 @@ class RegenerateSqliteDb extends Command
             if ($i < 3) {
                 /** @var SubdefSubstituer $substituer */
                 $substituer = $this->container['subdef.substituer'];
-                $substituer->substitute($story, 'preview', $media);
-                $substituer->substitute($story, 'thumbnail', $media);
+                $substituer->substituteSubdef($story, 'preview', $media);
+                $substituer->substituteSubdef($story, 'thumbnail', $media);
             }
             $DI['record_story_' . $i] = $story;
         }

lib/Alchemy/Phrasea/Command/Developer/SetupTestsDbs.php

@@ -13,6 +13,8 @@ namespace Alchemy\Phrasea\Command\Developer;
 
 use Alchemy\Phrasea\Command\Command;
 use Alchemy\Phrasea\Exception\RuntimeException;
+use Alchemy\Phrasea\Utilities\StringHelper;
+use Doctrine\DBAL\Connection;
 use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -44,20 +46,30 @@ class SetupTestsDbs extends Command
         $dbs[] = $settings['database']['ab_name'];
         $dbs[] = $settings['database']['db_name'];
 
-        $schema = $this->container['orm.em']->getConnection()->getSchemaManager();
+        /** @var Connection $connection */
+        $connection = $this->container['orm.em']->getConnection();
+        $schema = $connection->getSchemaManager();
 
         foreach($dbs as $name) {
             $output->writeln('Creating database "'.$name.'"...<info>OK</info>');
+            $name = StringHelper::SqlQuote($name, StringHelper::SQL_IDENTIFIER);    // quote as `identifier`
             $schema->dropAndCreateDatabase($name);
         }
 
-        $this->container['orm.em']->getConnection()->executeUpdate('
-            GRANT ALL PRIVILEGES ON '.$settings['database']['ab_name'].'.* TO \''.$settings['database']['user'].'\'@\''.$settings['database']['host'].'\' IDENTIFIED BY \''.$settings['database']['password'].'\' WITH GRANT OPTION
-        ');
+        $user = StringHelper::SqlQuote($settings['database']['user'], StringHelper::SQL_VALUE); // quote as 'value'
+        $host = StringHelper::SqlQuote($settings['database']['host'], StringHelper::SQL_VALUE);
+        $pass = StringHelper::SqlQuote($settings['database']['password'], StringHelper::SQL_VALUE);
 
-        $this->container['orm.em']->getConnection()->executeUpdate('
-            GRANT ALL PRIVILEGES ON '.$settings['database']['db_name'].'.* TO \''.$settings['database']['user'].'\'@\''.$settings['database']['host'].'\' IDENTIFIED BY \''.$settings['database']['password'].'\' WITH GRANT OPTION
-        ');
+        $ab_name = StringHelper::SqlQuote($settings['database']['ab_name'], StringHelper::SQL_IDENTIFIER);
+        $db_name = StringHelper::SqlQuote($settings['database']['db_name'], StringHelper::SQL_IDENTIFIER);
+
+        $this->container['orm.em']->getConnection()->executeUpdate(
+            'GRANT ALL PRIVILEGES ON '.$ab_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION'
+        );
+
+        $this->container['orm.em']->getConnection()->executeUpdate(
+            'GRANT ALL PRIVILEGES ON '.$db_name.'.* TO '.$user.'@'.$host.' IDENTIFIED BY '.$pass.' WITH GRANT OPTION'
+        );
 
         $this->container['orm.em']->getConnection()->executeUpdate('SET @@global.sql_mode= ""');
 

lib/Alchemy/Phrasea/Command/SearchEngine/IndexCreateCommand.php

@@ -24,7 +24,7 @@ class IndexCreateCommand extends Command
     {
         $this
             ->setName('searchengine:index:create')
-            ->setDescription('Creates search index')
+            ->setDescription('Creates search index <fg=yellow;>(Deprecated use searchengine:index instead)</>')
             ->addOption('drop', 'd', InputOption::VALUE_NONE, 'Drops the index if it already exists.');
     }
 

lib/Alchemy/Phrasea/Command/SearchEngine/IndexDropCommand.php

@@ -23,12 +23,12 @@ class IndexDropCommand extends Command
     {
         $this
             ->setName('searchengine:index:drop')
-            ->setDescription('Deletes the search index')
+            ->setDescription('Deletes the search index <fg=yellow;>(Deprecated use searchengine:index instead)</>')
             ->addOption(
                'force',
                null,
                InputOption::VALUE_NONE,
-               "Don't ask for for the dropping of the index, but force the operation to run."
+               "Don't ask for the dropping of the index, but force the operation to run."
             )
         ;
     }

lib/Alchemy/Phrasea/Command/SearchEngine/IndexManipulateCommand.php

@@ -0,0 +1,215 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2014 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Command\SearchEngine;
+
+use Alchemy\Phrasea\Command\Command;
+use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
+use Alchemy\Phrasea\SearchEngine\Elastic\Indexer;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class IndexManipulateCommand extends Command
+{
+    /** @var  OutputInterface */
+    private $output = null;
+
+    /**
+     * print a string if verbosity >= verbose (-v)
+     *
+     * @param string $s
+     * @param int $verbosity
+     */
+    private function verbose($s, $verbosity = OutputInterface::VERBOSITY_VERBOSE)
+    {
+        if ($this->output->getVerbosity() >= $verbosity) {
+            $this->output->writeln($s);
+        }
+    }
+
+    protected function configure()
+    {
+        $this
+            ->setName('searchengine:index')
+            ->setDescription('Manipulates search index')
+            ->addOption('drop',      'd', InputOption::VALUE_NONE, 'Drops the index.')
+            ->addOption('create',    'c', InputOption::VALUE_NONE, 'Creates the index.')
+            ->addOption('populate',  'p', InputOption::VALUE_NONE, 'Populates the index.')
+            ->addOption('temporary', 't', InputOption::VALUE_NONE, 'Populates using temporary index.')
+            ->addOption('name', null, InputOption::VALUE_REQUIRED, 'index name', null)
+            ->addOption('host', null, InputOption::VALUE_REQUIRED, 'host', null)
+            ->addOption('port', null, InputOption::VALUE_REQUIRED, 'port', null)
+            ->addOption('order', null, InputOption::VALUE_REQUIRED, 'order (record_id|modification_date)[.asc|.desc]', null)
+            ->addOption(
+                'databox_id',
+                null,
+                InputOption::VALUE_OPTIONAL | InputOption::VALUE_IS_ARRAY,
+                'Only populate chosen databox'
+            )->addOption(
+                'force',
+                null,
+                InputOption::VALUE_NONE,
+                "Don't ask for for the dropping of the index, but force the operation to run."
+            );
+
+    }
+
+    protected function doExecute(InputInterface $input, OutputInterface $output)
+    {
+        $this->output = $output;
+
+        /** @var Indexer $indexer */
+        $indexer = $this->container['elasticsearch.indexer'];
+        /** @var ElasticsearchOptions $options */
+        $options = $indexer->getIndex()->getOptions();
+
+        if($input->getOption('name')) {
+            $options->setIndexName($input->getOption('name'));
+        }
+        if($input->getOption('host')) {
+            $options->setHost($input->getOption('host'));
+        }
+        if($input->getOption('port')) {
+            $options->setPort($input->getOption('port'));
+        }
+
+        if($input->getOption('order')) {
+            $order = explode('.', $input->getOption('order'));
+            if (!$options->setPopulateOrder($order[0])) {
+                $output->writeln(sprintf('<error>bad order value for --order</error>'));
+
+                return 1;
+            }
+            if (count($order) > 1) {
+                if (!$options->setPopulateDirection($order[1])) {
+                    $output->writeln(sprintf('<error>bad direction value for --order</error>'));
+
+                    return 1;
+                }
+            }
+        }
+
+        $idx = sprintf("%s@%s:%s", $options->getIndexName(), $options->getHost(), $options->getPort());
+
+        $drop         = $input->getOption('drop');
+        $create       = $input->getOption('create');
+        $populate     = $input->getOption('populate');
+        $temporary    = $input->getOption('temporary');
+        $databoxes_id = $input->getOption('databox_id');
+
+        if($temporary && (!$populate || $databoxes_id)) {
+            $output->writeln(sprintf('<error>temporary must be used to populate all databoxes</error>', $idx));
+
+            return 1;
+        }
+
+        $indexExists = $indexer->indexExists();
+
+        if ($drop && $indexExists) {
+            if ($input->getOption('force')) {
+                $confirmation = true;
+            }
+            else {
+                $question = '<question>You are about to delete the index and all contained data. Are you sure you wish to continue? (y/n)</question>';
+                $confirmation = $this->getHelper('dialog')->askConfirmation($output, $question, false);
+            }
+
+            if ($confirmation) {
+                $indexer->deleteIndex();
+                $this->verbose(sprintf('<info>Search index "%s" was dropped.</info>', $idx));
+            }
+            else {
+                $this->verbose('Canceled.');
+
+                return 0;
+            }
+        }
+
+        $indexExists = $indexer->indexExists();
+
+        if ($create) {
+            if($indexExists) {
+                $output->writeln(sprintf('<error>The search index "%s" already exists.</error>', $idx));
+
+                return 1;
+            }
+            else {
+                $r = $indexer->createIndex();
+                $this->verbose(sprintf('<info>Search index "%s@%s:%s" -> "%s" was created</info>'
+                    , $r['alias']
+                    , $options->getHost()
+                    , $options->getPort()
+                    , $r['index']
+                ));
+            }
+        }
+
+        $indexExists = $indexer->indexExists();
+
+        if($populate) {
+            if(!$indexExists) {
+                $r = $indexer->createIndex();
+                $this->verbose(sprintf('<info>Search index "%s@%s:%s" -> "%s" was created</info>'
+                    , $r['alias']
+                    , $options->getHost()
+                    , $options->getPort()
+                    , $r['index']
+                ));
+            }
+
+            $oldAliasName = $indexer->getIndex()->getName();
+            $newAliasName = $newIndexName = null;
+            if($temporary) {
+                // change the name to create a new index
+                $now = explode(' ', microtime());
+                $now = sprintf("%X%X", $now[1], 1000000*$now[0]);
+                $indexer->getIndex()->getOptions()->setIndexName($oldAliasName . "_T" . $now);
+
+                $r = $indexer->createIndex($oldAliasName);
+                $newIndexName = $r['index'];
+                $newAliasName = $r['alias'];
+
+                $this->verbose(sprintf('<info>Temporary index "%s@%s:%s" -> "%s" was created</info>'
+                    , $r['alias']
+                    , $options->getHost()
+                    , $options->getPort()
+                    , $r['index']
+                ));
+            }
+
+            foreach ($this->container->getDataboxes() as $databox) {
+                if (!$databoxes_id || in_array($databox->get_sbas_id(), $databoxes_id)) {
+                    $r = $indexer->populateIndex(Indexer::THESAURUS | Indexer::RECORDS, $databox, false); // , $temporary);
+                    $output->writeln(sprintf(
+                            "Indexation of databox \"%s\" finished in %0.2f sec (Mem. %0.2f Mo)",
+                            $databox->get_dbname(),
+                            $r['duration']/1000,
+                            $r['memory']/1048576)
+                    );
+                }
+            }
+
+            if($temporary) {
+                $this->verbose('<info>Renaming temporary :</info>');
+
+                $indexer->getIndex()->getOptions()->setIndexName($oldAliasName);
+
+                $r = $indexer->replaceIndex($newIndexName, $newAliasName);
+                foreach($r as $action) {
+                    $this->verbose(sprintf('  <info>%s</info>', $action['msg']));
+                }
+            }
+        }
+
+        return 0;
+    }
+}

lib/Alchemy/Phrasea/Command/SearchEngine/IndexPopulateCommand.php

@@ -23,7 +23,7 @@ class IndexPopulateCommand extends Command
     {
         $this
             ->setName('searchengine:index:populate')
-            ->setDescription('Populate search index')
+            ->setDescription('Populate search index <fg=yellow;>(Deprecated use searchengine:index instead)</>')
             ->addOption(
                 'thesaurus',
                 null,

lib/Alchemy/Phrasea/Command/Setup/FixAutoincrements.php

@@ -0,0 +1,381 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Command\Setup;
+
+use Alchemy\Phrasea\Command\Command;
+use Symfony\Component\Console\Helper\Table;
+use Symfony\Component\Console\Helper\TableCell;
+use Symfony\Component\Console\Helper\TableSeparator;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class FixAutoincrements extends Command
+{
+    /** @var  InputInterface */
+    private $input;
+
+    /** @var  OutputInterface */
+    private $output;
+
+    /** @var  \appbox */
+    private $appBox;
+
+    /** @var  \databox[] */
+    private $databoxes;
+
+    public function __construct($name = null)
+    {
+        parent::__construct($name);
+
+        $this
+            ->setDescription("Fix autoincrements")
+            ->addOption('dry', null, InputOption::VALUE_NONE, 'Dry run : list but do not update.');
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function doExecute(InputInterface $input, OutputInterface $output)
+    {
+        $this->input     = $input;
+        $this->output    = $output;
+        $this->appBox    = $this->getContainer()->getApplicationBox();
+        $this->databoxes = [];
+        foreach ($this->getContainer()->getDataboxes() as $databox) {
+            $this->databoxes[] = $databox;
+        }
+
+        $this
+            ->ab_fix_bas()
+            ->ab_fix_BasketElements()
+            ->ab_fix_Baskets()
+            ->ab_fix_Sessions()
+            ->db_fix_coll()
+            ->db_fix_record()
+        ;
+    }
+
+    /**
+     * @param \Appbox|\databox $box
+     * @param $table
+     * @param $subtables
+     */
+    private function box_fixTable($box, $table, $subtables)
+    {
+        $ouputTable = new Table($this->output);
+
+        $title = sprintf("fixing table \"%s.%s\"", $box->get_dbname(), $table);
+        $ouputTable->setHeaders([new TableCell($title, ['colspan'=>2])]);
+
+        $max_id = $this->box_getMax($box, $table, $subtables, $ouputTable);
+        //if($this->output->getVerbosity())
+        $this->box_setAutoIncrement($box, $table, $max_id, $ouputTable);
+
+        $ouputTable->render();
+
+        $this->output->writeln("");
+    }
+
+    /**
+     * @param \Appbox|\databox $box
+     * @param $table
+     * @param $subtables
+     * @param Table $ouputTable
+     * @return int
+     */
+    private function box_getMax($box, $table, $subtables, $ouputTable)
+    {
+        $sql = "  SELECT '".$box->get_dbname().'.'.$table.".AUTO_INCREMENT' AS src, AUTO_INCREMENT AS `id` FROM information_schema.TABLES\n"
+            . "         WHERE TABLE_SCHEMA = :dbname AND TABLE_NAME = '" . $table . "'\n";
+
+        foreach ($subtables as $subtable => $fieldname) {
+            $sql .=
+                "    UNION\n"
+                . "  SELECT '".$box->get_dbname().'.'.$subtable.'.'.$fieldname."' AS src, MAX(`" . $fieldname . "`)+1 AS `id` FROM `" . $subtable . "`\n";
+        }
+
+        $stmt = $box->get_connection()->executeQuery($sql, [':dbname' => $box->get_dbname()]);
+        $max_id = 0;
+        $rows = $stmt->fetchAll();
+        foreach($rows as $row) {
+            $id = $row['id'];
+
+            $ouputTable->addRow([$row['src'], is_null($id) ? "null" : $id]);
+
+            if(!is_null($id)){
+                $id = (int)$id;
+                if ($id > $max_id) {
+                    $max_id = $id;
+                }
+            }
+        }
+        $stmt->closeCursor();
+
+        return $max_id;
+    }
+
+    /**
+     * @param \Appbox|\databox $box
+     * @param $table
+     * @param $max_id
+     * @param Table $ouputTable
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    private function box_setAutoIncrement($box, $table, $max_id, $ouputTable)
+    {
+        $sql = "ALTER TABLE `" . $table . "` AUTO_INCREMENT = " . $max_id;
+        $msg = [
+            sprintf("%s.%s.AUTO_INCREMENT set to", $box->get_dbname(), $table),
+            $max_id,
+        ];
+
+        $this->box_setSQL($box, $sql, [], $msg, $ouputTable);
+    }
+
+    /**
+     * @param \Appbox|\databox $box
+     * @param string $sql
+     * @param array $parms
+     * @param string $msg[]
+     * @param Table $ouputTable
+     */
+    private function box_setSQL($box, $sql, $parms, $msg, $ouputTable)
+    {
+        $ouputTable->addRow(New TableSeparator());
+        if($this->input->getOption("dry")) {
+            $msg[1] = sprintf("<comment>%s</comment> (dry-run : not done)", $msg[1]);
+            $ouputTable->addRow($msg);
+        }
+        else {
+            try {
+                $stmt = $box->get_connection()->executeQuery($sql, $parms);
+                $stmt->closeCursor();
+
+                $msg[1] = sprintf("<info>%s</info>", $msg[1]);
+                $ouputTable->addRow($msg);
+            }
+            catch(\Exception $e) {
+                $msg[1] = sprintf("<error>%s</error>)", $msg[1]);
+                $ouputTable->addRow($msg);
+            }
+        }
+    }
+
+
+    //
+    //
+    //============================= Databoxes =======================
+    //
+    //
+
+    /**
+     * fix every Databox "record" autoincrement
+     *
+     * @return $this
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    private function db_fix_coll()
+    {
+        foreach ($this->databoxes as $databox) {
+            $this->box_fixTable(
+                $databox,
+                'coll',
+                [
+                    'collusr'   => 'coll_id',
+                    'log_colls' => 'coll_id',
+                ]
+            );
+        }
+
+        return $this;
+    }
+
+    /**
+     * fix every Databox "record" autoincrement
+     *
+     * @return $this
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    private function db_fix_record()
+    {
+        foreach ($this->databoxes as $databox) {
+            $this->box_fixTable(
+                $databox,
+                'record',
+                [
+                    'log_docs' => 'record_id',
+                    'log_view' => 'record_id',
+                    'record'   => 'record_id',
+                ]
+            );
+        }
+
+        return $this;
+    }
+
+
+    //
+    //
+    //============================= ApplicationBox =======================
+    //
+    //
+
+    /**
+     * fix AppBox "bas" autoincrement
+     *
+     * @return $this
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    private function ab_fix_bas()
+    {
+        $this->box_fixTable(
+            $this->appBox,
+            'bas',
+            [
+                'basusr'            => 'base_id',
+                'demand'            => 'base_id',
+                'Feeds'             => 'base_id',
+                'FtpExportElements' => 'base_id',
+                'LazaretFiles'      => 'base_id',
+                'OrderElements'     => 'base_id',
+                'Registrations'     => 'base_id',
+            ]
+        );
+
+        return $this;
+    }
+
+
+    /**
+     * fix AppBox "BasketElements" autoincrement
+     *
+     * @return $this
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    private function ab_fix_BasketElements()
+    {
+        $this->box_fixTable(
+            $this->appBox,
+            'BasketElements',
+            [
+                'ValidationDatas' => 'basket_element_id',
+            ]
+        );
+
+        return $this;
+    }
+
+
+    /**
+     * fix AppBox "Baskets" autoincrement
+     *
+     * @return $this
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    private function ab_fix_Baskets()
+    {
+        $this->box_fixTable(
+            $this->appBox,
+            'Baskets',
+            [
+                'BasketElements'     => 'basket_id',
+                'Orders'             => 'basket_id',
+                'ValidationSessions' => 'basket_id',
+            ]
+        );
+
+        return $this;
+    }
+
+
+    /**
+     * fix AppBox "Sessions" autoincrement
+     *
+     * @return $this
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    private function ab_fix_Sessions()
+    {
+        $ouputTable = new Table($this->output);
+
+        $title = sprintf("fixing table \"%s.%s\"", $this->appBox->get_dbname(), "Sessions");
+        $ouputTable->setHeaders([new TableCell($title, ['colspan'=>2])]);
+
+        $site = $this->getContainer()['conf']->get(['main', 'key']);
+
+        // if autoincrement, get the current value as a minimum
+        $max_SessionId = $this->box_getMax(
+            $this->appBox,
+            'Sessions',
+            [],          // no sub-tables
+            $ouputTable
+        );
+
+        // get max session from databoxes, using the "log" table which refers to ab.Sessions ids
+        foreach ($this->databoxes as $databox) {
+            $db = $databox->get_connection();
+
+            $sql = "SELECT MAX(`sit_session`) FROM `log` WHERE `site` = :site";
+            $stmt = $db->executeQuery($sql, [':site' => $site]);
+            $id = $stmt->fetchColumn(0);
+
+            $ouputTable->addRow([sprintf("%s.log.sit_session", $databox->get_dbname()), sprintf("%s", is_null($id) ? 'null' : $id)]);
+
+            if(!is_null($id)) {
+                $id = (int)$id + 1;
+                if ($id > $max_SessionId) {
+                    $max_SessionId = $id;
+                }
+            }
+            $stmt->closeCursor();
+        }
+
+        // fix using different methods
+        foreach ([
+                    // 4.0 with autoincrement
+                    [
+                        'sql'   => "ALTER TABLE `Sessions` AUTO_INCREMENT = " . $max_SessionId,   // can't use parameter here
+                        'parms' => [],
+                        'msg'   => ["ab.Sessions.AUTO_INCREMENT set to", $max_SessionId],
+                    ],
+                    /* --- custon generators not yet implemented in phraseanet
+                    // 4.0 with custom generator already set
+                    [
+                        'sql'   => "UPDATE `Id` SET value = :v WHERE `id` = :k",
+                        'parms' => [':v' => $max_SessionId, ':k' => 'Alchemy\\Phrasea\\Model\\Entities\\Session'],
+                        'msg'   => ["ab.Id['Sessions']' set to", $max_SessionId],
+                    ],
+                    // 4.0 with custom generator not yet set
+                    [
+                        'sql'   => "INSERT INTO `Id` (`id`, `value`) VALUES (:k, :v)",
+                        'parms' => [':v' => $max_SessionId, ':k' => 'Alchemy\\Phrasea\\Model\\Entities\\Session'],
+                        'msg'   => ["ab.Id['Sessions']' set to", $max_SessionId],
+                    ]
+                    --- */
+                ] as $sql) {
+
+            // one or more sql will fail, no pb
+            $this->box_setSQL($this->appBox, $sql['sql'], $sql['parms'], $sql['msg'], $ouputTable);
+        }
+
+        $ouputTable->render();
+
+        $this->output->writeln("");
+
+        return $this;
+    }
+
+}
+

lib/Alchemy/Phrasea/Command/Setup/Install.php

@@ -12,6 +12,7 @@
 namespace Alchemy\Phrasea\Command\Setup;
 
 use Alchemy\Phrasea\Command\Command;
+use Alchemy\Phrasea\Core\Configuration\StructureTemplate;
 use Doctrine\DBAL\Driver\Connection;
 use Symfony\Component\Console\Helper\DialogHelper;
 use Symfony\Component\Console\Input\ArrayInput;
@@ -23,11 +24,18 @@ use Symfony\Component\Process\ExecutableFinder;
 class Install extends Command
 {
     private $executableFinder;
+    /** @var StructureTemplate StructureTemplate */
+    private $structureTemplate;
 
-    public function __construct($name = null)
+    /**
+     * @param null|string $name
+     * @param StructureTemplate $structureTemplate
+     */
+    public function __construct($name, $structureTemplate)
     {
         parent::__construct($name);
 
+        $this->structureTemplate = $structureTemplate;
         $this->executableFinder = new ExecutableFinder();
 
         $this
@@ -38,9 +46,9 @@ class Install extends Command
             ->addOption('db-port', null, InputOption::VALUE_OPTIONAL, 'MySQL server port', 3306)
             ->addOption('db-user', null, InputOption::VALUE_OPTIONAL, 'MySQL server user', 'phrasea')
             ->addOption('db-password', null, InputOption::VALUE_OPTIONAL, 'MySQL server password', null)
-            ->addOption('db-template', null, InputOption::VALUE_OPTIONAL, 'Metadata structure language template (available are fr (french) and en (english))', null)
-            ->addOption('databox', null, InputOption::VALUE_OPTIONAL, 'Database name for the DataBox', null)
             ->addOption('appbox', null, InputOption::VALUE_OPTIONAL, 'Database name for the ApplicationBox', null)
+            ->addOption('databox', null, InputOption::VALUE_OPTIONAL, 'Database name for the DataBox', null)
+            ->addOption('db-template', null, InputOption::VALUE_OPTIONAL, 'Databox template (' . $this->structureTemplate->toString() . ')', null)
             ->addOption('data-path', null, InputOption::VALUE_OPTIONAL, 'Path to data repository', realpath(__DIR__ . '/../../../../../datas'))
             ->addOption('server-name', null, InputOption::VALUE_OPTIONAL, 'Server name')
             ->addOption('indexer', null, InputOption::VALUE_OPTIONAL, 'Path to Phraseanet Indexer', 'auto')
@@ -49,11 +57,22 @@ class Install extends Command
         return $this;
     }
 
+    private function serverNameToAppBoxName($serverName)
+    {
+        return "ab_" . $serverName;
+    }
+
+    private function serverNameToDataBoxName($serverName)
+    {
+        return "db_" . $serverName;
+    }
+
     /**
      * {@inheritdoc}
      */
     protected function doExecute(InputInterface $input, OutputInterface $output)
     {
+        /** @var DialogHelper $dialog */
         $dialog = $this->getHelperSet()->get('dialog');
 
         $output->writeln("<comment>
@@ -91,12 +110,16 @@ class Install extends Command
             }
         }
 
-        $abConn = $this->getABConn($input, $output, $dialog);
+        $serverName = $this->getServerName($input, $output, $dialog);
 
-        list($dbConn, $template) = $this->getDBConn($input, $output, $abConn, $dialog);
+        $abConn = $this->getABConn($input, $output, $dialog, $serverName);
+        if(!$abConn) {
+            return 1;       // no ab is fatal
+        }
+
+        list($dbConn, $templateName) = $this->getDBConn($input, $output, $abConn, $dialog);
         list($email, $password) = $this->getCredentials($input, $output, $dialog);
         $dataPath = $this->getDataPath($input, $output, $dialog);
-        $serverName = $this->getServerName($input, $output, $dialog);
 
         if (!$input->getOption('yes')) {
             $continue = $dialog->askConfirmation($output, "<question>Phraseanet is going to be installed, continue ? (N/y)</question>", false);
@@ -108,32 +131,32 @@ class Install extends Command
             }
         }
 
-        $this->container['phraseanet.installer']->install($email, $password, $abConn, $serverName, $dataPath, $dbConn, $template, $this->detectBinaries());
+        $this->container['phraseanet.installer']->install($email, $password, $abConn, $serverName, $dataPath, $dbConn, $templateName, $this->detectBinaries());
 
         if (null !== $this->getApplication()) {
             $command = $this->getApplication()->find('crossdomain:generate');
-            $command->run(new ArrayInput(array(
+            $command->run(new ArrayInput([
                 'command' => 'crossdomain:generate'
-            )), $output);
+            ]), $output);
         }
 
         $output->writeln("<info>Install successful !</info>");
 
-        return;
+        return 0;
     }
 
     private function getABConn(InputInterface $input, OutputInterface $output, DialogHelper $dialog)
     {
         $abConn = $info = null;
         if (!$input->getOption('appbox')) {
-            $output->writeln("\n<info>--- Database credentials ---</info>\n");
+            $output->writeln("<info>--- Database credentials ---</info>");
 
             do {
-                $hostname = $dialog->ask($output, "DB hostname (localhost) : ", 'localhost');
-                $port = $dialog->ask($output, "DB port (3306) : ", 3306);
-                $dbUser = $dialog->ask($output, "DB user : ");
-                $dbPassword = $dialog->askHiddenResponse($output, "DB password (hidden) : ");
-                $abName = $dialog->ask($output, "DB name (phraseanet) : ", 'phraseanet');
+                $hostname = $dialog->ask($output, 'DB hostname <comment>[default: "localhost"]</comment> : ', 'localhost');
+                $port = $dialog->ask($output, 'DB port <comment>[default: "3306"]</comment> : ', '3306');
+                $dbUser = $dialog->ask($output, 'DB user : ');
+                $dbPassword = $dialog->askHiddenResponse($output, 'DB password (hidden) : ');
+                $abName = $dialog->ask($output, 'ApplicationBox name <comment>[default: "phraseanet"]</comment> : ', 'phraseanet');
 
                 $info = [
                     'host'     => $hostname,
@@ -145,9 +168,10 @@ class Install extends Command
                 try {
                     $abConn = $this->container['dbal.provider']($info);
                     $abConn->connect();
-                    $output->writeln("\n\t<info>Application-Box : Connection successful !</info>\n");
+                    $output->writeln("<info>Application-Box : Connection successful !</info>");
                 } catch (\Exception $e) {
-                    $output->writeln("\n\t<error>Invalid connection parameters</error>\n");
+                    $output->writeln("<error>Application-Box : Failed to connect, try again.</error>");
+                    $abConn = null;
                 }
             } while (!$abConn);
         } else {
@@ -161,7 +185,7 @@ class Install extends Command
 
             $abConn = $this->container['dbal.provider']($info);
             $abConn->connect();
-            $output->writeln("\n\t<info>Application-Box : Connection successful !</info>\n");
+            $output->writeln("<info>Application-Box : Connection successful !</info>");
         }
 
         // add dbs.option & orm.options services to use orm.em later
@@ -175,12 +199,13 @@ class Install extends Command
 
     private function getDBConn(InputInterface $input, OutputInterface $output, Connection $abConn, DialogHelper $dialog)
     {
-        $dbConn = $template = $info = null;
-        $templates = $this->container['phraseanet.structure-template']->getAvailable();
+        $dbConn = $info = null;
+        $templateName = null;
+
         if (!$input->getOption('databox')) {
             do {
                 $retry = false;
-                $dbName = $dialog->ask($output, 'DataBox name, will not be created if empty : ', null);
+                $dbName = $dialog->ask($output, 'Data-Box name, will not be created if empty : ', null);
 
                 if ($dbName) {
                     try {
@@ -194,19 +219,13 @@ class Install extends Command
 
                         $dbConn = $this->container['dbal.provider']($info);
                         $dbConn->connect();
-                        $output->writeln("\n\t<info>Data-Box : Connection successful !</info>\n");
-
-                        do {
-                            $template = $dialog->ask($output, "Choose a language template for metadata structure, available are {$templates->__toString()} : ", 'en');
-                        }
-                        while (!in_array($template, array_keys($templates->getTemplates())));
-
-                        $output->writeln("\n\tLanguage selected is <info>'$template'</info>\n");
+                        $output->writeln("<info>Data-Box : Connection successful !</info>");
                     } catch (\Exception $e) {
+                        $output->writeln("    <error>Data-Box : Failed to connect, try again.</error>");
                         $retry = true;
                     }
                 } else {
-                    $output->writeln("\n\tNo databox will be created\n");
+                    $output->writeln("No databox will be created");
                 }
             } while ($retry);
         } else {
@@ -220,17 +239,37 @@ class Install extends Command
 
             $dbConn = $this->container['dbal.provider']($info);
             $dbConn->connect();
-            $output->writeln("\n\t<info>Data-Box : Connection successful !</info>\n");
-            $template = $input->getOption('db-template') ? : 'en';
+            $output->writeln("<info>Data-Box : Connection successful !</info>");
         }
 
         // add dbs.option & orm.options services to use orm.em later
         if ($dbConn && $info) {
+            /** @var StructureTemplate $templates */
+            $templates = $this->container['phraseanet.structure-template'];
+
+            // if a template name is provided, check that this template exists
+            $templateName = $input->getOption('db-template');
+            if($templateName && !$templates->getByName($templateName)) {
+                throw new \Exception_InvalidArgument(sprintf("Databox template \"%s\" not found.", $templateName));
+            }
+            if(!$templateName) {
+                // propose a default template : the first available if "en-simple" does not exists.
+                $defaultDBoxTemplate = $this->structureTemplate->getDefault();
+
+                do {
+                    $templateName = $dialog->ask($output, 'Choose a template from ('.$templates->toString().') for metadata structure <comment>[default: "'.$defaultDBoxTemplate.'"]</comment> : ', $defaultDBoxTemplate);
+                    if(!$templates->getByName($templateName)) {
+                        $output->writeln("<error>Data-Box template : Template not found, try again.</error>");
+                    }
+                }
+                while (!$templates->getByName($templateName));
+            }
+
             $this->container['dbs.options'] = array_merge($this->container['db.options.from_info']($info), $this->container['dbs.options']);
             $this->container['orm.ems.options'] = array_merge($this->container['orm.em.options.from_info']($info), $this->container['orm.ems.options']);
         }
 
-        return [$dbConn, $template];
+        return [$dbConn, $templateName];
     }
 
     private function getCredentials(InputInterface $input, OutputInterface $output, DialogHelper $dialog)
@@ -238,7 +277,7 @@ class Install extends Command
         $email = $password = null;
 
         if (!$input->getOption('email') && !$input->getOption('password')) {
-            $output->writeln("\n<info>--- Account Informations ---</info>\n");
+            $output->writeln("<info>--- Account Informations ---</info>");
 
             do {
                 $email = $dialog->ask($output, 'Please provide a valid e-mail address : ');
@@ -248,7 +287,7 @@ class Install extends Command
                 $password = $dialog->askHiddenResponse($output, 'Please provide a password (hidden, 6 character min) : ');
             } while (strlen($password) < 6);
 
-            $output->writeln("\n\t<info>Email / Password successfully set</info>\n");
+            $output->writeln("<info>Email / Password successfully set</info>");
         } elseif ($input->getOption('email') && $input->getOption('password')) {
             if (!\Swift_Validate::email($input->getOption('email'))) {
                 throw new \RuntimeException('Invalid email addess');

lib/Alchemy/Phrasea/Command/Setup/XSendFileMappingGenerator.php

@@ -37,7 +37,7 @@ class XSendFileMappingGenerator extends Command
     protected function doExecute(InputInterface $input, OutputInterface $output)
     {
         $extractor = new DataboxPathExtractor($this->container->getApplicationBox());
-        $paths = $extractor->extractPaths();
+        $paths = $extractor->extractPaths('xsendfile');
         foreach ($paths as $path) {
             $this->container['filesystem']->mkdir($path);
         }

lib/Alchemy/Phrasea/Controller/AbstractDelivery.php

@@ -34,11 +34,14 @@ abstract class AbstractDelivery
     {
         $mediaSubdefinition = $record->get_subdef($subdef);
 
+        $filename = $request->get("filename") ?: $mediaSubdefinition->get_file();
+
         $pathOut = $this->tamperProofSubDefinition($mediaSubdefinition, $watermark, $stamp);
 
         $disposition = $request->query->get('download') ? DeliverDataInterface::DISPOSITION_ATTACHMENT : DeliverDataInterface::DISPOSITION_INLINE;
 
-        $response = $this->deliverFile($pathOut, $mediaSubdefinition->get_file(), $disposition, $mediaSubdefinition->get_mime());
+        // nb: $filename will be sanitized, no need to do it here
+        $response = $this->deliverFile($pathOut,  $filename, $disposition, $mediaSubdefinition->get_mime());
 
         if (in_array($subdef, array('document', 'preview'))) {
             $response->setPrivate();

lib/Alchemy/Phrasea/Controller/Admin/RootController.php

@@ -246,8 +246,8 @@ class RootController extends Controller
             'searchable' => $request->request->get('searchable') ? '1' : '0',
             'printable'  => $request->request->get('printable') ? '1' : '0',
             'name'       => $request->request->get('name', ''),
-            'labelon'    => $request->request->get('label_on', ''),
-            'labeloff'   => $request->request->get('label_off', ''),
+            'labelon'    => htmlentities($request->request->get('label_on', '')),
+            'labeloff'   => htmlentities($request->request->get('label_off', '')),
             'labels_on'  => $request->request->get('labels_on', []),
             'labels_off' => $request->request->get('labels_off', []),
         ];

lib/Alchemy/Phrasea/Controller/Admin/SubdefsController.php

@@ -208,10 +208,10 @@ class SubdefsController extends Controller
     protected function getSubviewsMapping()
     {
         $mapping = [
-            Type::TYPE_IMAGE    => [Subdef::TYPE_IMAGE],
+            Type::TYPE_IMAGE    => [Subdef::TYPE_IMAGE, Subdef::TYPE_PDF],
             Type::TYPE_VIDEO    => [Subdef::TYPE_IMAGE, Subdef::TYPE_VIDEO, Subdef::TYPE_ANIMATION],
             Type::TYPE_AUDIO    => [Subdef::TYPE_IMAGE, Subdef::TYPE_AUDIO],
-            Type::TYPE_DOCUMENT => [Subdef::TYPE_IMAGE, Subdef::TYPE_FLEXPAPER],
+            Type::TYPE_DOCUMENT => [Subdef::TYPE_IMAGE, Subdef::TYPE_FLEXPAPER, Subdef::TYPE_PDF],
             Type::TYPE_FLASH    => [Subdef::TYPE_IMAGE]
         ];
 
@@ -229,7 +229,7 @@ class SubdefsController extends Controller
                     "JPG"                               => null,
                     "160px JPG"                         => [
                         Image::OPTION_SIZE       => "160",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -238,7 +238,7 @@ class SubdefsController extends Controller
                     ],
                     "320 px JPG (thumbnail Phraseanet)" => [
                         Image::OPTION_SIZE       => "320",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -247,7 +247,7 @@ class SubdefsController extends Controller
                     ],
                     "640px JPG"                         => [
                         Image::OPTION_SIZE       => "640",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -256,7 +256,7 @@ class SubdefsController extends Controller
                     ],
                     "1280px JPG (preview Phraseanet)"   => [
                         Image::OPTION_SIZE       => "1280",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -265,7 +265,7 @@ class SubdefsController extends Controller
                     ],
                     "2560px JPG"                        => [
                         Image::OPTION_SIZE       => "2560",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -275,7 +275,7 @@ class SubdefsController extends Controller
                     "PNG"                               => null,
                     "160px PNG 8 bits"                  => [
                         Image::OPTION_SIZE       => "160",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -284,7 +284,7 @@ class SubdefsController extends Controller
                     ],
                     "320px PNG 8 bits"                  => [
                         Image::OPTION_SIZE       => "320",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -293,7 +293,7 @@ class SubdefsController extends Controller
                     ],
                     "640px PNG 8 bits"                  => [
                         Image::OPTION_SIZE       => "640",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -302,7 +302,7 @@ class SubdefsController extends Controller
                     ],
                     "1280px PNG 8 bits"                 => [
                         Image::OPTION_SIZE       => "1280",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -311,7 +311,7 @@ class SubdefsController extends Controller
                     ],
                     "2560px PNG 8 bits"                 => [
                         Image::OPTION_SIZE       => "2560",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -321,7 +321,7 @@ class SubdefsController extends Controller
                     "TIFF"                              => null,
                     "1280 TIFF"                         => [
                         Image::OPTION_SIZE       => "1280",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -330,7 +330,7 @@ class SubdefsController extends Controller
                     ],
                     "2560px TIFF"                       => [
                         Image::OPTION_SIZE       => "2560",
-                        Image::OPTION_RESOLUTION => "75",
+                        Image::OPTION_RESOLUTION => "72",
                         Image::OPTION_STRIP      => "yes",
                         Image::OPTION_FLATTEN    => "yes",
                         Image::OPTION_QUALITY    => "75",
@@ -554,6 +554,11 @@ class SubdefsController extends Controller
                 ],
                 "form"        => [],
             ],
+            Subdef::TYPE_PDF => array(
+                "definitions" => array(
+                ),
+                "form" => array(),
+            ),
         ];
 
         return $config;

lib/Alchemy/Phrasea/Controller/Admin/TaskManagerController.php

@@ -86,10 +86,6 @@ class TaskManagerController extends Controller
 
     public function getLiveInformation(Request $request)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         if ($request->getRequestFormat() !== "json") {
             $this->app->abort(406, 'Only JSON format is accepted.');
         }
@@ -108,23 +104,28 @@ class TaskManagerController extends Controller
 
     public function getScheduler(Request $request)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         if ($request->getRequestFormat() !== "json") {
             $this->app->abort(406, 'Only JSON format is accepted.');
         }
-
-        return $this->app->json([
+        $ret = [
             'name' => $this->app->trans('Task Scheduler'),
             'configuration' => $this->app['task-manager.status']->getStatus(),
-            'urls' => [
+        ];
+
+        if (($this->app['phraseanet.configuration']['main']['task-manager']['enabled'] === true)) {
+            $ret['urls'] = [
                 'start' => $this->app->path('admin_tasks_scheduler_start'),
-                'stop' => $this->app->path('admin_tasks_scheduler_stop'),
+                'stop'  => $this->app->path('admin_tasks_scheduler_stop'),
+                'log'   => $this->app->path('admin_tasks_scheduler_log'),
+            ];
+        }
+        else {
+            $ret['urls'] = [
                 'log' => $this->app->path('admin_tasks_scheduler_log'),
-            ]
-        ]);
+            ];
+        }
+
+        return $this->app->json($ret);
     }
 
     public function getTasks(Request $request)
@@ -212,10 +213,6 @@ class TaskManagerController extends Controller
 
     public function postTaskDelete(Task $task)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         $this->getTaskManipulator()->delete($task);
 
         return $this->app->redirectPath('admin_tasks_list');
@@ -223,10 +220,6 @@ class TaskManagerController extends Controller
 
     public function postStartTask(Task $task)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         $this->getTaskManipulator()->start($task);
 
         return $this->app->redirectPath('admin_tasks_list');
@@ -234,10 +227,6 @@ class TaskManagerController extends Controller
 
     public function postStopTask(Task $task)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         $this->getTaskManipulator()->stop($task);
 
         return $this->app->redirectPath('admin_tasks_list');
@@ -252,10 +241,6 @@ class TaskManagerController extends Controller
 
     public function postSaveTask(Request $request, Task $task)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         if (!$this->doValidateXML($request->request->get('settings'))) {
             return $this->app->json(['success' => false, 'message' => sprintf('Unable to load XML %s', $request->request->get('xml'))]);
         }
@@ -292,10 +277,6 @@ class TaskManagerController extends Controller
 
     public function getTask(Request $request, Task $task)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         if ('json' === $request->getContentType()) {
             return $this->app->json(array_replace([
                 'id' => $task->getId(),
@@ -322,10 +303,6 @@ class TaskManagerController extends Controller
 
     public function validateXML(Request $request)
     {
-        if (false === $this->app['phraseanet.configuration']['main']['task-manager']['enabled']) {
-            throw new RuntimeException('The use of the task manager is disabled on this instance.');
-        }
-
         return $this->app->json(['success' => $this->doValidateXML($request->getContent())]);
     }
 

lib/Alchemy/Phrasea/Controller/Api/OAuth2Controller.php

@@ -167,6 +167,103 @@ class OAuth2Controller extends Controller
         return '';
     }
 
+    public function authorizeWithProviderAction(Request $request, $providerId)
+    {
+        $context = new Context(Context::CONTEXT_OAUTH2_NATIVE);
+        $this->dispatch(PhraseaEvents::PRE_AUTHENTICATE, new PreAuthenticate($request, $context));
+
+        //Check for auth params, send error or redirect if not valid
+        $params = $this->oAuth2Adapter->getAuthorizationRequestParameters($request);
+
+        /** @var ApiApplicationRepository $appRepository */
+        $appRepository = $this->app['repo.api-applications'];
+        if (null === $client = $appRepository->findByClientId($params['client_id'])) {
+            throw new NotFoundHttpException(sprintf('Application with client id %s could not be found', $params['client_id']));
+        }
+
+        $provider = $this->findProvider($providerId);
+
+        return $provider->authenticate($request->query->all());
+    }
+
+    public function authorizeCallbackAction(Request $request, $providerId)
+    {
+        $context = new Context(Context::CONTEXT_OAUTH2_NATIVE);
+        $provider = $this->findProvider($providerId);
+        $params = $this->oAuth2Adapter->getAuthorizationRequestParameters($request);
+
+        // triggers what's necessary
+        try {
+            $provider->onCallback($request);
+            $token = $provider->getToken();
+        } catch (NotAuthenticatedException $e) {
+            $this->getSession()->getFlashBag()->add('error', $this->app->trans('Unable to authenticate with %provider_name%', ['%provider_name%' => $provider->getName()]));
+
+            return $this->app->redirectPath('oauth2_authorize', array_merge(array('error' => 'login'), $params));
+        }
+
+        $userAuthProvider = $this->getUserAuthProviderRepository()
+            ->findWithProviderAndId($token->getProvider()->getId(), $token->getId());
+
+        if($userAuthProvider == null){
+            unset($params['state']);
+
+            return $this->app->redirectPath('oauth2_authorize', array_merge(array('error' => 'login'), $params));
+        }
+
+        try {
+            $user = $this->getAuthenticationSuggestionFinder()->find($token);
+        } catch (NotAuthenticatedException $e) {
+            $this->app->addFlash('error', $this->app->trans('Unable to retrieve provider identity'));
+
+            return $this->app->redirectPath('oauth2_authorize', array_merge(array('error' => 'login'), $params));
+        }
+
+        $this->getAuthenticator()->openAccount($userAuthProvider->getUser());
+        $event = new PostAuthenticate($request, new Response(), $user, $context);
+        $this->dispatch(PhraseaEvents::POST_AUTHENTICATE, $event);
+
+        /** @var ApiApplicationRepository $appRepository */
+        $appRepository = $this->app['repo.api-applications'];
+        if (null === $client = $appRepository->findByClientId($params['client_id'])) {
+            throw new NotFoundHttpException(sprintf('Application with client id %s could not be found', $params['client_id']));
+        }
+
+        $this->oAuth2Adapter->setClient($client);
+
+        //check if current client is already authorized by current user
+        $clients = $appRepository->findAuthorizedAppsByUser($this->getAuthenticatedUser());
+        $appAuthorized = false;
+
+        foreach ($clients as $authClient) {
+            if ($client->getClientId() == $authClient->getClientId()) {
+                $appAuthorized = true;
+                break;
+            }
+        }
+
+        $account = $this->oAuth2Adapter->updateAccount($this->getAuthenticatedUser());
+
+        $params['account_id'] = $account->getId();
+
+        //if native app show template
+        if ($this->oAuth2Adapter->isNativeApp($params['redirect_uri'])) {
+            $params = $this->oAuth2Adapter->finishNativeClientAuthorization($appAuthorized, $params);
+
+            $r = new Response($this->render("api/auth/native_app_access_token.html.twig", $params));
+            $r->headers->set('Content-Type', 'text/html');
+
+            return $r;
+        }
+
+        $this->oAuth2Adapter->finishClientAuthorization($appAuthorized, $params);
+
+        // As OAuth2 library already outputs response content, we need to send an empty
+        // response to avoid breaking silex controller
+        return '';
+
+    }
+
     /**
      *  TOKEN ENDPOINT
      *  Token endpoint - used to exchange an authorization grant for an access token.
@@ -206,4 +303,41 @@ class OAuth2Controller extends Controller
     {
         return $this->app['manipulator.api-account'];
     }
+
+    /**
+     * @param  string $providerId
+     * @return ProviderInterface
+     */
+    private function findProvider($providerId)
+    {
+        try {
+            return $this->getAuthenticationProviders()->get($providerId);
+        } catch (InvalidArgumentException $e) {
+            throw new NotFoundHttpException('The requested provider does not exist');
+        }
+    }
+
+    /**
+     * @return ProvidersCollection
+     */
+    private function getAuthenticationProviders()
+    {
+        return $this->app['authentication.providers'];
+    }
+
+    /**
+     * @return UsrAuthProviderRepository
+     */
+    private function getUserAuthProviderRepository()
+    {
+        return $this->app['repo.usr-auth-providers'];
+    }
+
+    /**
+     * @return SuggestionFinder
+     */
+    private function getAuthenticationSuggestionFinder()
+    {
+        return $this->app['authentication.suggestion-finder'];
+    }
 }

lib/Alchemy/Phrasea/Controller/Api/SearchController.php

@@ -59,14 +59,14 @@ class SearchController extends Controller
 
         $result = $this->getSearchEngine()->query($query, $options);
 
-        $this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $result->getUserQuery());
+        $this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $result->getQueryText());
 
         // log array of collectionIds (from $options) for each databox
         $collectionsReferencesByDatabox = $options->getCollectionsReferencesByDatabox();
         foreach ($collectionsReferencesByDatabox as $sbid => $references) {
             $databox = $this->findDataboxById($sbid);
             $collectionsIds = array_map(function(CollectionReference $ref){return $ref->getCollectionId();}, $references);
-            $this->getSearchEngineLogger()->log($databox, $result->getUserQuery(), $result->getTotal(), $collectionsIds);
+            $this->getSearchEngineLogger()->log($databox, $result->getQueryText(), $result->getTotal(), $collectionsIds);
         }
 
         $this->getSearchEngine()->clearCache();

lib/Alchemy/Phrasea/Controller/Api/V1Controller.php

@@ -17,6 +17,7 @@ use Alchemy\Phrasea\Account\Command\UpdatePasswordCommand;
 use Alchemy\Phrasea\Account\RestrictedStatusExtractor;
 use Alchemy\Phrasea\Application\Helper\DataboxLoggerAware;
 use Alchemy\Phrasea\Application\Helper\DispatcherAware;
+use Alchemy\Phrasea\Application\Helper\FilesystemAware;
 use Alchemy\Phrasea\Application\Helper\JsonBodyAware;
 use Alchemy\Phrasea\Authentication\Exception\RegistrationException;
 use Alchemy\Phrasea\Authentication\RegistrationService;
@@ -78,6 +79,7 @@ use Alchemy\Phrasea\Search\TechnicalDataView;
 use Alchemy\Phrasea\Search\V1SearchCompositeResultTransformer;
 use Alchemy\Phrasea\Search\V1SearchRecordsResultTransformer;
 use Alchemy\Phrasea\Search\V1SearchResultTransformer;
+use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
 use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
 use Alchemy\Phrasea\SearchEngine\SearchEngineLogger;
 use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
@@ -88,6 +90,7 @@ use Alchemy\Phrasea\Utilities\NullableDateTime;
 use Doctrine\ORM\EntityManager;
 use JMS\TranslationBundle\Annotation\Ignore;
 use League\Fractal\Resource\Item;
+use media_subdef;
 use Symfony\Component\Form\Form;
 use Symfony\Component\HttpFoundation\File\UploadedFile;
 use Symfony\Component\HttpFoundation\Request;
@@ -96,10 +99,12 @@ use Symfony\Component\HttpFoundation\Session\Session;
 use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
 use Symfony\Component\Translation\TranslatorInterface;
 
+
 class V1Controller extends Controller
 {
     use DataboxLoggerAware;
     use DispatcherAware;
+    use FilesystemAware;
     use JsonBodyAware;
 
     const OBJECT_TYPE_USER = 'http://api.phraseanet.com/api/objects/user';
@@ -685,8 +690,18 @@ class V1Controller extends Controller
 
         $checks = array_map(function (LazaretCheck $checker) use ($manager, $translator) {
             $checkerFQCN = $checker->getCheckClassname();
+
             return $manager->getCheckerFromFQCN($checkerFQCN)->getMessage($translator);
-        }, iterator_to_array($file->getChecks()));
+        }, $file->getChecksWhithNameKey());
+
+        $recordsMatch = array_map(function ($recordsTab){
+            $record = $recordsTab['record'];
+            $matched['record_id'] = $record->getRecordId();
+            $matched['collection'] = $record->getCollectionName();
+            $matched['checks'] = $recordsTab['reasons'];
+
+            return $matched;
+        }, array_values($file->getRecordsToSubstitute($this->app, true)));
 
         $usr_id = $user = null;
         if ($file->getSession()->getUser()) {
@@ -705,10 +720,12 @@ class V1Controller extends Controller
             'quarantine_session' => $session,
             'base_id'            => $file->getBaseId(),
             'original_name'      => $file->getOriginalName(),
+            'collection'         => $file->getCollection($this->app)->get_label($this->app['locale']),
             'sha256'             => $file->getSha256(),
             'uuid'               => $file->getUuid(),
             'forced'             => $file->getForced(),
             'checks'             => $file->getForced() ? [] : $checks,
+            'records_match'      => $recordsMatch?:[],
             'created_on'         => $file->getCreated()->format(DATE_ATOM),
             'updated_on'         => $file->getUpdated()->format(DATE_ATOM),
         ];
@@ -913,7 +930,14 @@ class V1Controller extends Controller
             ))->createResponse();
         }
 
-        $media = $this->app->getMediaFromUri($file->getPathname());
+        // Add file extension
+        $uploadedFilename = $file->getRealPath();
+
+        $renamedFilename = $file->getRealPath() . '.' . pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION);
+
+        $this->getFilesystem()->rename($uploadedFilename, $renamedFilename);
+
+        $media = $this->app->getMediaFromUri($renamedFilename);
 
         $Package = new File($this->app, $media, $collection, $file->getClientOriginalName());
 
@@ -1000,7 +1024,15 @@ class V1Controller extends Controller
             return $this->getBadRequestAction($request, 'Missing name parameter');
         }
 
-        $media = $this->app->getMediaFromUri($file->getPathname());
+        // Add file extension
+        $uploadedFilename = $file->getRealPath();
+
+        $renamedFilename = $file->getRealPath() . '.' . pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION);
+
+        $this->getFilesystem()->rename($uploadedFilename, $renamedFilename);
+
+        $media = $this->app->getMediaFromUri($renamedFilename);
+
         $record = $this->findDataboxById($request->get('databox_id'))->get_record($request->get('record_id'));
         $base_id = $record->getBaseId();
         $collection = \collection::getByBaseId($this->app, $base_id);
@@ -1011,7 +1043,11 @@ class V1Controller extends Controller
         }
         $adapt = ($request->get('adapt')===null || !(\p4field::isno($request->get('adapt'))));
         $ret['adapt'] = $adapt;
-        $this->getSubdefSubstituer()->substitute($record, $request->get('name'), $media, $adapt);
+        if($request->get('name') == 'document') {
+            $this->getSubdefSubstituer()->substituteDocument($record, $media, $adapt);
+        } else {
+            $this->getSubdefSubstituer()->substituteSubdef($record, $request->get('name'), $media, $adapt);
+        }
         foreach ($record->get_embedable_medias() as $name => $media) {
             if ($name == $request->get('name') &&
                 null !== ($subdef = $this->listEmbeddableMedia($request, $record, $media))) {
@@ -1499,14 +1535,14 @@ class V1Controller extends Controller
 
         $search_result = $this->getSearchEngine()->query((string)$request->get('query'), $options);
 
-        $this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $search_result->getUserQuery());
+        $this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $search_result->getQueryText());
 
         // log array of collectionIds (from $options) for each databox
         $collectionsReferencesByDatabox = $options->getCollectionsReferencesByDatabox();
         foreach ($collectionsReferencesByDatabox as $sbid => $references) {
             $databox = $this->findDataboxById($sbid);
             $collectionsIds = array_map(function(CollectionReference $ref){return $ref->getCollectionId();}, $references);
-            $this->getSearchEngineLogger()->log($databox, $search_result->getUserQuery(), $search_result->getTotal(), $collectionsIds);
+            $this->getSearchEngineLogger()->log($databox, $search_result->getQueryText(), $search_result->getTotal(), $collectionsIds);
         }
 
         $this->getSearchEngine()->clearCache();
@@ -2613,7 +2649,7 @@ class V1Controller extends Controller
                 continue;
             }
             $media = $this->app->getMediaFromUri($value->getRealPath());
-            $this->getSubdefSubstituer()->substitute($story, $name, $media);
+            $this->getSubdefSubstituer()->substituteSubdef($story, $name, $media);  // name = thumbnail | preview
             $this->getDataboxLogger($story->getDatabox())->log(
                 $story,
                 \Session_Logger::EVENT_SUBSTITUTE,
@@ -2644,6 +2680,205 @@ class V1Controller extends Controller
         return Result::create($request, $ret)->createResponse();
     }
 
+    /**
+     * Returns all documentary fields available for user
+     * @param Request $request
+     * @return Response
+     */
+    public function getCurrentUserStructureAction(Request $request)
+    {
+        $ret = [
+          "meta_fields" => $this->listUserAuthorizedMetadataFields($this->getAuthenticatedUser()),
+          "aggregable_fields" => $this->buildUserFieldList(ElasticsearchOptions::getAggregableTechnicalFields(), ['choices']),
+          "technical_fields" => $this->buildUserFieldList(media_subdef::getTechnicalFieldsList()),
+        ];
+
+        return Result::create($request, $ret)->createResponse();
+    }
+
+    /**
+     * Returns all sub-definitions available for the user
+     * @param Request $request
+     * @return Response
+     */
+    public function getCurrentUserSubdefsAction(Request $request)
+    {
+        $ret = [
+          "subdefs" => $this->listUserAuthorizedSubdefs($this->getAuthenticatedUser()),
+        ];
+
+        return Result::create($request, $ret)->createResponse();
+    }
+
+    /**
+     * Returns all collections available for the user
+     * @param Request $request
+     * @return Response
+     */
+    public function getCurrentUserCollectionsAction(Request $request)
+    {
+        $ret = [
+            "collections" => $this->listUserAuthorizedCollections($this->getAuthenticatedUser()),
+        ];
+        return Result::create($request, $ret)->createResponse();
+    }
+
+    /**
+     * Returns list of Metadata Fields from the databoxes on which the user has rights
+     * @param User $user
+     * @return array
+     */
+    private function listUserAuthorizedMetadataFields(User $user)
+    {
+        $acl = $this->getAclForUser($user);
+        $ret = [];
+
+        foreach ($acl->get_granted_sbas() as $databox) {
+            $databoxId = $databox->get_sbas_id();
+            foreach ($databox->get_meta_structure() as $databox_field) {
+                $data = [
+                  'name'             => $databox_field->get_name(),
+                  'id'               => $databox_field->get_id(),
+                  'databox_id'       => $databoxId,
+                  'multivalue'       => $databox_field->is_multi(),
+                  'indexable'        => $databox_field->is_indexable(),
+                  'readonly'         => $databox_field->is_readonly(),
+                  'business'         => $databox_field->isBusiness(),
+                  'source'           => $databox_field->get_tag()->getTagname(),
+                  'labels'           => [
+                    'fr' => $databox_field->get_label('fr'),
+                    'en' => $databox_field->get_label('en'),
+                    'de' => $databox_field->get_label('de'),
+                    'nl' => $databox_field->get_label('nl'),
+                  ],
+                ];
+                $ret[] = $data;
+            }
+
+            if ($acl->can_see_business_fields($databox) === false) {
+                $ret = array_values($this->removeBusinessFields($ret));
+            }
+        }
+
+        return $ret;
+    }
+
+    /**
+     * Build the aggregable/technical fields array
+     * @param array $fields
+     * @param array $excludes
+     * @return array
+     */
+    private function buildUserFieldList(array $fields, array $excludes = [])
+    {
+        $ret = [];
+
+        foreach ($fields as $key => $field) {
+            $data['name'] = $key;
+
+            foreach ($field as $k => $i) {
+                if (in_array($k, $excludes)) {
+                    continue;
+                }
+
+                $data[$k] = $i;
+            }
+
+            $ret[] = $data;
+        }
+
+        return $ret;
+    }
+
+    /**
+     * Returns list of sub-definitions from the databoxes on which the user has rights
+     * @param User $user
+     * @return array
+     */
+    private function listUserAuthorizedSubdefs(User $user)
+    {
+        $acl = $this->getAclForUser($user);
+        $ret = [];
+
+        foreach ($acl->get_granted_sbas() as $databox) {
+            $databoxId = $databox->get_sbas_id();
+            $subdefs = $databox->get_subdef_structure();
+            foreach ($subdefs as $subGroup) {
+                foreach ($subGroup->getIterator() as $sub) {
+                    $opt = [];
+                    $data = [
+                      'name'             => $sub->get_name(),
+                      'databox_id'       => $databoxId,
+                      'class'            => $sub->get_class(),
+                      'preset'           => $sub->get_preset(),
+                      'downloadable'     => $sub->isDownloadable(),
+                      'devices'          => $sub->getDevices(),
+                      'labels'           => [
+                        'fr' => $sub->get_label('fr'),
+                        'en' => $sub->get_label('en'),
+                        'de' => $sub->get_label('de'),
+                        'nl' => $sub->get_label('nl'),
+                      ],
+                    ];
+                    $options = $sub->getOptions();
+                    foreach ($options as $option) {
+                        $opt[$option->getName()] = $option->getValue();
+                    }
+                    $data['options'] = $opt;
+                    $ret[$subGroup->getName()][$sub->get_name()] = $data;
+                }
+            }
+        }
+
+        return $ret;
+    }
+
+    /**
+     * Returns list of collection from the databoxes on which the user has rights
+     * @param User $user
+     * @return array
+     */
+    private function listUserAuthorizedCollections(User $user)
+    {
+        $acl = $this->getAclForUser($user);
+        $rights = $acl->get_bas_rights();
+        $bases = $acl->get_granted_base();
+
+        $grants = [];
+
+        $statusMapper = new RestrictedStatusExtractor($acl, $this->getApplicationBox());
+
+        foreach ($bases as $base) {
+            $baseGrants = [];
+
+            foreach ($rights as $right) {
+                if (!$acl->has_right_on_base($base->get_base_id(), $right)) {
+                    continue;
+                }
+
+                $baseGrants[] = $right;
+            }
+
+            $grants[] = [
+                'databox_id'    => $base->get_sbas_id(),
+                'base_id'       => $base->get_base_id(),
+                'collection_id' => $base->get_coll_id(),
+                'name'          => $base->get_name(),
+                'logo'          => $base->get_binary_minilogos() ? base64_encode($base->get_binary_minilogos()) : '',
+                'labels'        => [
+                    'fr' => $base->get_label('fr'),
+                    'en' => $base->get_label('en'),
+                    'de' => $base->get_label('de'),
+                    'nl' => $base->get_label('nl'),
+                ],
+                'rights'        => $baseGrants,
+                'statuses'      => $statusMapper->getRestrictedStatuses($base->get_base_id())
+            ];
+        }
+
+        return $grants;
+    }
+
     public function deleteCurrentUserAction(Request $request)
     {
         try {
@@ -3093,4 +3328,16 @@ class V1Controller extends Controller
             $recordView->setCaption($captionView);
         }
     }
+
+    /**
+     * Remove business metadata fields
+     * @param array $fields
+     * @return array
+     */
+    private function removeBusinessFields(array $fields)
+    {
+        return array_filter($fields, function ($field) {
+            return $field['business'] !== true;
+        });
+    }
 }

lib/Alchemy/Phrasea/Controller/LightboxController.php

@@ -78,17 +78,39 @@ class LightboxController extends Controller
     {
         /** @var BasketElementRepository $repository */
         $repository = $this->app['repo.basket-elements'];
-
         $basketElement = $repository->findUserElement($sselcont_id, $this->getAuthenticatedUser());
 
+        $basket = $basketElement->getBasket();
+
+        $elements = $basket->getElements();
+        for ($i = 0; $i < count($elements); ++$i) {
+            if ($sselcont_id == $elements[$i]->getId()) {
+                $nextKey = $i + 1;
+                $prevKey = $i - 1;
+                if ($nextKey < count($elements)) {
+                    $nextId = $elements[$nextKey]->getId();
+                }
+                else {
+                    $nextId = null;
+                }
+                if ($prevKey >= 0) {
+                    $prevId = $elements[$prevKey]->getId();
+                }
+                else {
+                    $prevId = null;
+                }
+            }
+        }
+
         if ($this->app['browser']->isMobile()) {
             return $this->renderResponse('lightbox/basket_element.html.twig', [
                 'basket_element' => $basketElement,
-                'module_name'    => $basketElement->getRecord($this->app)->get_title()
+                'module_name'    => $basketElement->getRecord($this->app)->get_title(),
+                'nextId'         => $nextId,
+                'prevId'         => $prevId
             ]);
         }
 
-        $basket = $basketElement->getBasket();
 
         $ret = [];
         $ret['number'] = $basketElement->getRecord($this->app)->getNumber();

lib/Alchemy/Phrasea/Controller/Prod/EditController.php

@@ -358,6 +358,10 @@ class EditController extends Controller
                 $this->dispatch(PhraseaEvents::RECORD_EDIT, new RecordEdit($record));
             }
 
+            if (isset($rec['technicalsdatas']) && is_array($rec['technicalsdatas'])){
+                $record->insertOrUpdateTechnicalDatas($rec['technicalsdatas']);
+            }
+
             $newstat = $record->getStatus();
             $statbits = ltrim($statbits, 'x');
             if (!in_array($statbits, ['', 'null'])) {

lib/Alchemy/Phrasea/Controller/Prod/LanguageController.php

@@ -36,16 +36,17 @@ class LanguageController
             'serverTimeout'           => $translator->trans('phraseanet::erreur: La connection au serveur Phraseanet semble etre indisponible'),
             'serverDisconnected'      => $translator->trans('phraseanet::erreur: Votre session est fermee, veuillez vous re-authentifier'),
             'hideMessage'             => $translator->trans('phraseanet::Ne plus afficher ce message'),
-            'confirmGroup'            => $translator->trans('Supprimer egalement les documents rattaches a ces regroupements'),
-            'confirmDelete'           => $translator->trans('reponses:: Ces enregistrements vont etre definitivement supprimes et ne pourront etre recuperes. Etes vous sur ?'),
-            'cancel'                  => $translator->trans('boutton::annuler'),
-            'deleteTitle'             => $translator->trans('boutton::supprimer'),
-            'deleteRecords'           => $translator->trans('Delete records'),
-            'edit_hetero'             => $translator->trans('prod::editing valeurs heterogenes, choisir \'remplacer\', \'ajouter\' ou \'annuler\''),
-            'confirm_abandon'         => $translator->trans('prod::editing::annulation: abandonner les modification ?'),
-            'loading'                 => $translator->trans('phraseanet::chargement'),
-            'valider'                 => $translator->trans('boutton::valider'),
-            'annuler'                 => $translator->trans('boutton::annuler'),
+            'confirmGroup'    => $translator->trans('Supprimer egalement les documents rattaches a ces regroupements'),
+            'confirmDelete'   => $translator->trans('reponses:: Ces enregistrements vont etre definitivement supprimes et ne pourront etre recuperes. Etes vous sur ?'),
+            'cancel'          => $translator->trans('boutton::annuler'),
+            'deleteTitle'     => $translator->trans('boutton::supprimer'),
+            'deleteRecords'   => $translator->trans('Delete records'),
+            'moveToTrash'     => $translator->trans('prod:app trash: title-trash'),
+            'edit_hetero'     => $translator->trans('prod::editing valeurs heterogenes, choisir \'remplacer\', \'ajouter\' ou \'annuler\''),
+            'confirm_abandon' => $translator->trans('prod::editing::annulation: abandonner les modification ?'),
+            'loading'         => $translator->trans('phraseanet::chargement'),
+            'valider'         => $translator->trans('boutton::valider'),
+            'annuler'         => $translator->trans('boutton::annuler'),
             'create'                  => $translator->trans('boutton::creer'),
             'rechercher'              => $translator->trans('boutton::rechercher'),
             'renewRss'                => $translator->trans('boutton::renouveller'),
@@ -96,33 +97,34 @@ class LanguageController
             'onlyOneRecord'           => $translator->trans('You can choose only one record'),
             'errorAjaxRequest'        => $translator->trans('An error occured, please retry'),
             'fileBeingDownloaded'     => $translator->trans('Some files are being downloaded'),
-            'warning'                 => $translator->trans('Attention'),
-            'browserFeatureSupport'   => $translator->trans('This feature is not supported by your browser'),
-            'noActiveBasket'          => $translator->trans('No active basket'),
-            'pushUserCanDownload'     => $translator->trans('User can download HD'),
-            'feedbackCanContribute'   => $translator->trans('User contribute to the feedback'),
-            'feedbackCanSeeOthers'    => $translator->trans('User can see others choices'),
-            'forceSendDocument'       => $translator->trans('Force sending of the document ?'),
-            'export'                  => $translator->trans('Export'),
-            'share'                   => $translator->trans('Share'),
-            'move'                    => $translator->trans('Move'),
-            'push'                    => $translator->trans('Push'),
-            'feedback'                => $translator->trans('Feedback'),
-            'toolbox'                 => $translator->trans('Tool box'),
-            'print'                   => $translator->trans('Print'),
-            'attention'               => $translator->trans('Attention !'),
-            'mapMarkerEdit'           => $translator->trans('Edit position'),
-            'mapMarkerAdd'            => $translator->trans('Add a position'),
-            'mapMarkerMoveLabel'      => $translator->trans('Drag and drop the pin to move position'),
-            'mapMarkerEditCancel'     => $translator->trans('Cancel'),
-            'mapMarkerEditSubmit'     => $translator->trans('Submit'),
-            'Keyboard shortcuts'      => $translator->trans('Keyboard shortcuts'),
-            'Play'                    => $translator->trans('Play'),
-            'Change play speed'       => $translator->trans('Change play speed'),
-            'Pause'                   => $translator->trans('Pause'),
-            'One frame forward'       => $translator->trans('One frame forward'),
-            'One frame backward'      => $translator->trans('One frame backward'),
-            'Add an entry point'      => $translator->trans('Add an entry point'),
+            'warning'               => $translator->trans('Attention'),
+            'browserFeatureSupport' => $translator->trans('This feature is not supported by your browser'),
+            'noActiveBasket'        => $translator->trans('No active basket'),
+            'pushUserCanDownload'   => $translator->trans('User can download HD'),
+            'feedbackCanContribute' => $translator->trans('User contribute to the feedback'),
+            'feedbackCanSeeOthers'  => $translator->trans('User can see others choices'),
+            'forceSendDocument'     => $translator->trans('Force sending of the document ?'),
+            'export'                => $translator->trans('Export'),
+            'share'                 => $translator->trans('Share'),
+            'move'                  => $translator->trans('Move'),
+            'push'                  => $translator->trans('Push'),
+            'feedback'              => $translator->trans('Feedback'),
+            'toolbox'               => $translator->trans('Tool box'),
+            'videoEditor'           => $translator->trans('prod:edit: video-editor'),
+            'print'                 => $translator->trans('Print'),
+            'attention'             => $translator->trans('Attention !'),
+            'mapMarkerEdit'         => $translator->trans('Edit position'),
+            'mapMarkerAdd'          => $translator->trans('Add a position'),
+            'mapMarkerMoveLabel'    => $translator->trans('Drag and drop the pin to move position'),
+            'mapMarkerEditCancel'   => $translator->trans('Cancel'),
+            'mapMarkerEditSubmit'   => $translator->trans('Submit'),
+            'Keyboard shortcuts'    => $translator->trans('Keyboard shortcuts'),
+            'Play'                  => $translator->trans('Play'),
+            'Change play speed'     => $translator->trans('Change play speed'),
+            'Pause'                 => $translator->trans('Pause'),
+            'One frame forward'     => $translator->trans('One frame forward'),
+            'One frame backward'    => $translator->trans('One frame backward'),
+            'Add an entry point'    => $translator->trans('Add an entry point'),
             'Add an end point'        => $translator->trans('Add an end point'),
             'Navigate to entry point' => $translator->trans('Navigate to entry point'),
             'Navigate to end point'   => $translator->trans('Navigate to end point'),
@@ -130,20 +132,28 @@ class LanguageController
             'Toggle loop'             => $translator->trans('Toggle loop'),
             'Shift'                   => $translator->trans('Shift'),
             'Ctrl'                    => $translator->trans('Ctrl'),
-            'Space bar'               => $translator->trans('Space bar'),
-            'or'                      => $translator->trans('or'),
-            'Suppr'                   => $translator->trans('Suppr'),
-            'Add new range'           => $translator->trans('Add new range'),
-            'Export ranges'           => $translator->trans('Export ranges'),
-            'Start Range'             => $translator->trans('Start Range'),
-            'End Range'               => $translator->trans('End Range'),
-            'Remove current Range'    => $translator->trans('Remove current Range'),
-            'Go to start point'       => $translator->trans('Go to start point'),
-            'Go 1 frame backward'     => $translator->trans('Go 1 frame backward'),
-            'Go 1 frame forward'      => $translator->trans('Go 1 frame forward'),
-            'Go to end point'         => $translator->trans('Go to end point'),
-            'Move up range'           => $translator->trans('Move up range'),
-            'Move down range'         => $translator->trans('Move down range'),
+            'Space bar'             => $translator->trans('Space bar'),
+            'or'                    => $translator->trans('or'),
+            'Suppr'                 => $translator->trans('Suppr'),
+            'Add new range'         => $translator->trans('Add new range'),
+            'Export ranges'         => $translator->trans('Export ranges'),
+            'Start Range'           => $translator->trans('Start Range'),
+            'End Range'             => $translator->trans('End Range'),
+            'Remove current Range'  => $translator->trans('Remove current Range'),
+            'Go to start point'     => $translator->trans('Go to start point'),
+            'Go 1 frame backward' => $translator->trans('Go 1 frame backward'),
+            'Go 1 frame forward'  => $translator->trans('Go 1 frame forward'),
+            'Go to end point'     => $translator->trans('Go to end point'),
+            'Move up range'       => $translator->trans('Move up range'),
+            'Move down range'     => $translator->trans('Move down range'),
+            'error video editor'  => $translator->trans('prod:edit: only a media of type video can be edited'),
+            'Chapters'            => $translator->trans('prod:edit: chapters'),
+            'No hover to chapter' => $translator->trans('prod:edit: no overlaps for chapters'),
+            'suggested_values'    => $translator->trans('prod:edit: suggested_values'),
+            'title notice'        => $translator->trans('prod:mapboxgl: title notice'),
+            'description notice'  => $translator->trans('prod:mapboxgl: description notice'),
+            'title-map-dialog'    => $translator->trans('prod:mapboxgl: title map dialog'),
+            'create new user'     => $translator->trans('prod:push: create new user'),
         ]);
     }
 }

lib/Alchemy/Phrasea/Controller/Prod/LazaretController.php

@@ -225,8 +225,7 @@ class LazaretController extends Controller
             $media = $this->app->getMediaFromUri($lazaretFileName);
 
             $record = $lazaretFile->getCollection($this->app)->get_databox()->get_record($recordId);
-            $this->getSubDefinitionSubstituer()
-                ->substitute($record, 'document', $media);
+            $this->getSubDefinitionSubstituer()->substituteDocument($record, $media);
             $this->getDataboxLogger($record->getDatabox())->log(
                 $record,
                 \Session_Logger::EVENT_SUBSTITUTE,

lib/Alchemy/Phrasea/Controller/Prod/MoveCollectionController.php

@@ -110,6 +110,9 @@ class MoveCollectionController extends Controller
                 return $this->app->json($datas);
             }
 
+            /** @var \collection[] $trashCollectionsBySbasId */
+            $trashCollectionsBySbasId = [];
+
             foreach ($records as $record) {
                 $record->move_to_collection($collection, $this->getApplicationBox());
 
@@ -121,6 +124,33 @@ class MoveCollectionController extends Controller
                         }
                     }
                 }
+
+                $sbasId = $record->getDatabox()->get_sbas_id();
+                if (!array_key_exists($sbasId, $trashCollectionsBySbasId)) {
+                    $trashCollectionsBySbasId[$sbasId] = $record->getDatabox()->getTrashCollection();
+                }
+                if ($trashCollectionsBySbasId[$sbasId] !== null) {
+                    if ($record->getCollection()->get_coll_id() == $trashCollectionsBySbasId[$sbasId]->get_coll_id() && $collection->get_coll_id() !== $trashCollectionsBySbasId[$sbasId]->get_coll_id()) {
+                        // record is already in trash so active it
+                        foreach ($record->get_subdefs() as $subdef) {
+                            if (($pl = $subdef->get_permalink())) {
+                                $pl->set_is_activated(true);
+                            }
+                        }
+                        if ($request->request->get("chg_coll_son") == "1") {
+                            /** @var \record_adapter $child */
+                            foreach ($record->getChildren() as $child) {
+                                if ($this->getAclForUser()->has_right_on_base($child->getBaseId(), \ACL::CANDELETERECORD)) {
+                                    foreach ($child->get_subdefs() as $childSubdef) {
+                                        if (($childPl = $childSubdef->get_permalink())) {
+                                            $childPl->set_is_activated(true);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
             }
 
             $ret = [

lib/Alchemy/Phrasea/Controller/Prod/QueryController.php

@@ -15,6 +15,7 @@ use Alchemy\Phrasea\Cache\Exception;
 use Alchemy\Phrasea\Collection\Reference\CollectionReference;
 use Alchemy\Phrasea\Controller\Controller;
 use Alchemy\Phrasea\Core\Configuration\DisplaySettingService;
+use Alchemy\Phrasea\Model\Entities\ElasticsearchRecord;
 use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
@@ -132,9 +133,9 @@ class QueryController extends Controller
         // since the query comes from a submited form, normalize crlf,cr,lf ...
         $query = StringHelper::crlfNormalize($query);
 
-        $json = array(
+        $json = [
             'query' => $query
-        );
+        ];
 
         $options = SearchEngineOptions::fromRequest($this->app, $request);
 
@@ -168,7 +169,7 @@ class QueryController extends Controller
             foreach ($collectionsReferencesByDatabox as $sbid => $references) {
                 $databox = $this->findDataboxById($sbid);
                 $collectionsIds = array_map(function(CollectionReference $ref){return $ref->getCollectionId();}, $references);
-                $this->getSearchEngineLogger()->log($databox, $result->getUserQuery(), $result->getTotal(), $collectionsIds);
+                $this->getSearchEngineLogger()->log($databox, $result->getQueryText(), $result->getTotal(), $collectionsIds);
             }
 
             $proposals = $firstPage ? $result->getProposals() : false;
@@ -177,6 +178,8 @@ class QueryController extends Controller
 
             $page = $result->getCurrentPage($perPage);
 
+            $queryESLib = $result->getQueryESLib();
+
             $string = '';
 
             if ($npages > 1) {
@@ -229,23 +232,19 @@ class QueryController extends Controller
             }
             $string .= '<div style="display:none;"><div id="NEXT_PAGE"></div><div id="PREV_PAGE"></div></div>';
 
-            $explain = "<div id=\"explainResults\" class=\"myexplain\">";
-
-            $explain .= "<img src=\"/assets/common/images/icons/answers.gif\" /><span><b>";
-
-            if ($result->getTotal() != $result->getAvailable()) {
-                $explain .= $this->app->trans('reponses:: %available% Resultats rappatries sur un total de %total% trouves', ['available' => $result->getAvailable(), '%total%' => $result->getTotal()]);
-            } else {
-                $explain .= $this->app->trans('reponses:: %total% Resultats', ['%total%' => $result->getTotal()]);
-            }
-
-            $explain .= " </b></span>";
-            $explain .= '<br><div>' . ($result->getDuration() / 1000) . ' s</div>dans index ' . $result->getIndexes();
-            $explain .= "</div>";
+            $explain = $this->render(
+                "prod/results/infos.html.twig",
+                [
+                    'results'=> $result,
+                    'esquery' => $this->getAclForUser()->is_admin() ?
+                        json_encode($queryESLib['body'], JSON_PRETTY_PRINT | JSON_HEX_TAG | JSON_HEX_QUOT | JSON_UNESCAPED_SLASHES) :
+                        null
+                ]
+            );
 
             $infoResult = '<div id="docInfo">'
                 . $this->app->trans('%number% documents<br/>selectionnes', ['%number%' => '<span id="nbrecsel"></span>'])
-                . '</div><a href="#" class="infoDialog search-display-info" data-infos="' . str_replace('"', '&quot;', $explain) . '">'
+                . '</div><a href="#" class="search-display-info" data-infos="' . str_replace('"', '&quot;', $explain) . '">'
                 . $this->app->trans('%total% reponses', ['%total%' => '<span>'.$result->getTotal().'</span>']) . '</a>';
 
             $json['infos'] = $infoResult;
@@ -274,29 +273,69 @@ class QueryController extends Controller
             } else {
                 $template = 'prod/results/records.html.twig';
             }
-
             $json['results'] = $this->render($template, ['results'=> $result]);
 
-            /** Debug */
-            $json['parsed_query'] = $result->getEngineQuery();
-            /** End debug */
 
-            $fieldLabels = [];
             // add technical fields
+            $fieldLabels = [];
             foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
                 $fieldLabels[$k] = $this->app->trans($f['label']);
             }
+
             // add databox fields
+            // get infos about fields, fusionned and by databox
+            $fieldsInfos = [];  // by databox
             foreach ($this->app->getDataboxes() as $databox) {
+                $sbasId = $databox->get_sbas_id();
+                $fieldsInfos[$sbasId] = [];
                 foreach ($databox->get_meta_structure() as $field) {
-                    if (!isset($fieldLabels[$field->get_name()])) {
-                        $fieldLabels[$field->get_name()] = $field->get_label($this->app['locale']);
+                    $name = $field->get_name();
+                    $fieldsInfos[$sbasId][$name] = [
+                      'label' => $field->get_label($this->app['locale']),
+                      'type' => $field->get_type(),
+                      'business' => $field->isBusiness(),
+                      'multi' => $field->is_multi(),
+                    ];
+                    if (!isset($fieldLabels[$name])) {
+                        $fieldLabels[$name] = $field->get_label($this->app['locale']);
                     }
                 }
             }
 
-            $facets = [];
+            // populates fileds infos
+            $json['fields'] = $fieldsInfos;
 
+            // populates rawresults
+            // need acl so the result will not include business fields where not allowed
+            $acl = $this->getAclForUser();
+            $json['rawResults'] = [];
+            /** @var ElasticsearchRecord $record */
+            foreach($result->getResults() as $record) {
+                $rawRecord = $record->asArray();
+
+                $sbasId = $record->getDataboxId();
+                $baseId = $record->getBaseId();
+
+                $caption = $rawRecord['caption'];
+                if($acl && $acl->has_right_on_base($baseId, \ACL::CANMODIFRECORD)) {
+                    $caption = array_merge($caption, $rawRecord['privateCaption']);
+                }
+
+                // read the fields following the structure order
+                $rawCaption = [];
+                foreach($fieldsInfos[$sbasId] as $fieldName=>$fieldInfos) {
+                    if(array_key_exists($fieldName, $caption)) {
+                        $rawCaption[$fieldName] = $caption[$fieldName];
+                    }
+                }
+                $rawRecord['caption'] = $rawCaption;
+                unset($rawRecord['privateCaption']);
+
+                $json['rawResults'][$record->getId()] = $rawRecord;
+            }
+
+            // populates facets (aggregates)
+            $facets = [];
             foreach ($result->getFacets() as $facet) {
                 $facetName = $facet['name'];
 
@@ -311,6 +350,9 @@ class QueryController extends Controller
             $json['next_page'] = ($page < $npages && $result->getAvailable() > 0) ? ($page + 1) : false;
             $json['prev_page'] = ($page > 1 && $result->getAvailable() > 0) ? ($page - 1) : false;
             $json['form'] = $options->serialize();
+            $json['queryCompiled'] = $result->getQueryCompiled();
+            $json['queryAST'] = $result->getQueryAST();
+            $json['queryESLib'] = $queryESLib;
         }
         catch(\Exception $e) {
             // we'd like a message from the parser so get all the exceptions messages
@@ -319,9 +361,9 @@ class QueryController extends Controller
                 $msg .= ($msg ? "\n":"") . $e->getMessage();
             }
             $template = 'prod/results/help.html.twig';
-            $result = array(
+            $result = [
                 'error' => $msg
-            );
+            ];
             $json['results'] = $this->render($template, ['results'=> $result]);
         }
 

lib/Alchemy/Phrasea/Controller/Prod/RecordController.php

@@ -14,6 +14,8 @@ use Alchemy\Phrasea\Application\Helper\EntityManagerAware;
 use Alchemy\Phrasea\Application\Helper\SearchEngineAware;
 use Alchemy\Phrasea\Controller\Controller;
 use Alchemy\Phrasea\Controller\RecordsRequest;
+use Alchemy\Phrasea\Core\Event\RecordEdit;
+use Alchemy\Phrasea\Core\PhraseaEvents;
 use Alchemy\Phrasea\Model\Entities\BasketElement;
 use Alchemy\Phrasea\Model\Repositories\BasketElementRepository;
 use Alchemy\Phrasea\Model\Repositories\StoryWZRepository;
@@ -86,6 +88,7 @@ class RecordController extends Controller
             // get field's values
             $recordCaptions[$field->get_name()] = $field->get_serialized_values();
         }
+        $recordCaptions["technicalInfo"] = $record->getPositionFromTechnicalInfos();
 
         return $this->app->json([
             "desc"          => $this->render('prod/preview/caption.html.twig', [
@@ -189,7 +192,12 @@ class RecordController extends Controller
 
         $deleted = [];
 
+        /** @var \collection[] $trashCollectionsBySbasId */
+        $trashCollectionsBySbasId = [];
+
         $manager = $this->getEntityManager();
+
+        /** @var \record_adapter $record */
         foreach ($records as $record) {
             try {
                 $basketElements = $basketElementsRepository->findElementsByRecord($record);
@@ -205,10 +213,34 @@ class RecordController extends Controller
                     $manager->remove($attachedStory);
                 }
 
-                $deleted[] = $record->getId();
-                $record->delete();
-            } catch (\Exception $e) {
+                foreach($record->get_grouping_parents() as $story) {
+                    $this->getEventDispatcher()->dispatch(PhraseaEvents::RECORD_EDIT, new RecordEdit($story));
+                }
 
+                $sbasId = $record->getDatabox()->get_sbas_id();
+                if(!array_key_exists($sbasId, $trashCollectionsBySbasId)) {
+                    $trashCollectionsBySbasId[$sbasId] = $record->getDatabox()->getTrashCollection();
+                }
+                $deleted[] = $record->getId();
+                if($trashCollectionsBySbasId[$sbasId] !== null) {
+                    if($record->getCollection()->get_coll_id() == $trashCollectionsBySbasId[$sbasId]->get_coll_id()) {
+                        // record is already in trash so delete it
+                        $record->delete();
+                    } else {
+                        // move to trash collection
+                        $record->move_to_collection($trashCollectionsBySbasId[$sbasId], $this->getApplicationBox());
+                        // disable permalinks
+                        foreach($record->get_subdefs() as $subdef) {
+                            if( ($pl = $subdef->get_permalink()) ) {
+                                    $pl->set_is_activated(false);
+                            }
+                        }
+                    }
+                } else {
+                    // no trash collection, delete
+                    $record->delete();
+                }
+            } catch (\Exception $e) {
             }
         }
 
@@ -248,9 +280,46 @@ class RecordController extends Controller
             [\ACL::CANDELETERECORD]
         );
 
-        return $this->render('prod/actions/delete_records_confirm.html.twig', [
-            'records'   => $records,
+        $filteredRecord = $this->filterRecordToDelete($records);
+
+        return $this->app->json([
+            'renderView'     => $this->render('prod/actions/delete_records_confirm.html.twig', [
+                'records'        => $records,
+                'filteredRecord' => $filteredRecord
+            ]),
+            'filteredRecord' => $filteredRecord
         ]);
+
+    }
+
+    private function filterRecordToDelete(RecordsRequest $records)
+    {
+        $trashCollectionsBySbasId = [];
+        $goingToTrash = [];
+        $delete = [];
+        foreach ($records as $record) {
+            $sbasId = $record->getDatabox()->get_sbas_id();
+            if (!array_key_exists($sbasId, $trashCollectionsBySbasId)) {
+                $trashCollectionsBySbasId[$sbasId] = $record->getDatabox()->getTrashCollection();
+            }
+            if ($trashCollectionsBySbasId[$sbasId] !== null) {
+                if ($record->getCollection()->get_coll_id() == $trashCollectionsBySbasId[$sbasId]->get_coll_id()) {
+                    // record is already in trash
+                    $delete[] = $record;
+                }
+                else {
+                    // will be moved to trash
+                    $goingToTrash[] = $record;
+                }
+            }
+            else {
+                // trash does not exist
+                $delete[] = $record;
+            }
+        }
+        //check if all values in array are true
+        //return (!in_array(false, $goingToTrash, true));
+        return ['trash' => $goingToTrash, 'delete' => $delete];
     }
 
     /**
@@ -271,4 +340,12 @@ class RecordController extends Controller
 
         return $this->app->json($renewed);
     }
+
+    /**
+     * @return EventDispatcherInterface
+     */
+    private function getEventDispatcher()
+    {
+        return $this->app['dispatcher'];
+    }
 }

lib/Alchemy/Phrasea/Controller/Prod/SubdefsController.php

@@ -0,0 +1,61 @@
+<?php
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace Alchemy\Phrasea\Controller\Prod;
+
+use Alchemy\Phrasea\Controller\Controller;
+use PHPExiftool\Driver\Metadata\Metadata;
+use PHPExiftool\Driver\Metadata\MetadataBag;
+use PHPExiftool\Driver\TagFactory;
+use PHPExiftool\Driver\Value\Mono;
+use PHPExiftool\Reader;
+
+class SubdefsController extends Controller
+{
+    public function metadataAction($databox_id, $record_id, $subdef_name)
+    {
+        $record = new \record_adapter($this->app, (int) $databox_id, (int) $record_id);
+        $metadataBag = new MetadataBag();
+
+        try {
+            $fileEntity = $this->getExifToolReader()
+                ->files($record->get_subdef($subdef_name)->getRealPath())
+                ->first();
+            $metadatas = $fileEntity->getMetadatas();
+            foreach($metadatas as $metadata){
+                $valuedata = $fileEntity->executeQuery($metadata->getTag()->getTagname()."[not(@rdf:datatype = 'http://www.w3.org/2001/XMLSchema#base64Binary')]");
+                if(empty($valuedata)){
+                    $valuedata = new Mono($this->app->trans('Binary data'));
+                    $tag = TagFactory::getFromRDFTagname($metadata->getTag()->getTagname());
+                    $metadataBagElement = new Metadata($tag, $valuedata);
+                    $metadataBag->set($metadata->getTag()->getTagname(), $metadataBagElement);
+                }else{
+                    $metadataBag->set($metadata->getTag()->getTagname(), $metadata);
+                }
+            }
+        } catch (PHPExiftoolException $e) {
+            // ignore
+        } catch (\Exception_Media_SubdefNotFound $e) {
+            // ignore
+        }
+
+        return $this->render('prod/actions/Tools/metadata.html.twig', [
+            'record'   => $record,
+            'metadatas' => $metadataBag,
+        ]);
+    }
+
+    /**
+     * @return Reader
+     */
+    private function getExifToolReader()
+    {
+        return $this->app['exiftool.reader'];
+    }
+}

lib/Alchemy/Phrasea/Controller/Prod/ToolsController.php

@@ -23,7 +23,6 @@ use Alchemy\Phrasea\Record\RecordWasRotated;
 use DataURI\Parser;
 use MediaAlchemyst\Alchemyst;
 use MediaVorus\MediaVorus;
-use PHPExiftool\Exception\ExceptionInterface as PHPExiftoolException;
 use PHPExiftool\Reader;
 use Symfony\Component\HttpFoundation\Request;
 
@@ -38,13 +37,14 @@ class ToolsController extends Controller
     {
         $records = RecordsRequest::fromRequest($this->app, $request, false);
 
-        $metadata = false;
+        $metadatas = false;
         $record = null;
         $recordAccessibleSubdefs = array();
 
         if (count($records) == 1) {
             /** @var \record_adapter $record */
             $record = $records->first();
+            $databox = $record->getDatabox();
 
             // fetch subdef list:
             $subdefs = $record->get_subdefs();
@@ -82,27 +82,17 @@ class ToolsController extends Controller
                     );
                 }
             }
-
             if (!$record->isStory()) {
-                try {
-                    $metadata = $this->getExifToolReader()
-                        ->files($record->get_subdef('document')->getRealPath())
-                        ->first()->getMetadatas();
-                } catch (PHPExiftoolException $e) {
-                    // ignore
-                } catch (\Exception_Media_SubdefNotFound $e) {
-                    // ignore
-                }
+                $metadatas = true;
             }
         }
         $conf = $this->getConf();
 
         return $this->render('prod/actions/Tools/index.html.twig', [
-            'records'   => $records,
-            'record'    => $record,
-            'videoEditorConfig' => $conf->get(['video-editor']),
-            'recordSubdefs' => $recordAccessibleSubdefs,
-            'metadatas' => $metadata,
+            'records'           => $records,
+            'record'            => $record,
+            'recordSubdefs'     => $recordAccessibleSubdefs,
+            'metadatas'         => $metadatas,
         ]);
     }
 
@@ -202,7 +192,7 @@ class ToolsController extends Controller
 
                     $media = $this->app->getMediaFromUri($tempoFile);
 
-                    $this->getSubDefinitionSubstituer()->substitute($record, 'document', $media);
+                    $this->getSubDefinitionSubstituer()->substituteDocument($record, $media);
                     $record->insertTechnicalDatas($this->getMediaVorus());
                     $this->getMetadataSetter()->replaceMetadata($this->getMetadataReader() ->read($media), $record);
 
@@ -262,7 +252,7 @@ class ToolsController extends Controller
 
             $media = $this->app->getMediaFromUri($tempoFile);
 
-            $this->getSubDefinitionSubstituer()->substitute($record, 'thumbnail', $media);
+            $this->getSubDefinitionSubstituer()->substituteSubdef($record, 'thumbnail', $media);
             $this->getDataboxLogger($record->getDatabox())
                 ->log($record, \Session_Logger::EVENT_SUBSTITUTE, 'thumbnail', '');
 
@@ -426,11 +416,81 @@ class ToolsController extends Controller
 
         $media = $this->app->getMediaFromUri($fileName);
 
-        $this->getSubDefinitionSubstituer()->substitute($record, $subDefName, $media);
+        if($subDefName == 'document') {
+            $this->getSubDefinitionSubstituer()->substituteDocument($record, $media);
+        } else {
+            $this->getSubDefinitionSubstituer()->substituteSubdef($record, $subDefName, $media);
+        }
         $this->getDataboxLogger($record->getDatabox())
           ->log($record, \Session_Logger::EVENT_SUBSTITUTE, $subDefName, '');
 
         unset($media);
         $this->getFilesystem()->remove($fileName);
     }
+
+    /**
+     * @param $request
+     * @return \Symfony\Component\HttpFoundation\JsonResponse
+     */
+    public function saveMetasAction(Request $request)
+    {
+        $record = new \record_adapter($this->app,
+            (int)$request->request->get("databox_id"),
+            (int)$request->request->get("record_id"));
+
+        $metadatas[0] = [
+            'meta_struct_id' => (int)$request->request->get("meta_struct_id"),
+            'meta_id'        => '',
+            'value'          => $request->request->get("value")
+        ];
+        try {
+            $record->set_metadatas($metadatas);
+        }
+        catch (Exception $e) {
+            return $this->app->json(['success' => false, 'errorMessage' => $e->getMessage()]);
+        }
+
+        return $this->app->json(['success' => true, 'errorMessage' => '']);
+    }
+
+    public function videoEditorAction(Request $request)
+    {
+        $records = RecordsRequest::fromRequest($this->app, $request, false);
+
+        $metadatas = false;
+        $record = null;
+        $JSFields = [];
+
+        if (count($records) == 1) {
+            /** @var \record_adapter $record */
+            $record = $records->first();
+            $databox = $record->getDatabox();
+
+
+            foreach ($databox->get_meta_structure() as $meta) {
+                /** @var \databox_field $meta */
+                $fields[] = $meta;
+
+                /** @Ignore */
+                $JSFields[$meta->get_id()] = [
+                    'id'     => $meta->get_id(),
+                    'name'   => $meta->get_name(),
+                    '_value' => $record->getCaption([$meta->get_name()]),
+                ];
+            }
+
+            if (!$record->isStory()) {
+                $metadatas = true;
+            }
+        }
+        $conf = $this->getConf();
+
+        return $this->render('prod/actions/Tools/videoEditor.html.twig', [
+            'records'           => $records,
+            'record'            => $record,
+            'videoEditorConfig' => $conf->get(['video-editor']),
+            'metadatas'         => $metadatas,
+            'JSonFields'        => json_encode($JSFields),
+        ]);
+    }
 }

lib/Alchemy/Phrasea/Controller/Prod/UploadController.php

@@ -193,7 +193,7 @@ class UploadController extends Controller
                         file_put_contents($fileName, $dataUri->getData());
                         $media = $this->app->getMediaFromUri($fileName);
 
-                        $this->getSubDefinitionSubstituer()->substitute($elementCreated, 'thumbnail', $media);
+                        $this->getSubDefinitionSubstituer()->substituteSubdef($elementCreated, 'thumbnail', $media);
                         $this->getDataboxLogger($elementCreated->getDatabox())
                             ->log($elementCreated, \Session_Logger::EVENT_SUBSTITUTE, 'thumbnail', '');
 

lib/Alchemy/Phrasea/Controller/Root/LoginController.php

@@ -552,6 +552,7 @@ class LoginController extends Controller
         } while (null !== $this->getUserRepository()->findOneBy(['login' => $login]));
 
         $user = $this->getUserManipulator()->createUser($login, $this->getStringGenerator()->generateString(128));
+        $user->setGuest(true);
         $invite_user = $this->getUserRepository()->findByLogin(User::USER_GUEST);
 
         $usr_base_ids = array_keys($this->getAclForUser($user)->get_granted_base());

lib/Alchemy/Phrasea/Controller/SetupController.php

@@ -11,6 +11,7 @@
 namespace Alchemy\Phrasea\Controller;
 
 use Alchemy\Phrasea\Application;
+use Alchemy\Phrasea\Core\Configuration\StructureTemplate;
 use Alchemy\Phrasea\Setup\RequirementCollectionInterface;
 use Alchemy\Phrasea\Setup\Requirements\BinariesRequirements;
 use Alchemy\Phrasea\Setup\Requirements\FilesystemRequirements;
@@ -74,10 +75,13 @@ class SetupController extends Controller
             $warnings[] = $this->app->trans('It is not recommended to install Phraseanet without HTTPS support');
         }
 
+        /** @var StructureTemplate $st */
+        $st = $this->app['phraseanet.structure-template'];
+
         return $this->render('/setup/step2.html.twig', [
             'locale'              => $this->app['locale'],
             'available_locales'   => Application::getAvailableLanguages(),
-            'available_templates' => $this->app['phraseanet.structure-template']->getAvailable()->getTemplates(),
+            'available_templates' => $st->getNames(),
             'warnings'            => $warnings,
             'error'               => $request->query->get('error'),
             'current_servername'  => $request->getScheme() . '://' . $request->getHttpHost() . '/',
@@ -92,7 +96,7 @@ class SetupController extends Controller
 
         $servername = $request->getScheme() . '://' . $request->getHttpHost() . '/';
 
-         $dbConn = null;
+        $dbConn = null;
 
         $database_host = $request->request->get('hostname');
         $database_port = $request->request->get('port');

lib/Alchemy/Phrasea/ControllerProvider/Api/OAuth2.php

@@ -18,9 +18,12 @@ use Silex\Application;
 use Silex\ControllerCollection;
 use Silex\ControllerProviderInterface;
 use Silex\ServiceProviderInterface;
+use Alchemy\Phrasea\ControllerProvider\ControllerProviderTrait;
 
 class OAuth2 extends Api implements ControllerProviderInterface, ServiceProviderInterface
 {
+    use ControllerProviderTrait;
+
     public function register(Application $app)
     {
         $app['controller.oauth2'] = $app->share(function (PhraseaApplication $app) {
@@ -35,6 +38,16 @@ class OAuth2 extends Api implements ControllerProviderInterface, ServiceProvider
 
     public function connect(Application $app)
     {
+        $firewall = $this->getFirewall($app);
+
+        $requireUnauthenticated = function () use ($firewall) {
+            if (null !== $response = $firewall->requireNotAuthenticated()) {
+                return $response;
+            }
+
+            return null;
+        };
+
         if (! $this->isApiEnabled($app)) {
             return $app['controllers_factory'];
         }
@@ -48,6 +61,15 @@ class OAuth2 extends Api implements ControllerProviderInterface, ServiceProvider
 
         $controllers->post('/token', 'controller.oauth2:tokenAction');
 
+        $controllers->get('/provider/{providerId}/authorize/', 'controller.oauth2:authorizeWithProviderAction')
+            ->before($requireUnauthenticated)
+            ->bind('oauth2_provider_authorize');
+
+        // AuthProviders callbacks
+        $controllers->get('/provider/{providerId}/callback/', 'controller.oauth2:authorizeCallbackAction')
+            ->before($requireUnauthenticated)
+            ->bind('login_authentication_provider_callback');
+
         return $controllers;
     }
 }

lib/Alchemy/Phrasea/ControllerProvider/Api/V1.php

@@ -37,6 +37,7 @@ class V1 extends Api implements ControllerProviderInterface, ServiceProviderInte
             return (new V1Controller($app))
                 ->setDataboxLoggerLocator($app['phraseanet.logger'])
                 ->setDispatcher($app['dispatcher'])
+                ->setFileSystemLocator(new LazyLocator($app, 'filesystem'))
                 ->setJsonBodyHelper(new LazyLocator($app, 'json.body_helper'));
         });
     }
@@ -263,6 +264,9 @@ class V1 extends Api implements ControllerProviderInterface, ServiceProviderInte
 
         $controllers->get('/me/', 'controller.api.v1:getCurrentUserAction');
         $controllers->delete('/me/', 'controller.api.v1:deleteCurrentUserAction');
+        $controllers->get('/me/structures/', 'controller.api.v1:getCurrentUserStructureAction');
+        $controllers->get('/me/subdefs/', 'controller.api.v1:getCurrentUserSubdefsAction');
+        $controllers->get('/me/collections/', 'controller.api.v1:getCurrentUserCollectionsAction');
 
         $controllers->post('/me/request-collections/', 'controller.api.v1:createCollectionRequests');
         $controllers->post('/me/update-account/', 'controller.api.v1:updateCurrentUserAction');

lib/Alchemy/Phrasea/ControllerProvider/ControllerProviderServiceProvider.php

@@ -80,6 +80,7 @@ class ControllerProviderServiceProvider implements ServiceProviderInterface
             Prod\Root::class => [],
             Prod\Share::class => [],
             Prod\Story::class => [],
+            Prod\Subdefs::class => [],
             Prod\Tools::class => [],
             Prod\Tooltip::class => [],
             Prod\TOU::class => [],

lib/Alchemy/Phrasea/ControllerProvider/Prod/Subdefs.php

@@ -0,0 +1,44 @@
+<?php
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace Alchemy\Phrasea\ControllerProvider\Prod;
+
+use Alchemy\Phrasea\Application as PhraseaApplication;
+use Alchemy\Phrasea\Controller\Prod\SubdefsController;
+use Alchemy\Phrasea\ControllerProvider\ControllerProviderTrait;
+use Silex\ControllerProviderInterface;
+use Silex\ServiceProviderInterface;
+use Silex\Application;
+
+class Subdefs implements ControllerProviderInterface, ServiceProviderInterface
+{
+    use ControllerProviderTrait;
+
+    public function register(Application $app)
+    {
+        $app['controller.prod.subdefs'] = $app->share(function (PhraseaApplication $app) {
+            return (new SubdefsController($app));
+        });
+    }
+
+    public function boot(Application $app)
+    {
+        // no-op
+    }
+
+    public function connect(Application $app)
+    {
+        $controllers = $this->createAuthenticatedCollection($app);
+
+        $controllers->get('/{databox_id}/{record_id}/metadatas/{subdef_name}/', 'controller.prod.subdefs:metadataAction')
+            ->bind('prod_subdefs_metadata');
+
+        return $controllers;
+    }
+}

lib/Alchemy/Phrasea/ControllerProvider/Prod/Tools.php

@@ -69,6 +69,11 @@ class Tools implements ControllerProviderInterface, ServiceProviderInterface
 
         $controllers->post('/sharing-editor/{base_id}/{record_id}/', 'controller.prod.tools:editRecordSharing');
 
+        $controllers->post('/metadata/save/', 'controller.prod.tools:saveMetasAction')
+            ->bind('prod_tools_metadata_save');
+
+        $controllers->get('/videoEditor', 'controller.prod.tools:videoEditorAction');
+
         return $controllers;
     }
 }

lib/Alchemy/Phrasea/Core/Configuration/DisplaySettingService.php

@@ -18,6 +18,8 @@ class DisplaySettingService
     const ORDER_ALPHA_ASC = "ORDER_ALPHA_ASC";
     const ORDER_ALPHA_DESC = "ORDER_ALPHA_DESC";
     const ORDER_BY_ADMIN = "ORDER_BY_ADMIN";
+    const ORDER_BY_BCT = "ORDER_BY_BCT";
+    const ORDER_BY_HITS = "ORDER_BY_HITS";
 
     /**
      * The default user settings.

lib/Alchemy/Phrasea/Core/Configuration/RegistryFormManipulator.php

@@ -82,13 +82,16 @@ class RegistryFormManipulator
 
     private function filterNullValues(array &$array)
     {
-        return array_filter($array, function (&$value) {
+        foreach ($array as $key => &$value) {
             if (is_array($value)) {
-                $value = $this->filterNullValues($value);
+                $this->filterNullValues($value);
             }
+            else if ($key !== 'geonames-server' && $value === null) {
+                unset($array[$key]);
+            }
+        }
 
-            return null !== $value;
-        });
+        return $array;
     }
 
     private function getDefaultData(array $config)

lib/Alchemy/Phrasea/Core/Configuration/StructureTemplate.php

@@ -10,8 +10,6 @@
 
 namespace Alchemy\Phrasea\Core\Configuration;
 
-use Alchemy\Phrasea\Application;
-
 /**
  * Class StructureTemplate
  * @package Alchemy\Phrasea\Core\Configuration
@@ -19,25 +17,36 @@ use Alchemy\Phrasea\Application;
 class StructureTemplate
 {
     const TEMPLATE_EXTENSION = 'xml';
-    private $templates;
+    const DEFAULT_TEMPLATE = 'en-simple';
 
-    public function __construct(Application $app)
-    {
-        $this->app = $app;
-    }
+    /** @var  string */
+    private $rootPath;
+
+    /** @var  \SplFileInfo[] */
+    private $templates;
+    /** @var  string[] */
+    private $names;
 
     /**
-     * @return $this
-     * @throws \Exception
+     * @param string $rootPath
      */
-    public function getAvailable()
+    public function __construct($rootPath)
     {
-        $templateList = new \DirectoryIterator($this->app['root.path'] . '/lib/conf.d/data_templates');
-        if (empty($templateList)) {
-            throw new \Exception('No available structure template');
+        $this->rootPath = $rootPath;
+        $this->names = $this->templates = null;    // lazy loaded, not yet set
+    }
+
+    private function load()
+    {
+        if(!is_null($this->templates)) {
+            return;     // already loaded
         }
-        $templates = [];
-        $abbreviationLength = 2;
+
+        $templateList = new \DirectoryIterator($this->rootPath . '/lib/conf.d/data_templates');
+
+        $this->templates = [];
+        $this->names     = [];
+
         foreach ($templateList as $template) {
             if ($template->isDot()
                 || !$template->isFile()
@@ -45,65 +54,64 @@ class StructureTemplate
             ) {
                 continue;
             }
-            $name = $template->getFilename();
-            $abbreviation = strtolower(substr($name, 0, $abbreviationLength));
-            if (array_key_exists($abbreviation, $templates)) {
-                $abbreviation = strtolower(substr($name, 0, ++$abbreviationLength));
-            }
-            $templates[$abbreviation] = $template->getBasename('.' . self::TEMPLATE_EXTENSION);
-        }
-        $this->templates = $templates;
 
-        return $this;
+            $name = $template->getBasename('.' . self::TEMPLATE_EXTENSION);
+            // beware that the directoryiterator returns a reference on a static, so clone()
+            $this->templates[$name] = clone($template);
+            $this->names[]          = $name;
+        }
+    }
+
+    /**
+     * @param $templateName
+     * @return null|\SplFileInfo
+     */
+    public function getByName($templateName)
+    {
+        $this->load();
+
+        if (!array_key_exists($templateName, $this->templates)) {
+            return null;
+        }
+
+        return $this->templates[$templateName];
+    }
+
+    /**
+     * @param $index
+     * @return null|\SplFileInfo
+     */
+    public function getNameByIndex($index)
+    {
+        $this->load();
+
+        return $this->names[$index];
+    }
+
+    /**
+     * @return \string[]
+     */
+    public function getNames()
+    {
+        $this->load();
+
+        return $this->names;
+    }
+
+    public function toString()
+    {
+        $this->load();
+
+        return implode(', ', $this->names);
     }
 
     /**
      * @return string
      */
-    public function __toString()
+    public function getDefault()
     {
-        if (!$this->templates) {
-            return '';
-        }
-        $templateToString = '';
-        $cpt = 1;
-        $templateLength = count($this->templates);
-        foreach ($this->templates as $key => $value) {
-            if (($templateLength - 1) == $cpt) {
-                $separator = ' and ';
-            }
-            elseif (end($this->templates) == $value) {
-                $separator = '';
-            }
-            else {
-                $separator = ', ';
-            }
-            $templateToString .= $key . ' (' . $value . ')' . $separator;
-            $cpt++;
-        }
+        $this->load();
 
-        return $templateToString;
-    }
-
-    /**
-     * @param $template
-     * @return mixed
-     * @throws \Exception
-     */
-    public function getTemplateName($template = 'en')
-    {
-        if (!array_key_exists($template, $this->templates)) {
-            throw new \Exception('Not found template : ' . $template);
-        }
-
-        return $this->templates[$template];
-    }
-
-    /**
-     * @return mixed
-     */
-    public function getTemplates()
-    {
-        return $this->templates;
+        return $this->getByName(self::DEFAULT_TEMPLATE) ? self::DEFAULT_TEMPLATE : $this->getNameByIndex(0);
     }
 }

lib/Alchemy/Phrasea/Core/Event/Subscriber/RegistrationSubscriber.php

@@ -109,7 +109,7 @@ class RegistrationSubscriber extends AbstractNotificationSubscriber
         $body .= sprintf("%s : %s\n", $this->app->trans('admin::compte-utilisateur nom'), $registeredUser->getFirstName());
         $body .= sprintf("%s : %s\n", $this->app->trans('admin::compte-utilisateur prenom'), $registeredUser->getLastName());
         $body .= sprintf("%s : %s\n", $this->app->trans('admin::compte-utilisateur email'), $registeredUser->getEmail());
-        $body .= sprintf("%s/%s\n", $registeredUser->get_job(), $registeredUser->getCompany());
+        $body .= sprintf("%s/%s\n", $registeredUser->getJob(), $registeredUser->getCompany());
 
         $readyToSend = false;
         try {

lib/Alchemy/Phrasea/Core/Provider/ConfigurationServiceProvider.php

@@ -75,7 +75,7 @@ class ConfigurationServiceProvider implements ServiceProviderInterface
         });
 
         $app['phraseanet.structure-template'] = $app->share(function (Application $app) {
-            return new StructureTemplate($app);
+            return new StructureTemplate($app['root.path']);
         });
     }
 

lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php

@@ -146,6 +146,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface
         $app['elasticsearch.indexer.databox_fetcher_factory'] = $app->share(function ($app) {
             return new DataboxFetcherFactory(
                 $app['elasticsearch.record_helper'],
+                $app['elasticsearch.options'],
                 $app,
                 'search_engine.structure',
                 'thesaurus'

lib/Alchemy/Phrasea/Core/Provider/TwigServiceProvider.php

@@ -127,6 +127,23 @@ class TwigServiceProvider implements ServiceProviderInterface
             );
         }, ['needs_environment' => true, 'is_safe' => ['html']]));
 
+        $twig->addFilter(new \Twig_SimpleFilter('parseColor', function (\Twig_Environment $twig, $string) use ($app) {
+            $re = '/^(.*)\[#([0-9a-fA-F]{6})]$/m';
+            $stringArr = explode(';', $string);
+
+            foreach ($stringArr as $key => $value) {
+                preg_match_all($re, trim($value), $matches);
+                if ($matches && $matches[1] != null && $matches[2] != null) {
+                    $colorCode = '#' . $matches[2][0];
+                    $colorName = $matches[1][0];
+
+                    $stringArr[$key] = '<span style="white-space: nowrap;"><span class="color-dot" style="margin-right: 4px; background-color: ' . $colorCode . '"></span>' . $colorName . '</span>';
+                }
+            }
+
+            return implode('; ', $stringArr);
+        }, ['needs_environment' => true, 'is_safe' => ['html']]));
+
         $twig->addFilter(new \Twig_SimpleFilter('bounce',
             function (\Twig_Environment $twig, $fieldValue, $fieldName, $searchRequest, $sbasId) {
                 // bounce value if it is present in thesaurus as well
@@ -143,6 +160,25 @@ class TwigServiceProvider implements ServiceProviderInterface
         $twig->addFilter(new \Twig_SimpleFilter('escapeDoubleQuote', function ($value) {
             return str_replace('"', '\"', $value);
         }));
+
+        $twig->addFilter(new \Twig_SimpleFilter('formatDuration',
+            function ($secondsInDecimals) {
+                $time = [];
+                $hours = floor($secondsInDecimals / 3600);
+                $secondsInDecimals -= $hours * 3600;
+                $minutes = floor($secondsInDecimals / 60);
+                $secondsInDecimals -= $minutes * 60;
+                $seconds = intVal($secondsInDecimals % 60, 10);
+                if ($hours > 0) {
+                    array_push($time, (strlen($hours) < 2) ? "0{$hours}" : $hours);
+                }
+                array_push($time, (strlen($minutes) < 2) ? "0{$minutes}" : $minutes);
+                array_push($time, (strlen($seconds) < 2) ? "0{$seconds}" : $seconds);
+                $formattedTime = implode(':', $time);
+
+                return $formattedTime;
+            }
+        ));
     }
 
     /**

lib/Alchemy/Phrasea/Databox/DataboxPathExtractor.php

@@ -22,11 +22,19 @@ class DataboxPathExtractor
         $this->appbox = $appbox;
     }
 
-    public function extractPaths()
+    /**
+     * @param string $type
+     * @return array
+     */
+    public function extractPaths($type = null)
     {
         $paths = [];
 
         foreach ($this->appbox->get_databoxes() as $databox) {
+            if ($type === 'xsendfile') {
+                $paths[] = (string) $databox->get_sxml_structure()->path;
+            }
+
             foreach ($databox->get_subdef_structure()->getSubdefGroup('video') as $subdef) {
                 $paths[] = $subdef->get_path();
             }

lib/Alchemy/Phrasea/Databox/DataboxService.php

@@ -4,7 +4,9 @@ namespace Alchemy\Phrasea\Databox;
 
 use Alchemy\Phrasea\Application;
 use Alchemy\Phrasea\Core\Configuration\PropertyAccess;
+use Alchemy\Phrasea\Core\Configuration\StructureTemplate;
 use Alchemy\Phrasea\Model\Entities\User;
+use Alchemy\Phrasea\Utilities\StringHelper;
 use Doctrine\DBAL\Connection;
 
 /**
@@ -72,26 +74,101 @@ class DataboxService
     }
 
     /**
+     * @param $databaseName
+     * @param DataboxConnectionSettings|null $connectionSettings
+     * @return bool
+     */
+    public function exists($databaseName, DataboxConnectionSettings $connectionSettings = null)
+    {
+        $connectionSettings = $connectionSettings ?: DataboxConnectionSettings::fromArray(
+            $this->configuration->get(['main', 'database'])
+        );
+        $factory = $this->connectionFactory;
+
+        // do not simply try to connect to the database, list
+        /** @var Connection $connection */
+        $connection = $factory([
+            'host' => $connectionSettings->getHost(),
+            'port' => $connectionSettings->getPort(),
+            'user' => $connectionSettings->getUser(),
+            'password' => $connectionSettings->getPassword(),
+            'dbname' => null,
+        ]);
+
+        $ret = false;
+        $databaseName = strtolower($databaseName);
+        $sm = $connection->getSchemaManager();
+        $databases = $sm->listDatabases();
+        foreach($databases as $database) {
+            if(strtolower($database) == $databaseName) {
+                $ret = true;
+                break;
+            }
+        }
+
+        return $ret;
+    }
+
+    /**
+     * @param Connection $connection
+     * @param \SplFileInfo $template
+     * @return \databox
+     */
+    public function createDataboxFromConnection($connection, $template)
+    {
+        return \databox::create($this->app, $connection, $template);
+    }
+
+    /**
+     * @param $databaseName
+     * @param $templateName
      * @param User $owner
-     * @param string $databaseName
-     * @param string $dataTemplate
      * @param DataboxConnectionSettings|null $connectionSettings
      * @return \databox
+     * @throws \Exception_InvalidArgument
      */
     public function createDatabox(
         $databaseName,
-        $dataTemplate,
+        $templateName,
         User $owner,
         DataboxConnectionSettings $connectionSettings = null
     ) {
         $this->validateDatabaseName($databaseName);
 
-        $dataTemplate = new \SplFileInfo($this->rootPath . '/lib/conf.d/data_templates/' . $dataTemplate . '.xml');
+        /** @var StructureTemplate $st */
+        $st = $this->app['phraseanet.structure-template'];
+
+        $template = $st->getByName($templateName);
+        if(is_null($template)) {
+            throw new \Exception_InvalidArgument(sprintf('Databox template "%s" not found.', $templateName));
+        }
+
+        // if no connectionSettings (host, user, ...) are provided, create dbox beside appBox
         $connectionSettings = $connectionSettings ?: DataboxConnectionSettings::fromArray(
             $this->configuration->get(['main', 'database'])
         );
 
         $factory = $this->connectionFactory;
+
+        if(!$this->exists($databaseName, $connectionSettings)) {
+
+            // use a tmp connection to create the database
+            /** @var Connection $connection */
+            $connection = $factory([
+                'host'     => $connectionSettings->getHost(),
+                'port'     => $connectionSettings->getPort(),
+                'user'     => $connectionSettings->getUser(),
+                'password' => $connectionSettings->getPassword(),
+                'dbname'   => null
+            ]);
+            // the schemeManager does NOT quote identifiers, we MUST do it
+            // see : http://docs.doctrine-project.org/projects/doctrine-dbal/en/latest/reference/schema-manager.html
+            $connection->getSchemaManager()->createDatabase(StringHelper::SqlQuote($databaseName, StringHelper::SQL_IDENTIFIER));
+
+            $connection->close();
+            unset($connection);
+        }
+
         /** @var Connection $connection */
         $connection = $factory([
             'host' => $connectionSettings->getHost(),
@@ -103,7 +180,8 @@ class DataboxService
 
         $connection->connect();
 
-        $databox = \databox::create($this->app, $connection, $dataTemplate);
+        $databox = $this->createDataboxFromConnection($connection, $template);
+
         $databox->registerAdmin($owner);
 
         $connection->close();

lib/Alchemy/Phrasea/Filesystem/FilesystemService.php

@@ -10,6 +10,7 @@
 
 namespace Alchemy\Phrasea\Filesystem;
 
+use Alchemy\Phrasea\Media\Subdef\Specification\PdfSpecification;
 use Alchemy\Phrasea\Model\RecordInterface;
 use MediaAlchemyst\Specification\SpecificationInterface;
 
@@ -66,6 +67,20 @@ class FilesystemService
         return $pathdest . $this->generateSubdefFilename($record, $subdef);
     }
 
+    public function generateTemporarySubdefPathname(\record_adapter $record, \databox_subdef $subdef, $tmpDir)
+    {
+        $tmpDir = \p4string::addEndSlash($tmpDir);
+        $tmpDir = $tmpDir.$subdef->getSpecs()->getType()."/";
+
+        if(!is_dir($tmpDir)){
+            $this->filesystem->mkdir($tmpDir);
+        }
+
+        $filenameSufix = "_".$record->getDataboxId()."_".$this->generateSubdefFilename($record, $subdef);
+
+        return $tmpDir . hash('sha256', $filenameSufix) . $filenameSufix;
+    }
+
     /**
      * @param RecordInterface $record
      * @param string|\SplFileInfo $source
@@ -163,6 +178,8 @@ class FilesystemService
                 return $this->getExtensionFromVideoCodec($spec->getVideoCodec());
             case SpecificationInterface::TYPE_SWF:
                 return 'swf';
+            case PdfSpecification::TYPE_PDF:
+                return 'pdf';
         }
 
         return null;

lib/Alchemy/Phrasea/Form/Configuration/GeneralFormType.php

@@ -56,6 +56,12 @@ class GeneralFormType extends AbstractType
             'attr'        => ['min' => -1],
             'constraints' => new GreaterThanOrEqual(['value' => -1]),
         ]);
+        $builder->add('personalize-logo-choice', new PersonalisationLogoForm(), [
+            'label' => 'Design of personalization logo section',
+            'attr'  => [
+                'id' => 'personalize-logo-container'
+            ]
+        ]);
     }
 
     public function getName()

lib/Alchemy/Phrasea/Form/Configuration/PersonalisationLogoForm.php

@@ -0,0 +1,44 @@
+<?php
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2014 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Form\Configuration;
+
+use Symfony\Component\Form\AbstractType;
+use Symfony\Component\Form\FormBuilderInterface;
+
+class PersonalisationLogoForm extends AbstractType
+{
+    public function buildForm(FormBuilderInterface $builder, array $options)
+    {
+        $builder->add('originalChoiceInput', 'choice', [
+            'label'    => false,
+            'choices'  => ['original' => 'original-choice-label'],
+            'expanded' => true,
+
+        ]);
+        $builder->add('personaliseChoiceInput', 'choice', [
+            'label'    => false,
+            'choices'  => ['personalise' => 'personalise-choice-label'],
+            'expanded' => true,
+
+        ]);
+        $builder->add('personalizeLogoInput', 'file', [
+            'label' => false,
+        ]);
+        $builder->add('logoChoice', 'hidden', [
+            'label' => false,
+        ]);
+    }
+
+    public function getName()
+    {
+        return null;
+    }
+}

lib/Alchemy/Phrasea/Helper/User/Edit.php

@@ -712,6 +712,8 @@ class Edit extends \Alchemy\Phrasea\Helper\Helper
             $user = $this->app['repo.users']->find($usr_id);
             
             $this->app->getAclForUser($user)->apply_model($template, $base_ids);
+
+            $this->app['manipulator.user']->updateUser($user);
         }
 
         return $this;

lib/Alchemy/Phrasea/Helper/User/Manage.php

@@ -91,6 +91,7 @@ class Manage extends Helper
             'sbas_id' => $this->request->get('sbas_id'),
             'base_id' => $this->request->get('base_id'),
             'last_model' => $this->request->get('last_model'),
+            'filter_guest_user' => $this->request->get('filter_guest_user') ? true : false,
             'srt' => $this->request->get("srt", \User_Query::SORT_CREATIONDATE),
             'ord' => $this->request->get("ord", \User_Query::ORD_DESC),
             'per_page' => $results_quantity,
@@ -109,6 +110,7 @@ class Manage extends Helper
             ->last_model_is($this->query_parms['last_model'])
             ->get_inactives($this->query_parms['inactives'])
             ->include_templates(true)
+            ->include_invite($this->query_parms['filter_guest_user'])
             ->on_bases_where_i_am($this->app->getAclForUser($this->app->getAuthenticatedUser()), [\ACL::CANADMIN])
             ->limit($offset_start, $results_quantity)
             ->execute();

lib/Alchemy/Phrasea/Media/Subdef/Pdf.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace Alchemy\Phrasea\Media\Subdef;
+
+use Symfony\Component\Translation\TranslatorInterface;
+use Alchemy\Phrasea\Media\Subdef\Specification\PdfSpecification;
+
+class Pdf extends Provider
+{
+    protected $options = [];
+
+    public function __construct(TranslatorInterface $translator)
+    {
+        $this->translator = $translator;
+    }
+
+    public function getType()
+    {
+        return self::TYPE_PDF;
+    }
+
+    public function getDescription()
+    {
+        return $this->translator->trans('Generates a pdf file');
+    }
+
+    public function getMediaAlchemystSpec()
+    {
+        if (! $this->spec) {
+            $this->spec = new PdfSpecification();
+        }
+
+        return $this->spec;
+    }
+
+}

lib/Alchemy/Phrasea/Media/Subdef/Specification/PdfSpecification.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace Alchemy\Phrasea\Media\Subdef\Specification;
+
+use MediaAlchemyst\Specification\AbstractSpecification;
+
+class PdfSpecification extends AbstractSpecification
+{
+    const TYPE_PDF = 'pdf';
+
+    public function getType()
+    {
+        return self::TYPE_PDF;
+    }
+}

lib/Alchemy/Phrasea/Media/Subdef/Subdef.php

@@ -22,6 +22,7 @@ interface Subdef
     const TYPE_VIDEO = 'video';
     const TYPE_AUDIO = 'audio';
     const TYPE_FLEXPAPER = 'flexpaper';
+    const TYPE_PDF = 'pdf';
     const TYPE_UNKNOWN = 'unknown';
 
     /**

lib/Alchemy/Phrasea/Media/SubdefGenerator.php

@@ -19,10 +19,19 @@ use Alchemy\Phrasea\Core\Event\Record\SubDefinitionsCreationEvent;
 use Alchemy\Phrasea\Core\Event\Record\SubDefinitionCreationFailedEvent;
 use Alchemy\Phrasea\Core\Event\Record\RecordEvents;
 use Alchemy\Phrasea\Filesystem\FilesystemService;
+use Alchemy\Phrasea\Media\Subdef\Specification\PdfSpecification;
 use MediaAlchemyst\Alchemyst;
+use MediaAlchemyst\Specification\Image;
+use MediaAlchemyst\Specification\Video;
 use MediaVorus\MediaVorus;
 use MediaAlchemyst\Exception\ExceptionInterface as MediaAlchemystException;
+use Neutron\TemporaryFilesystem\Manager;
 use Psr\Log\LoggerInterface;
+use Unoconv\Exception\ExceptionInterface as UnoconvException;
+use Unoconv\Exception\RuntimeException;
+use Unoconv\Unoconv;
+use MediaVorus\Exception\FileNotFoundException as MediaVorusFileNotFoundException;
+use MediaAlchemyst\Exception\FileNotFoundException;
 
 class SubdefGenerator
 {
@@ -36,6 +45,9 @@ class SubdefGenerator
      */
     private $logger;
     private $mediavorus;
+    private $tmpFilePath;
+    private $tmpFilesystem;
+    private $tmpDirectory;
 
     public function __construct(Application $app, Alchemyst $alchemyst, FilesystemService $filesystem, MediaVorus $mediavorus, LoggerInterface $logger)
     {
@@ -44,10 +56,31 @@ class SubdefGenerator
         $this->filesystem = $filesystem;
         $this->logger = $logger;
         $this->mediavorus = $mediavorus;
+        $this->tmpDirectory = $this->app['conf']->get(['main', 'storage', 'tmp_files']);;
     }
 
     public function generateSubdefs(\record_adapter $record, array $wanted_subdefs = null)
     {
+        if ($record->get_hd_file() !== null) {
+            $mediaSource = $this->mediavorus->guess($record->get_hd_file()->getPathname());
+            $metadatas = $mediaSource->getMetadatas();
+
+            if ($metadatas->containsKey('XMP-xmp:PageImage')) {
+                if(!isset($this->tmpFilesystem)){
+                    $this->tmpFilesystem = Manager::create();
+                }
+                $tmpDir = $this->tmpFilesystem->createTemporaryDirectory();
+
+                try {
+                    $this->app['filesystem']->dumpFile($tmpDir.'/file.jpg', $metadatas->get('XMP-xmp:PageImage')->getValue()->asString());
+                    $this->tmpFilePath = $tmpDir.'/file.jpg';
+                } catch (\Exception $e) {
+                    $this->logger->error(sprintf('Unable to write temporary file : %s', $e->getMessage()));
+                }
+
+            }
+        }
+
         if (null === $subdefs = $record->getDatabox()->get_subdef_structure()->getSubdefGroup($record->getType())) {
             $this->logger->info(sprintf('Nothing to do for %s', $record->getType()));
             $subdefs = [];
@@ -118,6 +151,13 @@ class SubdefGenerator
             $record->clearSubdefCache($subdefname);
         }
 
+        if(isset($this->tmpFilesystem)){
+            $this->tmpFilesystem->clean();
+        }
+        if(isset($this->tmpFilePath)){
+            unset($this->tmpFilePath);
+        }
+
         $this->dispatch(
             RecordEvents::SUB_DEFINITIONS_CREATED,
             new SubDefinitionsCreatedEvent(
@@ -136,9 +176,59 @@ class SubdefGenerator
                 return;
             }
 
-            $this->alchemyst->turnInto($record->get_hd_file()->getPathname(), $pathdest, $subdef_class->getSpecs());
+            $destFile = null;
+
+            if(!empty($this->tmpDirectory)){
+                $destFile = $pathdest;
+                $pathdest = $this->filesystem->generateTemporarySubdefPathname($record, $subdef_class, $this->tmpDirectory);
+            }
+
+            if (isset($this->tmpFilePath) && $subdef_class->getSpecs() instanceof Image) {
+
+                $this->alchemyst->turnInto($this->tmpFilePath, $pathdest, $subdef_class->getSpecs());
+
+            } elseif ($subdef_class->getSpecs() instanceof PdfSpecification){
+
+                $this->generatePdfSubdef($record->get_hd_file()->getPathname(), $pathdest);
+
+            } else {
+
+                $this->alchemyst->turnInto($record->get_hd_file()->getPathname(), $pathdest, $subdef_class->getSpecs());
+
+            }
+
+            if($destFile){
+                $this->filesystem->copy($pathdest, $destFile);
+                $this->app['filesystem']->remove($pathdest);
+            }
+
         } catch (MediaAlchemystException $e) {
             $this->logger->error(sprintf('Subdef generation failed for record %d with message %s', $record->getRecordId(), $e->getMessage()));
         }
     }
+
+    private function generatePdfSubdef($source, $pathdest)
+    {
+        try {
+            $mediafile = $this->app['mediavorus']->guess($source);
+        } catch (MediaVorusFileNotFoundException $e) {
+            throw new FileNotFoundException(sprintf('File %s not found', $source));
+        }
+
+        try {
+            if ($mediafile->getFile()->getMimeType() != 'application/pdf') {
+                $this->app['unoconv']->transcode(
+                    $mediafile->getFile()->getPathname(), Unoconv::FORMAT_PDF, $pathdest
+                );
+
+            } else {
+                copy($mediafile->getFile()->getPathname(), $pathdest);
+            }
+        } catch (UnoconvException $e) {
+            throw new RuntimeException('Unable to transmute document to pdf due to Unoconv', null, $e);
+        } catch (\Exception $e) {
+            throw $e;
+        }
+
+    }
 }

lib/Alchemy/Phrasea/Metadata/PhraseanetMetadataSetter.php

@@ -11,6 +11,7 @@
 
 namespace Alchemy\Phrasea\Metadata;
 
+use Alchemy\Phrasea\Border\File;
 use Alchemy\Phrasea\Databox\DataboxRepository;
 use Alchemy\Phrasea\Metadata\Tag\NoSource;
 use PHPExiftool\Driver\Metadata\Metadata;
@@ -119,8 +120,11 @@ class PhraseanetMetadataSetter
                 if (!isset($metadataPerField[$fieldName])) {
                     $metadataPerField[$fieldName] = [];
                 }
-
-                $metadataPerField[$fieldName] = array_merge($metadataPerField[$fieldName], $metadata->getValue()->asArray());
+                if(in_array($tagName, File::$xmpTag)){
+                    $metadataPerField[$fieldName] = array_merge($metadataPerField[$fieldName], (array) File::sanitizeXmpUuid($metadata->getValue()->asString()));
+                }else{
+                    $metadataPerField[$fieldName] = array_merge($metadataPerField[$fieldName], $metadata->getValue()->asArray());
+                }
             }
         }
 

lib/Alchemy/Phrasea/Model/Entities/ElasticsearchRecord.php

@@ -53,6 +53,42 @@ class ElasticsearchRecord implements RecordInterface, MutableRecordInterface
     private $flags = [];
     private $highlight = [];
 
+    public function asArray()
+    {
+        return [
+            '_index'         => $this->_index,
+            '_type'          => $this->_type,
+            '_id'            => $this->_id,
+            '_version'       => $this->_version,
+            '_score'         => $this->_score,
+            'databoxId'      => $this->databoxId,
+            'recordId'       => $this->recordId,
+            'collectionId'   => $this->collectionId,
+            'baseId'         => $this->baseId,
+            'collectionName' => $this->collectionName,
+            'mimeType'       => $this->mimeType,
+            'title'          => $this->title,
+            'originalName'   => $this->originalName,
+            'updated'        => $this->updated,
+            'created'        => $this->created,
+            'sha256'         => $this->sha256,
+            'width'          => $this->width,
+            'height'         => $this->height,
+            'size'           => $this->size,
+            'uuid'           => $this->uuid,
+            'position'       => $this->position,
+            'type'           => $this->type,
+            'status'         => $this->status,
+            'isStory'        => $this->isStory,
+            'caption'        => $this->caption,
+            'privateCaption' => $this->privateCaption,
+            'exif'           => $this->exif,
+            'subdefs'        => $this->subdefs,
+            'flags'          => $this->flags,
+            'highlight'      => $this->highlight,
+        ];
+    }
+
     /**
      * @param string $index
      * @param string $type

lib/Alchemy/Phrasea/Model/Entities/LazaretFile.php

@@ -386,6 +386,17 @@ class LazaretFile
         $this->checks->removeElement($checks);
     }
 
+    /**
+     * @param LazaretCheck $checks
+     * @return string
+     */
+    public function getCheckerName(LazaretCheck $checks)
+    {
+        $checkNameTab = explode('\\', $checks->getCheckClassname());
+
+        return $checkNameTab[4];
+    }
+
     /**
      * Get checks
      *
@@ -396,6 +407,19 @@ class LazaretFile
         return $this->checks;
     }
 
+    /**
+     * @return array $checkers
+     */
+    public function getChecksWhithNameKey()
+    {
+        $checkers = [];
+        foreach($this->checks as $check){
+            $checkers[$this->getCheckerName($check)] = $check;
+        }
+
+        return $checkers;
+    }
+
     /**
      * Set session
      *
@@ -439,7 +463,7 @@ class LazaretFile
                             'reasons' => []
                         ];
                     }
-                    $merged[$record->getRecordId()]['reasons'][] = $check->getReason($app['translator']);
+                    $merged[$record->getRecordId()]['reasons'][$this->getCheckerName($check)] = $check->getReason($app['translator']);
                 }
                 else {
                     $merged[$record->getRecordId()] = $record;

lib/Alchemy/Phrasea/Model/Manipulator/UserManipulator.php

@@ -390,4 +390,9 @@ class UserManipulator implements ManipulatorInterface
 
         return $var;
     }
+
+    public function updateUser(User $user)
+    {
+        $this->manager->update($user);
+    }
 }

lib/Alchemy/Phrasea/Order/Controller/ApiOrderController.php

@@ -179,7 +179,15 @@ class ApiOrderController extends BaseOrderController
 
         set_time_limit(0);
         ignore_user_abort(true);
-        $file = \set_export::build_zip($this->app, $token, $exportData, $token->getValue() . '.zip');
+
+        $tmpDownloadDir = \p4string::addEndSlash($this->app['tmp.download.path']);
+        if(is_null($tmpDownloadDir)){
+            $tmpDownloadDir = '';
+        }
+
+        $zipFile = $tmpDownloadDir.'order_'.$token->getValue() . '.zip';
+
+        $file = \set_export::build_zip($this->app, $token, $exportData, $zipFile);
 
         return $this->deliverFile($file, $exportName, DeliverDataInterface::DISPOSITION_INLINE, 'application/zip');
     }

lib/Alchemy/Phrasea/Out/Module/PDF.php

@@ -25,6 +25,7 @@ class PDF
     const LAYOUT_PREVIEWCAPTIONTDM = 'previewCaptionTdm';
     const LAYOUT_THUMBNAILLIST = 'thumbnailList';
     const LAYOUT_THUMBNAILGRID = 'thumbnailGrid';
+    const LAYOUT_CAPTION = 'caption';
 
     public function __construct(Application $app, array $records, $layout)
     {
@@ -73,6 +74,8 @@ class PDF
                         continue 2;
                     }
                     break;
+                case self::LAYOUT_CAPTION:
+                    break;
             }
 
             $record->setNumber(count($list) + 1);
@@ -106,6 +109,9 @@ class PDF
             case self::LAYOUT_THUMBNAILGRID:
                 $this->print_thumbnailGrid();
                 break;
+            case self::LAYOUT_CAPTION:
+                $this->print_caption();
+                break;
         }
 
         return $this;
@@ -315,6 +321,67 @@ class PDF
         $this->pdf->SetLeftMargin($lmargin);
     }
 
+    protected function print_caption()
+    {
+        $this->pdf->AddPage();
+        $oldMargins = $this->pdf->getMargins();
+
+        $lmargin = $oldMargins['left'];
+        $rmargin = $oldMargins['right'];
+
+        foreach ($this->records as $rec) {
+            $title = "record : " . $rec->get_title();
+
+            $y = $this->pdf->GetY();
+            if($this->pdf->getPageHeight() - $y < 20){ // height of the footer is 15
+                $this->pdf->AddPage();
+                $y = $oldMargins['top'];
+            }
+
+            $t = \phrasea::bas_labels($rec->getBaseId(), $this->app);
+            $this->pdf->SetFont(PhraseaPDF::FONT, '', 10);
+            $this->pdf->SetFillColor(220, 220, 220);
+            $this->pdf->SetLeftMargin($lmargin);
+            $this->pdf->SetRightMargin($rmargin);
+            $this->pdf->SetX($lmargin);
+            $this->pdf->SetY($y);
+
+            $this->pdf->out = false;
+            $this->pdf->MultiCell(140, 4, $title, "LTR", "L", 1);
+            $y2 = $this->pdf->GetY();
+            $h = $y2 - $y;
+            $this->pdf->out = true;
+            $this->pdf->SetX($lmargin);
+            $this->pdf->SetY($y);
+            $this->pdf->Cell(0, $h, "", "LTR", 1, "R", 1);
+            $this->pdf->SetX($lmargin);
+            $this->pdf->SetY($y);
+            $this->pdf->Cell(0, 4, $t, "", 1, "R");
+            $this->pdf->SetX($lmargin);
+            $this->pdf->SetY($y);
+            $this->pdf->MultiCell(140, 4, $title, "", "L");
+            $this->pdf->SetX($lmargin);
+            $this->pdf->SetY($y = $y2);
+            $this->pdf->SetY($y + 2);
+
+            foreach ($rec->get_caption()->get_fields() as $field) {
+                $this->pdf->SetFont(PhraseaPDF::FONT, 'B', 12);
+                $this->pdf->Write(5, $field->get_name() . " : ");
+
+                $this->pdf->SetFont(PhraseaPDF::FONT, '', 12);
+                $t = str_replace(
+                    ["&lt;", "&gt;", "&amp;"]
+                    , ["<", ">", "&"]
+                    , strip_tags($field->get_serialized_values())
+                );
+                $this->pdf->Write(5, $t);
+
+                $this->pdf->Write(6, "\n");
+            }
+            $this->pdf->SetY($this->pdf->GetY() + 10);
+        }
+    }
+
     protected function print_preview($withtdm, $write_caption)
     {
         if ($withtdm === true) {

lib/Alchemy/Phrasea/Search/V1SearchRecordsResultTransformer.php

@@ -26,4 +26,4 @@ class V1SearchRecordsResultTransformer extends V1SearchTransformer
     {
         return $this->collection($resultView->getRecords(), $this->recordTransformer);
     }
-}
+}

lib/Alchemy/Phrasea/Search/V1SearchTransformer.php

@@ -36,7 +36,7 @@ abstract class V1SearchTransformer extends TransformerAbstract
                     return $suggestion->toArray();
                 }, $result->getSuggestions()->toArray()),
             'facets' => $result->getFacets(),
-            'query' => $result->getEngineQuery(),
+            'query' => $result->getQueryText(),
         ];
     }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/EqualExpression.php

@@ -27,11 +27,16 @@ class EqualExpression extends Node
             throw new QueryException(sprintf('Value "%s" for key "%s" is not valid.', $this->value, $this->key));
         }
 
-        $query = [
-            'term' => [
-                $this->key->getIndexField($context, true) => $this->value
-            ]
-        ];
+        if(method_exists($this->key, "buildQuery")) {
+            $query = $this->key->buildQuery($this->value, $context);
+        }
+        else {
+            $query = [
+                'term' => [
+                    $this->key->getIndexField($context, true) => $this->value
+                ]
+            ];
+        }
 
         if ($this->key instanceof QueryPostProcessor) {
             return $this->key->postProcessQuery($query, $context);

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/GeolocationKey.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
+
+use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
+
+class GeolocationKey implements Key
+{
+    const TYPE_GEOLOCATION = 'geolocation';
+
+    private $type;
+    private $key;
+    private $matcher = '/^(-?\\d*(\\.\\d*)?)[\\s]+(-?\\d*(\\.\\d*)?)[\\s]+(\\d*(\\.\\d*)?)(\\D*)$/';
+    private $units = [
+        'mi', 'miles',
+        'yd', 'yards',
+        'ft', 'feet',
+        'in', 'inch',
+        'km', 'kilometers',
+        'm', 'meters',
+        'cm', 'centimeters',
+        'mm', 'millimeters',
+        'NM', 'nmi', 'nauticalmiles',
+    ];
+
+    public static function geolocation()
+    {
+        return new self(self::TYPE_GEOLOCATION, 'geolocation');
+    }
+
+    public function buildQuery($value, QueryContext $context)
+    {
+        $matches = [];
+        if(preg_match($this->matcher, trim($value), $matches) === 1) {
+            $lat = $matches[1];
+            $lon = $matches[3];
+            $dst = $matches[5];
+            $uni = trim($matches[7]);
+            if(in_array($uni, $this->units)) {
+                return [
+                    'geo_distance' => [
+                        'distance' => $dst . $uni,
+                        'location' => [
+                            'lat' => $lat,
+                            'lon' => $lon
+                        ]
+                    ]
+                ];
+            }
+        }
+        return null;
+    }
+
+
+    private function __construct($type, $key)
+    {
+        $this->type = $type;
+        $this->key = $key;
+    }
+
+    public function getIndexField(QueryContext $context, $raw = false)
+    {
+        return $this->key;
+    }
+
+    public function isValueCompatible($value, QueryContext $context)
+    {
+        return preg_match($this->matcher, trim($value), $matches) === 1;
+    }
+
+    public function __toString()
+    {
+        return $this->type;
+    }
+}

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php

@@ -8,6 +8,8 @@ class NativeKey implements Key
 {
     const TYPE_DATABASE = 'database';
     const TYPE_COLLECTION = 'collection';
+    const TYPE_SHA256 = 'sha256';
+    const TYPE_UUID = 'uuid';
     const TYPE_MEDIA_TYPE = 'media_type';
     const TYPE_RECORD_IDENTIFIER = 'record_identifier';
 
@@ -24,6 +26,16 @@ class NativeKey implements Key
         return new self(self::TYPE_COLLECTION, 'collection_name');
     }
 
+    public static function sha256()
+    {
+        return new self(self::TYPE_SHA256, 'sha256');
+    }
+
+    public static function uuid()
+    {
+        return new self(self::TYPE_UUID, 'uuid');
+    }
+
     public static function mediaType()
     {
         return new self(self::TYPE_MEDIA_TYPE, 'type');

lib/Alchemy/Phrasea/SearchEngine/Elastic/DataboxFetcherFactory.php

@@ -35,16 +35,21 @@ class DataboxFetcherFactory
      */
     private $recordHelper;
 
+    /** @var  ElasticsearchOptions */
+    private $options;
+
     /**
      * @param RecordHelper $recordHelper
+     * @param ElasticsearchOptions $options
      * @param \ArrayAccess $container
      * @param string $structureKey
      * @param string $thesaurusKey
      */
-    public function __construct(RecordHelper $recordHelper, \ArrayAccess $container, $structureKey, $thesaurusKey)
+    public function __construct(RecordHelper $recordHelper, ElasticsearchOptions $options, \ArrayAccess $container, $structureKey, $thesaurusKey)
     {
         $this->recordHelper = $recordHelper;
-        $this->container = $container;
+        $this->options      = $options;
+        $this->container    = $container;
         $this->structureKey = $structureKey;
         $this->thesaurusKey = $thesaurusKey;
     }
@@ -59,14 +64,19 @@ class DataboxFetcherFactory
         $connection = $databox->get_connection();
 
         $candidateTerms = new CandidateTerms($databox);
-        $fetcher = new Fetcher($databox, array(
-            new CoreHydrator($databox->get_sbas_id(), $databox->get_viewname(), $this->recordHelper),
-            new TitleHydrator($connection),
-            new MetadataHydrator($connection, $this->getStructure(), $this->recordHelper),
-            new FlagHydrator($this->getStructure(), $databox),
-            new ThesaurusHydrator($this->getStructure(), $this->getThesaurus(), $candidateTerms),
-            new SubDefinitionHydrator($connection)
-        ), $fetcherDelegate);
+        $fetcher = new Fetcher(
+            $databox,
+            $this->options,
+            [
+                new CoreHydrator($databox->get_sbas_id(), $databox->get_viewname(), $this->recordHelper),
+                new TitleHydrator($connection, $this->recordHelper),
+                new MetadataHydrator($connection, $this->getStructure(), $this->recordHelper),
+                new FlagHydrator($this->getStructure(), $databox),
+                new ThesaurusHydrator($this->getStructure(), $this->getThesaurus(), $candidateTerms),
+                new SubDefinitionHydrator($connection)
+            ],
+            $fetcherDelegate
+        );
 
         $fetcher->setBatchSize(200);
         $fetcher->onDrain(function() use ($candidateTerms) {

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php

@@ -139,6 +139,7 @@ class ElasticSearchEngine implements SearchEngineInterface
         return [
             SearchEngineOptions::SORT_RELEVANCE => $this->app->trans('pertinence'),
             SearchEngineOptions::SORT_CREATED_ON => $this->app->trans('date dajout'),
+            SearchEngineOptions::SORT_UPDATED_ON => $this->app->trans('date de modification'),
         ];
     }
 
@@ -201,6 +202,11 @@ class ElasticSearchEngine implements SearchEngineInterface
         $this->notImplemented();
     }
 
+    private function notImplemented()
+    {
+        throw new LogicException('Not implemented');
+    }
+
     /**
      * {@inheritdoc}
      */
@@ -241,11 +247,6 @@ class ElasticSearchEngine implements SearchEngineInterface
         $this->notImplemented();
     }
 
-    private function notImplemented()
-    {
-        throw new LogicException('Not implemented');
-    }
-
     /**
      * {@inheritdoc}
      */
@@ -273,35 +274,35 @@ class ElasticSearchEngine implements SearchEngineInterface
     /**
      * {@inheritdoc}
      */
-    public function query($string, SearchEngineOptions $options = null)
+    public function query($queryText, SearchEngineOptions $options = null)
     {
         $options = $options ?: new SearchEngineOptions();
         $context = $this->context_factory->createContext($options);
 
         /** @var QueryCompiler $query_compiler */
         $query_compiler = $this->app['query_compiler'];
-        $recordQuery = $query_compiler->compile($string, $context);
+        $queryAST      = $query_compiler->parse($queryText)->dump();
+        $queryCompiled = $query_compiler->compile($queryText, $context);
 
-        $params = $this->createRecordQueryParams($recordQuery, $options, null);
+        $queryESLib = $this->createRecordQueryParams($queryCompiled, $options, null);
 
         // ask ES to return field _version (incremental version number of document)
-        $params['body']['version'] = true;
+        $queryESLib['body']['version'] = true;
 
-        $params['body']['from'] = $options->getFirstResult();
-        $params['body']['size'] = $options->getMaxResults();
+        $queryESLib['body']['from'] = $options->getFirstResult();
+        $queryESLib['body']['size'] = $options->getMaxResults();
         if($this->options->getHighlight()) {
-            $params['body']['highlight'] = $this->buildHighlightRules($context);
+            $queryESLib['body']['highlight'] = $this->buildHighlightRules($context);
         }
 
         $aggs = $this->getAggregationQueryParams($options);
         if ($aggs) {
-            $params['body']['aggs'] = $aggs;
+            $queryESLib['body']['aggs'] = $aggs;
         }
 
-        $res = $this->client->search($params);
+        $res = $this->client->search($queryESLib);
 
         $results = new ArrayCollection();
-
         $n = 0;
         foreach ($res['hits']['hits'] as $hit) {
             $results[] = ElasticsearchRecordHydrator::hydrate($hit, $n++);
@@ -310,16 +311,13 @@ class ElasticSearchEngine implements SearchEngineInterface
         /** @var FacetsResponse $facets */
         $facets = $this->facetsResponseFactory->__invoke($res);
 
-        $query['ast'] = $query_compiler->parse($string)->dump();
-        $query['query_main'] = $recordQuery;
-        $query['query'] = $params['body'];
-        $query['query_string'] = json_encode($params['body'], JSON_PRETTY_PRINT);
-
         return new SearchEngineResult(
             $options,
-            $results,   // ArrayCollection of results
-            $string,    // the query as typed by the user
-            json_encode($query),
+            $results,      // ArrayCollection of results
+            $queryText,    // the query as typed by the user
+            $queryAST,
+            $queryCompiled,
+            $queryESLib,
             $res['took'],   // duration
             $options->getFirstResult(),
             $res['hits']['total'],  // available
@@ -333,131 +331,6 @@ class ElasticSearchEngine implements SearchEngineInterface
         );
     }
 
-    private function buildHighlightRules(QueryContext $context)
-    {
-        $highlighted_fields = [];
-        foreach ($context->getHighlightedFields() as $field) {
-            switch ($field->getType()) {
-                case FieldMapping::TYPE_STRING:
-                    $index_field = $field->getIndexField();
-                    $raw_index_field = $field->getIndexField(true);
-                    $highlighted_fields[$index_field] = [
-                        // Requires calling Mapping::enableTermVectors() on this field mapping
-                        'matched_fields' => [$index_field, $raw_index_field],
-                        'type' => 'fvh'
-                    ];
-                    break;
-                case FieldMapping::TYPE_FLOAT:
-                case FieldMapping::TYPE_DOUBLE:
-                case FieldMapping::TYPE_INTEGER:
-                case FieldMapping::TYPE_LONG:
-                case FieldMapping::TYPE_SHORT:
-                case FieldMapping::TYPE_BYTE:
-                    continue;
-                case FieldMapping::TYPE_DATE:
-                default:
-                    continue;
-            }
-        }
-
-        return [
-            'pre_tags' =>  ['[[em]]'],
-            'post_tags' =>  ['[[/em]]'],
-            'order' => 'score',
-            'fields' => $highlighted_fields
-        ];
-    }
-
-    /**
-     * {@inheritdoc}
-     */
-    public function autocomplete($query, SearchEngineOptions $options)
-    {
-        $params = $this->createCompletionParams($query, $options);
-
-        $res = $this->client->suggest($params);
-
-        $ret = [
-            'text' => [],
-            'byField' => []
-        ];
-        foreach(array_keys($params['body']) as $fname) {
-            $t = [];
-            foreach($res[$fname] as $suggest) {    // don't know why there is a sub-array level
-                foreach($suggest['options'] as $option) {
-                    $text = $option['text'];
-                    if(!in_array($text, $ret['text'])) {
-                        $ret['text'][] = $text;
-                    }
-                    $t[] = [
-                        'label' => $text,
-                        'query' => $fname.':'.$text
-                    ];
-                }
-            }
-            if(!empty($t)) {
-                $ret['byField'][$fname] = $t;
-            }
-        }
-
-        return $ret;
-    }
-
-    /**
-     * {@inheritdoc}
-     */
-    public function resetCache()
-    {
-    }
-
-    /**
-     * {@inheritdoc}
-     */
-    public function clearCache()
-    {
-    }
-
-    /**
-     * {@inheritdoc}
-     */
-    public function clearAllCache(\DateTime $date = null)
-    {
-    }
-
-    private function createCompletionParams($query, SearchEngineOptions $options)
-    {
-        $body = [];
-        $context = [
-            'record_type' => $options->getSearchType() === SearchEngineOptions::RECORD_RECORD ?
-                SearchEngineInterface::GEM_TYPE_RECORD : SearchEngineInterface::GEM_TYPE_STORY
-        ];
-
-        $base_ids = $options->getBasesIds();
-        if (count($base_ids) > 0) {
-            $context['base_id'] = $base_ids;
-        }
-
-        $search_context = $this->context_factory->createContext($options);
-        $fields = $search_context->getUnrestrictedFields();
-        foreach($fields as $field) {
-            if($field->getType() == FieldMapping::TYPE_STRING) {
-                $k = '' . $field->getName();
-                $body[$k] = [
-                    'text' => $query,
-                    'completion' => [
-                        'field' => "caption." . $field->getName() . ".suggest",
-                        'context' => &$context
-                    ]
-                ];
-            }
-        }
-
-        return [
-            'index' => $this->indexName,
-            'body'  => $body
-        ];
-    }
-
     private function createRecordQueryParams($ESQuery, SearchEngineOptions $options, \record_adapter $record = null)
     {
         $params = [
@@ -486,66 +359,31 @@ class ElasticSearchEngine implements SearchEngineInterface
         return $params;
     }
 
-    private function getAggregationQueryParams(SearchEngineOptions $options)
+    private function createSortQueryParams(SearchEngineOptions $options)
     {
-        $aggs = [];
-        // technical aggregates (enable + optional limit)
-        foreach (ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
-            $size = $this->options->getAggregableFieldLimit($k);
-            if ($size !== databox_field::FACET_DISABLED) {
-                if ($size === databox_field::FACET_NO_LIMIT) {
-                    $size = ESField::FACET_NO_LIMIT;
-                }
-                $agg = [
-                    'terms' => [
-                        'field' => $f['field'],
-                        'size'  => $size
-                    ]
-                ];
-                $aggs[$k] = $agg;
-            }
-        }
-        // fields aggregates
-        $structure = $this->context_factory->getLimitedStructure($options);
-        foreach ($structure->getFacetFields() as $name => $field) {
-            // 2015-05-26 (mdarse) Removed databox filtering.
-            // It was already done by the ACL filter in the query scope, so no
-            // document that shouldn't be displayed can go this far.
-            $agg = [
-                'terms' => [
-                    'field' => $field->getIndexField(true),
-                    'size'  => $field->getFacetValuesLimit()
-                ]
-            ];
-            $aggs[$name] = AggregationHelper::wrapPrivateFieldAggregation($field, $agg);
-        }
-        return $aggs;
-    }
+        $sort = [];
 
-    private function createACLFilters(SearchEngineOptions $options)
-    {
-        // No ACLs if no user
-        if (false === $this->app->getAuthenticator()->isAuthenticated()) {
-            return [];
+        if ($options->getSortBy() === null || $options->getSortBy() === SearchEngineOptions::SORT_RELEVANCE) {
+            $sort['_score'] = $options->getSortOrder();
+        }
+        elseif ($options->getSortBy() === SearchEngineOptions::SORT_CREATED_ON) {
+            $sort['created_on'] = $options->getSortOrder();
+        }
+        elseif ($options->getSortBy() === SearchEngineOptions::SORT_UPDATED_ON) {
+            $sort['updated_on'] = $options->getSortOrder();
+        }
+        elseif ($options->getSortBy() === 'recordid') {
+            $sort['record_id'] = $options->getSortOrder();
+        }
+        else {
+            $sort[sprintf('caption.%s', $options->getSortBy())] = $options->getSortOrder();
         }
 
-        $acl = $this->app->getAclForUser($this->app->getAuthenticatedUser());
-
-        $grantedCollections = array_keys($acl->get_granted_base([\ACL::ACTIF]));
-
-        if (count($grantedCollections) === 0) {
-            return ['bool' => ['must_not' => ['match_all' => new \stdClass()]]];
+        if (!array_key_exists('record_id', $sort)) {
+            $sort['record_id'] = $options->getSortOrder();
         }
 
-        $appbox = $this->app['phraseanet.appbox'];
-
-        $flagNamesMap = $this->getFlagsKey($appbox);
-        // Get flags rules
-        $flagRules = $this->getFlagsRules($appbox, $acl, $grantedCollections);
-        // Get intersection between collection ACLs and collection chosen by end user
-        $aclRules = $this->getACLsByCollection($flagRules, $flagNamesMap);
-
-        return $this->buildACLsFilters($aclRules, $options);
+        return $sort;
     }
 
     private function createQueryFilters(SearchEngineOptions $options)
@@ -603,27 +441,6 @@ class ElasticSearchEngine implements SearchEngineInterface
         return $filters;
     }
 
-    private function createSortQueryParams(SearchEngineOptions $options)
-    {
-        $sort = [];
-
-        if ($options->getSortBy() === null || $options->getSortBy() === SearchEngineOptions::SORT_RELEVANCE) {
-            $sort['_score'] = $options->getSortOrder();
-        } elseif ($options->getSortBy() === SearchEngineOptions::SORT_CREATED_ON) {
-            $sort['created_on'] = $options->getSortOrder();
-        } elseif ($options->getSortBy() === 'recordid') {
-            $sort['record_id'] = $options->getSortOrder();
-        } else {
-            $sort[sprintf('caption.%s', $options->getSortBy())] = $options->getSortOrder();
-        }
-
-        if (! array_key_exists('record_id', $sort)) {
-            $sort['record_id'] = $options->getSortOrder();
-        }
-
-        return $sort;
-    }
-
     private function getFlagsKey(\appbox $appbox)
     {
         $flags = [];
@@ -638,6 +455,32 @@ class ElasticSearchEngine implements SearchEngineInterface
         return $flags;
     }
 
+    private function createACLFilters(SearchEngineOptions $options)
+    {
+        // No ACLs if no user
+        if (false === $this->app->getAuthenticator()->isAuthenticated()) {
+            return [];
+        }
+
+        $acl = $this->app->getAclForUser($this->app->getAuthenticatedUser());
+
+        $grantedCollections = array_keys($acl->get_granted_base([\ACL::ACTIF]));
+
+        if (count($grantedCollections) === 0) {
+            return ['bool' => ['must_not' => ['match_all' => new \stdClass()]]];
+        }
+
+        $appbox = $this->app['phraseanet.appbox'];
+
+        $flagNamesMap = $this->getFlagsKey($appbox);
+        // Get flags rules
+        $flagRules = $this->getFlagsRules($appbox, $acl, $grantedCollections);
+        // Get intersection between collection ACLs and collection chosen by end user
+        $aclRules = $this->getACLsByCollection($flagRules, $flagNamesMap);
+
+        return $this->buildACLsFilters($aclRules, $options);
+    }
+
     private function getFlagsRules(\appbox $appbox, \ACL $acl, array $collections)
     {
         $rules = [];
@@ -769,4 +612,166 @@ class ElasticSearchEngine implements SearchEngineInterface
             return [];
         }
     }
+
+    private function buildHighlightRules(QueryContext $context)
+    {
+        $highlighted_fields = [];
+        foreach ($context->getHighlightedFields() as $field) {
+            switch ($field->getType()) {
+                case FieldMapping::TYPE_STRING:
+                    $index_field = $field->getIndexField();
+                    $raw_index_field = $field->getIndexField(true);
+                    $highlighted_fields[$index_field . ".light"] = [
+                        // Requires calling Mapping::enableTermVectors() on this field mapping
+//                        'matched_fields' => [$index_field, $raw_index_field],
+                        'type' => 'fvh',
+                    ];
+                    break;
+                case FieldMapping::TYPE_FLOAT:
+                case FieldMapping::TYPE_DOUBLE:
+                case FieldMapping::TYPE_INTEGER:
+                case FieldMapping::TYPE_LONG:
+                case FieldMapping::TYPE_SHORT:
+                case FieldMapping::TYPE_BYTE:
+                    continue;
+                case FieldMapping::TYPE_DATE:
+                default:
+                    continue;
+            }
+        }
+
+        return [
+            'pre_tags'  => ['[[em]]'],
+            'post_tags' => ['[[/em]]'],
+            'order'     => 'score',
+            'fields'    => $highlighted_fields
+        ];
+    }
+
+    private function getAggregationQueryParams(SearchEngineOptions $options)
+    {
+        $aggs = [];
+        // technical aggregates (enable + optional limit)
+        foreach (ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
+            $size = $this->options->getAggregableFieldLimit($k);
+            if ($size !== databox_field::FACET_DISABLED) {
+                if ($size === databox_field::FACET_NO_LIMIT) {
+                    $size = ESField::FACET_NO_LIMIT;
+                }
+                $agg = [
+                    'terms' => [
+                        'field' => $f['field'],
+                        'size'  => $size
+                    ]
+                ];
+                $aggs[$k] = $agg;
+            }
+        }
+        // fields aggregates
+        $structure = $this->context_factory->getLimitedStructure($options);
+        foreach ($structure->getFacetFields() as $name => $field) {
+            // 2015-05-26 (mdarse) Removed databox filtering.
+            // It was already done by the ACL filter in the query scope, so no
+            // document that shouldn't be displayed can go this far.
+            $agg = [
+                'terms' => [
+                    'field' => $field->getIndexField(true),
+                    'size'  => $field->getFacetValuesLimit()
+                ]
+            ];
+            $aggs[$name] = AggregationHelper::wrapPrivateFieldAggregation($field, $agg);
+        }
+
+        return $aggs;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function autocomplete($query, SearchEngineOptions $options)
+    {
+        $params = $this->createCompletionParams($query, $options);
+
+        $res = $this->client->suggest($params);
+
+        $ret = [
+            'text'    => [],
+            'byField' => []
+        ];
+        foreach (array_keys($params['body']) as $fname) {
+            $t = [];
+            foreach ($res[$fname] as $suggest) {    // don't know why there is a sub-array level
+                foreach ($suggest['options'] as $option) {
+                    $text = $option['text'];
+                    if (!in_array($text, $ret['text'])) {
+                        $ret['text'][] = $text;
+                    }
+                    $t[] = [
+                        'label' => $text,
+                        'query' => $fname . ':' . $text
+                    ];
+                }
+            }
+            if (!empty($t)) {
+                $ret['byField'][$fname] = $t;
+            }
+        }
+
+        return $ret;
+    }
+
+    private function createCompletionParams($query, SearchEngineOptions $options)
+    {
+        $body = [];
+        $context = [
+            'record_type' => $options->getSearchType() === SearchEngineOptions::RECORD_RECORD ?
+                SearchEngineInterface::GEM_TYPE_RECORD : SearchEngineInterface::GEM_TYPE_STORY
+        ];
+
+        $base_ids = $options->getBasesIds();
+        if (count($base_ids) > 0) {
+            $context['base_id'] = $base_ids;
+        }
+
+        $search_context = $this->context_factory->createContext($options);
+        $fields = $search_context->getUnrestrictedFields();
+        foreach ($fields as $field) {
+            if ($field->getType() == FieldMapping::TYPE_STRING) {
+                $k = '' . $field->getName();
+                $body[$k] = [
+                    'text'       => $query,
+                    'completion' => [
+                        'field'   => "caption." . $field->getName() . ".suggest",
+                        'context' => &$context
+                    ]
+                ];
+            }
+        }
+
+        return [
+            'index' => $this->indexName,
+            'body'  => $body
+        ];
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function resetCache()
+    {
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function clearCache()
+    {
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function clearAllCache(\DateTime $date = null)
+    {
+    }
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php

@@ -11,6 +11,10 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic;
 
 class ElasticsearchOptions
 {
+    const POPULATE_ORDER_RID = "RECORD_ID";
+    const POPULATE_ORDER_MODDATE = "MODIFICATION_DATE";
+    const POPULATE_DIRECTION_ASC = "ASC";
+    const POPULATE_DIRECTION_DESC = "DESC";
     /** @var string */
     private $host;
     /** @var int */
@@ -25,6 +29,10 @@ class ElasticsearchOptions
     private $minScore;
     /** @var  bool */
     private $highlight;
+    /** @var string */
+    private $populateOrder;
+    /** @var string */
+    private $populateDirection;
 
     /** @var  int[] */
     private $_customValues;
@@ -46,6 +54,8 @@ class ElasticsearchOptions
             'replicas' => 0,
             'minScore' => 4,
             'highlight' => true,
+            'populate_order'     => self::POPULATE_ORDER_RID,
+            'populate_direction' => self::POPULATE_DIRECTION_DESC,
             'activeTab' => null,
         ];
 
@@ -63,6 +73,8 @@ class ElasticsearchOptions
         $self->setReplicas($options['replicas']);
         $self->setMinScore($options['minScore']);
         $self->setHighlight($options['highlight']);
+        $self->setPopulateOrder($options['populate_order']);
+        $self->setPopulateDirection($options['populate_direction']);
         $self->setActiveTab($options['activeTab']);
         foreach(self::getAggregableTechnicalFields() as $k => $f) {
             $self->setAggregableFieldLimit($k, $options[$k.'_limit']);
@@ -85,6 +97,8 @@ class ElasticsearchOptions
             'replicas' => $this->replicas,
             'minScore' => $this->minScore,
             'highlight' => $this->highlight,
+            'populate_order'     => $this->populateOrder,
+            'populate_direction' => $this->populateDirection,
             'activeTab' => $this->activeTab
         ];
         foreach(self::getAggregableTechnicalFields() as $k => $f) {
@@ -258,7 +272,7 @@ class ElasticsearchOptions
             ],
             'camera_model_aggregate' => [
                 'label' => 'Camera Model',
-                'field' => 'metadata_tags.CameraModel.raw',
+                'field' => 'metadata_tags.CameraModel',
                 'query' => 'meta.CameraModel:%s',
             ],
             'iso_aggregate' => [
@@ -270,11 +284,20 @@ class ElasticsearchOptions
                 'label' => 'Aperture',
                 'field' => 'metadata_tags.Aperture',
                 'query' => 'meta.Aperture=%s',
+                'output_formatter' => function($value) {
+                    return round($value, 1);
+                },
             ],
             'shutterspeed_aggregate' => [
                 'label' => 'Shutter speed',
                 'field' => 'metadata_tags.ShutterSpeed',
                 'query' => 'meta.ShutterSpeed=%s',
+                'output_formatter' => function($value) {
+                    if($value < 1.0 && $value != 0) {
+                        $value = '1/' . round(1.0 / $value);
+                    }
+                    return $value . ' s.';
+                },
             ],
             'flashfired_aggregate' => [
                 'label' => 'FlashFired',
@@ -283,6 +306,10 @@ class ElasticsearchOptions
                 'choices' => [
                     "aggregated (2 values: fired = 0 or 1)" => -1,
                 ],
+                'output_formatter' => function($value) {
+                    static $map = ['0'=>"No flash", '1'=>"Flash"];
+                    return array_key_exists($value, $map) ? $map[$value] : $value;
+                },
             ],
             'framerate_aggregate' => [
                 'label' => 'FrameRate',
@@ -322,4 +349,60 @@ class ElasticsearchOptions
         ];
     }
 
+    /**
+     * @param string $order
+     * @return bool returns false if order is invalid
+     */
+    public function setPopulateOrder($order)
+    {
+        $order = strtoupper($order);
+        if (in_array($order, [self::POPULATE_ORDER_RID, self::POPULATE_ORDER_MODDATE])) {
+            $this->populateOrder = $order;
+
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * @return string
+     */
+    public function getPopulateOrderAsSQL()
+    {
+        static $orderAsColumn = [
+            self::POPULATE_ORDER_RID     => "`record_id`",
+            self::POPULATE_ORDER_MODDATE => "`moddate`",
+        ];
+
+        // populateOrder IS one of the keys (ensured by setPopulateOrder)
+        return $orderAsColumn[$this->populateOrder];
+    }
+
+    /**
+     * @param string $direction
+     * @return bool returns false if direction is invalid
+     */
+    public function setPopulateDirection($direction)
+    {
+        $direction = strtoupper($direction);
+        if (in_array($direction, [self::POPULATE_DIRECTION_DESC, self::POPULATE_DIRECTION_ASC])) {
+            $this->populateDirection = $direction;
+
+            return true;
+        }
+
+        return false;
+    }
+
+
+    /**
+     * @return string
+     */
+    public function getPopulateDirectionAsSQL()
+    {
+        // already a SQL word
+        return $this->populateDirection;
+    }
+
 }

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchRecordHydrator.php

@@ -32,7 +32,10 @@ class ElasticsearchRecordHydrator
             if (substr($key, 0, strlen($prefix)) == $prefix) {
                 $key = substr($key, strlen($prefix));
             }
-            $highlight[$key] = $value;
+            if (substr($key, -6) == '.light') {
+                $key = substr($key, 0, strlen($key) - 6);
+                $highlight[$key] = $value;
+            }
         }
 
         $record = new ElasticsearchRecord();
@@ -62,7 +65,7 @@ class ElasticsearchRecordHydrator
         $record->setTitles((array) igorw\get_in($data, ['title'], []));
         $record->setCaption((array) igorw\get_in($data, ['caption'], []));
         $record->setPrivateCaption((array) igorw\get_in($data, ['private_caption'], []));
-        $record->setExif((array) igorw\get_in($data, ['exif'], []));
+        $record->setExif((array)igorw\get_in($data, ['metadata_tags'], []));
         $record->setSubdefs((array) igorw\get_in($data, ['subdefs'], []));
         $record->setFlags((array) igorw\get_in($data, ['flags'], []));
         $record->setHighlight((array) $highlight);

lib/Alchemy/Phrasea/SearchEngine/Elastic/FieldMapping.php

@@ -26,13 +26,14 @@ class FieldMapping
     const TYPE_COMPLETION = 'completion';
 
     // Number core types
-    const TYPE_FLOAT   = 'float';
-    const TYPE_DOUBLE  = 'double';
-    const TYPE_INTEGER = 'integer';
-    const TYPE_LONG    = 'long';
-    const TYPE_SHORT   = 'short';
-    const TYPE_BYTE    = 'byte';
-    const TYPE_IP      = 'ip';
+    const TYPE_FLOAT     = 'float';
+    const TYPE_DOUBLE    = 'double';
+    const TYPE_INTEGER   = 'integer';
+    const TYPE_LONG      = 'long';
+    const TYPE_SHORT     = 'short';
+    const TYPE_BYTE      = 'byte';
+    const TYPE_IP        = 'ip';
+    const TYPE_GEO_POINT = 'geo_point';
 
     // Compound types
     const TYPE_OBJECT  = 'object';
@@ -48,6 +49,7 @@ class FieldMapping
         self::TYPE_SHORT,
         self::TYPE_BYTE,
         self::TYPE_IP,
+        self::TYPE_GEO_POINT,
         self::TYPE_OBJECT,
         self::TYPE_COMPLETION
     );

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php

@@ -16,13 +16,11 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\BulkOperation;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\RecordIndexer;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\TermIndexer;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\RecordQueuer;
-use appbox;
 use Closure;
 use Elasticsearch\Client;
 use Psr\Log\LoggerInterface;
 use igorw;
 use Psr\Log\NullLogger;
-use record_adapter;
 use Symfony\Component\Stopwatch\Stopwatch;
 use SplObjectStorage;
 
@@ -84,20 +82,64 @@ class Indexer
         $this->deleteQueue = new SplObjectStorage();
     }
 
-    public function createIndex($withMapping = true)
+    /**
+     * @return Index
+     */
+    public function getIndex()
     {
-        $params = array();
-        $params['index'] = $this->index->getName();
-        $params['body']['settings']['number_of_shards'] = $this->index->getOptions()->getShards();
-        $params['body']['settings']['number_of_replicas'] = $this->index->getOptions()->getReplicas();
-        $params['body']['settings']['analysis'] = $this->index->getAnalysis();
+        return $this->index;
+    }
 
-        if ($withMapping) {
-            $params['body']['mappings'][RecordIndexer::TYPE_NAME] = $this->index->getRecordIndex()->getMapping()->export();
-            $params['body']['mappings'][TermIndexer::TYPE_NAME]   = $this->index->getTermIndex()->getMapping()->export();
+    public function createIndex($indexName = null)
+    {
+        $aliasName = $this->index->getName();
+        if($indexName === null) {
+            $indexName = $aliasName;
         }
 
+        $now = sprintf("%s.%06d", Date('YmdHis'), 1000000*explode(' ', microtime())[0]) ;
+        $indexName .=  ('_' . $now);
+
+        $params = [
+            'index' => $indexName,
+            'body'  => [
+                'settings' => [
+                    'number_of_shards'   => $this->index->getOptions()->getShards(),
+                    'number_of_replicas' => $this->index->getOptions()->getReplicas(),
+                    'analysis'           => $this->index->getAnalysis()
+                ],
+                'mappings' => [
+                    RecordIndexer::TYPE_NAME => $this->index->getRecordIndex()->getMapping()->export(),
+                    TermIndexer::TYPE_NAME   => $this->index->getTermIndex()->getMapping()->export()
+                ]
+                //    ,
+                //    'aliases' => [
+                //        $aliasName => []
+                //    ]
+            ]
+        ];
+
         $this->client->indices()->create($params);
+
+        $params = [
+            'body' => [
+                'actions' => [
+                    [
+                        'add' => [
+                            'index' => $indexName,
+                            'alias' => $aliasName
+                        ]
+                    ]
+                ]
+            ]
+        ];
+
+        $this->client->indices()->updateAliases($params);
+
+        return [
+            'index' => $indexName,
+            'alias' => $aliasName
+        ];
     }
 
     public function updateMapping()
@@ -126,38 +168,129 @@ class Indexer
         ]);
     }
 
+    /**
+     * @param string $newIndexName
+     * @param string $newAliasName
+     * @return array
+     */
+    public function replaceIndex($newIndexName, $newAliasName)
+    {
+        $ret = [];
+
+        $oldIndexes = $this->client->indices()->getAlias(
+            [
+                'index' => $this->index->getName()
+            ]
+        );
+
+        // delete old alias(es), only one alias on one index should exist
+        foreach($oldIndexes as $oldIndexName => $data) {
+            foreach($data['aliases'] as $oldAliasName => $data2) {
+                $params['body']['actions'][] = [
+                    'remove' => [
+                        'alias' => $oldAliasName,
+                        'index' => $oldIndexName,
+                    ]
+                ];
+                $ret[] = [
+                    'action' => "ALIAS_REMOVE",
+                    'msg'    => sprintf('alias "%s" -> "%s" removed', $oldAliasName, $oldIndexName),
+                    'alias'  => $oldAliasName,
+                    'index'  => $oldIndexName,
+                ];
+            }
+        }
+
+        // create new alias
+        $params['body']['actions'][] = [
+            'add' => [
+                'alias' => $this->index->getName(),
+                'index' => $newIndexName,
+            ]
+        ];
+        $ret[] = [
+            'action' => "ALIAS_ADD",
+            'msg'   => sprintf('alias "%s" -> "%s" added', $this->index->getName(), $newIndexName),
+            'alias' => $this->index->getName(),
+            'index' => $newIndexName,
+        ];
+
+        //
+        $params['body']['actions'][] = [
+            'remove' => [
+                'alias' => $newAliasName,
+                'index' => $newIndexName,
+            ]
+        ];
+        $ret[] = [
+            'action' => "ALIAS_REMOVE",
+            'msg'    => sprintf('alias "%s" -> "%s" removed', $newAliasName, $newIndexName),
+            'alias'  => $newAliasName,
+            'index'  => $newIndexName,
+        ];
+
+
+        $this->client->indices()->updateAliases($params);
+
+        // delete old index(es), only one index should exist
+        $params = [
+            'index' => []
+        ];
+        foreach($oldIndexes as $oldIndexName => $data) {
+            $params['index'][] = $oldIndexName;
+            $ret[] = [
+                'action' => "INDEX_DELETE",
+                'msg'    => sprintf('index "%s" deleted', $oldIndexName),
+                'index'  => $oldIndexName,
+            ];
+        }
+        $this->client->indices()->delete(
+            $params
+        );
+
+        return $ret;
+    }
+
     public function populateIndex($what, \databox $databox)
     {
         $stopwatch = new Stopwatch();
         $stopwatch->start('populate');
 
-        $this->apply(function (BulkOperation $bulk) use ($what, $databox) {
-            if ($what & self::THESAURUS) {
-                $this->termIndexer->populateIndex($bulk, $databox);
+        $this->apply(
+            function (BulkOperation $bulk) use ($what, $databox) {
+                if ($what & self::THESAURUS) {
+                    $this->termIndexer->populateIndex($bulk, $databox);
 
-                // Record indexing depends on indexed terms so we need to make
-                // everything ready to search
-                $bulk->flush();
-                $this->client->indices()->refresh();
-                $this->client->indices()->clearCache();
-                $this->client->indices()->flushSynced();
-            }
+                    // Record indexing depends on indexed terms so we need to make
+                    // everything ready to search
+                    $bulk->flush();
+                    $this->client->indices()->refresh();
+                }
 
-            if ($what & self::RECORDS) {
-                $databox->clearCandidates();
-                $this->recordIndexer->populateIndex($bulk, $databox);
+                if ($what & self::RECORDS) {
+                    $databox->clearCandidates();
+                    $this->recordIndexer->populateIndex($bulk, $databox);
 
-                // Final flush
-                $bulk->flush();
-            }
-        }, $this->index);
+                    // Final flush
+                    $bulk->flush();
+                }
+            },
+            $this->index
+        );
 
         // Optimize index
-        $params = array('index' => $this->index->getName());
-        $this->client->indices()->optimize($params);
+        $this->client->indices()->optimize(
+            [
+                'index' => $this->index->getName()
+            ]
+        );
 
         $event = $stopwatch->stop('populate');
-        printf("Indexation finished in %s min (Mem. %s Mo)", ($event->getDuration()/1000/60), bcdiv($event->getMemory(), 1048576, 2));
+
+        return [
+            'duration' => $event->getDuration(),
+            'memory'   => $event->getMemory()
+        ];
     }
 
     public function migrateMappingForDatabox($databox)

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php

@@ -12,6 +12,7 @@
 namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record;
 
 use Alchemy\Phrasea\Core\PhraseaTokens;
+use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegate;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegateInterface;
@@ -24,6 +25,7 @@ use PDO;
 class Fetcher
 {
     private $databox;
+    private $options;
     private $connection;
     private $statement;
     private $delegate;
@@ -36,9 +38,10 @@ class Fetcher
     private $postFetch;
     private $onDrain;
 
-    public function __construct(databox $databox, array $hydrators, FetcherDelegateInterface $delegate = null)
+    public function __construct(databox $databox,ElasticsearchOptions $options, array $hydrators, FetcherDelegateInterface $delegate = null)
     {
         $this->databox = $databox;
+        $this->options = $options;
         $this->connection = $databox->get_connection();;
         $this->hydrators  = $hydrators;
         $this->delegate   = $delegate ?: new FetcherDelegate();
@@ -136,7 +139,7 @@ class Fetcher
                  . " FROM (record r INNER JOIN coll c ON (c.coll_id = r.coll_id))"
                  . " LEFT JOIN subdef ON subdef.record_id=r.record_id AND subdef.name='document'"
                  . " -- WHERE"
-                 . " ORDER BY r.record_id DESC"
+                 . " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL()
                  . " LIMIT :offset, :limit";
 
             $where = $this->delegate->buildWhereClause();

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/GpsPosition.php

@@ -81,6 +81,30 @@ class GpsPosition
             && $this->latitude_ref !== null;
     }
 
+    public function isCompleteComposite()
+    {
+        return $this->longitude !== null
+            && $this->latitude !== null;
+    }
+
+    public function getCompositeLongitude()
+    {
+        if ($this->longitude === null) {
+            return null;
+        }
+
+        return $this->longitude ;
+    }
+
+    public function getCompositeLatitude()
+    {
+        if ($this->latitude === null) {
+            return null;
+        }
+
+        return $this->latitude;
+    }
+
     public function getSignedLongitude()
     {
         if ($this->longitude === null) {

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php

@@ -121,6 +121,9 @@ SQL;
     private function sanitizeValue($value, $type)
     {
         switch ($type) {
+            case FieldMapping::TYPE_STRING:
+                return str_replace("\0", "", $value);
+
             case FieldMapping::TYPE_DATE:
                 return $this->helper->sanitizeDate($value);
 
@@ -152,9 +155,18 @@ SQL;
         // Push this tag into object
         $position->set($tag_name, $value);
         // Try to output complete position
-        if ($position->isComplete()) {
-            $records[$id]['metadata_tags']['Longitude'] = $position->getSignedLongitude();
-            $records[$id]['metadata_tags']['Latitude'] = $position->getSignedLatitude();
+        if ($position->isCompleteComposite()) {
+            $lon = $position->getCompositeLongitude();
+            $lat = $position->getCompositeLatitude();
+
+            $records[$id]['metadata_tags']['Longitude'] = $lon;
+            $records[$id]['metadata_tags']['Latitude'] = $lat;
+
+            $records[$id]["location"] = [
+                "lat" => $lat,
+                "lon" => $lon
+            ];
+
             unset($this->gps_position_buffer[$id]);
         }
     }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndex.php

@@ -61,8 +61,8 @@ class RecordIndex implements MappingProvider
         // Collection name (still indexed for facets)
         $mapping->addStringField('collection_name')->disableAnalysis();
 
-        $mapping->addStringField('uuid')->disableIndexing();
-        $mapping->addStringField('sha256')->disableIndexing();
+        $mapping->addStringField('uuid')->disableAnalysis();
+        $mapping->addStringField('sha256')->disableAnalysis();
         $mapping->addStringField('original_name')->disableIndexing();
         $mapping->addStringField('mime')->disableAnalysis();
         $mapping->addStringField('type')->disableAnalysis();
@@ -73,6 +73,8 @@ class RecordIndex implements MappingProvider
         $mapping->addIntegerField('height')->disableIndexing();
         $mapping->addIntegerField('size')->disableIndexing();
 
+        $mapping->addGeoPointField('location');
+
         $mapping->addDateField('created_on', FieldMapping::DATE_FORMAT_MYSQL_OR_CAPTION);
         $mapping->addDateField('updated_on', FieldMapping::DATE_FORMAT_MYSQL_OR_CAPTION);
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/MappingBuilder.php

@@ -45,6 +45,15 @@ class MappingBuilder
         return $this->mapping->addField(new FieldMapping($name, FieldMapping::TYPE_INTEGER));
     }
 
+    /**
+     * @param string $name
+     * @return FieldMapping
+     */
+    public function addGeoPointField($name)
+    {
+        return $this->mapping->addField(new FieldMapping($name, FieldMapping::TYPE_GEO_POINT));
+    }
+
     /**
      * @param string $name
      * @return FieldMapping

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/FacetsResponse.php

@@ -7,6 +7,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
 use Alchemy\Phrasea\SearchEngine\SearchEngineSuggestion;
 use Doctrine\Common\Collections\ArrayCollection;
+use igorw;
 
 class FacetsResponse
 {
@@ -55,12 +56,21 @@ class FacetsResponse
     private function buildBucketsValues($name, $buckets)
     {
         $values = array();
+
+        // does this aggregate has an output_formatter ? if not use a equality formatter
+        /** @var callable $formatter */
+        $formatter = igorw\get_in(
+            ElasticsearchOptions::getAggregableTechnicalFields(), [$name, 'output_formatter'],
+            function($v){return $v;}
+        );
+
         foreach ($buckets as $bucket) {
             if (!isset($bucket['key']) || !isset($bucket['doc_count'])) {
                 $this->throwAggregationResponseError();
             }
+
             $values[] = array(
-                'value' => $bucket['key'],
+                'value' => $formatter($bucket['key']),
                 'count' => $bucket['doc_count'],
                 'query' => $this->buildQuery($name, $bucket['key']),
             );