Merge branch 'master' into PHRAS-2196-api-stories-pagination

lib/Alchemy/Phrasea/Application.php

@@ -92,9 +92,12 @@ use Alchemy\WorkerProvider\WorkerServiceProvider;
 use Doctrine\DBAL\Event\ConnectionEventArgs;
 use MediaVorus\Media\MediaInterface;
 use MediaVorus\MediaVorus;
+use Monolog\Handler\ErrorLogHandler;
 use Monolog\Handler\RotatingFileHandler;
+use Monolog\Handler\StreamHandler;
 use Monolog\Logger;
 use Neutron\ReCaptcha\ReCaptchaServiceProvider;
+use Psr\Log\LoggerInterface;
 use Silex\Application as SilexApplication;
 use Silex\Application\TranslationTrait;
 use Silex\Application\UrlGeneratorTrait;
@@ -112,6 +115,7 @@ use Symfony\Component\Form\FormBuilderInterface;
 use Symfony\Component\Form\FormInterface;
 use Symfony\Component\Form\FormTypeInterface;
 use Symfony\Component\HttpFoundation\RedirectResponse;
+use Symfony\Component\Process\ExecutableFinder;
 use Unoconv\UnoconvServiceProvider;
 use XPDF\PdfToText;
 use XPDF\XPDFServiceProvider;
@@ -234,8 +238,19 @@ class Application extends SilexApplication
 
         $this->register(new UnicodeServiceProvider());
         $this->register(new ValidatorServiceProvider());
-        $this->register(new XPDFServiceProvider());
-        $this->setupXpdf();
+
+        if ($this['configuration.store']->isSetup()) {
+            $binariesConfig = $this['conf']->get(['main', 'binaries']);
+            $executableFinder = new ExecutableFinder();
+            $this->register(new XPDFServiceProvider(), [
+                'xpdf.configuration' => [
+                    'pdftotext.binaries' => isset($binariesConfig['pdftotext_binary']) ? $binariesConfig['pdftotext_binary'] : $executableFinder->find('pdftotext'),
+                ]
+            ]);
+
+            $this->setupXpdf();
+        }
+
         $this->register(new FileServeServiceProvider());
         $this->register(new ManipulatorServiceProvider());
         $this->register(new PluginServiceProvider());
@@ -253,6 +268,23 @@ class Application extends SilexApplication
         $this->register(new OrderServiceProvider());
         $this->register(new WebhookServiceProvider());
 
+        $this['monolog'] = $this->share(
+            $this->extend('monolog', function (LoggerInterface $logger, Application $app) {
+
+                $logger->pushHandler(new ErrorLogHandler(
+                    ErrorLogHandler::SAPI,
+                    Logger::ERROR
+                ));
+
+                $logger->pushHandler(new StreamHandler(
+                    fopen('php://stderr', 'w'),
+                    Logger::ERROR
+                ));
+
+                return $logger;
+            })
+        );
+
         $this['phraseanet.exception_handler'] = $this->share(function ($app) {
             /** @var PhraseaExceptionHandler $handler */
             $handler =  PhraseaExceptionHandler::register($app['debug']);
@@ -633,7 +665,7 @@ class Application extends SilexApplication
     private function setupGeonames()
     {
         $this['geonames.server-uri'] = $this->share(function (Application $app) {
-            return $app['conf']->get(['registry', 'webservices', 'geonames-server'], 'http://geonames.alchemyasp.com/');
+            return $app['conf']->get(['registry', 'webservices', 'geonames-server'], 'https://geonames.alchemyasp.com/');
         });
     }
 

lib/Alchemy/Phrasea/Application/ApiApplicationLoader.php

@@ -24,9 +24,11 @@ use Alchemy\Phrasea\Core\Event\Subscriber\ApiExceptionHandlerSubscriber;
 use Alchemy\Phrasea\Core\Event\Subscriber\ApiOauth2ErrorsSubscriber;
 use Alchemy\Phrasea\Core\PhraseaEvents;
 use Alchemy\Phrasea\Core\Provider\JsonSchemaServiceProvider;
+use Alchemy\Phrasea\Report\ControllerProvider\ApiReportControllerProvider;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 
+
 class ApiApplicationLoader extends BaseApplicationLoader
 {
     protected function doPrePluginServiceRegistration(Application $app)
@@ -34,6 +36,7 @@ class ApiApplicationLoader extends BaseApplicationLoader
         $app->register(new OAuth2());
         $app->register(new V1());
         $app->register(new V2());
+        $app->register(new ApiReportControllerProvider());
         $app->register(new JsonSchemaServiceProvider());
     }
 
@@ -132,6 +135,7 @@ class ApiApplicationLoader extends BaseApplicationLoader
         $app->mount('/datafiles/', new Datafiles());
         $app->mount('/api/v1', new V1());
         $app->mount('/api/v2', new V2());
+        $app->mount('/api/report', new ApiReportControllerProvider());
         $app->mount('/permalink/', new Permalink());
         $app->mount($app['controller.media_accessor.route_prefix'], new MediaAccessor());
         $app->mount('/include/minify/', new Minifier());

lib/Alchemy/Phrasea/Application/RouteLoader.php

@@ -5,9 +5,11 @@ namespace Alchemy\Phrasea\Application;
 use Alchemy\EmbedProvider\EmbedServiceProvider;
 use Alchemy\Phrasea\Application;
 use Alchemy\Phrasea\ControllerProvider as Providers;
+use Alchemy\Phrasea\Report\ControllerProvider\ProdReportControllerProvider;
 use Assert\Assertion;
 use Silex\ControllerProviderInterface;
 
+
 class RouteLoader
 {
 
@@ -53,6 +55,7 @@ class RouteLoader
         '/prod/records/edit'           => Providers\Prod\Edit::class,
         '/prod/records/movecollection' => Providers\Prod\MoveCollection::class,
         '/prod/records/property'       => Providers\Prod\Property::class,
+        '/prod/report/'                => ProdReportControllerProvider::class,
         '/prod/share/'                 => Providers\Prod\Share::class,
         '/prod/story'                  => Providers\Prod\Story::class,
         '/prod/subdefs'                => Providers\Prod\Subdefs::class,
@@ -62,8 +65,6 @@ class RouteLoader
         '/prod/upload/'                => Providers\Prod\Upload::class,
         '/prod/WorkZone'               => Providers\Prod\WorkZone::class,
         '/prod/'                       => Providers\Prod\Root::class,
-        '/report/activity'             => Providers\Report\Activity::class,
-        '/report/informations'         => Providers\Report\Information::class,
         '/report/'                     => Providers\Report\Root::class,
         '/session/'                    => Providers\Root\Session::class,
         '/setup'                       => Providers\Setup::class,

lib/Alchemy/Phrasea/CLI.php

@@ -73,9 +73,26 @@ class CLI extends Application
 
         $this->bindRoutes();
 
-        $this['logger'] = $this->extend('logger', function () {
-            return new Console\Logger\ConsoleLogger(new Console\Output\ConsoleOutput(Console\Output\ConsoleOutput::VERBOSITY_DEBUG));
-        });
+        $logger = false;
+
+        if ($this['configuration.store']->isSetup()){
+
+            $config = $this['configuration.store']->getConfig();
+
+            if ((isset($config['console_logger_enabled_environments']) && in_array($environment, $config['console_logger_enabled_environments']))){
+                $logger = true;
+            }
+        }
+
+        if ($environment == self::ENV_DEV){
+            $logger = true;
+        }
+
+        if ($logger){
+            $this['logger'] = $this->extend('logger', function () {
+                return new Console\Logger\ConsoleLogger(new Console\Output\ConsoleOutput(Console\Output\ConsoleOutput::VERBOSITY_DEBUG));
+            });
+        }
 
         error_reporting(-1);
         ErrorHandler::register();

lib/Alchemy/Phrasea/Cache/RedisCache.php

@@ -73,7 +73,7 @@ class RedisCache extends CacheProvider implements Cache
      */
     protected function doDelete($id)
     {
-        return $this->_redis->delete($id);
+        return $this->_redis->del($id);
     }
 
     /**

lib/Alchemy/Phrasea/Cache/WinCacheCache.php

@@ -11,7 +11,7 @@
 
 namespace Alchemy\Phrasea\Cache;
 
-use Doctrine\Common\Cache\WincacheCache as DoctrineWinCache;
+use Doctrine\Common\Cache\WinCacheCache as DoctrineWinCache;
 
 class WinCacheCache extends DoctrineWinCache implements Cache
 {

lib/Alchemy/Phrasea/Collection/CollectionRepositoryRegistry.php

@@ -84,6 +84,15 @@ class CollectionRepositoryRegistry
         throw new \OutOfBoundsException('No repository available for given base [baseId: ' . $baseId . ' ].');
     }
 
+    public function getBaseIdMap()
+    {
+        if ($this->baseIdMap === null) {
+            $this->loadBaseIdMap();
+        }
+
+        return $this->baseIdMap;
+    }
+
     public function purgeRegistry()
     {
         $this->baseIdMap = null;

lib/Alchemy/Phrasea/Collection/CollectionService.php

@@ -319,7 +319,9 @@ class CollectionService
 
             $result = $userQuery->on_base_ids([ $reference->getBaseId()] )
                 ->who_have_right([\ACL::ORDER_MASTER])
-                ->execute()->get_results();
+                ->include_templates(true)
+                ->execute()
+                ->get_results();
 
             /** @var ACLProvider $acl */
             $acl = $this->app['acl'];

lib/Alchemy/Phrasea/Command/Plugin/AbstractPluginCommand.php

@@ -51,4 +51,41 @@ abstract class AbstractPluginCommand extends Command
         $this->container['plugins.autoloader-generator']->write($manifests);
         $output->writeln(" <comment>OK</comment>");
     }
+
+    protected function doInstallPlugin($source, InputInterface $input, OutputInterface $output)
+    {
+        $temporaryDir = $this->container['temporary-filesystem']->createTemporaryDirectory();
+
+        $output->write("Importing <info>$source</info>...");
+        $this->container['plugins.importer']->import($source, $temporaryDir);
+        $output->writeln(" <comment>OK</comment>");
+
+        $output->write("Validating plugin...");
+        $manifest = $this->container['plugins.plugins-validator']->validatePlugin($temporaryDir);
+        $output->writeln(" <comment>OK</comment> found <info>".$manifest->getName()."</info>");
+
+        $targetDir  = $this->container['plugin.path'] . DIRECTORY_SEPARATOR . $manifest->getName();
+
+        $output->write("Setting up composer...");
+        $this->container['plugins.composer-installer']->install($temporaryDir);
+        $output->writeln(" <comment>OK</comment>");
+
+        $output->write("Installing plugin <info>".$manifest->getName()."</info>...");
+        $this->container['filesystem']->mirror($temporaryDir, $targetDir);
+        $output->writeln(" <comment>OK</comment>");
+
+        $output->write("Copying public files <info>".$manifest->getName()."</info>...");
+        $this->container['plugins.assets-manager']->update($manifest);
+        $output->writeln(" <comment>OK</comment>");
+
+        $output->write("Removing temporary directory...");
+        $this->container['filesystem']->remove($temporaryDir);
+        $output->writeln(" <comment>OK</comment>");
+
+        $output->write("Activating plugin...");
+        $this->container['conf']->set(['plugins', $manifest->getName(), 'enabled'], true);
+        $output->writeln(" <comment>OK</comment>");
+
+        $this->updateConfigFiles($input, $output);
+    }
 }

lib/Alchemy/Phrasea/Command/Plugin/AddPlugin.php

@@ -14,6 +14,7 @@ namespace Alchemy\Phrasea\Command\Plugin;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\ArrayInput;
 
 class AddPlugin extends AbstractPluginCommand
 {
@@ -29,41 +30,36 @@ class AddPlugin extends AbstractPluginCommand
     protected function doExecutePluginAction(InputInterface $input, OutputInterface $output)
     {
         $source = $input->getArgument('source');
+        $shouldDownload = $this->shouldDownloadPlugin($source);
 
-        $temporaryDir = $this->container['temporary-filesystem']->createTemporaryDirectory();
+        if ($shouldDownload){
+            $command = $this->getApplication()->find('plugins:download');
+            $arguments = [
+                'command' => 'plugins:download',
+                'source'  => $source,
+                'shouldInstallPlugin' => true
+            ];
 
-        $output->write("Importing <info>$source</info>...");
-        $this->container['plugins.importer']->import($source, $temporaryDir);
-        $output->writeln(" <comment>OK</comment>");
+            $downloadInput = new ArrayInput($arguments);
+            $command->run($downloadInput, $output);
 
-        $output->write("Validating plugin...");
-        $manifest = $this->container['plugins.plugins-validator']->validatePlugin($temporaryDir);
-        $output->writeln(" <comment>OK</comment> found <info>".$manifest->getName()."</info>");
+        } else {
 
-        $targetDir  = $this->container['plugin.path'] . DIRECTORY_SEPARATOR . $manifest->getName();
-
-        $output->write("Setting up composer...");
-        $this->container['plugins.composer-installer']->install($temporaryDir);
-        $output->writeln(" <comment>OK</comment>");
-
-        $output->write("Installing plugin <info>".$manifest->getName()."</info>...");
-        $this->container['filesystem']->mirror($temporaryDir, $targetDir);
-        $output->writeln(" <comment>OK</comment>");
-
-        $output->write("Copying public files <info>".$manifest->getName()."</info>...");
-        $this->container['plugins.assets-manager']->update($manifest);
-        $output->writeln(" <comment>OK</comment>");
-
-        $output->write("Removing temporary directory...");
-        $this->container['filesystem']->remove($temporaryDir);
-        $output->writeln(" <comment>OK</comment>");
-
-        $output->write("Activating plugin...");
-        $this->container['conf']->set(['plugins', $manifest->getName(), 'enabled'], true);
-        $output->writeln(" <comment>OK</comment>");
-
-        $this->updateConfigFiles($input, $output);
+            $this->doInstallPlugin($source, $input, $output);
+        }
 
         return 0;
     }
+
+    protected function shouldDownloadPlugin($source)
+    {
+        $allowedScheme = array('https','ssh');
+
+        $scheme =  parse_url($source, PHP_URL_SCHEME);
+        if (in_array($scheme, $allowedScheme)){
+            return true;
+        } else{
+            return false;
+        }
+    }
 }

lib/Alchemy/Phrasea/Command/Plugin/DownloadPlugin.php

@@ -0,0 +1,157 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Command\Plugin;
+
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\ArrayInput;
+use Cz\Git\GitRepository as GitRepository;
+
+class DownloadPlugin extends AbstractPluginCommand
+{
+
+    public function __construct()
+    {
+        parent::__construct('plugins:download');
+
+        $this
+            ->setDescription('Downloads a plugin to Phraseanet')
+            ->addArgument('source', InputArgument::REQUIRED, 'The source is a remote url (.zip or .git)')
+            ->addArgument('destination', InputArgument::OPTIONAL, 'Download destination')
+            ->addArgument('shouldInstallPlugin', InputArgument::OPTIONAL, 'True or false, determines if plugin should be installed after download');
+    }
+
+    protected function doExecutePluginAction(InputInterface $input, OutputInterface $output)
+    {
+        $source = $input->getArgument('source');
+        $destination = $input->getArgument('destination');
+        $shouldInstallPlugin = false;
+        $shouldInstallPlugin = $input->getArgument('shouldInstallPlugin');
+
+        $destinationSubdir = '/plugin-'.md5($source);
+
+        if ($destination){
+
+            $destination = trim($destination);
+            $destination = rtrim($destination, '/');
+
+            $localDownloadPath = $destination;
+
+        } else {
+
+            $localDownloadPath = '/tmp/plugin-download' . $destinationSubdir;
+        }
+
+        if (!is_dir($localDownloadPath)) {
+            mkdir($localDownloadPath, 0755, true);
+        }
+
+        $extension = $this->getURIExtension($source);
+
+        if ($extension){
+
+            switch ($extension){
+
+                case 'zip':
+
+                    $localUnpackPath = '/tmp/plugin-zip'. $destinationSubdir;
+
+                    if (!is_dir($localUnpackPath)) {
+                        mkdir($localUnpackPath, 0755, true);
+                    }
+
+                    $localArchiveFile = $localUnpackPath . '/plugin-downloaded.zip';
+
+                    // download
+                    $output->writeln("Downloading <info>$source</info>...");
+                    set_time_limit(0);
+                    $fp = fopen ($localArchiveFile, 'w+');
+                    $ch = curl_init($source);;
+                    curl_setopt($ch, CURLOPT_FILE, $fp);
+                    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+                    curl_exec($ch);
+                    curl_close($ch);
+                    fclose($fp);
+
+                    // unpack
+                    $output->writeln("Unpacking <info>$source</info>...");
+                    $zip = new \ZipArchive();
+                    $errorUnpack = false;
+
+                    if ($zip->open($localArchiveFile)) {
+                        for ($i = 0; $i < $zip->numFiles; $i++) {
+                            if (!($zip->extractTo($localDownloadPath, array($zip->getNameIndex($i))))) {
+                                $errorUnpack = true;
+                            }
+                        }
+                        $zip->close();
+                    }
+
+                    if ($errorUnpack){
+                        $output->writeln("Failed unzipping <info>$source</info>");
+                    } else {
+                        $output->writeln("Plugin downloaded to <info>$localDownloadPath</info>");
+                        if ($shouldInstallPlugin) $this->doInstallPlugin($localDownloadPath, $input, $output);
+                    }
+
+                    // remove zip archive
+                    $this->delDirTree($localUnpackPath);
+
+                    break;
+
+                case 'git':
+                    $output->writeln("Downloading <info>$source</info>...");
+                    $repo = GitRepository::cloneRepository($source, $localDownloadPath);
+                    $output->writeln("Plugin downloaded to <info>$localDownloadPath</info>");
+                    if ($shouldInstallPlugin) $this->doInstallPlugin($localDownloadPath, $input, $output);
+                    break;
+
+            }
+
+        } else {
+
+            $output->writeln("The source <info>$source</info> is not supported. Only .zip and .git are supported.");
+        }
+
+        return 0;
+    }
+
+    protected function getURIExtension($source)
+    {
+        $validExtension = false;
+        $allowedExtension = array('zip','git');
+
+        $path = parse_url($source, PHP_URL_PATH);
+        if (strpos($path, '.') !== false) {
+            $pathParts = explode('.', $path);
+            $extension = $pathParts[1];
+            if (in_array($extension, $allowedExtension)){
+                $validExtension = true;
+            }
+        }
+
+        if ($validExtension){
+            return $extension;
+        } else {
+            return false;
+        }
+    }
+
+    protected static function delDirTree($dir) {
+        $files = array_diff(scandir($dir), array('.','..'));
+        foreach ($files as $file) {
+            (is_dir("$dir/$file")) ? self::delDirTree("$dir/$file") : unlink("$dir/$file");
+        }
+        return rmdir($dir);
+    }
+}

lib/Alchemy/Phrasea/Command/Setup/FixAutoincrements.php

@@ -193,8 +193,7 @@ class FixAutoincrements extends Command
                 $databox,
                 'coll',
                 [
-                    'collusr'   => 'coll_id',
-                    'log_colls' => 'coll_id',
+                    'collusr' => 'coll_id',
                 ]
             );
         }

lib/Alchemy/Phrasea/Command/Setup/FixLogCollId.php

@@ -0,0 +1,396 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Command\Setup;
+
+use Alchemy\Phrasea\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Doctrine\DBAL\Driver\ResultStatement;
+
+
+class FixLogCollId extends Command
+{
+    const OPTION_DISTINT_VALUES = 0;
+    const OPTION_ALL_VALUES     = 1;
+
+    const PLAYDRY_NONE = 0;
+    const PLAYDRY_SQL  = 1;
+    const PLAYDRY_CODE = 2;
+
+    /** @var InputInterface */
+    private $input;
+    /** @var OutputInterface */
+    private $output;
+    /** @var \Databox[] */
+    private $databoxes;
+
+    /** @var int */
+    private $batch_size;
+    /** @var bool */
+    private $dry;
+    /** @var bool */
+    private $show_sql;
+    /** @var bool */
+    private $keep_tmp_table;
+
+
+    public function __construct($name = null)
+    {
+        parent::__construct("patch:log_coll_id");
+
+        $this->setDescription('Fix empty (null) coll_id in "log_docs" and "log_view" tables.');
+        $this->addOption('databox',            null, InputOption::VALUE_OPTIONAL, 'Mandatory : The id (or dbname or viewname) of the databox');
+        $this->addOption('batch_size',         null, InputOption::VALUE_OPTIONAL, 'work on a batch of n entries (default=100000)');
+        $this->addOption('dry',                null, InputOption::VALUE_NONE,     'dry run, list but don\'t act');
+        $this->addOption('show_sql',           null, InputOption::VALUE_NONE,     'show sql pre-selecting records');
+        $this->addOption('keep_tmp_table',     null, InputOption::VALUE_NONE,     'keep the working "tmp_coll" table (help debug)');
+
+        $this->setHelp("help");
+    }
+
+    /**
+     * sanity check the cmd line options
+     *
+     * @param InputInterface $input
+     * @param OutputInterface $output
+     * @return bool
+     */
+    protected function sanitizeArgs(InputInterface $input, OutputInterface $output)
+    {
+        $argsOK = true;
+
+        // find the databox / collection by id or by name
+        $this->databoxes = [];
+        if(!is_null($d = $input->getOption('databox'))) {
+            $d = trim($d);
+        }
+        foreach ($this->container->getDataboxes() as $db) {
+            if(is_null($d)){
+                $this->databoxes[] = $db;
+            }
+            else {
+                if ($db->get_sbas_id() == (int)$d || $db->get_viewname() == $d || $db->get_dbname() == $d) {
+                    $this->databoxes[] = $db;
+                }
+            }
+        }
+        if (empty($this->databoxes)) {
+            if(is_null($d)) {
+                $output->writeln(sprintf("<error>No databox found</error>", $d));
+            }
+            else {
+                $output->writeln(sprintf("<error>Unknown databox \"%s\"</error>", $d));
+            }
+            $argsOK = false;
+        }
+
+        // get options
+        $this->batch_size      = $input->getOption('batch_size');
+        $this->show_sql       = $input->getOption('show_sql') ? true : false;
+        $this->dry            = $input->getOption('dry') ? true : false;
+        $this->keep_tmp_table = $input->getOption('keep_tmp_table') ? true : false;
+
+        if(is_null($this->batch_size)) {
+            $this->batch_size = 100000;
+        }
+        if($this->batch_size < 1) {
+            $output->writeln(sprintf('<error>batch_size must be > 0</error>'));
+            $argsOK = false;
+        }
+
+        return $argsOK;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function doExecute(InputInterface $input, OutputInterface $output)
+    {
+        // $time_start = new \DateTime();
+
+        if(!$this->sanitizeArgs($input, $output)) {
+            return -1;
+        }
+
+        $this->input  = $input;
+        $this->output = $output;
+
+
+        foreach($this->databoxes as $databox) {
+
+            $this->output->writeln("");
+            $this->output->writeln(sprintf("<info>================================ Working on databox %s (id:%s) ================================</info>", $databox->get_dbname(), $databox->get_sbas_id()));
+
+            if (!$this->showCount($databox)) {
+                // databox not patched
+                break;
+            }
+
+            $this->createWorkingTable($databox);
+
+            // loop to compute coll_id from top to bottom
+            do {
+                $n = $this->computeCollId($databox);   // in dry mode, n=0
+            }
+            while ($n > 0);
+
+            // set the "from_coll_id"
+            $this->computeCollIdFrom($databox);
+
+            // copy results back to the log_docs
+            $this->copyReults($databox);
+
+
+            if (!$this->keep_tmp_table) {
+                $this->dropWorkingTable($databox);
+            }
+
+            $this->output->writeln("");
+        }
+
+        return 0;
+    }
+
+    private function createWorkingTable(\databox $databox)
+    {
+        $this->output->writeln("");
+        $this->output->writeln(sprintf("<info> ----------------- Creating working %s table -----------------</info>", ($this->keep_tmp_table ? "(temporary)" : "")));
+        $this->output->writeln("");
+
+        $sql = "CREATE " . ($this->keep_tmp_table ? "TABLE IF NOT EXISTS" : "TEMPORARY TABLE") . " `tmp_colls` (\n"
+            . " `record_id` int(11) unsigned NOT NULL,\n"
+            . " `from_id` int(11) unsigned NOT NULL,\n"
+            . " `from_date` datetime NOT NULL,\n"
+            . " `to_id` int(11) unsigned DEFAULT NULL,\n"
+            . " `to_date` datetime DEFAULT NULL,\n"
+            . " `coll_id_from` int(10) unsigned DEFAULT NULL,\n"
+            . " `coll_id` int(10) unsigned DEFAULT NULL,\n"
+            . " KEY `record_id` (`record_id`),\n"
+            . " KEY `from_id` (`from_id`),\n"
+            . " KEY `from_date` (`from_date`),\n"
+            . " KEY `to_id` (`to_id`),\n"
+            . " KEY `to_date` (`to_date`)\n"
+            . ") ENGINE=InnoDB;";
+
+        if($this->show_sql) {
+            $this->output->writeln($sql);
+            $this->output->writeln("");
+        }
+        $stmt = $databox->get_connection()->prepare($sql);
+        $stmt->execute();
+        $stmt->closeCursor();
+
+        $sql = "TRUNCATE TABLE `tmp_colls`";
+
+        if($this->show_sql) {
+            $this->output->writeln($sql);
+            $this->output->writeln("");
+        }
+        $stmt = $databox->get_connection()->prepare($sql);
+        $stmt->execute();
+        $stmt->closeCursor();
+    }
+
+    private function dropWorkingTable(\databox $databox)
+    {
+        $this->output->writeln(sprintf("<info> ----------------- Drop working table -----------------</info>"));
+        $sql = "DROP TABLE `tmp_colls`";
+        if($this->show_sql) {
+            $this->output->writeln($sql);
+        }
+        $stmt = $databox->get_connection()->prepare($sql);
+        $stmt->execute();
+        $stmt->closeCursor();
+    }
+
+    private function showCount(\databox $databox)
+    {
+        $ret = true;
+        $this->playSQL(
+            $databox,
+            [
+                'msg' => "Count work to do",
+                'sql' => "SELECT\n"
+                    . "  SUM(IF(ISNULL(`coll_id`), 0, 1)) AS `n`,\n"
+                    . "  COUNT(*) AS `t`,\n"
+                    . "  SUM(IF(`coll_id`>0, 1, 0)) AS `p`,\n"
+                    . "  SUM(IF(`coll_id`=0, 1, 0)) AS `z`\n"
+                    . "  FROM `log_docs`",
+                'code' => function(ResultStatement $stmt) use($ret) {
+                    $row = $stmt->fetch();
+                    if(is_null($row['n'])) {
+                        // no coll_id ?
+                        $this->output->writeln(sprintf("<error>The \"log_docs\" table has no \"coll_id\" column ? Please apply patch 410alpha12a</error>"));
+                        $ret = false;
+                    }
+                    $this->output->writeln("");
+                    $this->output->writeln(sprintf("done: %s / %s  (fixed: %s ; can't fix: %s)", $row['n'], $row['t'], $row['p'], $row['z']));
+                },
+                'playdry' =>  self::PLAYDRY_SQL | self::PLAYDRY_CODE,
+            ]
+        );
+
+        return $ret;
+    }
+
+    private function computeCollId(\databox $databox)
+    {
+        static $sql_lastid = null;
+        static $stmt_lastid = null;
+
+        static $sql_insert = null;
+        static $stmt_insert = null;
+
+        $ret = 0;
+        if(!$stmt_lastid) {
+            $sql_lastid = "SELECT @m:=COALESCE(MAX(`from_id`), 0) AS `lastid` FROM `tmp_colls`";
+
+            $stmt_lastid = $databox->get_connection()->prepare($sql_lastid);
+
+            $sql_insert =  "INSERT INTO `tmp_colls`\n"
+                . "    SELECT `r1`.`record_id`, `r1`.`id` AS `from_id`, `r1`.`date` AS `from_date`, MIN(`r2`.`id`) AS `to_id`, MIN(`r2`.`date`) AS `to_date`, NULL AS `coll_id_from`, `r1`.`final` AS `coll_id`\n"
+                . "    FROM (\n"
+                . "      SELECT `id`, `date`, `record_id`, `action`, `final` FROM `log_docs`\n"
+                . "      WHERE `id` > @m AND `action` IN('add', 'collection')\n"
+                . "      ORDER BY `id` ASC\n"
+                . "      LIMIT " . $this->batch_size . "\n"
+                . "    ) AS `r1`\n"
+                . "    LEFT JOIN `log_docs` AS `r2`\n"
+                . "    ON `r2`.`record_id`=`r1`.`record_id` AND `r2`.`action`='collection' AND `r2`.`id`>`r1`.`id`\n"
+                . "    GROUP BY `r1`.`id`\n"
+                // . "    ORDER BY `record_id` ASC, `from_id` ASC"
+            ;
+
+            $stmt_insert = $databox->get_connection()->prepare($sql_insert);
+
+            $this->output->writeln("");
+            $this->output->writeln(sprintf("<info> ----------------- Compute coll_id to working table ----------------- %s</info>",
+                    $this->dry ? " -- NOT PLAYED IN DRY MODE --" : ""
+                )
+            );
+            $this->output->writeln("");
+            if ($this->show_sql) {
+                $this->output->writeln($sql_lastid);
+                $this->output->writeln($sql_insert);
+            }
+        }
+
+        if(!$this->dry) {
+            $stmt_lastid->execute();
+            $stmt_lastid->closeCursor();
+
+            $stmt_insert->execute();
+            $ret = $stmt_insert->rowCount();
+            $stmt_insert->closeCursor();
+        }
+
+        return $ret;
+    }
+
+    private function computeCollIdFrom(\databox $databox)
+    {
+        static $sql = null;
+        static $stmt = null;
+
+        $ret = 0;
+        if(!$stmt) {
+            $sql =  "UPDATE `tmp_colls` AS `t1` INNER JOIN `tmp_colls` AS `t2` ON `t2`.`from_id`=`t1`.`to_id`\n"
+                . " SET `t2`.`coll_id_from` = `t1`.`coll_id`";
+
+            $this->output->writeln("");
+            $this->output->writeln(sprintf("<info> ----------------- Compute coll_id_from to working table ----------------- %s</info>",
+                    $this->dry ? " -- NOT PLAYED IN DRY MODE --" : ""
+                )
+            );
+            $this->output->writeln("");
+            if ($this->show_sql) {
+                $this->output->writeln($sql);
+            }
+
+            $stmt = $databox->get_connection()->prepare($sql);
+        }
+
+        if(!$this->dry) {
+            $stmt->execute();
+            $ret = $stmt->rowCount();
+            $stmt->closeCursor();
+        }
+
+        return $ret;
+    }
+
+    private function copyReults(\databox $databox)
+    {
+        $this->playSQL(
+            $databox,
+            [
+                'msg' => "Copy result back to \"log_docs\"",
+                'sql' => "UPDATE `tmp_colls` AS `t` INNER JOIN `log_docs` AS `d`\n"
+                    . " ON ISNULL(`d`.`coll_id`)\n"
+                    . "    AND `t`.`record_id` = `d`.`record_id`\n"
+                    . "    AND `d`.`id` >= `t`.`from_id`\n"
+                    . "    AND (`d`.`id` < `t`.`to_id` OR ISNULL(`t`.`to_id`))\n"
+                    . " SET `d`.`coll_id_from` = IF(`action`='collection', `t`.`coll_id_from`, NULL),\n"
+                    . "     `d`.`coll_id` = `t`.`coll_id`",
+                'code' => null,
+                'playdry' => self::PLAYDRY_NONE,
+            ]
+        );
+
+        $this->playSQL(
+            $databox,
+            [
+                'msg' => "Copy result back to \"log_view\"",
+                'sql' => "UPDATE `log_view` AS `v` INNER JOIN `tmp_colls` AS `t`\n"
+                    . " ON ISNULL(`v`.`coll_id`)\n"
+                    . "    AND `t`.`record_id` = `v`.`record_id`\n"
+                    . "    AND `v`.`date` >= `t`.`from_date`\n"
+                    . "    AND (`v`.`date` < `t`.`to_date` OR ISNULL(`t`.`to_date`))\n"
+                    . " SET `v`.`coll_id` = `t`.`coll_id`",
+                'code' => null,
+                'playdry' => self::PLAYDRY_NONE,
+            ]
+        );
+
+    }
+
+
+    private function playSQL(\databox $databox, Array $work)
+    {
+        $this->output->writeln("");
+        $this->output->writeln(sprintf("<info> ----------------- %s ----------------- %s</info>",
+                $work['msg'],
+                $this->dry && !($work['playdry'] & self::PLAYDRY_SQL) ? " -- NOT PLAYED IN DRY MODE --" : ""
+            )
+        );
+        $this->output->writeln("");
+
+        if ($this->show_sql) {
+            $this->output->writeln($work['sql']);
+        }
+        $stmt = null;
+        if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) {
+            $stmt = $databox->get_connection()->prepare($work['sql']);
+            $stmt->execute();
+        }
+        if($work['code'] && (!$this->dry || ($work['playdry'] & self::PLAYDRY_CODE))) {
+            $code = $work['code'];
+            $code($stmt);
+        }
+        if(!$this->dry || ($work['playdry'] & self::PLAYDRY_SQL)) {
+            $stmt->closeCursor();
+        }
+    }
+}
+

lib/Alchemy/Phrasea/Controller/Admin/CollectionController.php

@@ -50,6 +50,7 @@ class CollectionController extends Controller
             $query = $this->createUserQuery();
             $admins = $query->on_base_ids([$bas_id])
                 ->who_have_right([\ACL::ORDER_MASTER])
+                ->include_templates(true)
                 ->execute()
                 ->get_results();
         }

lib/Alchemy/Phrasea/Controller/Admin/DataboxesController.php

@@ -160,6 +160,9 @@ class DataboxesController extends Controller
 
         try {
             $connectionSettings = $this->buildSettingsFromRequest($request);
+
+            \phrasea::clear_sbas_params($this->app);
+
             $databox = $databoxService->mountDatabox($dbName, $this->app->getAuthenticatedUser(), $connectionSettings);
 
             return $this->app->redirectPath('admin_database', [
@@ -174,7 +177,8 @@ class DataboxesController extends Controller
         catch (\Exception $exception) {
             return $this->app->redirectPath('admin_databases', [
                 'success' => 0,
-                'error' => 'mount-failed'
+                // 'error' => 'mount-failed'
+                'error' => $exception->getMessage()
             ]);
         }
     }

lib/Alchemy/Phrasea/Controller/Admin/FieldsController.php

@@ -314,6 +314,9 @@ class FieldsController extends Controller
             ->set_readonly($data['readonly'])
             ->set_type($data['type'])
             ->set_tbranch($data['tbranch'])
+            ->set_generate_cterms($data['generate_cterms'])
+            ->set_gui_editable($data['gui_editable'])
+            ->set_gui_visible($data['gui_visible'])
             ->set_report($data['report'])
             ->setVocabularyControl(null)
             ->setVocabularyRestricted(false);
@@ -349,7 +352,7 @@ class FieldsController extends Controller
     {
         return [
             'name', 'multi', 'thumbtitle', 'tag', 'business', 'indexable', 'aggregable',
-            'required', 'separator', 'readonly', 'type', 'tbranch', 'report',
+            'required', 'separator', 'readonly', 'gui_editable', 'gui_visible' , 'type', 'tbranch', 'generate_cterms', 'report',
             'vocabulary-type', 'vocabulary-restricted', 'dces-element', 'labels'
         ];
     }

lib/Alchemy/Phrasea/Controller/Admin/SubdefsController.php

@@ -145,6 +145,7 @@ class SubdefsController extends Controller
                         $options[Audio::OPTION_AUDIOBITRATE] = $config["audio"]["definitions"][$preset][Audio::OPTION_AUDIOBITRATE];
                         $options[Audio::OPTION_AUDIOSAMPLERATE] = $config["audio"]["definitions"][$preset][Audio::OPTION_AUDIOSAMPLERATE];
                         $options[Audio::OPTION_ACODEC] = $config["audio"]["definitions"][$preset][Audio::OPTION_ACODEC];
+                        $options[Audio::OPTION_AUDIOCHANNEL] = $config["audio"]["definitions"][$preset][Audio::OPTION_AUDIOCHANNEL];
                         foreach ($config["audio"]["definitions"][$preset][Subdef::OPTION_DEVICE] as $devices) {
                             $options[Subdef::OPTION_DEVICE][] = $devices;
                         }
@@ -211,7 +212,7 @@ class SubdefsController extends Controller
     {
         $mapping = [
             Type::TYPE_IMAGE    => [Subdef::TYPE_IMAGE, Subdef::TYPE_PDF],
-            Type::TYPE_VIDEO    => [Subdef::TYPE_IMAGE, Subdef::TYPE_VIDEO, Subdef::TYPE_ANIMATION],
+            Type::TYPE_VIDEO    => [Subdef::TYPE_IMAGE, Subdef::TYPE_VIDEO, Subdef::TYPE_ANIMATION, Subdef::TYPE_AUDIO],
             Type::TYPE_AUDIO    => [Subdef::TYPE_IMAGE, Subdef::TYPE_AUDIO],
             Type::TYPE_DOCUMENT => [Subdef::TYPE_IMAGE, Subdef::TYPE_FLEXPAPER, Subdef::TYPE_PDF],
             Type::TYPE_FLASH    => [Subdef::TYPE_IMAGE]
@@ -525,29 +526,44 @@ class SubdefsController extends Controller
             ],
             Subdef::TYPE_AUDIO     => [
                 "definitions" => [
-                    "Low AAC 96 kbit/s"     => [
+                    "Low MP3 96 kbit/s"     => [
                         Audio::OPTION_AUDIOBITRATE    => "100",
                         Audio::OPTION_AUDIOSAMPLERATE => "8000",
                         Audio::OPTION_ACODEC          => "libmp3lame",
                         Subdef::OPTION_DEVICE         => ["all"]
                     ],
-                    "Normal AAC 128 kbit/s" => [
+                    "Normal MP3 128 kbit/s" => [
                         Audio::OPTION_AUDIOBITRATE    => "180",
                         Audio::OPTION_AUDIOSAMPLERATE => "44100",
                         Audio::OPTION_ACODEC          => "libmp3lame",
                         Subdef::OPTION_DEVICE         => ["all"]
                     ],
-                    "High AAC 320 kbit/s"   => [
+                    "High MP3 320 kbit/s"   => [
                         Audio::OPTION_AUDIOBITRATE    => "230",
                         Audio::OPTION_AUDIOSAMPLERATE => "50000",
                         Audio::OPTION_ACODEC          => "libmp3lame",
                         Subdef::OPTION_DEVICE         => ["all"]
                     ],
+                    "Wave Mono 16 kHz"   => [
+                        Audio::OPTION_AUDIOBITRATE    => "256",
+                        Audio::OPTION_AUDIOSAMPLERATE => "16000",
+                        Audio::OPTION_ACODEC          => "pcm_s16le",
+                        Audio::OPTION_AUDIOCHANNEL    => "mono",
+                        Subdef::OPTION_DEVICE         => ["all"]
+                    ],
+                    "Wave Mono 8 kHz"   => [
+                        Audio::OPTION_AUDIOBITRATE    => "128",
+                        Audio::OPTION_AUDIOSAMPLERATE => "8000",
+                        Audio::OPTION_ACODEC          => "pcm_s16le",
+                        Audio::OPTION_AUDIOCHANNEL    => "mono",
+                        Subdef::OPTION_DEVICE         => ["all"]
+                    ],
                 ],
                 "form"        => [
                     Audio::OPTION_AUDIOBITRATE    => "slide",
                     Audio::OPTION_AUDIOSAMPLERATE => "select",
                     Audio::OPTION_ACODEC          => "select",
+                    Audio::OPTION_AUDIOCHANNEL    => "select",
                     Subdef::OPTION_DEVICE         => "checkbox",
                 ],
             ],

lib/Alchemy/Phrasea/Controller/Admin/UserController.php

@@ -233,6 +233,7 @@ class UserController extends Controller
             ->who_have_right($have_right)
             ->who_have_not_right($have_not_right)
             ->on_base_ids($on_base)
+            ->include_templates(true)
             ->execute()
             ->get_results();
 

lib/Alchemy/Phrasea/Controller/Api/V1Controller.php

@@ -89,9 +89,10 @@ use Alchemy\Phrasea\Status\StatusStructure;
 use Alchemy\Phrasea\TaskManager\LiveInformation;
 use Alchemy\Phrasea\Utilities\NullableDateTime;
 use Doctrine\ORM\EntityManager;
-use JMS\TranslationBundle\Annotation\Ignore;
+use Guzzle\Http\Client as Guzzle;
 use League\Fractal\Resource\Item;
 use media_subdef;
+use Neutron\TemporaryFilesystem\TemporaryFilesystemInterface;
 use Symfony\Component\Form\Form;
 use Symfony\Component\HttpFoundation\File\UploadedFile;
 use Symfony\Component\HttpFoundation\Request;
@@ -594,6 +595,9 @@ class V1Controller extends Controller
                 ],
                 'separator'        => $databox_field->get_separator(),
                 'thesaurus_branch' => $databox_field->get_tbranch(),
+                'generate_cterms'  => $databox_field->get_generate_cterms(),
+                'gui_editable'     => $databox_field->get_gui_editable(),
+                'gui_visible'      => $databox_field->get_gui_visible(),
                 'type'             => $databox_field->get_type(),
                 'indexable'        => $databox_field->is_indexable(),
                 'multivalue'       => $databox_field->is_multi(),
@@ -906,19 +910,6 @@ class V1Controller extends Controller
 
     public function addRecordAction(Request $request)
     {
-        if (count($request->files->get('file')) == 0) {
-            return $this->getBadRequestAction($request, 'Missing file parameter');
-        }
-
-        $file = $request->files->get('file');
-        if (!$file instanceof UploadedFile) {
-            return $this->getBadRequestAction($request, 'You can upload one file at time');
-        }
-
-        if (!$file->isValid()) {
-            return $this->getBadRequestAction($request, 'Data corrupted, please try again');
-        }
-
         if (!$request->get('base_id')) {
             return $this->getBadRequestAction($request, 'Missing base_id parameter');
         }
@@ -931,16 +922,57 @@ class V1Controller extends Controller
             ))->createResponse();
         }
 
-        // Add file extension
-        $uploadedFilename = $file->getRealPath();
+        if (count($request->files->get('file')) == 0) {
+            if(count($request->get('url')) == 0) {
+                return $this->getBadRequestAction($request, 'Missing file parameter');
+            }
+            else {
+                // upload via url
+                $url = $request->get('url');
+                $pi = pathinfo($url);   // filename, extension
 
-        $renamedFilename = $file->getRealPath() . '.' . pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION);
+                /** @var TemporaryFilesystemInterface $tmpFs */
+                $tmpFs = $this->app['temporary-filesystem'];
+                $tempfile = $tmpFs->createTemporaryFile('download_', null, $pi['extension']);
 
-        $this->getFilesystem()->rename($uploadedFilename, $renamedFilename);
+                try {
+                    $guzzle = new Guzzle($url);
+                    $res = $guzzle->get("", [], ['save_to' => $tempfile])->send();
+                }
+                catch (\Exception $e) {
+                    return $this->getBadRequestAction($request, sprintf('Error "%s" downloading "%s"', $e->getMessage(), $url));
+                }
 
-        $media = $this->app->getMediaFromUri($renamedFilename);
+                if($res->getStatusCode() !== 200) {
+                    return $this->getBadRequestAction($request, sprintf('Error %s downloading "%s"', $res->getStatusCode(), $url));
+                }
 
-        $Package = new File($this->app, $media, $collection, $file->getClientOriginalName());
+                $originalName = $pi['filename'] . '.' . $pi['extension'];
+                $uploadedFilename = $newPathname = $tempfile;
+            }
+        }
+        else {
+            // upload via file
+            $file = $request->files->get('file');
+            if (!$file instanceof UploadedFile) {
+                return $this->getBadRequestAction($request, 'You can upload one file at time');
+            }
+            if (!$file->isValid()) {
+                return $this->getBadRequestAction($request, 'Data corrupted, please try again');
+            }
+
+            $uploadedFilename = $file->getPathname();
+            $originalName = $file->getClientOriginalName();
+            $newPathname = $file->getPathname() . '.' . $file->getClientOriginalExtension();
+
+            if (false === rename($file->getPathname(), $newPathname)) {
+                return Result::createError($request, 403, 'Error while renaming file')->createResponse();
+            }
+        }
+
+        $media = $this->app->getMediaFromUri($newPathname);
+
+        $Package = new File($this->app, $media, $collection, $originalName);
 
         if ($request->get('status')) {
             $Package->addAttribute(new Status($this->app, $request->get('status')));
@@ -985,6 +1017,11 @@ class V1Controller extends Controller
         $nosubdef = $request->get('nosubdefs') === '' || \p4field::isyes($request->get('nosubdefs'));
         $this->getBorderManager()->process($session, $Package, $callback, $behavior, $nosubdef);
 
+        // remove $newPathname on temporary directory
+        if ($newPathname !== $uploadedFilename) {
+            @rename($newPathname, $uploadedFilename);
+        }
+
         $ret = ['entity' => null];
 
         if ($output instanceof \record_adapter) {
@@ -1056,6 +1093,11 @@ class V1Controller extends Controller
             }
         }
 
+        // remove $newPathname on temporary directory
+        if ($renamedFilename !== $uploadedFilename) {
+            @rename($renamedFilename, $uploadedFilename);
+        }
+
         return Result::create($request, $ret)->createResponse();
     }
 
@@ -1966,7 +2008,7 @@ class V1Controller extends Controller
                 return $this->getBadRequestAction($request);
             }
 
-            $datas = substr($datas, 0, ($n)) . $value . substr($datas, ($n + 2));
+            $datas = substr($datas, 0, ($n)) . $value . substr($datas, ($n + 1));
         }
 
         $record->setStatus(strrev($datas));
@@ -2570,8 +2612,18 @@ class V1Controller extends Controller
         foreach ($recordsData as $data) {
             $records[] = $this->addOrDelStoryRecord($story, $data, $action);
             if($action === 'ADD' && !$cover_set && isset($data->{'use_as_cover'}) && $data->{'use_as_cover'} === true) {
+                $coverSource = [];
+
+                if (isset($data->{'thumbnail_cover_source'})) {
+                    $coverSource['thumbnail_cover_source'] = $data->{'thumbnail_cover_source'};
+                }
+
+                if (isset($data->{'preview_cover_source'})) {
+                    $coverSource['preview_cover_source'] = $data->{'preview_cover_source'};
+                }
+
                 // because we can try many records as cover source, we let it fail
-                $cover_set = ($this->setStoryCover($story, $data->{'record_id'}, true) !== false);
+                $cover_set = ($this->setStoryCover($story, $data->{'record_id'}, true, $coverSource) !== false);
             }
         }
 
@@ -2625,14 +2677,26 @@ class V1Controller extends Controller
 
         $story = new \record_adapter($this->app, $databox_id, $story_id);
 
+        $coverSource = [];
+
+        if (isset($data->{'thumbnail_cover_source'})) {
+            $coverSource['thumbnail_cover_source'] = $data->{'thumbnail_cover_source'};
+        }
+
+        if (isset($data->{'preview_cover_source'})) {
+            $coverSource['preview_cover_source'] = $data->{'preview_cover_source'};
+        }
+
         // we do NOT let "setStoryCover()" fail : pass false as last arg
-        $record_key = $this->setStoryCover($story, $data->{'record_id'}, false);
+        $record_key = $this->setStoryCover($story, $data->{'record_id'}, false, $coverSource);
 
         return Result::create($request, array($record_key))->createResponse();
     }
 
-    protected function setStoryCover(\record_adapter $story, $record_id, $can_fail=false)
+    protected function setStoryCover(\record_adapter $story, $record_id, $can_fail=false, $coverSource = [])
     {
+        $coverSource = array_merge(['thumbnail_cover_source' => 'thumbnail', 'preview_cover_source' => 'preview'], $coverSource);
+
         try {
             $record = new \record_adapter($this->app, $story->getDataboxId(), $record_id);
         } catch (\Exception_Record_AdapterNotFound $e) {
@@ -2644,18 +2708,22 @@ class V1Controller extends Controller
             $this->app->abort(404, sprintf('Record identified by databox_id %s and record_id %s is not in the story', $story->getDataboxId(), $record_id));
         }
 
-        if ($record->getType() !== 'image' && $record->getType() !== 'video') {
-            // this can fail so we can loop on many records during story creation...
-            if($can_fail) {
-                return false;
-            }
-            $this->app->abort(403, sprintf('Record identified by databox_id %s and record_id %s is not an image nor a video', $story->getDataboxId(), $record_id));
-        }
+        // taking account all record type as a cover
+//        if ($record->getType() !== 'image' && $record->getType() !== 'video') {
+//            // this can fail so we can loop on many records during story creation...
+//            if($can_fail) {
+//                return false;
+//            }
+//            $this->app->abort(403, sprintf('Record identified by databox_id %s and record_id %s is not an image nor a video', $story->getDataboxId(), $record_id));
+//        }
 
         foreach ($record->get_subdefs() as $name => $value) {
-            if (!in_array($name, array('thumbnail', 'preview'))) {
+            if (!($key = array_search($name, $coverSource))) {
                 continue;
             }
+
+            $name = ($key == 'thumbnail_cover_source') ? 'thumbnail': 'preview';
+
             $media = $this->app->getMediaFromUri($value->getRealPath());
             $this->getSubdefSubstituer()->substituteSubdef($story, $name, $media);  // name = thumbnail | preview
             $this->getDataboxLogger($story->getDatabox())->log(

lib/Alchemy/Phrasea/Controller/PermalinkController.php

@@ -74,10 +74,10 @@ class PermalinkController extends AbstractDelivery
 
     public function deliverPermaview(Request $request, $sbas_id, $record_id, $subdef)
     {
-        return $this->doDeliverPermaview($sbas_id, $record_id, $request->query->get('token'), $subdef);
+        return $this->doDeliverPermaview($sbas_id, $record_id, $request->query->get('token'), $subdef, $request->query->get('t'));
     }
 
-    private function doDeliverPermaview($sbas_id, $record_id, $token, $subdefName)
+    private function doDeliverPermaview($sbas_id, $record_id, $token, $subdefName, $currentTime = null)
     {
         $databox = $this->findDataboxById($sbas_id);
         $record = $this->retrieveRecord($databox, $token, $record_id, $subdefName);
@@ -105,6 +105,7 @@ class PermalinkController extends AbstractDelivery
             'token'       => $token,
             'record'      => $record,
             'recordUrl'   => $information->getUrl(),
+            'currentTime' => $currentTime
         ]);
     }
 

lib/Alchemy/Phrasea/Controller/Prod/EditController.php

@@ -75,6 +75,9 @@ class EditController extends Controller
                     'format'               => '',
                     'explain'              => '',
                     'tbranch'              => $meta->get_tbranch(),
+                    'generate_cterms'       => $meta->get_generate_cterms(),
+                    'gui_editable'         => $meta->get_gui_editable(),
+                    'gui_visible'         => $meta->get_gui_visible(),
                     'maxLength'            => $meta->get_tag()
                         ->getMaxLength(),
                     'minLength'            => $meta->get_tag()

lib/Alchemy/Phrasea/Controller/Prod/LazaretController.php

@@ -38,7 +38,7 @@ class LazaretController extends Controller
      *
      * @param Request     $request The current request
      *
-     * @return Response
+     * @return String
      */
     public function listElement(Request $request)
     {
@@ -70,7 +70,7 @@ class LazaretController extends Controller
     public function getElement($file_id)
     {
         $ret = ['success' => false, 'message' => '', 'result'  => []];
-        
+
         /* @var LazaretFile $lazaretFile */
         $lazaretFile = $this->getLazaretFileRepository()->find($file_id);
 
@@ -126,6 +126,16 @@ class LazaretController extends Controller
 
         $ret = $lazaretManipulator->add($file_id, $keepAttributes, $attributesToKeep);
 
+        try{
+            // get the new record
+            $record = \Collection::getByBaseId($this->app, $request->request->get('bas_id'))->get_databox()->get_record($ret['result']['record_id']);
+            $postStatus = (array) $request->request->get('status');
+            // update status
+            $this->updateRecordStatus($record, $postStatus);
+        }catch(\Exception $e){
+            $ret['message'] = $this->app->trans('An error occured when wanting to change status!');
+        }
+
         return $this->app->json($ret);
     }
 
@@ -216,6 +226,7 @@ class LazaretController extends Controller
 
             return $this->app->json($ret);
         }
+        $postStatus = (array) $request->request->get('status');
 
         $path = $this->app['tmp.lazaret.path'] . '/';
         $lazaretFileName = $path .$lazaretFile->getFilename();
@@ -233,6 +244,9 @@ class LazaretController extends Controller
                 ''
             );
 
+            // update status
+            $this->updateRecordStatus($record, $postStatus);
+
             //Delete lazaret file
             $manager = $this->getEntityManager();
             $manager->remove($lazaretFile);
@@ -278,6 +292,35 @@ class LazaretController extends Controller
         );
     }
 
+    /**
+     * @param Request $request
+     * @param $databox_id
+     * @param $record_id
+     * @return \Symfony\Component\HttpFoundation\JsonResponse
+     */
+    public function getDestinationStatus(Request $request, $databox_id, $record_id)
+    {
+        if (!$request->isXmlHttpRequest()) {
+            $this->app->abort(400);
+        }
+        $record = new \record_adapter($this->app, (int) $databox_id, (int) $record_id);
+        $databox = $this->findDataboxById($databox_id);
+        $statusStructure = $databox->getStatusStructure();
+        $recordsStatuses = [];
+        foreach ($statusStructure as $status) {
+            //  make the key as a string for the json usage in javascript
+            $bit = "'".$status['bit']."'";
+            if (!isset($recordsStatuses[$bit])) {
+                $recordsStatuses[$bit] = $status;
+            }
+            $statusSet = \databox_status::bitIsSet($record->getStatusBitField(), $status['bit']);
+            if (!isset($recordsStatuses[$bit]['flag'])) {
+                $recordsStatuses[$bit]['flag'] = (int) $statusSet;
+            }
+        }
+        return $this->app->json(['status' => $recordsStatuses]);
+    }
+
     /**
      * @return LazaretFileRepository
      */
@@ -293,4 +336,32 @@ class LazaretController extends Controller
     {
         return $this->app['border-manager'];
     }
+
+    /**
+     * Set new status to selected record
+     *
+     * @param  \record_adapter $record
+     * @param  array           $postStatus
+     * @return array|null
+     */
+    private function updateRecordStatus(\record_adapter $record, array $postStatus)
+    {
+        $sbasId = $record->getDataboxId();
+        if (isset($postStatus[$sbasId]) && is_array($postStatus[$sbasId])) {
+            $postStatus = $postStatus[$sbasId];
+            $currentStatus = strrev($record->getStatus());
+            $newStatus = '';
+            foreach (range(0, 31) as $i) {
+                $newStatus .= isset($postStatus[$i]) ? ($postStatus[$i] ? '1' : '0') : $currentStatus[$i];
+            }
+            $record->setStatus(strrev($newStatus));
+            $this->getDataboxLogger($record->getDatabox())
+                ->log($record, \Session_Logger::EVENT_STATUS, '', '');
+            return [
+                'current_status' => $currentStatus,
+                'new_status'     => $newStatus,
+            ];
+        }
+        return null;
+    }
 }

lib/Alchemy/Phrasea/Controller/Prod/MoveCollectionController.php

@@ -114,6 +114,7 @@ class MoveCollectionController extends Controller
             $trashCollectionsBySbasId = [];
 
             foreach ($records as $record) {
+                $oldCollectionId = $record->getCollection()->get_coll_id();
                 $record->move_to_collection($collection, $this->getApplicationBox());
 
                 if ($request->request->get("chg_coll_son") == "1") {
@@ -130,7 +131,7 @@ class MoveCollectionController extends Controller
                     $trashCollectionsBySbasId[$sbasId] = $record->getDatabox()->getTrashCollection();
                 }
                 if ($trashCollectionsBySbasId[$sbasId] !== null) {
-                    if ($record->getCollection()->get_coll_id() == $trashCollectionsBySbasId[$sbasId]->get_coll_id() && $collection->get_coll_id() !== $trashCollectionsBySbasId[$sbasId]->get_coll_id()) {
+                    if ($oldCollectionId == $trashCollectionsBySbasId[$sbasId]->get_coll_id() && $collection->get_coll_id() !== $trashCollectionsBySbasId[$sbasId]->get_coll_id()) {
                         // record is already in trash so active it
                         foreach ($record->get_subdefs() as $subdef) {
                             if (($pl = $subdef->get_permalink())) {

lib/Alchemy/Phrasea/Controller/Prod/PropertyController.php

@@ -9,6 +9,7 @@
  */
 namespace Alchemy\Phrasea\Controller\Prod;
 
+use Alchemy\Phrasea\Application\Helper\DataboxLoggerAware;
 use Alchemy\Phrasea\Controller\Controller;
 use Alchemy\Phrasea\Controller\RecordsRequest;
 use Symfony\Component\HttpFoundation\Request;
@@ -16,6 +17,8 @@ use Symfony\Component\HttpFoundation\Response;
 
 class PropertyController extends Controller
 {
+    use DataboxLoggerAware;
+
     /**
      *  Display Status property
      *
@@ -198,6 +201,9 @@ class PropertyController extends Controller
 
             $record->setStatus(strrev($newStatus));
 
+            $this->getDataboxLogger($record->getDatabox())
+                ->log($record, \Session_Logger::EVENT_STATUS, '', '');
+
             return [
                 'current_status' => $currentStatus,
                 'new_status'     => $newStatus,

lib/Alchemy/Phrasea/Controller/Prod/PushController.php

@@ -463,6 +463,8 @@ class PushController extends Controller
         }
 
         try {
+            $manager = $this->getEntityManager();
+
             $password = $this->getRandomGenerator()->generateString(128);
 
             $user = $this->getUserManipulator()->createUser($email, $password, $email);
@@ -476,12 +478,15 @@ class PushController extends Controller
                 $user->setCompany($request->request->get('company'));
             }
             if ($request->request->get('job')) {
-                $user->setCompany($request->request->get('job'));
+                $user->setJob($request->request->get('job'));
             }
-            if ($request->request->get('form_geonameid')) {
-                $this->getUserManipulator()->setGeonameId($user, $request->request->get('form_geonameid'));
+            if ($request->request->get('city')) {
+                $this->getUserManipulator()->setGeonameId($user, $request->request->get('city'));
             }
 
+            $manager->persist($user);
+            $manager->flush();
+
             $result['message'] = $this->app->trans('User successfully created');
             $result['success'] = true;
             $result['user'] = $this->formatUser($user);

lib/Alchemy/Phrasea/Controller/Prod/QueryController.php

@@ -161,7 +161,32 @@ class QueryController extends Controller
             $result = $engine->query($query, $options);
 
             if ($this->getSettings()->getUserSetting($user, 'start_page') === 'LAST_QUERY') {
-                $userManipulator->setUserSetting($user, 'start_page_query', $query);
+                // try to save the "fulltext" query which will be restored on next session
+                try {
+                    // local code to find "FULLTEXT" value from jsonQuery
+                    $findFulltext = function($clause) use(&$findFulltext) {
+                        if(array_key_exists('_ux_zone', $clause) && $clause['_ux_zone']=='FULLTEXT') {
+                            return $clause['value'];
+                        }
+                        if($clause['type']=='CLAUSES') {
+                            foreach($clause['clauses'] as $c) {
+                                if(($r = $findFulltext($c)) !== null) {
+                                    return $r;
+                                }
+                            }
+                        }
+                        return null;
+                    };
+
+                    $userManipulator->setUserSetting($user, 'last_jsonquery', (string)$request->request->get('jsQuery'));
+                    $jsQuery = @json_decode((string)$request->request->get('jsQuery'), true);
+                    if(($ft = $findFulltext($jsQuery['query'])) !== null) {
+                        $userManipulator->setUserSetting($user, 'start_page_query', $ft);
+                    }
+                }
+                catch(\Exception $e) {
+                    // no-op
+                }
             }
 
             // log array of collectionIds (from $options) for each databox
@@ -189,7 +214,7 @@ class QueryController extends Controller
                 if (min($d2top, $d2bottom) < 4) {
                     if ($d2bottom < 4) {
                         if($page != 1){
-                            $string .= "<a id='PREV_PAGE' class='btn btn-primary btn-mini'></a>";
+                            $string .= "<a id='PREV_PAGE' class='btn btn-primary btn-mini icon-baseline-chevron_left-24px'></a>";
                         }
                         for ($i = 1; ($i <= 4 && (($i <= $npages) === true)); $i++) {
                             if ($i == $page)
@@ -198,13 +223,13 @@ class QueryController extends Controller
                                 $string .= '<a class="btn btn-primary btn-mini search-navigate-action" data-page="'.$i.'">' . $i . '</a>';
                         }
                         if ($npages > 4)
-                            $string .= "<a id='NEXT_PAGE' class='btn btn-primary btn-mini'></a>";
-                        $string .= '<a href="#" class="btn btn-primary btn-mini search-navigate-action" data-page="' . $npages . '" id="last"></a>';
+                            $string .= "<a id='NEXT_PAGE' class='btn btn-primary btn-mini icon icon-baseline-chevron_right-24px'></a>";
+                        $string .= '<a href="#" class="btn btn-primary btn-mini search-navigate-action icon icon-double-arrows" data-page="' . $npages . '" id="last"></a>';
                     } else {
                         $start = $npages - 4;
                         if (($start) > 0){
-                            $string .= '<a class="btn btn-primary btn-mini search-navigate-action" data-page="1" id="first"></a>';
-                            $string .= '<a id="PREV_PAGE" class="btn btn-primary btn-mini"></a>';
+                            $string .= '<a class="btn btn-primary btn-mini search-navigate-action" data-page="1" id="first"><span class="icon icon-double-arrows icon-inverse"></span></a>';
+                            $string .= '<a id="PREV_PAGE" class="btn btn-primary btn-mini icon icon-baseline-chevron_left-24px"></a>';
                         }else
                             $start = 1;
                         for ($i = ($start); $i <= $npages; $i++) {
@@ -214,11 +239,11 @@ class QueryController extends Controller
                                 $string .= '<a class="btn btn-primary btn-mini search-navigate-action" data-page="'.$i.'">' . $i . '</a>';
                         }
                         if($page < $npages){
-                            $string .= "<a id='NEXT_PAGE' class='btn btn-primary btn-mini'></a>";
+                            $string .= "<a id='NEXT_PAGE' class='btn btn-primary btn-mini icon icon-baseline-chevron_right-24px'></a>";
                         }
                     }
                 } else {
-                    $string .= '<a class="btn btn-primary btn-mini btn-mini search-navigate-action" data-page="1" id="first"></a>';
+                    $string .= '<a class="btn btn-primary btn-mini search-navigate-action" data-page="1" id="first"><span class="icon icon-double-arrows icon-inverse"></span></a>';
 
                     for ($i = ($page - 2); $i <= ($page + 2); $i++) {
                         if ($i == $page)
@@ -227,10 +252,10 @@ class QueryController extends Controller
                             $string .= '<a class="btn btn-primary btn-mini search-navigate-action" data-page="'.$i.'">' . $i . '</a>';
                     }
 
-                    $string .= '<a href="#" class="btn btn-primary btn-mini search-navigate-action" data-page="' . $npages . '" id="last"></a>';
+                    $string .= '<a href="#" class="btn btn-primary btn-mini search-navigate-action icon icon-double-arrows" data-page="' . $npages . '" id="last"></a>';
                 }
             }
-            $string .= '<div style="display:none;"><div id="NEXT_PAGE"></div><div id="PREV_PAGE"></div></div>';
+            $string .= '<div style="display:none;"><div id="NEXT_PAGE" class="icon icon-baseline-chevron_right-24px"></div><div id="PREV_PAGE" class="icon icon-baseline-chevron_left-24px"></div></div>';
 
             $explain = $this->render(
                 "prod/results/infos.html.twig",
@@ -291,7 +316,7 @@ class QueryController extends Controller
                         </tfoot>
                     </table></div></div>'
                 . '</div><a href="#" class="search-display-info" data-infos="' . str_replace('"', '&quot;', $explain) . '">'
-                . $this->app->trans('%total% reponses', ['%total%' => '<span>'.$result->getTotal().'</span>']) . '</a>';
+                . $this->app->trans('%total% reponses', ['%total%' => '<span>'.number_format($result->getTotal(),null, null, ' ').'</span>']) . '</a>';
 
             $json['infos'] = $infoResult;
             $json['navigationTpl'] = $string;
@@ -323,9 +348,14 @@ class QueryController extends Controller
 
 
             // add technical fields
-            $fieldLabels = [];
+            $fieldsInfosByName = [];
             foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
-                $fieldLabels[$k] = $this->app->trans($f['label']);
+                $fieldsInfosByName[$k] = $f;
+                $fieldsInfosByName[$k]['trans_label'] = $this->app->trans($f['label']);
+                $fieldsInfosByName[$k]['labels'] = [];
+                foreach($this->app->getAvailableLanguages() as $locale => $lng) {
+                    $fieldsInfosByName[$k]['labels'][$locale] = $this->app->trans($f['label'], [], "messages", $locale);
+                }
             }
 
             // add databox fields
@@ -337,13 +367,24 @@ class QueryController extends Controller
                 foreach ($databox->get_meta_structure() as $field) {
                     $name = $field->get_name();
                     $fieldsInfos[$sbasId][$name] = [
-                      'label' => $field->get_label($this->app['locale']),
-                      'type' => $field->get_type(),
+                      'label'    => $field->get_label($this->app['locale']),
+                      'labels'   => $field->get_labels(),
+                      'type'     => $field->get_type(),
                       'business' => $field->isBusiness(),
-                      'multi' => $field->is_multi(),
+                      'multi'    => $field->is_multi(),
                     ];
-                    if (!isset($fieldLabels[$name])) {
-                        $fieldLabels[$name] = $field->get_label($this->app['locale']);
+
+                    // infos on the "same" field (by name) on multiple databoxes !!!
+                    // label(s) can be inconsistants : the first databox wins
+                    if (!isset($fieldsInfosByName[$name])) {
+                        $fieldsInfosByName[$name] = [
+                            'label'       => $field->get_label($this->app['locale']),
+                            'labels'      => $field->get_labels(),
+                            'type'        => $field->get_type(),
+                            'field'       => $field->get_name(),
+                            'trans_label' => $field->get_label($this->app['locale']),
+                        ];
+                        $field->get_label($this->app['locale']);
                     }
                 }
             }
@@ -382,14 +423,29 @@ class QueryController extends Controller
 
             // populates facets (aggregates)
             $facets = [];
+            // $facetClauses = [];
             foreach ($result->getFacets() as $facet) {
                 $facetName = $facet['name'];
 
-                $facet['label'] = isset($fieldLabels[$facetName]) ? $fieldLabels[$facetName] : $facetName;
+                if(array_key_exists($facetName, $fieldsInfosByName)) {
 
-                $facets[] = $facet;
+                    $f = $fieldsInfosByName[$facetName];
+
+                    $facet['label'] = $f['trans_label'];
+                    $facet['labels'] = $f['labels'];
+                    $facet['type'] = strtoupper($f['type']) . "-AGGREGATE";
+                    $facets[] = $facet;
+
+                    // $facetClauses[] = [
+                    //    'type'  => strtoupper($f['type']) . "-AGGREGATE",
+                    //    'field' => $f['field'],
+                    //    'facet' => $facet
+                    // ];
+                }
             }
 
+            // $json['jsq'] = $facetClauses;
+
             $json['facets'] = $facets;
             $json['phrasea_props'] = $proposals;
             $json['total_answers'] = (int) $result->getAvailable();
@@ -413,7 +469,6 @@ class QueryController extends Controller
             $json['results'] = $this->render($template, ['results'=> $result]);
         }
 
-
         return $this->app->json($json);
     }
 

lib/Alchemy/Phrasea/Controller/Prod/RecordController.php

@@ -90,35 +90,44 @@ class RecordController extends Controller
         }
         $recordCaptions["technicalInfo"] = $record->getPositionFromTechnicalInfos();
 
+        //  escape record title before rendering
+        $recordTitle = explode("</span>", $record->get_title());
+        if (count($recordTitle) >1) {
+            $recordTitle[1] = htmlspecialchars($recordTitle[1]);
+            $recordTitle = implode("</span>", $recordTitle);
+        } else {
+            $recordTitle = htmlspecialchars($record->get_title());
+        }
+
         return $this->app->json([
-            "desc"          => $this->render('prod/preview/caption.html.twig', [
+            "desc"            => $this->render('prod/preview/caption.html.twig', [
                 'record'        => $record,
                 'highlight'     => $query,
                 'searchEngine'  => $searchEngine,
                 'searchOptions' => $options,
             ]),
-            "recordCaptions"=> $recordCaptions,
-            "html_preview"  => $this->render('common/preview.html.twig', [
+            "recordCaptions"  => $recordCaptions,
+            "html_preview"    => $this->render('common/preview.html.twig', [
                 'record'        => $record
             ]),
-            "others"        => $this->render('prod/preview/appears_in.html.twig', [
+            "others"          => $this->render('prod/preview/appears_in.html.twig', [
                 'parents'       => $record->get_grouping_parents(),
                 'baskets'       => $record->get_container_baskets($this->getEntityManager(), $this->getAuthenticatedUser()),
             ]),
-            "current"       => $train,
-            "record"        => $currentRecord,
-            "history"       => $this->render('prod/preview/short_history.html.twig', [
+            "current"         => $train,
+            "record"          => $currentRecord,
+            "history"         => $this->render('prod/preview/short_history.html.twig', [
                 'record'        => $record,
             ]),
-            "popularity"    => $this->render('prod/preview/popularity.html.twig', [
+            "popularity"      => $this->render('prod/preview/popularity.html.twig', [
                 'record'        => $record,
             ]),
-            "tools"         => $this->render('prod/preview/tools.html.twig', [
+            "tools"           => $this->render('prod/preview/tools.html.twig', [
                 'record'        => $record,
             ]),
-            "pos"           => $record->getNumber(),
-            "title"         => $record->get_title(),
-            "databox_name" => $record->getDatabox()->get_dbname(),
+            "pos"             => $record->getNumber(),
+            "title"           => $recordTitle,
+            "databox_name"    => $record->getDatabox()->get_dbname(),
             "collection_name" => $record->getCollection()->get_name(),
             "collection_logo" => $record->getCollection()->getLogo($record->getBaseId(), $this->app),
         ]);

lib/Alchemy/Phrasea/Controller/Prod/RootController.php

@@ -15,12 +15,11 @@ use Alchemy\Phrasea\Core\Configuration\DisplaySettingService;
 use Alchemy\Phrasea\Exception\SessionNotFound;
 use Alchemy\Phrasea\Feed\Aggregate;
 use Alchemy\Phrasea\Helper;
-use Alchemy\Phrasea\Model\Entities\UserSetting;
+use Alchemy\Phrasea\Helper\WorkZone as WorkzoneHelper;
 use Alchemy\Phrasea\Model\Repositories\FeedRepository;
-use Symfony\Component\Finder\Finder;
-use Symfony\Component\Finder\SplFileInfo;
 use Symfony\Component\HttpFoundation\Request;
 
+
 class RootController extends Controller
 {
     use Application\Helper\FirewallAware;
@@ -41,12 +40,11 @@ class RootController extends Controller
     public function indexAction(Request $request) {
         try {
             \Session_Logger::updateClientInfos($this->app, 1);
-        } catch (SessionNotFound $e) {
+        }
+        catch (SessionNotFound $e) {
             return $this->app->redirectPath('logout');
         }
 
-        $css = [];
-
         $user = $this->getAuthenticatedUser();
         $cssfile = $this->getSettings()->getUserSetting($user, 'css');
 
@@ -85,6 +83,22 @@ class RootController extends Controller
         /** @var \Closure $filter */
         $filter = $this->app['plugin.filter_by_authorization'];
 
+        /* prepare work to extend whole taskbar... later
+        $menus = [
+            'push' => ['native'=>true, 'n'=>0],
+            'tools' => ['native'=>true, 'n'=>0],
+        ];
+        / ** @var ActionBarPluginInterface $plugin * /
+        foreach($filter('actionbar') as $kplugin=>$plugin) {
+            foreach($plugin->getActionBar() as $kmenu=>$menu) {
+                if(!array_key_exists($kmenu, $menus)) {
+                    $menus[$kmenu] = ['native'=>false, 'n'=>0];
+                }
+                $menus[$kmenu]['n']++;
+            }
+        }
+        */
+
         $plugins = [
             'workzone' => $filter('workzone'),
             'actionbar' => $filter('actionbar'),
@@ -92,7 +106,7 @@ class RootController extends Controller
 
         return $this->render('prod/index.html.twig', [
             'module_name'          => 'Production',
-            'WorkZone'             => new Helper\WorkZone($this->app, $request),
+            'WorkZone'             => new WorkzoneHelper($this->app, $request),
             'module_prod'          => $helper,
             'search_datas'         => $helper->get_search_datas(),
             'cssfile'              => $cssfile,
@@ -105,7 +119,7 @@ class RootController extends Controller
             'feeds'                => $feeds,
             'aggregate'            => $aggregate,
             'GV_google_api'        => $conf->get(['registry', 'webservices', 'google-charts-enabled']),
-            'geocodingProviders'    => $conf->get(['geocoding-providers']),
+            'geocodingProviders'   => $conf->get(['geocoding-providers']),
             'search_status'        => \databox_status::getSearchStatus($this->app),
             'thesau_js_list'       => $thjslist,
             'thesau_json_sbas'     => json_encode($sbas),

lib/Alchemy/Phrasea/Controller/Prod/UploadController.php

@@ -26,6 +26,7 @@ use Alchemy\Phrasea\Model\Entities\LazaretFile;
 use Alchemy\Phrasea\Model\Entities\LazaretSession;
 use DataURI\Exception\Exception as DataUriException;
 use DataURI\Parser;
+use Guzzle\Http\Client as Guzzle;
 use Symfony\Component\HttpFoundation\File\UploadedFile;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
@@ -76,6 +77,30 @@ class UploadController extends Controller
         ]);
     }
 
+    public function getHead(Request $request)
+    {
+        $response = [
+            'content-type' => null,
+            'content-length' => null,
+            'basename' => null
+        ];
+        try {
+            $url = $request->get('url');
+            $basename = pathinfo($url, PATHINFO_BASENAME);
+
+            $guzzle = new Guzzle($url);
+            $res = $guzzle->head("")->send();
+            $response['content-type'] = $res->getContentType();
+            $response['content-length'] = $res->getContentLength();
+            $response['basename'] = $basename;
+        }
+        catch (\Exception $e) {
+            // no-op : head will return no info but will not crash
+        }
+
+        return $this->app->json($response);
+    }
+
     /**
      * Upload processus
      *
@@ -98,7 +123,7 @@ class UploadController extends Controller
             'message' => '',
             'element' => '',
             'reasons' => [],
-            'id' => '',
+            'id'      => '',
         ];
 
         if (null === $request->files->get('files')) {
@@ -119,18 +144,44 @@ class UploadController extends Controller
             throw new AccessDeniedHttpException('User is not allowed to add record on this collection');
         }
 
+        /** @var UploadedFile $file */
         $file = current($request->files->get('files'));
 
         if (!$file->isValid()) {
             throw new BadRequestHttpException('Uploaded file is invalid');
         }
 
-        try {
+        if ($file->getClientOriginalName() === "blob" && $file->getClientMimeType() === "application/json") {
+
+            // a "upload by url" was done, we receive a tiny json that contains url.
+            $json = json_decode(file_get_contents($file->getRealPath()), true);
+            $url = $json['url'];
+            $pi = pathinfo($url);   // filename, extension
+
+            $tempfile = $this->getTemporaryFilesystem()->createTemporaryFile('download_', null, $pi['extension']);
+
+            try {
+                $guzzle = new Guzzle($url);
+                $res = $guzzle->get("", [], ['save_to' => $tempfile])->send();
+            }
+            catch (\Exception $e) {
+                throw new BadRequestHttpException(sprintf('Error "%s" downloading "%s"', $e->getMessage(), $url));
+            }
+
+            if($res->getStatusCode() !== 200) {
+                throw new BadRequestHttpException(sprintf('Error %s downloading "%s"', $res->getStatusCode(), $url));
+            }
+
+            $uploadedFilename = $renamedFilename = $tempfile;
+
+            $originalName = $pi['filename'] . '.' . $pi['extension'];
+
+        } else {
             // Add file extension, so mediavorus can guess file type for octet-stream file
             $uploadedFilename = $file->getRealPath();
             $renamedFilename = null;
 
-            if(!empty($this->app['conf']->get(['main', 'storage', 'tmp_files']))){
+            if(!empty($this->app['conf']->get(['main', 'storage', 'tmp_files']))) {
                 $tmpStorage = \p4string::addEndSlash($this->app['conf']->get(['main', 'storage', 'tmp_files'])).'upload/';
 
                 if(!is_dir($tmpStorage)){
@@ -139,12 +190,16 @@ class UploadController extends Controller
 
                 $renamedFilename = $tmpStorage. pathinfo($file->getRealPath(), PATHINFO_FILENAME) .'.' . pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION);
 
-            }else{
+            } else {
                 $renamedFilename = $file->getRealPath() . '.' . pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION);
             }
 
             $this->getFilesystem()->rename($uploadedFilename, $renamedFilename);
 
+            $originalName = $file->getClientOriginalName();
+        }
+
+        try {
             $media = $this->app->getMediaFromUri($renamedFilename);
             $collection = \collection::getByBaseId($this->app, $base_id);
 
@@ -153,7 +208,7 @@ class UploadController extends Controller
 
             $this->getEntityManager()->persist($lazaretSession);
 
-            $packageFile = new File($this->app, $media, $collection, $file->getClientOriginalName());
+            $packageFile = new File($this->app, $media, $collection, $originalName);
 
             $postStatus = $request->request->get('status');
 
@@ -184,7 +239,9 @@ class UploadController extends Controller
 
             $code = $this->getBorderManager()->process( $lazaretSession, $packageFile, $callback, $forceBehavior);
 
-            $this->getFilesystem()->rename($renamedFilename, $uploadedFilename);
+            if($renamedFilename !== $uploadedFilename) {
+                $this->getFilesystem()->rename($renamedFilename, $uploadedFilename);
+            }
 
             if (!!$forceBehavior) {
                 $reasons = [];
@@ -310,15 +367,17 @@ class UploadController extends Controller
             $postMaxSize = PHP_INT_MAX;
         }
 
+        $r = 0;
         switch (strtolower(substr($postMaxSize, -1))) {
             /** @noinspection PhpMissingBreakStatementInspection */
             case 'g':
-                $postMaxSize *= 1024;
+                $r += 10;
             /** @noinspection PhpMissingBreakStatementInspection */
             case 'm':
-                $postMaxSize *= 1024;
+                $r += 10;
             case 'k':
-                $postMaxSize *= 1024;
+                $r += 10;
+                $postMaxSize = ((int)($postMaxSize))<<$r;
         }
 
         return min(UploadedFile::getMaxFilesize(), (int) $postMaxSize);

lib/Alchemy/Phrasea/Controller/Report/ActivityController.php

@@ -1,802 +0,0 @@
-<?php
-/*
- * This file is part of Phraseanet
- *
- * (c) 2005-2016 Alchemy
- *
- * For the full copyright and license information, please view the LICENSE
- * file that was distributed with this source code.
- */
-namespace Alchemy\Phrasea\Controller\Report;
-
-use Alchemy\Phrasea\Controller\Controller;
-use Alchemy\Phrasea\Core\Response\CSVFileResponse;
-use Goodby\CSV\Export\Standard\Collection\CallbackCollection;
-use Goodby\CSV\Export\Standard\Exporter;
-use Symfony\Component\HttpFoundation\JsonResponse;
-use Symfony\Component\HttpFoundation\Request;
-use Symfony\Component\HttpFoundation\Response;
-
-class ActivityController extends Controller
-{
-    /**
-     * Display connexions report group by user
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportConnexionsByUsers(Request $request)
-    {
-        $activity = new \module_report_activity(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-        $activity->setBound("user", true);
-
-        //set Limit
-        if ($activity->getEnableLimit()
-            && ('' !== $page = $request->request->get('page', ''))
-            && ('' !== $limit = $request->request->get('limit', ''))) {
-            $activity->setLimit($page, $limit);
-        } else {
-            $activity->setLimit(false, false);
-        }
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->getConnexionBase(false, $request->request->get('on', 'user'));
-
-            return $this->getCSVResponse($activity, 'activity_connection_base');
-        }
-
-        $report = $activity->getConnexionBase(false, $request->request->get('on', 'user'));
-
-        return $this->app->json([
-            'rs' =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => false,
-            'title'       => false,
-        ]);
-    }
-
-    /**
-     * Display download report group by user
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportDownloadsByUsers(Request $request)
-    {
-        $conf = [
-            'user'      => [$this->app->trans('report:: utilisateur'), 0, 1, 0, 0],
-            'nbdoc'     => [$this->app->trans('report:: nombre de documents'), 0, 0, 0, 0],
-            'nbprev'    => [$this->app->trans('report:: nombre de preview'), 0, 0, 0, 0],
-        ];
-
-        $activity = new \module_report_activity(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-
-        //set Limit
-        if ($activity->getEnableLimit()
-            && ('' !== $page = $request->request->get('page', ''))
-            && ('' !== $limit = $request->request->get('limit', ''))) {
-            $activity->setLimit($page, $limit);
-        } else {
-            $activity->setLimit(false, false);
-        }
-
-        $report = $activity->getDetailDownload($conf, $request->request->get('on'));
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-
-            return $this->getCSVResponse($activity, 'activity_detail_download');
-        }
-
-        return $this->app->json([
-            'rs' =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => false,
-            'title'       => false,
-        ]);
-    }
-
-    /**
-     * Display the most asked question
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportBestOfQuestions(Request $request)
-    {
-        $conf = [
-            'search'    => [$this->app->trans('report:: question'), 0, 0, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0],
-            'nb_rep'    => [$this->app->trans('report:: nombre de reponses'), 0, 0, 0, 0]
-        ];
-
-        $activity = new \module_report_activity(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setLimit(1, $request->request->get('limit', 20));
-        $activity->setTop(20);
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $activity->getTopQuestion($conf);
-
-            return $this->getCSVResponse($activity, 'activity_questions_best_of');
-        }
-
-        $report = $activity->getTopQuestion($conf);
-
-        return $this->app->json([
-            'rs' =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]),
-            'display_nav' => false,
-            'title'       => false
-        ]);
-    }
-
-    /**
-     * Display report about questions that return no result
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportNoBestOfQuestions(Request $request)
-    {
-        $conf = [
-            'search'    => [$this->app->trans('report:: question'), 0, 0, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0],
-            'nb_rep'    => [$this->app->trans('report:: nombre de reponses'), 0, 0, 0, 0]
-        ];
-
-        $activity = new \module_report_activity(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        //set Limit
-        if ($activity->getEnableLimit()
-            && ('' !== $page = $request->request->get('page', ''))
-            && ('' !== $limit = $request->request->get('limit', ''))) {
-            $activity->setLimit($page, $limit);
-        } else {
-            $activity->setLimit(false, false);
-        }
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $activity->getTopQuestion($conf, true);
-
-            return $this->getCSVResponse($activity, 'activity_top_ten_questions');
-        }
-
-        $report = $activity->getTopQuestion($conf, true);
-
-        return $this->app->json([
-            'rs' =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]),
-            'display_nav' => false,
-            'title'       => false
-        ]);
-    }
-
-    /**
-     * Display an overview of connexion among hours of the da
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportSiteActiviyPerHours(Request $request)
-    {
-        $activity = new \module_report_activity(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $activity->getActivityPerHours();
-
-            return $this->getCSVResponse($activity, 'activity_per_hours');
-        }
-
-        $report = $activity->getActivityPerHours();
-
-        return $this->app->json([
-            'rs' =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => true,
-                'is_doc'      => false
-            ]),
-            'display_nav' => false,
-            'title'       => false
-        ]);
-    }
-
-    /**
-     * Display an overview of downloaded document grouped by day
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportSiteActivityPerDays(Request $request)
-    {
-        $conf = [
-            'ddate'     => [$this->app->trans('report:: jour'), 0, 0, 0, 0],
-            'total'     => [$this->app->trans('report:: total des telechargements'), 0, 0, 0, 0],
-            'preview'   => [$this->app->trans('report:: preview'), 0, 0, 0, 0],
-            'document'  => [$this->app->trans('report:: document'), 0, 0, 0, 0]
-        ];
-
-        $activity = new \module_report_activity(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        //set Limit
-        if ($activity->getEnableLimit()
-            && ('' !== $page = $request->request->get('page', ''))
-            && ('' !== $limit = $request->request->get('limit', ''))) {
-            $activity->setLimit($page, $limit);
-        } else {
-            $activity->setLimit(false, false);
-        }
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $activity->getDownloadByBaseByDay($conf);
-
-            return $this->getCSVResponse($activity, 'activity_db_by_base_by_day');
-        }
-
-        $report = $activity->getDownloadByBaseByDay($conf);
-
-        return $this->app->json([
-            'rs' =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => false,
-            'title'       => false,
-        ]);
-    }
-
-    /**
-     * Display report about pushed documents
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportPushedDocuments(Request $request)
-    {
-        $conf = [
-            'user'      => ['', 1, 0, 1, 1],
-            'getter'    => ["Destinataire", 1, 0, 1, 1],
-            'date'      => ['', 1, 0, 1, 1],
-            'record_id' => ['', 1, 1, 1, 1],
-            'file'      => ['', 1, 0, 1, 1],
-            'mime'      => ['', 1, 0, 1, 1],
-        ];
-
-        $activity = new \module_report_push(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $this->doReport($request, $activity, $conf);
-
-            return $this->getCSVResponse($activity, 'activity_pushed_documents');
-        }
-
-        $report = $this->doReport($request, $activity, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record'],
-        ]);
-    }
-
-    /**
-     * Display report about added documents
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportAddedDocuments(Request $request)
-    {
-        $conf = [
-            'user'      => ['', 1, 0, 1, 1],
-            'date'      => ['', 1, 0, 1, 1],
-            'record_id' => ['', 1, 1, 1, 1],
-            'file'      => ['', 1, 0, 1, 1],
-            'mime'      => ['', 1, 0, 1, 1],
-        ];
-
-        $activity = new \module_report_add(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $this->doReport($request, $activity, $conf);
-
-            return $this->getCSVResponse($activity, 'activity_added_documents');
-        }
-
-        $report = $this->doReport($request, $activity, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record'],
-        ]);
-    }
-
-    /**
-     * Display report about edited documents
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportEditedDocuments(Request $request)
-    {
-        $conf = [
-            'user'      => ['', 1, 0, 1, 1],
-            'date'      => ['', 1, 0, 1, 1],
-            'record_id' => ['', 1, 1, 1, 1],
-            'file'      => ['', 1, 0, 1, 1],
-            'mime'      => ['', 1, 0, 1, 1],
-        ];
-
-        $activity = new \module_report_edit(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $this->doReport($request, $activity, $conf);
-
-            return $this->getCSVResponse($activity, 'activity_edited_documents');
-        }
-
-        $report = $this->doReport($request, $activity, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record'],
-        ]);
-    }
-
-    /**
-     * Display report about validated documents
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportValidatedDocuments(Request $request)
-    {
-        $conf = [
-            'user'      => ['', 1, 0, 1, 1],
-            'getter'    => ["Destinataire", 1, 0, 1, 1],
-            'date'      => ['', 1, 0, 1, 1],
-            'record_id' => ['', 1, 1, 1, 1],
-            'file'      => ['', 1, 0, 1, 1],
-            'mime'      => ['', 1, 0, 1, 1],
-        ];
-
-        $activity = new \module_report_validate(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $this->doReport($request, $activity, $conf);
-
-            return $this->getCSVResponse($activity, 'activity_validated_documents');
-        }
-
-        $report = $this->doReport($request, $activity, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record'],
-        ]);
-    }
-
-    /**
-     * Display report about documents sent by mail
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportSentDocuments(Request $request)
-    {
-        $conf = [
-            'user'      => ['', 1, 0, 1, 1],
-            'date'      => ['', 1, 0, 1, 1],
-            'record_id' => ['', 1, 1, 1, 1],
-            'file'      => ['', 1, 0, 1, 1],
-            'mime'      => ['', 1, 0, 1, 1],
-            'comment'   => [$this->app->trans('Receiver'), 1, 0, 1, 1],
-        ];
-
-        $activity = new \module_report_sent(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $activity->setConfig(false);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $activity->setHasLimit(false);
-            $activity->setPrettyString(false);
-
-            $this->doReport($request, $activity, $conf);
-
-            return $this->getCSVResponse($activity, 'activity_send_documents');
-        }
-
-        $report = $this->doReport($request, $activity, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false,
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record'],
-        ]);
-    }
-
-    /**
-     * Set Report configuration according to request parameters
-     *
-     * @param  Request        $request A request instance
-     * @param  \module_report $report  A report instance
-     * @param  Array          $conf    A report column configuration
-     * @param  Boolean        $what    Whether to group on a particular field or not
-     * @return Array
-     */
-    private function doReport(Request $request, \module_report $report, $conf, $what = false)
-    {
-        if ($this->getConf()->get(['registry', 'modules', 'anonymous-report'])) {
-            if (isset($conf['user'])) {
-                unset($conf['user']);
-            }
-
-            if (isset($conf['ip'])) {
-                unset($conf['ip']);
-            }
-        }
-        //save initial conf
-        $base_conf = $conf;
-        //format conf according user preferences
-        if ('' !== $columnsList = $request->request->get('list_column', '')) {
-            $new_conf = $conf;
-            $columns = explode(",", $columnsList);
-
-            foreach (array_keys($conf) as $col) {
-                if (!in_array($col, $columns)) {
-                    unset($new_conf[$col]);
-                }
-            }
-
-            $conf = $new_conf;
-        }
-
-        //display content of a table column when user click on it
-        if ($request->request->get('conf') == 'on') {
-            return $this->app->json(['liste' => $this->render('report/listColumn.html.twig', [
-                'conf'  => $base_conf
-            ]), "title" => $this->app->trans("configuration")]);
-        }
-
-        //set order
-        if (('' !== $order = $request->request->get('order', '')) && ('' !== $field = $request->request->get('champ', ''))) {
-            $report->setOrder($field, $order);
-        }
-
-        //work on filters
-        $mapColumnTitleToSqlField = $report->getTransQueryString();
-
-        $currentfilter = [];
-
-        if ('' !== $serializedFilter = $request->request->get('liste_filter', '')) {
-            $currentfilter = @unserialize(urldecode($serializedFilter));
-        }
-
-        $filter = new \module_report_filter($this->app, $currentfilter, $mapColumnTitleToSqlField);
-
-        if ('' !== $filterColumn = $request->request->get('filter_column', '')) {
-            $field = current(explode(' ', $filterColumn));
-            $value = $request->request->get('filter_value', '');
-
-            if ($request->request->get('liste') == 'on') {
-                return $this->app->json(['diag'  => $this->render('report/colFilter.html.twig', [
-                    'result' => $report->colFilter($field),
-                    'field'  => $field
-                ]), "title"  => $this->app->trans('filtrer les resultats sur la colonne %colonne%', ['%colonne%' => $field])]);
-            }
-
-            if ($field === $value) {
-                $filter->removeFilter($field);
-            } else {
-                $filter->addFilter($field, '=', $value);
-            }
-        }
-
-        //set new request filter if user asking for them
-        if ($request->request->get('precise') == 1) {
-            $filter->addFilter('xml', 'LIKE', $request->request->get('word', ''));
-        } elseif ($request->request->get('precise') == 2) {
-            $filter->addFilter('record_id', '=', $request->request->get('word', ''));
-        }
-
-        //set filters to current report
-        $report->setFilter($filter->getTabFilter());
-        $report->setActiveColumn($filter->getActiveColumn());
-        $report->setPostingFilter($filter->getPostingFilter());
-
-        // display a new arraywhere results are group
-        if ('' !== $groupby = $request->request->get('groupby', '')) {
-            $report->setConfig(false);
-            $groupby = current(explode(' ', $groupby));
-
-            $reportArray = $report->buildReport(false, $groupby);
-
-            if (count($reportArray['allChamps']) > 0 && count($reportArray['display']) > 0) {
-                $groupField = isset($reportArray['display'][$reportArray['allChamps'][0]]['title']) ? $reportArray['display'][$reportArray['allChamps'][0]]['title'] : '';
-            } else {
-                $groupField = isset($conf[strtolower($groupby)]['title']) ? $conf[strtolower($groupby)]['title'] : '';
-            }
-
-            return $this->app->json([
-                'rs' => $this->render('report/ajax_data_content.html.twig', [
-                    'result'      => isset($reportArray['report']) ? $reportArray['report'] : $reportArray,
-                    'is_infouser' => false,
-                    'is_nav'      => false,
-                    'is_groupby'  => true,
-                    'is_plot'     => false,
-                    'is_doc'      => false,
-                ]),
-                'display_nav' => false,
-                'title'       => $this->app->trans('Groupement des resultats sur le champ %name%', ['%name%' => $groupField]),
-            ]);
-        }
-
-        //set Limit
-        if ($report->getEnableLimit()
-            && ('' !== $page = $request->request->get('page', ''))
-            && ('' !== $limit = $request->request->get('limit', ''))) {
-            $report->setLimit($page, $limit);
-        } else {
-            $report->setLimit(false, false);
-        }
-
-        //time to build our report
-        if (false === $what) {
-            $reportArray = $report->buildReport($conf);
-        } else {
-            $reportArray = $report->buildReport($conf, $what, $request->request->get('tbl', false));
-        }
-
-        return $reportArray;
-    }
-
-    private function getCSVResponse(\module_report $report, $type)
-    {
-        // set headers
-        $headers = [];
-        foreach (array_keys($report->getDisplay()) as $k) {
-            $headers[$k] = $k;
-        }
-        // set headers as first row
-        $result = $report->getResult();
-        array_unshift($result, $headers);
-
-        $collection = new CallbackCollection($result, function ($row) use ($report) {
-            // restrict to displayed fields
-            return array_map('strip_tags', array_intersect_key($row, $report->getDisplay()));
-        });
-
-        $filename = sprintf('report_export_%s_%s.csv', $type, date('Ymd'));
-        /** @var Exporter $exporter */
-        $exporter = $this->app['csv.exporter'];
-        $response = new CSVFileResponse($filename, function () use ($exporter, $collection) {
-            $exporter->export('php://output', $collection);
-        });
-
-        return $response;
-    }
-}

lib/Alchemy/Phrasea/Controller/Report/InformationController.php

@@ -1,502 +0,0 @@
-<?php
-/*
- * This file is part of Phraseanet
- *
- * (c) 2005-2016 Alchemy
- *
- * For the full copyright and license information, please view the LICENSE
- * file that was distributed with this source code.
- */
-namespace Alchemy\Phrasea\Controller\Report;
-
-use Alchemy\Phrasea\Controller\Controller;
-use Alchemy\Phrasea\Core\Response\CSVFileResponse;
-use Goodby\CSV\Export\Standard\Collection\CallbackCollection;
-use Goodby\CSV\Export\Standard\Exporter;
-use Symfony\Component\HttpFoundation\JsonResponse;
-use Symfony\Component\HttpFoundation\Request;
-
-class InformationController extends Controller
-{
-    /**
-     * Display information about a user
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportInformationUser(Request $request)
-    {
-        $conf = [
-            'config'    => [
-                'photo'     => [$this->app->trans('report:: document'), 0, 0, 0, 0],
-                'record_id' => [$this->app->trans('report:: record id'), 0, 0, 0, 0],
-                'date'      => [$this->app->trans('report:: date'), 0, 0, 0, 0],
-                'type'      => [$this->app->trans('phrseanet:: sous definition'), 0, 0, 0, 0],
-                'titre'     => [$this->app->trans('report:: titre'), 0, 0, 0, 0],
-                'taille'    => [$this->app->trans('report:: poids'), 0, 0, 0, 0]
-            ],
-            'conf'  => [
-                'identifiant'   => [$this->app->trans('report:: identifiant'), 0, 0, 0, 0],
-                'nom'           => [$this->app->trans('report:: nom'), 0, 0, 0, 0],
-                'mail'          => [$this->app->trans('report:: email'), 0, 0, 0, 0],
-                'adresse'       => [$this->app->trans('report:: adresse'), 0, 0, 0, 0],
-                'tel'           => [$this->app->trans('report:: telephone'), 0, 0, 0, 0]
-            ],
-            'config_cnx'    => [
-                'ddate'     => [$this->app->trans('report:: date'), 0, 0, 0, 0],
-                'appli'     => [$this->app->trans('report:: modules'), 0, 0, 0, 0],
-            ],
-            'config_dl' => [
-                'ddate'     => [$this->app->trans('report:: date'), 0, 0, 0, 0],
-                'record_id' => [$this->app->trans('report:: record id'), 0, 1, 0, 0],
-                'final'     => [$this->app->trans('phrseanet:: sous definition'), 0, 0, 0, 0],
-                'coll_id'   => [$this->app->trans('report:: collections'), 0, 0, 0, 0],
-                'comment'   => [$this->app->trans('report:: commentaire'), 0, 0, 0, 0],
-            ],
-            'config_ask' => [
-                'search'    => [$this->app->trans('report:: question'), 0, 0, 0, 0],
-                'ddate'     => [$this->app->trans('report:: date'), 0, 0, 0, 0]
-            ]
-        ];
-
-        $report = null;
-        $html = $html_info = '';
-        $from = $request->request->get('from', '');
-        $on = $request->request->get('on', '');
-        $selectValue = $request->request->get('user', '');
-
-        if ('' === $selectValue) {
-            $this->app->abort(400);
-        }
-
-        if ('' !== $on && $this->getConf()->get(['registry', 'modules', 'anonymous-report']) == true) {
-            $conf['conf'] = [
-                $on   => [$on, 0, 0, 0, 0],
-                'nb'   => [$this->app->trans('report:: nombre'), 0, 0, 0, 0]
-            ];
-        }
-
-        if ($from == 'CNXU' || $from == 'CNX') {
-            $report = new \module_report_connexion(
-                $this->app,
-                $request->request->get('dmin'),
-                $request->request->get('dmax'),
-                $request->request->get('sbasid'),
-                $request->request->get('collection')
-            );
-            $conf_array = $conf['config_cnx'];
-            $title = $this->app->trans('report:: historique des connexions');
-        } elseif ($from == 'USR' || $from == 'GEN') {
-            $report = new \module_report_download(
-                $this->app,
-                $request->request->get('dmin'),
-                $request->request->get('dmax'),
-                $request->request->get('sbasid'),
-                $request->request->get('collection')
-            );
-            $conf_array = $conf['config_dl'];
-            $title = $this->app->trans('report:: historique des telechargements');
-        } elseif ($from == 'ASK') {
-            $report = new \module_report_question(
-                $this->app,
-                $request->request->get('dmin'),
-                $request->request->get('dmax'),
-                $request->request->get('sbasid'),
-                $request->request->get('collection')
-            );
-            $conf_array = $conf['config_ask'];
-            $title = $this->app->trans('report:: historique des questions');
-        }
-
-        if ($report) {
-            $mapColumnTitleToSqlField = $report->getTransQueryString();
-
-            $currentfilter = [];
-
-            if ('' !== $serializedFilter = $request->request->get('liste_filter', '')) {
-                $currentfilter = @unserialize(urldecode($serializedFilter));
-            }
-
-            $filter = new \module_report_filter($this->app, $currentfilter, $mapColumnTitleToSqlField);
-
-            if ('' !== $filterColumn = $request->request->get('filter_column', '')) {
-                $field = current(explode(' ', $filterColumn));
-                $value = $request->request->get('filter_value', '');
-
-                if ($request->request->get('liste') == 'on') {
-                    return $this->app->json([
-                        'diag'  => $this->render('report/colFilter.html.twig', [
-                            'result' => $report->colFilter($field),
-                            'field'  => $field
-                        ]),
-                        'title'  => $this->app->trans('filtrer les resultats sur la colonne %colonne%', ['%colonne%' => $field])]);
-                }
-
-                if ($field === $value) {
-                    $filter->removeFilter($field);
-                } else {
-                    $filter->addFilter($field, '=', $value);
-                }
-            }
-
-            if ('' !== $selectValue && '' !== $from) {
-                $filter->addfilter('usrid', '=', $selectValue);
-            } elseif ('' !== $on && '' !== $selectValue) {
-                $filter->addfilter($on, '=', $selectValue);
-            }
-
-            if ($report instanceof \module_report_download) {
-                $report->setIsInformative(true);
-            }
-
-            $report->setFilter($filter->getTabFilter());
-            $report->setOrder('ddate', 'DESC');
-            $report->setConfig(false);
-            $report->setTitle($title);
-            $report->setHasLimit(false);
-
-            $reportArray = $report->buildReport($conf_array);
-
-            if ($request->request->get('printcsv') == 'on') {
-                $report->setPrettyString(false);
-
-                return $this->getCSVResponse($report, 'info_user');
-            }
-
-            $html = $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($reportArray['report']) ? $reportArray['report'] : $reportArray,
-                'is_infouser' => $report instanceof \module_report_download,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]);
-        }
-
-        $info = new \module_report_nav(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $info->setPeriode('');
-        $info->setCsv(false);
-
-        $infoArray = $info->buildTabGrpInfo(
-            null !== $report ? $report->getReq() : '',
-            null !== $report ? $report->getParams() : [],
-            $selectValue,
-            $conf['conf'],
-            $on
-        );
-
-        if (false == $this->app['conf']->get(['registry', 'modules', 'anonymous-report'])) {
-            $html_info = $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($infoArray['report']) ? $infoArray['report'] : $infoArray,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]);
-
-            $title = ('' === $on && isset($infoArray['result'])) ? $infoArray['result'][0]['identifiant'] : $selectValue;
-        } else {
-            $title = $selectValue;
-        }
-
-        return $this->app->json([
-            'rs'          => sprintf('%s%s', $html_info, $html),
-            'display_nav' => false,
-            'title'       => $title
-        ]);
-    }
-
-    /**
-     * Display a browser version
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportInformationBrowser(Request $request)
-    {
-        $conf = [
-            'version'   => [$this->app->trans('report::version'), 0, 0, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0]
-        ];
-
-        $info = new \module_report_nav(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $info->setCsv(false);
-        $info->setConfig(false);
-
-        if ('' === $browser = $request->request->get('user', '')) {
-            $this->app->abort(400);
-        }
-
-        $reportArray = $info->buildTabInfoNav($conf, $browser);
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($reportArray['report']) ? $reportArray['report'] : $reportArray,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]),
-            'display_nav' => false,
-            'title'       => $browser
-        ]);
-    }
-
-    /**
-     * Display information about a document
-     *
-     * @param  Request      $request
-     * @return JsonResponse
-     */
-    public function doReportInformationDocument(Request $request)
-    {
-        $config = [
-            'photo'     => [$this->app->trans('report:: document'), 0, 0, 0, 0],
-            'record_id' => [$this->app->trans('report:: record id'), 0, 0, 0, 0],
-            'date'      => [$this->app->trans('report:: date'), 0, 0, 0, 0],
-            'type'      => [$this->app->trans('phrseanet:: sous definition'), 0, 0, 0, 0],
-            'titre'     => [$this->app->trans('report:: titre'), 0, 0, 0, 0],
-            'taille'    => [$this->app->trans('report:: poids'), 0, 0, 0, 0]
-        ];
-
-        $config_dl = [
-            'ddate'     => [$this->app->trans('report:: date'), 0, 0, 0, 0],
-            'user'      => [$this->app->trans('report:: utilisateurs'), 0, 0, 0, 0],
-            'final'     => [$this->app->trans('phrseanet:: sous definition'), 0, 0, 0, 0],
-            'coll_id'   => [$this->app->trans('report:: collections'), 0, 0, 0, 0],
-            'comment'   => [$this->app->trans('report:: commentaire'), 0, 0, 0, 0],
-            'fonction'  => [$this->app->trans('report:: fonction'), 0, 0, 0, 0],
-            'activite'  => [$this->app->trans('report:: activite'), 0, 0, 0, 0],
-            'pays'      => [$this->app->trans('report:: pays'), 0, 0, 0, 0],
-            'societe'   => [$this->app->trans('report:: societe'), 0, 0, 0, 0]
-        ];
-
-        //format conf according user preferences
-        if ('' !== $columnsList = $request->request->get('list_column', '')) {
-            $new_conf = $config_dl;
-            $columns = explode(',', $columnsList);
-
-            foreach (array_keys($config_dl) as $col) {
-                if (!in_array($col, $columns)) {
-                    unset($new_conf[$col]);
-                }
-            }
-
-            $config_dl = $new_conf;
-        }
-
-        try {
-            $record = new \record_adapter(
-                $this->app,
-                $request->request->get('sbasid'),
-                $request->request->get('rid')
-            );
-        } catch (\Exception $e) {
-            $this->app->abort(404);
-        }
-
-        $what = new \module_report_nav(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $what->setPeriode('');
-        $what->setCsv(false);
-        $what->setPrint(false);
-
-        /** @var \record_adapter $record */
-        $reportArray = $what->buildTabUserWhat(
-            $record->getBaseId(),
-            $record->getRecordId(),
-            $config
-        );
-
-        $title = $what->getTitle();
-
-        $html = $this->render('report/ajax_data_content.html.twig', [
-            'result'      => isset($reportArray['report']) ? $reportArray['report'] : $reportArray,
-            'is_infouser' => false,
-            'is_nav'      => false,
-            'is_groupby'  => false,
-            'is_plot'     => false,
-            'is_doc'      => false
-        ]);
-
-        $from = $request->request->get('from', '');
-
-        if ('TOOL' === $from) {
-            $what->setTitle('');
-
-            return $this->app->json([
-                'rs'          => $html,
-                'display_nav' => false,
-                'title'       => $title
-            ]);
-        }
-
-        if ('DASH' !== $from && 'PUSHDOC' !== $from) {
-            $download = new \module_report_download(
-                $this->app,
-                $request->request->get('dmin'),
-                $request->request->get('dmax'),
-                $request->request->get('sbasid'),
-                $request->request->get('collection')
-            );
-
-            $mapColumnTitleToSqlField = $download->getTransQueryString();
-
-            $currentfilter = [];
-
-            if ('' !== $serializedFilter = $request->request->get('liste_filter', '')) {
-                $currentfilter = @unserialize(urldecode($serializedFilter));
-            }
-
-            $filter = new \module_report_filter($this->app, $currentfilter, $mapColumnTitleToSqlField);
-
-            if ('' !== $filterColumn = $request->request->get('filter_column', '')) {
-                $field = current(explode(' ', $filterColumn));
-                $value = $request->request->get('filter_value', '');
-
-                if ($request->request->get('liste') == 'on') {
-                    return $this->app->json([
-                        'diag'  => $this->render('report/colFilter.html.twig', [
-                            'result' => $download->colFilter($field),
-                            'field'  => $field
-                        ]),
-                        'title'  => $this->app->trans('filtrer les resultats sur la colonne %colonne%', ['%colonne%' => $field])
-                    ]);
-                }
-
-                if ($field === $value) {
-                    $filter->removeFilter($field);
-                } else {
-                    $filter->addFilter($field, '=', $value);
-                }
-            }
-
-            $filter->addfilter('record_id', '=', $record->getRecordId());
-
-            $download->setFilter($filter->getTabFilter());
-            $download->setOrder('ddate', 'DESC');
-            $download->setTitle($this->app->trans('report:: historique des telechargements'));
-            $download->setConfig(false);
-
-            $reportArray = $download->buildReport($config_dl);
-
-            if ($request->request->get('printcsv') == 'on') {
-                $download->setPrettyString(false);
-
-                return $this->getCSVResponse($download, 'info_document');
-            }
-
-            $html .= $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($reportArray['report']) ? $reportArray['report'] : $reportArray,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]);
-
-            return $this->app->json([
-                'rs'          => $html,
-                'display_nav' => false,
-                'title'       => $title
-            ]);
-        }
-
-        if ($this->getConf()->get(['registry', 'modules', 'anonymous-report']) == false && $from !== 'DOC' && $from !== 'DASH' && $from !== 'GEN' && $from !== 'PUSHDOC') {
-            $conf = [
-                'identifiant'   => [$this->app->trans('report:: identifiant'), 0, 0, 0, 0],
-                'nom'           => [$this->app->trans('report:: nom'), 0, 0, 0, 0],
-                'mail'          => [$this->app->trans('report:: email'), 0, 0, 0, 0],
-                'adresse'       => [$this->app->trans('report:: adresse'), 0, 0, 0, 0],
-                'tel'           => [$this->app->trans('report:: telephone'), 0, 0, 0, 0]
-            ];
-
-            $info = new \module_report_nav(
-                $this->app,
-                $request->request->get('dmin'),
-                $request->request->get('dmax'),
-                $request->request->get('sbasid'),
-                $request->request->get('collection')
-            );
-
-            $info->setPeriode('');
-            $info->setConfig(false);
-            $info->setTitle($this->app->trans('report:: utilisateur'));
-
-            $reportArray = $info->buildTabGrpInfo(false, [],  $request->request->get('user'), $conf, false);
-
-            if ($request->request->get('printcsv') == 'on' && isset($download)) {
-                return $this->getCSVResponse($this->app, $info, 'info_user');
-            }
-
-            $html .= $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($reportArray['report']) ? $reportArray['report'] : $reportArray,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]);
-
-            return $this->app->json([
-                'rs'          => $html,
-                'display_nav' => false,
-                'title'       => $title
-            ]);
-        }
-
-        return $this->app->json([
-            'rs'          => $html,
-            'display_nav' => false,
-            'title'       => $title
-        ]);
-    }
-
-    private function getCSVResponse(\module_report $report, $type)
-    {
-        // set headers
-        $headers = [];
-        foreach (array_keys($report->getDisplay()) as $k) {
-            $headers[$k] = $k;
-        }
-        // set headers as first row
-        $result = $report->getResult();
-        array_unshift($result, $headers);
-
-        $collection = new CallbackCollection($result, function ($row) use ($report) {
-            // restrict fields to the displayed ones
-            return array_map('strip_tags', array_intersect_key($row, $report->getDisplay()));
-        });
-
-        /** @var Exporter $exporter */
-        $exporter = $this->app['csv.exporter'];
-        $filename = sprintf('report_export_%s_%s.csv', $type, date('Ymd'));
-        $response = new CSVFileResponse($filename, function () use ($exporter, $collection) {
-            $exporter->export('php://output', $collection);
-        });
-
-        return $response;
-    }
-}

lib/Alchemy/Phrasea/Controller/Report/RootController.php

@@ -10,12 +10,8 @@
 namespace Alchemy\Phrasea\Controller\Report;
 
 use Alchemy\Phrasea\Controller\Controller;
-use Alchemy\Phrasea\Core\Response\CSVFileResponse;
-use Goodby\CSV\Export\Standard\Collection\CallbackCollection;
-use Goodby\CSV\Export\Standard\Exporter;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\Request;
-use Symfony\Component\HttpFoundation\Response;
 
 class RootController extends Controller
 {
@@ -53,22 +49,34 @@ class RootController extends Controller
 
         $granted = [];
 
-        foreach ($this->getAclForUser()->get_granted_base([\ACL::CANREPORT]) as $collection) {
-            if (!isset($granted[$collection->get_sbas_id()])) {
-                $granted[$collection->get_sbas_id()] = [
-                    'id' => $collection->get_sbas_id(),
-                    'name' => $collection->get_databox()->get_viewname(),
-                    'collections' => []
+        $acl = $this->getAclForUser();
+        foreach ($acl->get_granted_base([\ACL::CANREPORT]) as $collection) {
+            $sbas_id = $collection->get_sbas_id();
+            if (!isset($granted[$sbas_id])) {
+                $granted[$sbas_id] = [
+                    'id'          => $sbas_id,
+                    'name'        => $collection->get_databox()->get_viewname(),
+                    'collections' => [],
+                    'metas'       => []
                 ];
+
+                foreach ($collection->get_databox()->get_meta_structure() as $meta) {
+                    // skip the fields that can't be reported
+                    if (!$meta->is_report() || ($meta->isBusiness() && !$acl->can_see_business_fields($collection->get_databox()))) {
+                        continue;
+                    }
+                    $granted[$sbas_id]['metas'][] = $meta->get_name();
+                }
             }
-            $granted[$collection->get_sbas_id()]['collections'][] = [
-                'id' => $collection->get_coll_id(),
+            $granted[$sbas_id]['collections'][] = [
+                'id'      => $collection->get_coll_id(),
                 'base_id' => $collection->get_base_id(),
-                'name' => $collection->get_name()
+                'name'    => $collection->get_name(),
             ];
         }
 
         $conf = $this->getConf();
+
         return $this->render('report/report_layout_child.html.twig', [
             'ajax_dash'     => true,
             'dashboard'     => null,
@@ -82,594 +90,4 @@ class RootController extends Controller
             'ajax_chart'    => false
         ]);
     }
-
-    /**
-     * Gets available collections where current user can see report and
-     * format date
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function initReport(Request $request)
-    {
-        $popbases = $request->request->get('popbases', []);
-
-        if ('' === $dmin = $request->request->get('dmin', '')) {
-            $dmin = date('Y') . '-' . date('m') . '-01';
-        }
-
-        if ('' === $dmax = $request->request->get('dmax', '')) {
-            $dmax = date('Y') . '-' . date('m') . '-' . date('d');
-        }
-
-        $dmin = \DateTime::createFromFormat('Y-m-d H:i:s', sprintf('%s 00:00:00', $dmin));
-        $dmax = \DateTime::createFromFormat('Y-m-d H:i:s', sprintf('%s 23:59:59', $dmax));
-
-        //get user's sbas & collections selection from popbases
-        $selection = [];
-        $liste = $id_sbas = '';
-        $i = 0;
-        foreach (array_fill_keys($popbases, 0) as $key => $val) {
-            $exp = explode('_', $key);
-            if ($exp[0] != $id_sbas && $i != 0) {
-                $selection[$id_sbas]['liste'] = $liste;
-                $liste = '';
-            }
-            $selection[$exp[0]][] = $exp[1];
-            $liste .= (empty($liste) ? '' : ',') . $exp[1];
-            $id_sbas = $exp[0];
-            $i ++;
-        }
-        //fill the last entry
-        $selection[$id_sbas]['liste'] = $liste;
-
-        return $this->render('report/ajax_report_content.html.twig', [
-            'selection' => $selection,
-            'anonymous' => $this->getConf()->get(['registry', 'modules', 'anonymous-report']),
-            'ajax'      => true,
-            'dmin'      => $dmin->format('Y-m-d H:i:s'),
-            'dmax'      => $dmax->format('Y-m-d H:i:s'),
-        ]);
-    }
-
-    /**
-     * Display instance connexion report
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportConnexions(Request $request)
-    {
-        $cnx = new \module_report_connexion(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $conf = [
-            'user'      => [$this->app->trans('phraseanet::utilisateurs'), 1, 1, 1, 1],
-            'ddate'     => [$this->app->trans('report:: date'), 1, 0, 1, 1],
-            'ip'        => [$this->app->trans('report:: IP'), 1, 0, 0, 0],
-            'appli'     => [$this->app->trans('report:: modules'), 1, 0, 0, 0],
-            'fonction'  => [$this->app->trans('report::fonction'), 1, 1, 1, 1],
-            'activite'  => [$this->app->trans('report::activite'), 1, 1, 1, 1],
-            'pays'      => [$this->app->trans('report::pays'), 1, 1, 1, 1],
-            'societe'   => [$this->app->trans('report::societe'), 1, 1, 1, 1]
-        ];
-
-        if ($request->request->get('printcsv') == 'on') {
-            $cnx->setHasLimit(false);
-            $cnx->setPrettyString(false);
-
-            $this->doReport($request, $cnx, $conf);
-
-            return $this->getCSVResponse($cnx, 'connections');
-        }
-
-        $report = $this->doReport($request, $cnx, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record']
-        ]);
-    }
-
-    /**
-     * Display instance questions report
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportQuestions(Request $request)
-    {
-        $questions = new \module_report_question(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $conf = [
-            'user'      => [$this->app->trans('report:: utilisateur'), 1, 1, 1, 1],
-            'search'    => [$this->app->trans('report:: question'), 1, 0, 1, 1],
-            'ddate'     => [$this->app->trans('report:: date'), 1, 0, 1, 1],
-            'fonction'  => [$this->app->trans('report:: fonction'), 1, 1, 1, 1],
-            'activite'  => [$this->app->trans('report:: activite'), 1, 1, 1, 1],
-            'pays'      => [$this->app->trans('report:: pays'), 1, 1, 1, 1],
-            'societe'   => [$this->app->trans('report:: societe'), 1, 1, 1, 1]
-        ];
-
-        if ($request->request->get('printcsv') == 'on') {
-            $questions->setHasLimit(false);
-            $questions->setPrettyString(false);
-
-            $this->doReport($request, $questions, $conf);
-
-            return $this->getCSVResponse($questions, 'questions');
-        }
-
-        $report = $this->doReport($request, $questions, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record']
-        ]);
-    }
-
-    /**
-     * Display instance download report
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportDownloads(Request $request)
-    {
-        $download = new \module_report_download(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $conf_pref = [];
-
-        foreach (\module_report::getPreff($this->app, $request->request->get('sbasid')) as $field) {
-            $conf_pref[strtolower($field)] = [$field, 0, 0, 0, 0];
-        }
-
-        $conf = array_merge([
-            'user'      => [$this->app->trans('report:: utilisateurs'), 1, 1, 1, 1],
-            'ddate'     => [$this->app->trans('report:: date'), 1, 0, 1, 1],
-            'record_id' => [$this->app->trans('report:: record id'), 1, 1, 1, 1],
-            'final'     => [$this->app->trans('phrseanet:: sous definition'), 1, 0, 1, 1],
-            'coll_id'   => [$this->app->trans('report:: collections'), 1, 0, 1, 1],
-            'comment'   => [$this->app->trans('report:: commentaire'), 1, 0, 0, 0],
-            'fonction'  => [$this->app->trans('report:: fonction'), 1, 1, 1, 1],
-            'activite'  => [$this->app->trans('report:: activite'), 1, 1, 1, 1],
-            'pays'      => [$this->app->trans('report:: pays'), 1, 1, 1, 1],
-            'societe'   => [$this->app->trans('report:: societe'), 1, 1, 1, 1]
-        ], $conf_pref);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $download->setHasLimit(false);
-            $download->setPrettyString(false);
-
-            $this->doReport($request, $download, $conf);
-
-            $r = $this->getCSVResponse($download, 'download');
-
-            return $r;
-        }
-
-        $report = $this->doReport($request, $download, $conf);
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record']
-        ]);
-    }
-
-    /**
-     * Display instance document report
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportDocuments(Request $request)
-    {
-        $document = new \module_report_download(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $conf_pref = [];
-
-        foreach (\module_report::getPreff($this->app, $request->request->get('sbasid')) as $field) {
-            $conf_pref[$field] = array($field, 0, 0, 0, 0);
-        }
-
-        $conf = array_merge([
-            'telechargement'    => [$this->app->trans('report:: telechargements'), 1, 0, 0, 0],
-            'record_id'         => [$this->app->trans('report:: record id'), 1, 1, 1, 0],
-            'final'             => [$this->app->trans('phraseanet:: sous definition'), 1, 0, 1, 1],
-            'file'              => [$this->app->trans('report:: fichier'), 1, 0, 0, 1],
-            'mime'              => [$this->app->trans('report:: type'), 1, 0, 1, 1],
-            'size'              => [$this->app->trans('report:: taille'), 1, 0, 1, 1]
-        ], $conf_pref);
-
-        if ($request->request->get('printcsv') == 'on') {
-            $document->setHasLimit(false);
-            $document->setPrettyString(false);
-
-            $this->doReport($request, $document, $conf, 'record_id');
-
-            $r = $this->getCSVResponse($document, 'documents');
-
-            return $r;
-        }
-
-        $report = $this->doReport($request, $document, $conf, 'record_id');
-
-        if ($report instanceof Response) {
-            return $report;
-        }
-
-        return $this->app->json([
-            'rs'          =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => false,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => true
-            ]),
-            'display_nav' => $report['display_nav'], // do we display the prev and next button ?
-            'next'        => $report['next_page'], //Number of the next page
-            'prev'        => $report['previous_page'], //Number of the previoous page
-            'page'        => $report['page'], //The current page
-            'filter'      => ((sizeof($report['filter']) > 0) ? serialize($report['filter']) : ''), //the serialized filters
-            'col'         => $report['active_column'], //all the columns where a filter is applied
-            'limit'       => $report['nb_record']
-        ]);
-    }
-
-    /**
-     * Display information about client (browser, resolution etc ...)
-     *
-     * @param  Request $request
-     * @return JsonResponse
-     */
-    public function doReportClients(Request $request)
-    {
-        $nav = new \module_report_nav(
-            $this->app,
-            $request->request->get('dmin'),
-            $request->request->get('dmax'),
-            $request->request->get('sbasid'),
-            $request->request->get('collection')
-        );
-
-        $conf_nav = [
-            'nav'       => [$this->app->trans('report:: navigateur'), 0, 1, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0],
-            'pourcent'  => [$this->app->trans('report:: pourcentage'), 0, 0, 0, 0]
-        ];
-
-        $conf_combo = [
-            'combo'     => [$this->app->trans('report:: navigateurs et plateforme'), 0, 0, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0],
-            'pourcent'  => [$this->app->trans('report:: pourcentage'), 0, 0, 0, 0]
-        ];
-        $conf_os = [
-            'os'        => [$this->app->trans('report:: plateforme'), 0, 0, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0],
-            'pourcent'  => [$this->app->trans('report:: pourcentage'), 0, 0, 0, 0]
-        ];
-        $conf_res = [
-            'res'       => [$this->app->trans('report:: resolution'), 0, 0, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0],
-            'pourcent'  => [$this->app->trans('report:: pourcentage'), 0, 0, 0, 0]
-        ];
-        $conf_mod = [
-            'appli'     => [$this->app->trans('report:: module'), 0, 0, 0, 0],
-            'nb'        => [$this->app->trans('report:: nombre'), 0, 0, 0, 0],
-            'pourcent'  => [$this->app->trans('report:: pourcentage'), 0, 0, 0, 0]
-        ];
-
-        $report = [
-            'nav'   => $nav->buildTabNav($conf_nav),
-            'os'    => $nav->buildTabOs($conf_os),
-            'res'   => $nav->buildTabRes($conf_res),
-            'mod'   => $nav->buildTabModule($conf_mod),
-            'combo' => $nav->buildTabCombo($conf_combo)
-        ];
-
-        if ($request->request->get('printcsv') == 'on') {
-            $result = [];
-
-            $result[] = array_keys($conf_nav);
-            foreach ($report['nav']['result'] as $row) {
-                $result[] =  array_values($row);
-            };
-            $result[] = array_keys($conf_os);
-            foreach ($report['os']['result'] as $row) {
-                $result[] =  array_values($row);
-            };
-            $result[] = array_keys($conf_res);
-            foreach ($report['res']['result'] as $row) {
-                $result[] =  array_values($row);
-            };
-            $result[] = array_keys($conf_mod);
-            foreach ($report['mod']['result'] as $row) {
-                $result[] =  array_values($row);
-            };
-            $result[] = array_keys($conf_combo);
-            foreach ($report['combo']['result'] as $row) {
-                $result[] =  array_values($row);
-            };
-
-            /** @var Exporter $exporter */
-            $exporter = $this->app['csv.exporter'];
-            $filename = sprintf('report_export_info_%s.csv', date('Ymd'));
-            $response = new CSVFileResponse($filename, function () use ($exporter, $result) {
-                $exporter->export('php://output', $result);
-            });
-
-            return $response;
-        }
-
-        return $this->app->json([
-            'rs' =>  $this->render('report/ajax_data_content.html.twig', [
-                'result'      => isset($report['report']) ? $report['report'] : $report,
-                'is_infouser' => false,
-                'is_nav'      => true,
-                'is_groupby'  => false,
-                'is_plot'     => false,
-                'is_doc'      => false
-            ]),
-            'display_nav' => false,
-            'title'       => false
-        ]);
-    }
-
-    /**
-     * Set Report configuration according to request parameters
-     *
-     * @param  Request        $request A request instance
-     * @param  \module_report $report  A report instance
-     * @param  Array          $conf    A report column configuration
-     * @param  Boolean        $what    Whether to group on a particular field or not
-     * @return Array
-     */
-    private function doReport(Request $request, \module_report $report, $conf, $what = false)
-    {
-        if ($this->getConf()->get(['registry', 'modules', 'anonymous-report']) == true) {
-            if (isset($conf['user'])) {
-                unset($conf['user']);
-            }
-
-            if (isset($conf['ip'])) {
-                unset($conf['ip']);
-            }
-        }
-
-        //save initial conf
-        $base_conf = $conf;
-        //format conf according user preferences
-        if ('' !== $columnsList = $request->request->get('list_column', '')) {
-            $new_conf = $conf;
-            $columns = explode(',', $columnsList);
-
-            foreach (array_keys($conf) as $col) {
-                if (!in_array($col, $columns)) {
-                    unset($new_conf[$col]);
-                }
-            }
-
-            $conf = $new_conf;
-        }
-
-        //display content of a table column when user click on it
-        if ($request->request->get('conf') == 'on') {
-            return $this->app->json(['liste' => $this->render('report/listColumn.html.twig', [
-                'conf'  => $base_conf
-            ]), 'title' => $this->app->trans('configuration')]);
-        }
-
-        //set order
-        if (('' !== $order = $request->request->get('order', '')) && ('' !== $field = $request->request->get('champ', ''))) {
-            $report->setOrder($field, $order);
-        }
-
-        //work on filters
-        $mapColumnTitleToSqlField = $report->getTransQueryString();
-
-        $currentfilter = [];
-
-        if ('' !== $serializedFilter = $request->request->get('liste_filter', '')) {
-            $currentfilter = @unserialize(urldecode($serializedFilter));
-        }
-
-        $filter = new \module_report_filter($this->app, $currentfilter, $mapColumnTitleToSqlField);
-
-        if ('' !== $filterColumn = $request->request->get('filter_column', '')) {
-            $field = current(explode(' ', $filterColumn));
-            $value = $request->request->get('filter_value', '');
-
-            if ($request->request->get('liste') == 'on') {
-                return $this->app->json(['diag'  => $this->render('report/colFilter.html.twig', [
-                    'result' => $report->colFilter($field),
-                    'field'  => $field
-                ]), 'title'  => $this->app->trans('filtrer les resultats sur la colonne %colonne%', ['%colonne%' => $field])]);
-            }
-
-            if ($field === $value) {
-                $filter->removeFilter($field);
-            } else {
-                $filter->addFilter($field, '=', $value);
-            }
-        }
-
-        //set new request filter if user asking for them
-        if ($request->request->get('precise') == 1) {
-            $filter->addFilter('xml', 'LIKE', $request->request->get('word', ''));
-        } elseif ($request->request->get('precise') == 2) {
-            $filter->addFilter('record_id', '=', $request->request->get('word', ''));
-        }
-
-        //set filters to current report
-        $report->setFilter($filter->getTabFilter());
-        $report->setActiveColumn($filter->getActiveColumn());
-        $report->setPostingFilter($filter->getPostingFilter());
-
-        // display a new arraywhere results are group
-        if ('' !== $groupby = $request->request->get('groupby', '')) {
-
-            $report->setConfig(false);
-            $groupby = current(explode(' ', $groupby));
-
-            $reportArray = $report->buildReport(false, $groupby);
-
-            if (count($reportArray['allChamps']) > 0 && count($reportArray['display']) > 0) {
-                $groupField = isset($reportArray['display'][$reportArray['allChamps'][0]]['title']) ? $reportArray['display'][$reportArray['allChamps'][0]]['title'] : '';
-            } else {
-                $groupField = isset($conf[strtolower($groupby)]['title']) ? $conf[strtolower($groupby)]['title'] : '';
-            }
-
-            return $this->app->json([
-                'rs' => $this->render('report/ajax_data_content.html.twig', [
-                    'result'      => isset($reportArray['report']) ? $reportArray['report'] : $reportArray,
-                    'is_infouser' => false,
-                    'is_nav'      => false,
-                    'is_groupby'  => true,
-                    'is_plot'     => false,
-                    'is_doc'      => false
-                ]),
-                'display_nav' => false,
-                'title'       => $this->app->trans('Groupement des resultats sur le champ %name%', ['%name%' => $groupField])
-            ]);
-        }
-
-        //set Limit
-        if ($report->getEnableLimit()
-            && ('' !== $page = $request->request->get('page', ''))
-            && ('' !== $limit = $request->request->get('limit', ''))) {
-            $report->setLimit($page, $limit);
-        } else {
-            $report->setLimit(false, false);
-        }
-
-        //time to build our report
-        if (false === $what) {
-            $reportArray = $report->buildReport($conf);
-        } else {
-            $reportArray = $report->buildReport($conf, $what, $request->request->get('tbl', false));
-        }
-
-        return $reportArray;
-    }
-
-    /**
-     * Prefix the method to call with the controller class name
-     *
-     * @param  string $method The method to call
-     * @return string
-     */
-    private function call($method)
-    {
-        return sprintf('%s::%s', __CLASS__, $method);
-    }
-
-    private function getCSVResponse(\module_report $report, $type)
-    {
-        // set headers
-        $headers = [];
-        foreach (array_keys($report->getDisplay()) as $k) {
-            $headers[$k] = $k;
-        }
-        // set headers as first row
-        $result = $report->getResult();
-
-        array_unshift($result, $headers);
-
-        $collection = new CallbackCollection($result, function ($row) use ($headers) {
-            // restrict fields to the displayed ones
-            //    return array_map("strip_tags", array_intersect_key($row, $report->getDisplay()));
-            $ret = array();
-            foreach($headers as $f) {
-                $ret[$f] = array_key_exists($f, $row) ? strip_tags($row[$f]) : '';
-            }
-            return $ret;
-        });
-
-        $filename = sprintf('report_export_%s_%s.csv', $type, date('Ymd'));
-
-        /** @var Exporter $exporter */
-        $exporter = $this->app['csv.exporter'];
-        $cb = function () use ($exporter, $collection) {
-            $exporter->export('php://output', $collection);
-        };
-
-        $response = new CSVFileResponse($filename, $cb);
-
-        return $response;
-    }
 }

lib/Alchemy/Phrasea/Controller/Root/AccountController.php

@@ -17,20 +17,27 @@ use Alchemy\Phrasea\Application\Helper\EntityManagerAware;
 use Alchemy\Phrasea\Application\Helper\NotifierAware;
 use Alchemy\Phrasea\Authentication\Phrasea\PasswordEncoder;
 use Alchemy\Phrasea\Controller\Controller;
-use Alchemy\Phrasea\ControllerProvider\Root\Login;
 use Alchemy\Phrasea\Core\Configuration\RegistrationManager;
 use Alchemy\Phrasea\Exception\InvalidArgumentException;
 use Alchemy\Phrasea\Form\Login\PhraseaRenewPasswordForm;
 use Alchemy\Phrasea\Model\Entities\ApiApplication;
 use Alchemy\Phrasea\Model\Entities\FtpCredential;
 use Alchemy\Phrasea\Model\Entities\Session;
+use Alchemy\Phrasea\Model\Entities\User;
 use Alchemy\Phrasea\Model\Manipulator\ApiAccountManipulator;
+use Alchemy\Phrasea\Model\Manipulator\ApiApplicationManipulator;
+use Alchemy\Phrasea\Model\Manipulator\BasketManipulator;
 use Alchemy\Phrasea\Model\Manipulator\TokenManipulator;
 use Alchemy\Phrasea\Model\Manipulator\UserManipulator;
 use Alchemy\Phrasea\Model\Repositories\ApiAccountRepository;
 use Alchemy\Phrasea\Model\Repositories\ApiApplicationRepository;
+use Alchemy\Phrasea\Model\Repositories\BasketRepository;
+use Alchemy\Phrasea\Model\Repositories\FeedPublisherRepository;
 use Alchemy\Phrasea\Model\Repositories\TokenRepository;
+use Alchemy\Phrasea\Model\Repositories\ValidationSessionRepository;
+use Alchemy\Phrasea\Notification\Mail\MailRequestAccountDelete;
 use Alchemy\Phrasea\Notification\Mail\MailRequestEmailUpdate;
+use Alchemy\Phrasea\Notification\Mail\MailSuccessAccountDelete;
 use Alchemy\Phrasea\Notification\Receiver;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\RedirectResponse;
@@ -201,6 +208,7 @@ class AccountController extends Controller
      */
     public function accountAccess()
     {
+        //var_dump($this->getRegistrationManager()->getRegistrationSummary($this->getAuthenticatedUser()));die;
         return $this->render('account/access.html.twig', [
             'inscriptions' => $this->getRegistrationManager()->getRegistrationSummary($this->getAuthenticatedUser())
         ]);
@@ -298,13 +306,102 @@ class AccountController extends Controller
         $manager = $this->getEventManager();
         $user = $this->getAuthenticatedUser();
 
+        $repo_baskets = $this->getBasketRepository();
+        $baskets = $repo_baskets->findActiveValidationAndBasketByUser($user);
+
+        $apiAccounts = $this->getApiAccountRepository()->findByUser($user);
+
+        $ownedFeeds = $this->getFeedPublisherRepository()->findBy(['user' => $user, 'owner' => true]);
+
+        $initiatedValidations = $this->getValidationSessionRepository()->findby(['initiator' => $user, ]);
+
         return $this->render('account/account.html.twig', [
-            'user'          => $user,
-            'evt_mngr'      => $manager,
-            'notifications' => $manager->list_notifications_available($user),
+            'user'                  => $user,
+            'evt_mngr'              => $manager,
+            'notifications'         => $manager->list_notifications_available($user),
+            'baskets'               => $baskets,
+            'api_accounts'          => $apiAccounts,
+            'owned_feeds'           => $ownedFeeds,
+            'initiated_validations' => $initiatedValidations,
         ]);
     }
 
+    /**
+     * @param Request $request
+     * @return RedirectResponse
+     */
+    public function processDeleteAccount(Request $request)
+    {
+        $user = $this->getAuthenticatedUser();
+
+        if($this->app['conf']->get(['user_account', 'deleting_policies', 'email_confirmation'])) {
+
+            // send email confirmation
+
+            try {
+                $receiver = Receiver::fromUser($user);
+            } catch (InvalidArgumentException $e) {
+                $this->app->addFlash('error', $this->app->trans('phraseanet::erreur: echec du serveur de mail'));
+
+                return $this->app->redirectPath('account');
+            }
+
+            $token = $this->getTokenManipulator()->createAccountDeleteToken($user, $user->getEmail());
+            $url = $this->app->url('account_confirm_delete', ['token' => $token->getValue()]);
+
+
+            $mail = MailRequestAccountDelete::create($this->app, $receiver);
+            $mail->setUserOwner($user);
+            $mail->setButtonUrl($url);
+            $mail->setExpiration($token->getExpiration());
+
+            $this->deliver($mail);
+
+            $this->app->addFlash('info', $this->app->trans('phraseanet::account: A confirmation e-mail has been sent. Please follow the instructions contained to continue account deletion'));
+
+            return $this->app->redirectPath('account');
+
+        } else {
+            $this->doDeleteAccount($user);
+
+            $response = $this->app->redirectPath('homepage', [
+                'redirect' => $request->query->get("redirect")
+            ]);
+
+            $response->headers->clearCookie('persistent');
+            $response->headers->clearCookie('last_act');
+
+            return $response;
+        }
+
+    }
+
+    public function confirmDeleteAccount(Request $request)
+    {
+        if (($tokenValue = $request->query->get('token')) !== null ) {
+            if (null === $token = $this->getTokenRepository()->findValidToken($tokenValue)) {
+                $this->app->addFlash('error', $this->app->trans('Token not found'));
+
+                return $this->app->redirectPath('account');
+            }
+
+            $user = $token->getUser();
+            // delete account and datas
+            $this->doDeleteAccount($user);
+
+            $this->getTokenManipulator()->delete($token);
+        }
+
+        $response = $this->app->redirectPath('homepage', [
+            'redirect' => $request->query->get("redirect")
+        ]);
+
+        $response->headers->clearCookie('persistent');
+        $response->headers->clearCookie('last_act');
+
+        return $response;
+    }
+
     /**
      * Update account information
      *
@@ -405,6 +502,57 @@ class AccountController extends Controller
         return $this->app->redirectPath('account');
     }
 
+    /**
+     * @param User $user
+     */
+    private function doDeleteAccount(User $user)
+    {
+        // basket
+        $repo_baskets = $this->getBasketRepository();
+        $baskets = $repo_baskets->findActiveByUser($user);
+        $this->getBasketManipulator()->removeBaskets($baskets);
+
+        // application
+        $applications = $this->getApiApplicationRepository()->findByUser($user);
+
+        $this->getApiApplicationManipulator()->deleteApiApplications($applications);
+
+
+        //  revoke access and delete phraseanet user account
+
+        $list = array_keys($this->app['repo.collections-registry']->getBaseIdMap());
+
+        try {
+            $this->app->getAclForUser($user)->revoke_access_from_bases($list);
+        }
+        catch (\Exception $e) {
+            // one or more access could not be revoked ? the user will not be phantom
+            $this->app->addFlash('error', $this->app->trans('phraseanet::error: failed to revoke some user access'));
+        }
+
+        if ($this->app->getAclForUser($user)->is_phantom()) {
+            // send confirmation email: the account has been deleted
+
+            try {
+                $receiver = Receiver::fromUser($user);
+                $mail = MailSuccessAccountDelete::create($this->app, $receiver);
+            }
+            catch (InvalidArgumentException $e) {
+                $this->app->addFlash('error', $this->app->trans('phraseanet::erreur: echec du serveur de mail'));
+                $mail = null;
+            }
+
+            $this->app['manipulator.user']->delete($user);
+
+            if($mail) {
+                $this->deliver($mail);
+            }
+
+            $this->getAuthenticator()->closeAccount();
+            $this->app->addFlash('info', $this->app->trans('phraseanet::account The account has been deleted'));
+        }
+    }
+
     /**
      * @return PasswordEncoder
      */
@@ -500,4 +648,44 @@ class AccountController extends Controller
     {
         return $this->app['events-manager'];
     }
+
+    /**
+     * @return BasketManipulator
+     */
+    private function getBasketManipulator()
+    {
+        return $this->app['manipulator.basket'];
+    }
+
+    /**
+     * @return BasketRepository
+     */
+    private function getBasketRepository()
+    {
+        return $this->app['repo.baskets'];
+    }
+
+    /**
+     * @return ApiApplicationManipulator
+     */
+    private function getApiApplicationManipulator()
+    {
+        return $this->app['manipulator.api-application'];
+    }
+
+    /**
+     * @return FeedPublisherRepository
+     */
+    private function getFeedPublisherRepository()
+    {
+        return $this->app['repo.feed-publishers'];
+    }
+
+    /**
+     * @return ValidationSessionRepository
+     */
+    private function getValidationSessionRepository()
+    {
+        return $this->app['repo.validation-session'];
+    }
 }

lib/Alchemy/Phrasea/Controller/Root/LoginController.php

@@ -265,7 +265,7 @@ class LoginController extends Controller
         return $this->render('login/register-classic.html.twig', array_merge(
             $this->getDefaultTemplateVariables($request),
             [
-                'geonames_server_uri' => str_replace(sprintf('%s:', parse_url($url, PHP_URL_SCHEME)), '', $url),
+                'geonames_server_uri' => $url,
                 'form' => $form->createView()
             ]));
     }

lib/Alchemy/Phrasea/Controller/Root/SessionController.php

@@ -94,10 +94,9 @@ class SessionController extends Controller
     }
 
     /**
-     * Check session state
-     *
-     * @param  Request $request
+     * @param Request $request
      * @return JsonResponse
+     * @throws \Exception       in case "new \DateTime()" fails ?
      */
     public function updateSession(Request $request)
     {
@@ -120,7 +119,8 @@ class SessionController extends Controller
 
                 return $this->app->json($ret);
             }
-        } else {
+        }
+        else {
             $ret['status'] = 'disconnected';
 
             return $this->app->json($ret);
@@ -128,7 +128,8 @@ class SessionController extends Controller
 
         try {
             $this->getApplicationBox()->get_connection();
-        } catch (\Exception $e) {
+        }
+        catch (\Exception $e) {
             return $this->app->json($ret);
         }
 
@@ -148,8 +149,9 @@ class SessionController extends Controller
             $module->setModuleId($moduleId);
             $module->setSession($session);
             $manager->persist($module);
-        } else {
-            $manager->persist($session->getModuleById($moduleId)->setUpdated(new \DateTime()));
+        }
+        else {
+            $manager->persist($session->getModuleById($moduleId)->setUpdated($now));
         }
 
         $manager->persist($session);
@@ -231,7 +233,10 @@ class SessionController extends Controller
      */
     private function getBasketRepository()
     {
-        return $this->getEntityManager()->getRepository('Phraseanet:Basket');
+        /** @var BasketRepository $ret */
+        $ret = $this->getEntityManager()->getRepository('Phraseanet:Basket');
+
+        return $ret;
     }
 
     /**

lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusController.php

@@ -806,7 +806,7 @@ class ThesaurusController extends Controller
                         if (!$t) {
                             $t = "...";
                         }
-                        $fullBranch = " / " . $t . $fullBranch;
+                        $fullBranch = " / " . htmlspecialchars($t) . $fullBranch;
                     }
                 }
                 $nodes = $xpathstruct->query("/record/description/*");
@@ -1159,7 +1159,7 @@ class ThesaurusController extends Controller
             '1',
             null
         );
-        $fullpath = $dom->getElementsByTagName("fullpath_html")->item(0)->firstChild->nodeValue;
+        $fullpathHtml = $dom->getElementsByTagName("fullpath_html")->item(0)->firstChild->nodeValue;
         $hits = $dom->getElementsByTagName("allhits")->item(0)->firstChild->nodeValue;
 
         $languages = $synonyms = [];
@@ -1180,6 +1180,16 @@ class ThesaurusController extends Controller
             $languages[$lng_code[0]] = $language;
         }
 
+        //  Escape path  between span tag in fullpath_html
+        preg_match_all("'(<[^><]*>)(.*?)(<[^><]*>)'", $fullpathHtml, $matches, PREG_SET_ORDER);
+
+        $safeFullpath = '';
+        foreach($matches as $match) {
+            unset($match[0]);  // full match result not used
+            $match[2] = htmlspecialchars($match[2]);
+            $safeFullpath .= implode('', $match);
+        }
+
         return $this->render('thesaurus/properties.html.twig', [
             'typ' => $request->get('typ'),
             'bid' => $request->get('bid'),
@@ -1187,7 +1197,7 @@ class ThesaurusController extends Controller
             'id' => $request->get('id'),
             'dlg' => $request->get('dlg'),
             'languages' => $languages,
-            'fullpath' => $fullpath,
+            'fullpath' => $safeFullpath,
             'hits' => $hits,
             'synonyms' => $synonyms,
         ]);

lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusXmlHttpController.php

@@ -1426,12 +1426,7 @@ class ThesaurusXmlHttpController extends Controller
 
     public function searchTermJson(Request $request)
     {
-        if (null === $lng = $request->get('lng')) {
-            $data = explode('_', $this->app['locale']);
-            if (count($data) > 0) {
-                $lng = $data[0];
-            }
-        }
+        $lng = $request->get('lng');
 
         $html = '';
         $sbid = (int) $request->get('sbid');
@@ -1485,7 +1480,10 @@ class ThesaurusXmlHttpController extends Controller
                         $q2 .= ' and starts-with(@k, \'' . \thesaurus::xquery_escape($unicode->remove_indexer_chars($t[1])) . '\')';
                     }
 
-					$q2 .= ' and @lng=\'' . \thesaurus::xquery_escape($lng) . '\'';
+                    if($lng != null){
+                        $q2 .= ' and @lng=\'' . \thesaurus::xquery_escape($lng) . '\'';
+                    }
+
                     $q .= ('//sy[' . $q2 . ']');
 
                     $nodes = $xpath->query($q);

lib/Alchemy/Phrasea/ControllerProvider/ControllerProviderServiceProvider.php

@@ -77,6 +77,7 @@ class ControllerProviderServiceProvider implements ServiceProviderInterface
             Prod\Push::class => [],
             Prod\Query::class => [],
             Prod\Record::class => [],
+            \Alchemy\Phrasea\Report\ControllerProvider\ProdReportControllerProvider::class => [],
             Prod\Root::class => [],
             Prod\Share::class => [],
             Prod\Story::class => [],
@@ -87,8 +88,6 @@ class ControllerProviderServiceProvider implements ServiceProviderInterface
             Prod\Upload::class => [],
             Prod\UsrLists::class => [],
             Prod\WorkZone::class => [],
-            Report\Activity::class => [],
-            Report\Information::class => [],
             Report\Root::class => [],
             Root\Account::class => [],
             Root\Developers::class => [],

lib/Alchemy/Phrasea/ControllerProvider/Prod/Lazaret.php

@@ -82,6 +82,11 @@ class Lazaret implements ControllerProviderInterface, ServiceProviderInterface
             ->assert('file_id', '\d+')
             ->bind('lazaret_thumbnail');
 
+        $controllers->get('/{databox_id}/{record_id}/status', 'controller.prod.lazaret:getDestinationStatus')
+            ->assert('databox_id', '\d+')
+            ->assert('record_id', '\d+')
+            ->bind('lazaret_destination_status');
+
         return $controllers;
     }
 }

lib/Alchemy/Phrasea/ControllerProvider/Prod/Property.php

@@ -25,7 +25,9 @@ class Property implements ControllerProviderInterface, ServiceProviderInterface
     public function register(Application $app)
     {
         $app['controller.prod.property'] = $app->share(function (PhraseaApplication $app) {
-            return (new PropertyController($app));
+            return (new PropertyController($app))
+                ->setDataboxLoggerLocator($app['phraseanet.logger'])
+            ;
         });
     }
 

lib/Alchemy/Phrasea/ControllerProvider/Prod/Upload.php

@@ -66,6 +66,9 @@ class Upload implements ControllerProviderInterface, ServiceProviderInterface
         $controllers->get('/html5-version/', 'controller.prod.upload:getHtml5UploadForm')
             ->bind('upload_html5_form');
 
+        $controllers->get('/head/', 'controller.prod.upload:getHead')
+            ->bind('upload_head');
+
         $controllers->post('/', 'controller.prod.upload:upload')
             ->bind('upload');
 

lib/Alchemy/Phrasea/ControllerProvider/Report/Activity.php

@@ -1,81 +0,0 @@
-<?php
-
-/*
- * This file is part of Phraseanet
- *
- * (c) 2005-2016 Alchemy
- *
- * For the full copyright and license information, please view the LICENSE
- * file that was distributed with this source code.
- */
-
-namespace Alchemy\Phrasea\ControllerProvider\Report;
-
-use Alchemy\Phrasea\Application as PhraseaApplication;
-use Alchemy\Phrasea\Controller\Report\ActivityController;
-use Alchemy\Phrasea\ControllerProvider\ControllerProviderTrait;
-use Silex\Application;
-use Silex\ControllerProviderInterface;
-use Silex\ServiceProviderInterface;
-
-class Activity implements ControllerProviderInterface, ServiceProviderInterface
-{
-    use ControllerProviderTrait;
-
-    public function register(Application $app)
-    {
-        $app['controller.report.activity'] = $app->share(function (PhraseaApplication $app) {
-            return new ActivityController($app);
-        });
-    }
-
-    public function boot(Application $app)
-    {
-        // no-op
-    }
-
-    public function connect(Application $app)
-    {
-        $controllers = $this->createAuthenticatedCollection($app);
-        $firewall = $this->getFirewall($app);
-
-        $controllers->before(function () use ($firewall) {
-            $firewall->requireAccessToModule('report');
-        });
-
-        $controllers->post('/users/connexions', 'controller.report.activity:doReportConnexionsByUsers')
-            ->bind('report_activity_users_connexions');
-
-        $controllers->post('/users/downloads', 'controller.report.activity:doReportDownloadsByUsers')
-            ->bind('report_activity_users_downloads');;
-
-        $controllers->post('/questions/best-of', 'controller.report.activity:doReportBestOfQuestions')
-            ->bind('report_activity_questions_bestof');
-
-        $controllers->post('/questions/no-best-of', 'controller.report.activity:doReportNoBestOfQuestions')
-            ->bind('report_activity_questions_nobestof');
-
-        $controllers->post('/instance/hours', 'controller.report.activity:doReportSiteActiviyPerHours')
-            ->bind('report_activity_instance_hours');
-
-        $controllers->post('/instance/days', 'controller.report.activity:doReportSiteActivityPerDays')
-            ->bind('report_activity_instance_days');
-
-        $controllers->post('/documents/pushed', 'controller.report.activity:doReportPushedDocuments')
-            ->bind('report_activity_documents_pushed');
-
-        $controllers->post('/documents/added', 'controller.report.activity:doReportAddedDocuments')
-            ->bind('report_activity_documents_added');
-
-        $controllers->post('/documents/edited', 'controller.report.activity:doReportEditedDocuments')
-            ->bind('report_activity_documents_edited');
-
-        $controllers->post('/documents/validated', 'controller.report.activity:doReportValidatedDocuments')
-            ->bind('report_activity_documents_validated');
-
-        $controllers->post('/documents/sent', 'controller.report.activity:doReportSentDocuments')
-            ->bind('report_activity_documents_sent');
-
-        return $controllers;
-    }
-}

lib/Alchemy/Phrasea/ControllerProvider/Report/Information.php

@@ -1,57 +0,0 @@
-<?php
-
-/*
- * This file is part of Phraseanet
- *
- * (c) 2005-2016 Alchemy
- *
- * For the full copyright and license information, please view the LICENSE
- * file that was distributed with this source code.
- */
-
-namespace Alchemy\Phrasea\ControllerProvider\Report;
-
-use Alchemy\Phrasea\Application as PhraseaApplication;
-use Alchemy\Phrasea\Controller\Report\InformationController;
-use Alchemy\Phrasea\ControllerProvider\ControllerProviderTrait;
-use Silex\Application;
-use Silex\ControllerProviderInterface;
-use Silex\ServiceProviderInterface;
-
-class Information implements ControllerProviderInterface, ServiceProviderInterface
-{
-    use ControllerProviderTrait;
-
-    public function register(Application $app)
-    {
-        $app['controller.report.information'] = $app->share(function (PhraseaApplication $app) {
-            return new InformationController($app);
-        });
-    }
-
-    public function boot(Application $app)
-    {
-        // no-op
-    }
-
-    public function connect(Application $app)
-    {
-        $controllers = $this->createAuthenticatedCollection($app);
-        $firewall = $this->getFirewall($app);
-
-        $controllers->before(function () use ($firewall) {
-            $firewall->requireAccessToModule('report');
-        });
-
-        $controllers->post('/user', 'controller.report.information:doReportInformationUser')
-            ->bind('report_infomations_user');
-
-        $controllers->post('/browser', 'controller.report.information:doReportInformationBrowser')
-            ->bind('report_infomations_browser');
-
-        $controllers->post('/document', 'controller.report.information:doReportInformationDocument')
-            ->bind('report_infomations_document');
-
-        return $controllers;
-    }
-}

lib/Alchemy/Phrasea/ControllerProvider/Root/Account.php

@@ -52,6 +52,14 @@ class Account implements ControllerProviderInterface, ServiceProviderInterface
         $controllers->get('/', 'account.controller:displayAccount')
             ->bind('account');
 
+        // allow to delete phraseanet account
+        $controllers->get('/delete/process', 'account.controller:processDeleteAccount')
+            ->bind('account_process_delete');
+
+        $controllers->get('/delete/confirm', 'account.controller:confirmDeleteAccount')
+            ->bind('account_confirm_delete');
+
+
         // Updates current logged in user account
         $controllers->post('/', 'account.controller:updateAccount')
             ->bind('submit_update_account');

lib/Alchemy/Phrasea/Core/Configuration/DisplaySettingService.php

@@ -41,7 +41,8 @@ class DisplaySettingService
         'css'                     => '000000',
         'start_page_query'        => '',
         'order_collection_by'     => self::ORDER_BY_ADMIN,
-        'start_page'              => 'QUERY',
+        'start_page'              => 'LAST_QUERY',
+        'last_jsonquery'          => '',
         'rollover_thumbnail'      => 'caption',
         'technical_display'       => '1',
         'doctype_display'         => '1',

lib/Alchemy/Phrasea/Core/Configuration/RegistrationManager.php

@@ -176,7 +176,7 @@ class RegistrationManager
         }
 
         if ($isTimeLimited && $isOutDated && $isPending) {
-            $userRegistration['type'] = 'out-time';
+            $userRegistration['type'] = 'out-dated';
 
             return  $userRegistration;
         }

lib/Alchemy/Phrasea/Core/Configuration/RegistryFormManipulator.php

@@ -146,7 +146,7 @@ class RegistryFormManipulator
             ],
             'webservices'  => [
                 'google-charts-enabled' => true,
-                'geonames-server' => 'http://geonames.alchemyasp.com/',
+                'geonames-server' => 'https://geonames.alchemyasp.com/',
                 'captchas-enabled' => false,
                 'recaptcha-public-key' => '',
                 'recaptcha-private-key' => '',

lib/Alchemy/Phrasea/Core/Database/DatabaseMaintenanceService.php

@@ -66,6 +66,8 @@ class DatabaseMaintenanceService
 
     public function upgradeDatabase(\base $base, $applyPatches)
     {
+       $this->reconnect();
+
         $recommends = [];
         $allTables = [];
 
@@ -126,6 +128,8 @@ class DatabaseMaintenanceService
      */
     public function alterTableEngine($tableName, $engine, array & $recommends)
     {
+        $this->reconnect();
+
         $sql = 'ALTER TABLE `' . $tableName . '` ENGINE = ' . $engine;
 
         try {
@@ -145,6 +149,8 @@ class DatabaseMaintenanceService
      */
     public function createTable(\SimpleXMLElement $table)
     {
+        $this->reconnect();
+
         $field_stmt = $defaults_stmt = [];
 
         $create_stmt = "CREATE TABLE IF NOT EXISTS `" . $table['name'] . "` (";
@@ -270,6 +276,8 @@ class DatabaseMaintenanceService
 
     public function upgradeTable(\SimpleXMLElement $table)
     {
+        $this->reconnect();
+
         $correct_table = ['fields' => [], 'indexes' => [], 'collation' => []];
         $alter = $alter_pre = $return = [];
 
@@ -476,6 +484,8 @@ class DatabaseMaintenanceService
         }
 
         foreach ($alter_pre as $a) {
+            $this->reconnect();
+
             try {
                 $this->connection->exec($a);
             } catch (\Exception $e) {
@@ -488,6 +498,8 @@ class DatabaseMaintenanceService
         }
 
         foreach ($alter as $a) {
+            $this->reconnect();
+
             try {
                 $this->connection->exec($a);
             } catch (\Exception $e) {
@@ -561,6 +573,7 @@ class DatabaseMaintenanceService
         $this->app['swiftmailer.transport'] = null;
 
         foreach ($list_patches as $patch) {
+
             // Gets doctrine migrations required for current patch
             foreach ($patch->getDoctrineMigrations() as $doctrineVersion) {
                 /** @var \Doctrine\DBAL\Migrations\Version $version */
@@ -579,6 +592,8 @@ class DatabaseMaintenanceService
 
                     // Execute migration if not marked as migrated and not already applied by an older patch
                     if (!$migration->isAlreadyApplied()) {
+                        $this->reconnect();
+
                         $version->execute('up');
                         continue;
                     }
@@ -586,10 +601,14 @@ class DatabaseMaintenanceService
                     // Or mark it as migrated
                     $version->markMigrated();
                 } else {
+                    $this->reconnect();
+
                     $version->execute('up');
                 }
             }
 
+            $this->reconnect();
+
             if (false === $patch->apply($base, $this->app)) {
                 $success = false;
             }
@@ -597,4 +616,12 @@ class DatabaseMaintenanceService
 
         return $success;
     }
+
+    private function reconnect()
+    {
+        if($this->connection->ping() === false) {
+            $this->connection->close();
+            $this->connection->connect();
+        }
+    }
 }

lib/Alchemy/Phrasea/Core/Provider/RepositoriesServiceProvider.php

@@ -66,6 +66,9 @@ class RepositoriesServiceProvider implements ServiceProviderInterface
         $app['repo.validation-participants'] = $app->share(function (PhraseaApplication $app) {
             return $app['orm.em']->getRepository('Phraseanet:ValidationParticipant');
         });
+        $app['repo.validation-session'] = $app->share(function (PhraseaApplication $app) {
+            return $app['orm.em']->getRepository('Phraseanet:ValidationSession');
+        });
         $app['repo.story-wz'] = $app->share(function (PhraseaApplication $app) {
             return $app['orm.em']->getRepository('Phraseanet:StoryWZ');
         });

lib/Alchemy/Phrasea/Core/Version.php

@@ -16,7 +16,7 @@ class Version
     /**
      * @var string
      */
-    private $number = '4.1.0-alpha.12';
+    private $number = '4.1.0-alpha.18a';
 
     /**
      * @var string

lib/Alchemy/Phrasea/Databox/Field/DbalDataboxFieldRepository.php

@@ -36,6 +36,9 @@ final class DbalDataboxFieldRepository implements DataboxFieldRepository
         'label_fr',
         'label_de',
         'label_nl',
+        'generate_cterms',
+        'gui_editable',
+        'gui_visible',
     ];
 
     /** @var DataboxFieldFactory */

lib/Alchemy/Phrasea/Filesystem/FilesystemService.php

@@ -201,6 +201,8 @@ class FilesystemService
                 return 'ogg';
             case 'libmp3lame':
                 return 'mp3';
+            case 'pcm_s16le':
+                return 'wav';
         }
 
         return null;

lib/Alchemy/Phrasea/Helper/WorkZone.php

@@ -11,28 +11,29 @@
 
 namespace Alchemy\Phrasea\Helper;
 
-use Doctrine\Common\Collections\ArrayCollection;
 use Alchemy\Phrasea\Model\Entities\Basket as BasketEntity;
+use Alchemy\Phrasea\Model\Repositories\BasketRepository;
+use Alchemy\Phrasea\Model\Repositories\StoryWZRepository;
+use Doctrine\Common\Collections\ArrayCollection;
 
 class WorkZone extends Helper
 {
-    const BASKETS = 'baskets';
-    const STORIES = 'stories';
+    const BASKETS     = 'baskets';
+    const STORIES     = 'stories';
     const VALIDATIONS = 'validations';
 
     /**
-     *
      * Returns an ArrayCollection containing three keys :
-     *    - self::BASKETS : an ArrayCollection of the actives baskets
-     *     (Non Archived)
+     *    - self::BASKETS : an ArrayCollection of the actives baskets (Non Archived)
      *    - self::STORIES : an ArrayCollection of working stories
      *    - self::VALIDATIONS : the validation people are waiting from me
      *
-     * @return \Doctrine\Common\Collections\ArrayCollection
+     * @param  null|string $sort     "date"|"name"
+     * @return ArrayCollection
      */
-    public function getContent($sort)
+    public function getContent($sort = null)
     {
-        /* @var $repo_baskets Alchemy\Phrasea\Model\Repositories\BasketRepository */
+        /* @var $repo_baskets BasketRepository */
         $repo_baskets = $this->app['repo.baskets'];
 
         $sort = in_array($sort, ['date', 'name']) ? $sort : 'name';
@@ -42,7 +43,7 @@ class WorkZone extends Helper
         $baskets = $repo_baskets->findActiveByUser($this->app->getAuthenticatedUser(), $sort);
 
         // force creation of a default basket
-        if (0 === count($baskets)) {
+        if (count($baskets) === 0) {
             $basket = new BasketEntity();
 
             $basket->setName($this->app->trans('Default basket'));
@@ -55,7 +56,7 @@ class WorkZone extends Helper
 
         $validations = $repo_baskets->findActiveValidationByUser($this->app->getAuthenticatedUser(), $sort);
 
-        /* @var $repo_stories Alchemy\Phrasea\Model\Repositories\StoryWZRepository */
+        /* @var $repo_stories StoryWZRepository */
         $repo_stories = $this->app['repo.story-wz'];
 
         $stories = $repo_stories->findByUser($this->app, $this->app->getAuthenticatedUser(), $sort);

lib/Alchemy/Phrasea/Media/Subdef/Audio.php

@@ -19,6 +19,7 @@ class Audio extends Provider
     const OPTION_THREADS = 'threads';
     const OPTION_ACODEC = 'acodec';
     const OPTION_AUDIOSAMPLERATE = 'audiosamplerate';
+    const OPTION_AUDIOCHANNEL = 'audiochannel';
 
     public function __construct(TranslatorInterface $translator)
     {
@@ -29,9 +30,12 @@ class Audio extends Provider
             47250, 48000, 50000, 50400, 88200, 96000
         ];
 
+        $audioChannel = ['mono', 'stereo'];
+
         $this->registerOption(new OptionType\Range($this->translator->trans('Audio Birate'), self::OPTION_AUDIOBITRATE, 32, 320, 128, 32));
         $this->registerOption(new OptionType\Enum($this->translator->trans('AudioSamplerate'), self::OPTION_AUDIOSAMPLERATE, $AVaudiosamplerate));
-        $this->registerOption(new OptionType\Enum($this->translator->trans('Audio Codec'), self::OPTION_ACODEC, ['libmp3lame', 'flac'], 'libmp3lame'));
+        $this->registerOption(new OptionType\Enum($this->translator->trans('Audio Codec'), self::OPTION_ACODEC, ['libmp3lame', 'flac', 'pcm_s16le'], 'libmp3lame'));
+        $this->registerOption(new OptionType\Enum($this->translator->trans('Audio channel'), self::OPTION_AUDIOCHANNEL, $audioChannel));
     }
 
     public function getType()
@@ -53,7 +57,21 @@ class Audio extends Provider
         $this->spec->setAudioCodec($this->getOption(self::OPTION_ACODEC)->getValue());
         $this->spec->setAudioSampleRate($this->getOption(self::OPTION_AUDIOSAMPLERATE)->getValue());
         $this->spec->setAudioKiloBitrate($this->getOption(self::OPTION_AUDIOBITRATE)->getValue());
+        $this->spec->setAudioChannels($this->getChannelNumber($this->getOption(self::OPTION_AUDIOCHANNEL)->getValue()));
 
         return $this->spec;
     }
+
+    private function getChannelNumber($audioChannel)
+    {
+        switch($audioChannel)
+        {
+            case 'mono':
+                return 1;
+            case 'stereo':
+                return 2;
+            default:
+                return null;
+        }
+    }
 }

lib/Alchemy/Phrasea/Media/SubdefGenerator.php

@@ -61,7 +61,7 @@ class SubdefGenerator
 
     public function generateSubdefs(\record_adapter $record, array $wanted_subdefs = null)
     {
-        if ($record->get_hd_file() !== null) {
+        if ($record->get_hd_file() !== null && $record->get_hd_file()->getMimeType() == "application/x-indesign") {
             $mediaSource = $this->mediavorus->guess($record->get_hd_file()->getPathname());
             $metadatas = $mediaSource->getMetadatas();
 
@@ -69,15 +69,27 @@ class SubdefGenerator
                 if(!isset($this->tmpFilesystem)){
                     $this->tmpFilesystem = Manager::create();
                 }
-                $tmpDir = $this->tmpFilesystem->createTemporaryDirectory();
+                $tmpDir = $this->tmpFilesystem->createTemporaryDirectory(0777, 500);
 
-                try {
-                    $this->app['filesystem']->dumpFile($tmpDir.'/file.jpg', $metadatas->get('XMP-xmp:PageImage')->getValue()->asString());
-                    $this->tmpFilePath = $tmpDir.'/file.jpg';
-                } catch (\Exception $e) {
-                    $this->logger->error(sprintf('Unable to write temporary file : %s', $e->getMessage()));
+                $files = $this->app['exiftool.preview-extractor']->extract($record->get_hd_file()->getPathname(), $tmpDir);
+
+                $selected = null;
+                $size = null;
+
+                foreach ($files as $file) {
+                    if ($file->isDir() || $file->isDot()) {
+                        continue;
+                    }
+
+                    if (is_null($selected) || $file->getSize() > $size) {
+                        $selected = $file->getPathname();
+                        $size = $file->getSize();
+                    }
                 }
 
+                if ($selected) {
+                    $this->tmpFilePath =  $selected;
+                }
             }
         }
 

lib/Alchemy/Phrasea/Media/SubdefSubstituer.php

@@ -73,6 +73,9 @@ class SubdefSubstituer
 
         $this->createMediaSubdef($record, 'document', $media);
 
+        $record->setMimeType($media->getFile()->getMimeType());
+        $record->setType($media->getType());
+
         $record->write_metas();
 
         if ($shouldSubdefsBeRebuilt) {

lib/Alchemy/Phrasea/Metadata/PhraseanetMetadataSetter.php

@@ -14,6 +14,7 @@ namespace Alchemy\Phrasea\Metadata;
 use Alchemy\Phrasea\Border\File;
 use Alchemy\Phrasea\Databox\DataboxRepository;
 use Alchemy\Phrasea\Metadata\Tag\NoSource;
+use DateTime;
 use PHPExiftool\Driver\Metadata\Metadata;
 
 class PhraseanetMetadataSetter
@@ -66,8 +67,16 @@ class PhraseanetMetadataSetter
                     continue;
                 }
 
-                $data['value'] = $value;
+                if ($field->get_type() == 'date') {
+                    try {
+                        $dateTime = new DateTime($value);
+                        $value = $dateTime->format('Y/m/d H:i:s');
+                    } catch (\Exception $e) {
+                        // $value unchanged
+                    }
+                }
 
+                $data['value'] = $value;
                 $metadataInRecordFormat[] = $data;
             }
         }

lib/Alchemy/Phrasea/Model/Entities/LazaretFile.php

@@ -12,6 +12,7 @@
 namespace Alchemy\Phrasea\Model\Entities;
 
 use Alchemy\Phrasea\Application;
+use Alchemy\Phrasea\Border\Attribute\AttributeInterface;
 use Doctrine\ORM\Mapping as ORM;
 use Gedmo\Mapping\Annotation as Gedmo;
 use \record_adapter;
@@ -474,4 +475,32 @@ class LazaretFile
         return $merged;
     }
 
+    /**
+     * @param Application $app
+     * @return array|null
+     */
+    public function getStatus(Application $app)
+    {
+        /**@var LazaretAttribute $atribute*/
+        foreach ($this->attributes as $atribute) {
+            if ($atribute->getName() == AttributeInterface::NAME_STATUS) {
+                $databox = $this->getCollection($app)->get_databox();
+                $statusStructure = $databox->getStatusStructure();
+                $recordsStatuses = [];
+                foreach ($statusStructure as $status) {
+                    $bit = $status['bit'];
+                    if (!isset($recordsStatuses[$bit])) {
+                        $recordsStatuses[$bit] = $status;
+                    }
+                    $statusSet = \databox_status::bitIsSet(bindec($atribute->getValue()), $bit);
+                    if (!isset($recordsStatuses[$bit]['flag'])) {
+                        $recordsStatuses[$bit]['flag'] = (int) $statusSet;
+                    }
+                }
+                return $recordsStatuses;
+            }
+        }
+        return null;
+    }
+
 }

lib/Alchemy/Phrasea/Model/Entities/ValidationSession.php

@@ -18,7 +18,7 @@ use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
 
 /**
  * @ORM\Table(name="ValidationSessions")
- * @ORM\Entity
+ * @ORM\Entity(repositoryClass="Alchemy\Phrasea\Model\Repositories\ValidationSessionRepository")
  */
 class ValidationSession
 {

lib/Alchemy/Phrasea/Model/Manipulator/ApiApplicationManipulator.php

@@ -57,6 +57,14 @@ class ApiApplicationManipulator implements ManipulatorInterface
         $this->om->flush();
     }
 
+    public function deleteApiApplications(array $applications)
+    {
+        foreach ($applications as $application) {
+            $this->om->remove($application);
+        }
+        $this->om->flush();
+    }
+
     public function update(ApiApplication $application)
     {
         $this->om->persist($application);

lib/Alchemy/Phrasea/Model/Manipulator/BasketManipulator.php

@@ -118,4 +118,12 @@ class BasketManipulator
         $this->manager->remove($basket);
         $this->manager->flush();
     }
+
+    public function removeBaskets(array $baskets)
+    {
+        foreach ($baskets as $basket) {
+            $this->manager->remove($basket);
+        }
+        $this->manager->flush();
+    }
 }

lib/Alchemy/Phrasea/Model/Manipulator/LazaretManipulator.php

@@ -226,6 +226,8 @@ class LazaretManipulator
             $this->entityManager->remove($lazaretFile);
             $this->entityManager->flush();
 
+            $ret['result']['record_id'] = $record->getRecordId();
+
             $ret['success'] = true;
         } catch (\Exception $e) {
             $ret['message'] = $this->app->trans('An error occured');

lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php

@@ -26,6 +26,7 @@ class TokenManipulator implements ManipulatorInterface
     const TYPE_FEED_ENTRY = 'FEED_ENTRY';
     const TYPE_PASSWORD = 'password';
     const TYPE_ACCOUNT_UNLOCK = 'account-unlock';
+    const TYPE_ACCOUNT_DELETE = 'account-delete';
     const TYPE_DOWNLOAD = 'download';
     const TYPE_MAIL_DOWNLOAD = 'mail-download';
     const TYPE_EMAIL = 'email';
@@ -167,6 +168,16 @@ class TokenManipulator implements ManipulatorInterface
         return $this->create($user, self::TYPE_ACCOUNT_UNLOCK, new \DateTime('+3 days'));
     }
 
+    /**
+     * @param User $user
+     *
+     * @return Token
+     */
+    public function createAccountDeleteToken(User $user, $email)
+    {
+        return $this->create($user, self::TYPE_ACCOUNT_DELETE, new \DateTime('+1 hour'), $email);
+    }
+
     /**
      * @param User $user
      *

lib/Alchemy/Phrasea/Model/Repositories/BasketRepository.php

@@ -54,21 +54,24 @@ class BasketRepository extends EntityRepository
     /**
      * Returns all basket for a given user that are not marked as archived
      *
-     * @param  User $user
+     * @param User $user
+     * @param null|string $sort
      * @return Basket[]
      */
     public function findActiveByUser(User $user, $sort = null)
     {
-        $dql = 'SELECT b
-            FROM Phraseanet:Basket b
-            LEFT JOIN b.elements e
-            WHERE b.user = :usr_id
-            AND b.archived = false';
+        // checked : 4 usages, "b.elements" is useless
+        $dql = "SELECT b\n"
+            . " FROM Phraseanet:Basket b\n"
+            // . " LEFT JOIN b.elements e\n"    //
+            . " WHERE b.user = :usr_id\n"
+            . " AND b.archived = false";
 
         if ($sort == 'date') {
-            $dql .= ' ORDER BY b.created DESC';
-        } elseif ($sort == 'name') {
-            $dql .= ' ORDER BY b.name ASC';
+            $dql .= "\n ORDER BY b.created DESC";
+        }
+        elseif ($sort == 'name') {
+            $dql .= "\n ORDER BY b.name ASC";
         }
 
         $query = $this->_em->createQuery($dql);
@@ -80,24 +83,27 @@ class BasketRepository extends EntityRepository
     /**
      * Returns all unread basket for a given user that are not marked as archived
      *
-     * @param  User                                         $user
+     * @param  User $user
      * @return Basket[]
      */
     public function findUnreadActiveByUser(User $user)
     {
-        $dql = 'SELECT b
-            FROM Phraseanet:Basket b
-            JOIN b.elements e
-            LEFT JOIN b.validation s
-            LEFT JOIN s.participants p
-            WHERE b.archived = false
-            AND (
-              (b.user = :usr_id_owner AND b.isRead = false)
-              OR (b.user != :usr_id_ownertwo
-                  AND p.user = :usr_id_participant
-                  AND p.is_aware = false)
-              )
-            AND (s.expires IS NULL OR s.expires > CURRENT_TIMESTAMP())';
+        // checked : 2 usages, "b.elements" is useless
+        $dql = "SELECT b\n"
+            . " FROM Phraseanet:Basket b\n"
+            // . "  JOIN b.elements e\n"
+            . "  LEFT JOIN b.validation s\n"
+            . "  LEFT JOIN s.participants p\n"
+            . " WHERE b.archived = false\n"
+            . " AND (\n"
+            . "   (b.user = :usr_id_owner AND b.isRead = false)\n"
+            . "    OR \n"
+            . "   (b.user != :usr_id_ownertwo\n"
+            . "      AND p.user = :usr_id_participant\n"
+            . "      AND p.is_aware = false\n"
+            . "      AND s.expires > CURRENT_TIMESTAMP()\n"
+            . "   )\n"
+            . " )";
 
         $params = [
             'usr_id_owner'       => $user->getId(),
@@ -115,11 +121,22 @@ class BasketRepository extends EntityRepository
      * Returns all baskets that are in validation session not expired  and
      * where a specified user is participant (not owner)
      *
-     * @param  User                                         $user
+     * @param  User         $user
+     * @param  null|string  $sort
      * @return Basket[]
      */
     public function findActiveValidationByUser(User $user, $sort = null)
     {
+        // checked : 2 usages, "b.elements" seems useless.
+        $dql = "SELECT b\n"
+            . "FROM Phraseanet:Basket b\n"
+            // . "  JOIN b.elements e\n"
+            // . "  JOIN e.validation_datas v\n"
+            . "  JOIN b.validation s\n"
+            . "  JOIN s.participants p\n"
+            . "WHERE b.user != ?1 AND p.user = ?2\n"
+            . "  AND (s.expires IS NULL OR s.expires > CURRENT_TIMESTAMP())";
+
         $dql = 'SELECT b
             FROM Phraseanet:Basket b
             JOIN b.elements e
@@ -130,9 +147,9 @@ class BasketRepository extends EntityRepository
              AND (s.expires IS NULL OR s.expires > CURRENT_TIMESTAMP()) ';
 
         if ($sort == 'date') {
-            $dql .= ' ORDER BY b.created DESC';
+            $dql .= "\nORDER BY b.created DESC";
         } elseif ($sort == 'name') {
-            $dql .= ' ORDER BY b.name ASC';
+            $dql .= "\nORDER BY b.name ASC";
         }
 
         $query = $this->_em->createQuery($dql);
@@ -152,10 +169,11 @@ class BasketRepository extends EntityRepository
      */
     public function findUserBasket($basket_id, User $user, $requireOwner)
     {
-        $dql = 'SELECT b
-            FROM Phraseanet:Basket b
-            LEFT JOIN b.elements e
-            WHERE b.id = :basket_id';
+        // checked : 3 usages, "b.elements e" seems useless
+        $dql = "SELECT b\n"
+            . " FROM Phraseanet:Basket b\n"
+            // . " LEFT JOIN b.elements e\n"
+            . " WHERE b.id = :basket_id";
 
         $query = $this->_em->createQuery($dql);
         $query->setParameters(['basket_id' => $basket_id]);
@@ -188,7 +206,7 @@ class BasketRepository extends EntityRepository
 
     public function findContainingRecordForUser(\record_adapter $record, User $user)
     {
-
+        // todo : check "e.sbas_id = e.sbas_id" ???
         $dql = 'SELECT b
             FROM Phraseanet:Basket b
             JOIN b.elements e
@@ -210,30 +228,31 @@ class BasketRepository extends EntityRepository
     {
         switch ($type) {
             case self::RECEIVED:
-                $dql = 'SELECT b
-                FROM Phraseanet:Basket b
-                JOIN b.elements e
-                WHERE b.user = :usr_id AND b.pusher_id IS NOT NULL';
+                // todo : check when called, and if "LEFT JOIN b.elements e" is usefull
+                $dql = "SELECT b\n"
+                    . "FROM Phraseanet:Basket b\n"
+                    . "  JOIN b.elements e\n"
+                    . "WHERE b.user = :usr_id AND b.pusher_id IS NOT NULL";
                 $params = [
                     'usr_id' => $user->getId()
                 ];
                 break;
             case self::VALIDATION_DONE:
-                $dql = 'SELECT b
-                FROM Phraseanet:Basket b
-                JOIN b.elements e
-                JOIN b.validation s
-                JOIN s.participants p
-                WHERE b.user != ?1 AND p.user = ?2';
+                // todo : check when called, and if "LEFT JOIN b.elements e" is usefull
+                $dql = "SELECT b\n"
+                    . "FROM Phraseanet:Basket b\n"
+                    . "  JOIN b.elements e\n"
+                    . "  JOIN b.validation s\n"
+                    . "  JOIN s.participants p\n"
+                    . "WHERE b.user != ?1 AND p.user = ?2";
                 $params = [
-                    1       => $user->getId()
-                    , 2       => $user->getId()
+                    1 => $user->getId(),
+                    2 => $user->getId()
                 ];
                 break;
             case self::VALIDATION_SENT:
                 $dql = 'SELECT b
                 FROM Phraseanet:Basket b
-                JOIN b.elements e
                 JOIN b.validation v
                 WHERE b.user = :usr_id';
                 $params = [
@@ -243,7 +262,6 @@ class BasketRepository extends EntityRepository
             case self::MYBASKETS:
                 $dql = 'SELECT b
                 FROM Phraseanet:Basket b
-                LEFT JOIN b.elements e
                 LEFT JOIN b.validation s
                 LEFT JOIN s.participants p
                 WHERE (b.user = :usr_id)';
@@ -252,6 +270,7 @@ class BasketRepository extends EntityRepository
                 ];
                 break;
             default:
+                // todo : check when called, and if "LEFT JOIN b.elements e" is usefull
                 $dql = 'SELECT b
                 FROM Phraseanet:Basket b
                 LEFT JOIN b.elements e
@@ -297,6 +316,7 @@ class BasketRepository extends EntityRepository
      */
     public function findActiveValidationAndBasketByUser(User $user, $sort = null)
     {
+        // todo : check caller and if "LEFT JOIN b.elements e" is usefull
         $dql = 'SELECT b
             FROM Phraseanet:Basket b
             LEFT JOIN b.elements e

lib/Alchemy/Phrasea/Model/Repositories/ValidationSessionRepository.php

@@ -0,0 +1,24 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2014 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Model\Repositories;
+
+use Doctrine\ORM\EntityRepository;
+
+/**
+ * ValidationSessionRepository
+ *
+ * This class was generated by the Doctrine ORM. Add your own custom
+ * repository methods below.
+ */
+class ValidationSessionRepository extends EntityRepository
+{
+}

lib/Alchemy/Phrasea/Model/Types/BinaryString.php

@@ -12,6 +12,7 @@
 namespace Alchemy\Phrasea\Model\Types;
 
 use Doctrine\DBAL\Types\Type;
+use Doctrine\DBAL\Platforms\MySqlPlatform;
 use Doctrine\DBAL\Platforms\AbstractPlatform;
 
 class BinaryString extends Type
@@ -23,10 +24,21 @@ class BinaryString extends Type
         return static::BINARY_STRING;
     }
 
+    /**
+     * {@inheritdoc}
+     *
+     * @see: https://blog.vandenbrand.org/2015/06/25/creating-a-custom-doctrine-dbal-type-the-right-way/
+     *     about the reason of the COMMENT in the column
+     */
     public function getSQLDeclaration(array $fieldDeclaration, AbstractPlatform $platform)
     {
         if ($platform->getName() === 'mysql') {
-            return $platform->getVarcharTypeDeclarationSQL($fieldDeclaration)." ". $platform->getCollationFieldDeclaration('utf8_bin');
+            /** @var MySqlPlatform $platform */
+            return $platform->getVarcharTypeDeclarationSQL($fieldDeclaration)
+                // . " CHARACTER SET utf8"
+                . " " . $platform->getColumnCollationDeclarationSQL('utf8_bin')
+                . " COMMENT '(DC2Type:binary_string)'"
+                ;
         }
 
         return $platform->getVarcharTypeDeclarationSQL($fieldDeclaration);
@@ -39,4 +51,12 @@ class BinaryString extends Type
     {
         return $platform->getVarcharDefaultLength();
     }
+
+    /**
+     * @inheritdoc
+     */
+    public function requiresSQLCommentHint(AbstractPlatform $platform)
+    {
+        return true;
+    }
 }

lib/Alchemy/Phrasea/Notification/Mail/AbstractMail.php

@@ -17,6 +17,8 @@ use Alchemy\Phrasea\Notification\ReceiverInterface;
 
 abstract class AbstractMail implements MailInterface
 {
+    const MAIL_SKIN = 'default';
+
     /** @var Application */
     protected $app;
     /** @var EmitterInterface */
@@ -59,6 +61,7 @@ abstract class AbstractMail implements MailInterface
             'expiration'        => $this->getExpiration(),
             'buttonUrl'         => $this->getButtonURL(),
             'buttonText'        => $this->getButtonText(),
+            'mailSkin'          => $this->getMailSkin(),
         ]);
     }
 
@@ -166,6 +169,14 @@ abstract class AbstractMail implements MailInterface
         $this->url = $url;
     }
 
+    /**
+     * @return string
+     */
+    public function getMailSkin()
+    {
+        return self::MAIL_SKIN;
+    }
+
     /**
      * {@inheritdoc}
      */

lib/Alchemy/Phrasea/Notification/Mail/MailRequestAccountDelete.php

@@ -0,0 +1,105 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Notification\Mail;
+
+use Alchemy\Phrasea\Exception\LogicException;
+use Alchemy\Phrasea\Model\Entities\User;
+
+class MailRequestAccountDelete extends AbstractMailWithLink
+{
+    const MAIL_SKIN = 'warning';
+
+    /** @var User */
+    private $user;
+
+    /**
+     * Set the user owner
+     *
+     * @param User $userOwner
+     */
+    public function setUserOwner(User $userOwner)
+    {
+        $this->user = $userOwner;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getSubject()
+    {
+        return $this->app->trans('Email:deletion:request:subject Delete account confirmation');
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getMessage()
+    {
+        if (!$this->user) {
+            throw new LogicException('You must set a user before calling getMessage');
+        }
+
+        return $this->app->trans("Email:deletion:request:message Hello %civility% %firstName% %lastName%.
+            We have received an account deletion request for your account on %urlInstance%, please confirm this deletion by clicking on the link below.
+            If you are not at the origin of this request, please change your password as soon as possible %resetPassword%
+            Link is valid for one hour.", [
+            '%civility%' => $this->getOwnerCivility(),
+            '%firstName%'=> $this->user->getFirstName(),
+            '%lastName%' => $this->user->getLastName(),
+            '%urlInstance%' => '<a href="'.$this->getPhraseanetURL().'">'.$this->getPhraseanetURL().'</a>',
+            '%resetPassword%' => '<a href="'.$this->app->url('reset_password').'">'.$this->app->url('reset_password').'</a>',
+        ]);
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getButtonText()
+    {
+        return $this->app->trans('Email:deletion:request:textButton Delete my account');
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getButtonURL()
+    {
+        return $this->url;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getMailSkin()
+    {
+        return self::MAIL_SKIN;
+    }
+
+    private function getOwnerCivility()
+    {
+        if (!$this->user) {
+            throw new LogicException('You must set a user before calling getMessage');
+        }
+
+        $civilities = [
+            User::GENDER_MISS => 'Miss',
+            User::GENDER_MRS  => 'Mrs',
+            User::GENDER_MR   => 'Mr',
+        ];
+
+        if (array_key_exists($this->user->getGender(), $civilities)) {
+            return $civilities[$this->user->getGender()];
+        } else {
+            return '';
+        }
+    }
+}

lib/Alchemy/Phrasea/Notification/Mail/MailSuccessAccountDelete.php

@@ -0,0 +1,45 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Notification\Mail;
+
+class MailSuccessAccountDelete extends AbstractMail
+{
+    /**
+     * {@inheritdoc}
+     */
+    public function getSubject()
+    {
+        return $this->app->trans('Delete account successfull');
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getMessage()
+    {
+        return $this->app->trans('Your phraseanet account on %urlInstance% has been deleted!', ['%urlInstance%' => '<a href="'.$this->getPhraseanetURL().'">'.$this->getPhraseanetURL().'</a>']);
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getButtonText()
+    {
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getButtonURL()
+    {
+    }
+}

lib/Alchemy/Phrasea/Out/Module/Excel.php

@@ -0,0 +1,86 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Out\Module;
+
+use Box\Spout\Writer;
+use Box\Spout\Writer\WriterFactory;
+use Box\Spout\Common\Type;
+
+
+class Excel
+{
+    const FORMAT_CSV  = 'format_csv';
+    const FORMAT_ODS  = 'format_ods';
+    const FORMAT_XLSX = 'format_xlsx';
+
+    private $format;
+
+    /** @var \Box\Spout\Writer\WriterInterface  */
+    private $writer;
+
+
+    public function __construct($format, $filename)
+    {
+        $this->format = $format;
+
+        switch($format) {
+            case self::FORMAT_CSV:
+                /** @var Writer\CSV\Writer $writer */
+                $writer = WriterFactory::create(Type::CSV);
+                $writer->setFieldDelimiter(';')
+                       ->setShouldAddBOM(false);
+                break;
+            case self::FORMAT_ODS:
+                /** @var Writer\ODS\Writer $writer */
+                $writer = WriterFactory::create(Type::ODS);
+                break;
+            case self::FORMAT_XLSX:
+                /** @var Writer\XLSX\Writer $writer */
+                $writer = WriterFactory::create(Type::XLSX);
+                break;
+            default:
+                throw new \InvalidArgumentException(sprintf("format \"%s\" is not handled by Spout"));
+                break;
+        }
+
+        $writer->openToBrowser($filename);
+        $this->writer = $writer;
+    }
+
+    public function __destruct()
+    {
+        $this->writer->close();
+    }
+
+    public function getActiveSheet()
+    {
+        if($this->format == self::FORMAT_CSV) {
+            return "_unique_sheet_";
+        }
+        /** @var Writer\XLSX\Writer $w */
+        $w = $this->writer;
+        $sheetIndex = $w->getCurrentSheet()->getIndex();
+
+        return $sheetIndex;
+    }
+
+    public function addRow($row)
+    {
+        $this->writer->addRow($row);
+    }
+
+    public function render()
+    {
+        $this->writer->close();
+    }
+
+}

lib/Alchemy/Phrasea/Out/Module/Excel_bad.php

@@ -0,0 +1,88 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+/*
+
+namespace Alchemy\Phrasea\Out\Module;
+
+use PhpOffice\PhpSpreadsheet\Spreadsheet;
+use PhpOffice\PhpSpreadsheet\Writer\Xlsx as XlsxWriter;
+use PhpOffice\PhpSpreadsheet\IOFactory;
+
+class Excel
+{
+    const FORMAT_CSV  = 'format_csv';
+    const FORMAT_XLS  = 'format_xls';
+    const FORMAT_XLSX = 'format_xlsx';
+
+    private $spreadsheet;
+
+    /** @var int[] * /
+    private $currentRowBySheet;
+
+    public function __construct()
+    {
+        $this->currentRowBySheet = [];
+        $this->spreadsheet = new Spreadsheet();
+    }
+
+    public function getActiveSheet()
+    {
+        $sheetIndex = $this->spreadsheet->getActiveSheetIndex();
+        if(!array_key_exists($sheetIndex, $this->currentRowBySheet)) {
+            $this->currentRowBySheet[$sheetIndex] = 1;
+        }
+
+        return $this->spreadsheet->getActiveSheet();
+    }
+
+    public function addRow($row)
+    {
+        $sheet = $this->getActiveSheet();
+        $sheetIndex = $this->spreadsheet->getActiveSheetIndex();
+        /** @var int $r * /
+        $r = $this->currentRowBySheet[$sheetIndex];
+        $c = 1;
+        foreach($row as $v) {
+            $sheet->setCellValueByColumnAndRow($c++, $r, $v);
+        }
+        $this->currentRowBySheet[$sheetIndex] = $r+1;
+    }
+
+    public function fill()
+    {
+        $sheet = $this->getActiveSheet();
+        $sheet->setCellValue('A1', 'Hello World !');
+    }
+
+    public function render($format)
+    {
+        switch($format) {
+            case self::FORMAT_XLS:
+                header('Content-Type: application/vnd.ms-excel');
+                $writer = IOFactory::createWriter($this->spreadsheet, 'Xls');
+                break;
+            case self::FORMAT_XLSX:
+                header('Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+                $writer = IOFactory::createWriter($this->spreadsheet, 'Xlsx');
+                break;
+        }
+        header('Content-Disposition: attachment;filename="myfile.xls"');
+        header('Cache-Control: max-age=0');
+
+        $writer = IOFactory::createWriter($this->spreadsheet, 'Xls');
+
+        $writer->save('php://output');
+    }
+
+}
+
+*/

lib/Alchemy/Phrasea/Report/Controller/ApiReportController.php

@@ -0,0 +1,145 @@
+<?php
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report\Controller;
+
+use Alchemy\Phrasea\Application\Helper\JsonBodyAware;
+use Alchemy\Phrasea\Controller\Api\Result;
+use Alchemy\Phrasea\Report\ReportConnections;
+use Alchemy\Phrasea\Report\ReportDownloads;
+use Alchemy\Phrasea\Report\ReportFactory;
+use Alchemy\Phrasea\Report\ReportRecords;
+use Alchemy\Phrasea\Report\ReportService;
+use Doctrine\Common\Collections\ArrayCollection;
+use Symfony\Component\HttpFoundation\Request;
+
+
+class ApiReportController
+{
+    use JsonBodyAware;
+
+    private $reportFactory;
+    private $reportService;
+    private $anonymousReport;
+    private $acl;
+
+
+    /**
+     * @param ReportFactory $reportFactory
+     * @param ReportService $reportService
+     * @param Bool $anonymousReport
+     * @param \ACL $acl
+     */
+    public function __construct(ReportFactory $reportFactory, ReportService $reportService, $anonymousReport, \ACL $acl)
+    {
+        $this->reportFactory   = $reportFactory;
+        $this->reportService   = $reportService;
+        $this->anonymousReport = $anonymousReport;
+        $this->acl             = $acl;
+    }
+
+    /**
+     * route api/report
+     *
+     * @param Request $request
+     * @return \Symfony\Component\HttpFoundation\Response
+     */
+    public function rootAction(Request $request)
+    {
+        $ret = [
+            'granted' => $this->reportService->getGranted()
+        ];
+
+        $result = Result::create($request, $ret);
+
+        return $result->createResponse();
+    }
+
+    /**
+     * route api/report/connections
+     *
+     * @param Request $request
+     * @param $sbasId
+     * @return \Symfony\Component\HttpFoundation\Response
+     */
+    public function connectionsAction(Request $request, $sbasId)
+    {
+        /** @var ReportConnections $report */
+        $report = $this->reportFactory->createReport(
+            ReportFactory::CONNECTIONS,
+            $sbasId,
+            [
+                'dmin'      => $request->get('dmin'),
+                'dmax'      => $request->get('dmax'),
+                'group'     => $request->get('group'),
+                'anonymize' => $this->anonymousReport,
+            ]
+        );
+
+        $result = Result::create($request, $report->getContent());
+
+        return $result->createResponse();
+    }
+
+    /**
+     * route api/report/downloads
+     *
+     * @param Request $request
+     * @param $sbasId
+     * @return \Symfony\Component\HttpFoundation\Response
+     */
+    public function downloadsAction(Request $request, $sbasId)
+    {
+        /** @var ReportDownloads $report */
+        $report = $this->reportFactory->createReport(
+            ReportFactory::DOWNLOADS,
+            $sbasId,
+            [
+                'dmin'      => $request->get('dmin'),
+                'dmax'      => $request->get('dmax'),
+                'group'     => $request->get('group'),
+                'bases'     => $request->get('base'),
+                'anonymize' => $this->anonymousReport,
+            ]
+        );
+
+        $result = Result::create($request, $report->getContent());
+
+        return $result->createResponse();
+    }
+
+    /**
+     * route api/report/records
+     *
+     * @param Request $request
+     * @param $sbasId
+     * @return \Symfony\Component\HttpFoundation\Response
+     */
+    public function recordsAction(Request $request, $sbasId)
+    {
+        /** @var ReportRecords $report */
+        $report = $this->reportFactory->createReport(
+            ReportFactory::RECORDS,
+            $sbasId,
+            [
+                'dmin'  => $request->get('dmin'),
+                'dmax'  => $request->get('dmax'),
+                'group' => $request->get('group'),
+                'base'  => $request->get('base'),
+                'meta'  => $request->get('meta'),
+            ]
+        );
+
+        $result = Result::create($request, $report->getContent());
+
+        return $result->createResponse();
+    }
+
+}

lib/Alchemy/Phrasea/Report/Controller/ProdReportController.php

@@ -0,0 +1,210 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace Alchemy\Phrasea\Report\Controller;
+
+use Alchemy\Phrasea\Controller\Controller;
+use Alchemy\Phrasea\Report\Report;
+use Alchemy\Phrasea\Report\ReportConnections;
+use Alchemy\Phrasea\Report\ReportDownloads;
+use Alchemy\Phrasea\Report\ReportFactory;
+use Alchemy\Phrasea\Report\ReportRecords;
+use Doctrine\Common\Collections\ArrayCollection;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\HttpFoundation\StreamedResponse;
+
+
+class ProdReportController extends Controller
+{
+    private static $mapFromExtension = [
+        'csv' => [
+            'contentType' => 'text/csv',
+            'format'      => Report::FORMAT_CSV,
+        ],
+        'ods' => [
+            'contentType' => 'application/vnd.oasis.opendocument.spreadsheet',
+            'format'      => Report::FORMAT_ODS,
+        ],
+        'xlsx' => [
+            'contentType' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            'format'      => Report::FORMAT_XLSX,
+        ],
+    ];
+
+    private $reportFactory;
+    private $anonymousReport;
+    private $acl;
+
+    private $extension = null;
+
+
+    /**
+     * @param ReportFactory $reportFactory
+     * @param Bool $anonymousReport
+     * @param \ACL $acl
+     */
+    public function __construct(ReportFactory $reportFactory, $anonymousReport, \ACL $acl)
+    {
+        $this->reportFactory   = $reportFactory;
+        $this->anonymousReport = $anonymousReport;
+        $this->acl             = $acl;
+    }
+
+    /**
+     * route prod/report/connections
+     *
+     * @param Request $request
+     * @return Response
+     */
+    public function indexAction(Request $request)
+    {
+        return new Response($this->render('prod/report/index.html.twig', [
+            'truc' => "hello"
+        ]));
+    }
+
+    /**
+     * route prod/report/connections
+     *
+     * @param Request $request
+     * @param $sbasId
+     * @return StreamedResponse
+     */
+    public function connectionsAction(Request $request, $sbasId)
+    {
+        if(!($extension = $request->get('format'))) {
+            $extension = 'csv';
+        }
+        if(!array_key_exists($extension, self::$mapFromExtension)) {
+            throw new \InvalidArgumentException(sprintf("bad format \"%s\" for report", $extension));
+        }
+        $this->extension = $extension;
+
+        /** @var ReportConnections $report */
+        $report = $this->reportFactory->createReport(
+            ReportFactory::CONNECTIONS,
+            $sbasId,
+            [
+                'dmin'      => $request->get('dmin'),
+                'dmax'      => $request->get('dmax'),
+                'group'     => $request->get('group'),
+                'anonymize' => $this->anonymousReport,
+            ]
+        );
+
+        $report->setFormat(self::$mapFromExtension[$this->extension]['format']);
+
+        $response = new StreamedResponse();
+
+        $this->setHeadersFromFormat($response, $report);
+
+        $response->setCallback(function() use($report) {
+            $report->render();
+        });
+
+        return $response;
+    }
+
+    /**
+     * route prod/report/downloads
+     *
+     * @param Request $request
+     * @param $sbasId
+     * @return StreamedResponse
+     */
+    public function downloadsAction(Request $request, $sbasId)
+    {
+        if(!($extension = $request->get('format'))) {
+            $extension = 'csv';
+        }
+        if(!array_key_exists($extension, self::$mapFromExtension)) {
+            throw new \InvalidArgumentException(sprintf("bad format \"%s\" for report", $extension));
+        }
+        $this->extension = $extension;
+
+        /** @var ReportDownloads $report */
+        $report = $this->reportFactory->createReport(
+            ReportFactory::DOWNLOADS,
+            $sbasId,
+            [
+                'dmin'      => $request->get('dmin'),
+                'dmax'      => $request->get('dmax'),
+                'group'     => $request->get('group'),
+                'bases'     => $request->get('base'),
+                'anonymize' => $this->anonymousReport,
+            ]
+        );
+
+        $report->setFormat(self::$mapFromExtension[$this->extension]['format']);
+
+        $response = new StreamedResponse();
+
+        $this->setHeadersFromFormat($response, $report);
+
+        $response->setCallback(function() use($report) {
+            $report->render();
+        });
+
+        return $response;
+    }
+
+    /**
+     * route prod/report/records
+     *
+     * @param Request $request
+     * @param $sbasId
+     * @return StreamedResponse
+     */
+    public function recordsAction(Request $request, $sbasId)
+    {
+        if(!($extension = $request->get('format'))) {
+            $extension = 'csv';
+        }
+        if(!array_key_exists($extension, self::$mapFromExtension)) {
+            throw new \InvalidArgumentException(sprintf("bad format \"%s\" for report", $extension));
+        }
+        $this->extension = $extension;
+
+        /** @var ReportRecords $report */
+        $report = $this->reportFactory->createReport(
+            ReportFactory::RECORDS,
+            $sbasId,
+            [
+                'dmin'  => $request->get('dmin'),
+                'dmax'  => $request->get('dmax'),
+                'group' => $request->get('group'),
+                'base'  => $request->get('base'),
+                'meta'  => $request->get('meta'),
+            ]
+        );
+
+        $report->setFormat(self::$mapFromExtension[$this->extension]['format']);
+
+set_time_limit(600);
+        $response = new StreamedResponse();
+
+        $this->setHeadersFromFormat($response, $report);
+
+        $response->setCallback(function() use($report) {
+            $report->render();
+        });
+
+        return $response;
+    }
+
+
+    private function setHeadersFromFormat($response, Report $report)
+    {
+        $response->headers->set('Content-Type', self::$mapFromExtension[$this->extension]['contentType']);
+        $response->headers->set('Content-Disposition', 'attachment;filename="' . $report->getName() . '"');
+        $response->headers->set('Cache-Control', 'max-age=0');
+    }
+
+}

lib/Alchemy/Phrasea/Report/ControllerProvider/ApiReportControllerProvider.php

@@ -0,0 +1,112 @@
+<?php
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace Alchemy\Phrasea\Report\ControllerProvider;
+
+use Alchemy\Phrasea\Application as PhraseaApplication;
+use Alchemy\Phrasea\ControllerProvider\Api\Api;
+use Alchemy\Phrasea\ControllerProvider\ControllerProviderTrait;
+use Alchemy\Phrasea\Core\Event\Listener\OAuthListener;
+use Alchemy\Phrasea\Report\Controller\ApiReportController;
+use Alchemy\Phrasea\Report\ReportFactory;
+use Alchemy\Phrasea\Report\ReportService;
+use Silex\Application;
+use Silex\Controller;
+use Silex\ControllerProviderInterface;
+use Silex\ServiceProviderInterface;
+
+
+class ApiReportControllerProvider extends Api implements ControllerProviderInterface, ServiceProviderInterface
+{
+    use ControllerProviderTrait;
+
+
+    const VERSION = '2.0.0';
+
+    public function register(Application $app)
+    {
+        $app['controller.api.v2.report'] = $app->share(
+            function (PhraseaApplication $app) {
+                return (new ApiReportController(
+                    $app['report.factory'],
+                    $app['report.service'],
+                    $app['conf']->get(['registry', 'modules', 'anonymous-report']),
+                    $app->getAclForUser($app->getAuthenticatedUser())
+                ));
+            }
+        );
+
+        $app['report.factory'] = $app->share(
+            function (PhraseaApplication $app) {
+                return (new ReportFactory(
+                    $app['conf']->get(['main', 'key']),
+                    $app['phraseanet.appbox'],
+                    $app->getAclForUser($app->getAuthenticatedUser())
+                ));
+            }
+        );
+
+        $app['report.service'] = $app->share(
+            function (PhraseaApplication $app) {
+                return (new ReportService(
+                    $app['conf']->get(['main', 'key']),
+                    $app['phraseanet.appbox'],
+                    $app->getAclForUser($app->getAuthenticatedUser())
+                ));
+            }
+        );
+    }
+
+    public function boot(Application $app)
+    {
+        // Intentionally left empty
+    }
+
+    public function connect(Application $app)
+    {
+        if (! $this->isApiEnabled($app)) {
+            return $app['controllers_factory'];
+        }
+
+        $controllers = $this->createCollection($app);
+        /*
+        $firewall = $this->getFirewall($app);
+
+        $controllers->before(function () use ($firewall) {
+            $firewall->requireAccessToModule('report');
+        });
+        */
+
+        $controllers->before(new OAuthListener());
+        $controllers
+            ->match('/', 'controller.api.v2.report:rootAction')
+            ->method('GET|POST')
+        ;
+
+        $controllers
+            ->match('/connections/{sbasId}/', 'controller.api.v2.report:connectionsAction')
+            ->assert('sbasId', '\d+')
+            ->method('GET|POST')
+        ;
+
+        $controllers
+            ->match('/downloads/{sbasId}/', 'controller.api.v2.report:downloadsAction')
+            ->assert('sbasId', '\d+')
+            ->method('GET|POST')
+        ;
+
+        $controllers
+            ->match('/records/{sbasId}/', 'controller.api.v2.report:recordsAction')
+            ->assert('sbasId', '\d+')
+            ->method('GET|POST')
+        ;
+
+        return $controllers;
+    }
+}

lib/Alchemy/Phrasea/Report/ControllerProvider/ProdReportControllerProvider.php

@@ -0,0 +1,85 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report\ControllerProvider;
+
+use Alchemy\Phrasea\Application as PhraseaApplication;
+use Alchemy\Phrasea\ControllerProvider\ControllerProviderTrait;
+use Alchemy\Phrasea\Report\Controller\ProdReportController;
+use Alchemy\Phrasea\Report\ReportFactory;
+use Silex\Application;
+use Silex\ControllerProviderInterface;
+use Silex\ServiceProviderInterface;
+
+
+class ProdReportControllerProvider implements ControllerProviderInterface, ServiceProviderInterface
+{
+    use ControllerProviderTrait;
+
+    public function register(Application $app)
+    {
+        $app['controller.prod.report'] = $app->share(
+            function (PhraseaApplication $app) {
+                return (new ProdReportController(
+                    $app['report.factory'],
+                    $app['conf']->get(['registry', 'modules', 'anonymous-report']),
+                    $app->getAclForUser($app->getAuthenticatedUser())
+                ));
+            }
+        );
+
+        $app['report.factory'] = $app->share(
+            function (PhraseaApplication $app) {
+                return (new ReportFactory(
+                    $app['conf']->get(['main', 'key']),
+                    $app['phraseanet.appbox'],
+                    $app->getAclForUser($app->getAuthenticatedUser())
+                ));
+            }
+        );
+    }
+
+    public function boot(Application $app)
+    {
+        // no-op
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function connect(Application $app)
+    {
+        $controllers = $this->createAuthenticatedCollection($app);
+        
+        $controllers
+            ->match('/connections/{sbasId}/', 'controller.prod.report:connectionsAction')
+            ->assert('sbasId', '\d+')
+            ->bind('report2_connections')
+            ->method('GET|POST')
+        ;
+
+        $controllers
+            ->match('/downloads/{sbasId}/', 'controller.prod.report:downloadsAction')
+            ->assert('sbasId', '\d+')
+            ->bind('report2_downloads')
+            ->method('GET|POST')
+        ;
+
+        $controllers
+            ->match('/records/{sbasId}/', 'controller.prod.report:recordsAction')
+            ->assert('sbasId', '\d+')
+            ->bind('report2_records')
+            ->method('GET|POST')
+        ;
+
+        return $controllers;
+    }
+}

lib/Alchemy/Phrasea/Report/Report.php

@@ -0,0 +1,157 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report;
+
+use Alchemy\Phrasea\Application;
+use Alchemy\Phrasea\Out\Module\Excel;
+
+
+abstract class Report
+{
+    const FORMAT_CSV  = 'format_csv';
+    const FORMAT_ODS  = 'format_ods';
+    // const FORMAT_XLS  = 'format_xls';
+    const FORMAT_XLSX = 'format_xlsx';
+
+    private $format = self::FORMAT_CSV;
+
+    /** @var  \databox */
+    protected $databox;
+    protected $parms;
+
+    public function __construct(\databox $databox, $parms)
+    {
+        $this->databox = $databox;
+        $this->parms = $parms;
+
+        $this->databox->get_connection()->getWrappedConnection()->setAttribute(\PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, FALSE);
+    }
+
+    abstract function getName();
+
+    abstract function getColumnTitles();
+
+    abstract function getKeyName();
+
+    abstract function getAllRows($callback);
+
+    protected function getDatabox()
+    {
+        return $this->databox;
+    }
+
+    public function getContent()
+    {
+        $ret = [];
+        $this->getAllRows(
+            function($row) use($ret) {
+                $ret[] = $row;
+            }
+        );
+
+        return $ret;
+    }
+
+    /**
+     * get quoted coll id's granted for report, possibly filtered by
+     * baseIds : only from this list of bases
+     *
+     * @param \ACL $acl
+     * @param int[]|null $baseIds
+     * @return array
+     */
+    protected function getCollIds(\ACL $acl, $baseIds)
+    {
+        $ret = [];
+        /** @var \collection $collection */
+        foreach($acl->get_granted_base([\ACL::CANREPORT]) as $collection) {
+            if($collection->get_sbas_id() != $this->databox->get_sbas_id()) {
+                continue;
+            }
+            if(!is_null($baseIds) && !in_array($collection->get_base_id(), $baseIds)) {
+                continue;
+            }
+            $ret[] = $this->databox->get_connection()->quote($collection->get_coll_id());
+        }
+
+        return $ret;
+    }
+
+    public function setFormat($format)
+    {
+        if(!in_array($format, [
+            //self::FORMAT_XLS,
+            self::FORMAT_CSV,
+            self::FORMAT_ODS,
+            self::FORMAT_XLSX,
+        ])) {
+            throw new \InvalidArgumentException(sprintf("bad format \"%s\" for report", $format));
+        }
+        $this->format = $format;
+
+        return $this;
+    }
+
+    public function getFormat()
+    {
+        return $this->format;
+    }
+
+    public function render()
+    {
+        switch($this->format) {
+            //case self::FORMAT_XLS:
+            case self::FORMAT_CSV:
+            case self::FORMAT_ODS:
+            case self::FORMAT_XLSX:
+                $this->renderAsExcel();
+                break;
+            default:
+                // should not happen since format is checked before
+                break;
+        }
+    }
+
+    private function renderAsExcel()
+    {
+        switch($this->format) {
+            //case self::FORMAT_XLS:
+            //    $excel = new Excel(Excel::FORMAT_XLS);
+            //    header('Content-Type: application/vnd.ms-excel');
+            //    break;
+            case self::FORMAT_XLSX:
+                $excel = new Excel(Excel::FORMAT_XLSX, $this->getName() . ".xlsx");
+                break;
+            case self::FORMAT_ODS:
+                $excel = new Excel(Excel::FORMAT_ODS, $this->getName() . ".ods");
+                break;
+            case self::FORMAT_CSV:
+            default:
+                $excel = new Excel(Excel::FORMAT_CSV, $this->getName() . ".csv");
+                break;
+        }
+
+        $excel->addRow($this->getColumnTitles());
+
+        $n = 0;
+        $this->getAllRows(
+            function($row) use($excel, $n) {
+                $excel->addRow($row);
+                if($n++ % 10000 === 0) {
+                    flush();
+                }
+            }
+        );
+
+        $excel->render();
+    }
+
+}

lib/Alchemy/Phrasea/Report/ReportConnections.php

@@ -0,0 +1,149 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report;
+
+use Alchemy\Phrasea\Application;
+use Alchemy\Phrasea\Exception\InvalidArgumentException;
+
+
+class ReportConnections extends Report
+{
+    private $appKey;
+
+    /* those vars will be set once by computeVars() */
+    private $name = null;
+    private $sql = null;
+    private $columnTitles = [];
+    private $keyName = null;
+
+
+    public function getColumnTitles()
+    {
+        $this->computeVars();
+        return $this->columnTitles;
+    }
+
+    public function getKeyName()
+    {
+        $this->computeVars();
+        return $this->keyName;
+    }
+
+    public function getName()
+    {
+        $this->computeVars();
+        return $this->name;
+    }
+
+    public function setAppKey($appKey)
+    {
+        $this->appKey = $appKey;
+
+        return $this;
+    }
+
+    public function getAllRows($callback)
+    {
+        $this->computeVars();
+        $stmt = $this->databox->get_connection()->executeQuery($this->sql, []);
+        while (($row = $stmt->fetch())) {
+            $callback($row);
+        }
+        $stmt->closeCursor();
+    }
+
+    private function computeVars()
+    {
+        if(!is_null($this->name)) {
+            // vars already computed
+            return;
+        }
+
+        switch($this->parms['group']) {
+            case null:
+                $this->name = "Connections";
+                $this->columnTitles = ['id', 'date', 'usrid', 'user', 'fonction', 'societe', 'activite', 'pays', 'nav', 'version', 'os', 'res', 'ip', 'user_agent'];
+                if($this->parms['anonymize']) {
+                    $sql = "SELECT `id`, `date`,\n"
+                        . "        `usrid`, '-' AS `user`, '-' AS `fonction`, '-' AS `societe`, '-' AS `activite`, '-' AS `pays`,\n"
+                        . "        `nav`, `version`, `os`, `res`, `ip`, `user_agent` FROM `log`\n"
+                        . " WHERE {{GlobalFilter}}";
+                }
+                else {
+                    $sql = "SELECT `id`, `date`,\n"
+                        . "        `usrid`, `user`, `fonction`, `societe`, `activite`, `pays`,\n"
+                        . "        `nav`, `version`, `os`, `res`, `ip`, `user_agent` FROM `log`\n"
+                        . " WHERE {{GlobalFilter}}";
+                }
+                $this->keyName = null;
+                break;
+            case 'user':
+                $this->name = "Connections per user";
+                $this->columnTitles = ['user_id', 'user', 'fonction', 'societe', 'activite', 'pays', 'min_date', 'max_date', 'nb'];
+                if($this->parms['anonymize']) {
+                    $sql = "SELECT `usrid`, '-' AS `user`, '-' AS `fonction`, '-' AS `societe`, '-' AS `activite`, '-' AS `pays`,\n"
+                        . "        MIN(`date`) AS `dmin`, MAX(`date`) AS `dmax`, SUM(1) AS `nb` FROM `log`\n"
+                        . " WHERE {{GlobalFilter}}\n"
+                        . " GROUP BY `usrid`\n"
+                        . " ORDER BY `nb` DESC";
+                }
+                else {
+                    $sql = "SELECT `usrid`, `user`, `fonction`, `societe`, `activite`, `pays`,\n"
+                        . "        MIN(`date`) AS `dmin`, MAX(`date`) AS `dmax`, SUM(1) AS `nb` FROM `log`\n"
+                        . " WHERE {{GlobalFilter}}\n"
+                        . " GROUP BY `usrid`\n"
+                        . " ORDER BY `nb` DESC";
+                }
+                $this->keyName = 'usrid';
+                break;
+            case 'nav':
+            case 'nav,version':
+            case 'os':
+            case 'os,nav':
+            case 'os,nav,version':
+            case 'res':
+                $this->name = "Connections per " . $this->parms['group'];
+                $groups  = explode(',', $this->parms['group']);
+                $qgroups = implode(
+                    ',',
+                    array_map(function($g) {return '`'.$g.'`';}, $groups)
+                );
+                $this->columnTitles = $groups;
+                $this->columnTitles[] = 'nb';
+                $sql = "SELECT " . $qgroups . ", SUM(1) AS `nb` FROM `log`\n"
+                    . " WHERE {{GlobalFilter}}\n"
+                    . " GROUP BY " . $qgroups . "\n"
+                    . " ORDER BY `nb` DESC"
+                ;
+                $this->keyName = null;
+                break;
+            default:
+                throw new InvalidArgumentException('invalid "group" argument');
+                break;
+        }
+
+        $filter = "`usrid`>0";
+        // next line : comment to disable "site", to test on an imported dataset from another instance
+        $filter .= "  AND `site` = " . $this->databox->get_connection()->quote($this->appKey);
+
+        if($this->parms['dmin']) {
+            $filter .= "\n  AND `log`.`date` >= " . $this->databox->get_connection()->quote($this->parms['dmin']);
+        }
+        if($this->parms['dmax']) {
+            $filter .= "\n  AND `log`.`date` <= " . $this->databox->get_connection()->quote($this->parms['dmax']);
+        }
+
+        $this->sql = str_replace('{{GlobalFilter}}', $filter, $sql);
+
+        // file_put_contents("/tmp/phraseanet-log.txt", sprintf("%s (%d) %s\n", __FILE__, __LINE__, var_export($this->sql, true)), FILE_APPEND);
+    }
+
+}

lib/Alchemy/Phrasea/Report/ReportDownloads.php

@@ -0,0 +1,179 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report;
+
+use Alchemy\Phrasea\Application;
+use Alchemy\Phrasea\Exception\InvalidArgumentException;
+
+
+class ReportDownloads extends Report
+{
+    private $appKey;
+
+    /** @var  \ACL */
+    private $acl;
+
+    /* those vars will be set once by computeVars() */
+    private $name = null;
+    private $sql = null;
+    private $columnTitles = [];
+    private $keyName = null;
+
+
+    public function getColumnTitles()
+    {
+        $this->computeVars();
+        return $this->columnTitles;
+    }
+
+    public function getKeyName()
+    {
+        $this->computeVars();
+        return $this->keyName;
+    }
+
+    public function getName()
+    {
+        $this->computeVars();
+        return $this->name;
+    }
+
+    public function setAppKey($appKey)
+    {
+        $this->appKey = $appKey;
+
+        return $this;
+    }
+
+    public function setACL($acl)
+    {
+        $this->acl = $acl;
+
+        return $this;
+    }
+
+    public function getAllRows($callback)
+    {
+        $this->computeVars();
+        $stmt = $this->databox->get_connection()->executeQuery($this->sql, []);
+        while (($row = $stmt->fetch())) {
+            $callback($row);
+        }
+        $stmt->closeCursor();
+    }
+
+    private function computeVars()
+    {
+        if(!is_null($this->name)) {
+            // vars already computed
+            return;
+        }
+
+        switch ($this->parms['group']) {
+            case null:
+                $this->name = "Downloads";
+                $this->columnTitles = ['id', 'usrid', 'user', 'fonction', 'societe', 'activite', 'pays', 'date', 'record_id', 'coll_id', 'subdef'];
+                if($this->parms['anonymize']) {
+                    $sql = "SELECT `ld`.`id`, `l`.`usrid`, '-' AS `user`, '-' AS `fonction`, '-' AS `societe`, '-' AS `activite`, '-' AS `pays`,\n"
+                        . "        `ld`.`date`, `ld`.`record_id`, `ld`.`coll_id`, `ld`.`final`"
+                        . " FROM `log_docs` AS `ld` INNER JOIN `log` AS `l` ON `l`.`id`=`ld`.`log_id`\n"
+                        . " WHERE {{GlobalFilter}}";
+                }
+                else {
+                    $sql = "SELECT `ld`.`id`, `l`.`usrid`, `l`.`user`, `l`.`fonction`, `l`.`societe`, `l`.`activite`, `l`.`pays`,\n"
+                        . "        `ld`.`date`, `ld`.`record_id`, `ld`.`coll_id`, `ld`.`final`"
+                        . " FROM `log_docs` AS `ld` INNER JOIN `log` AS `l` ON `l`.`id`=`ld`.`log_id`\n"
+                        . " WHERE {{GlobalFilter}}";
+                }
+                $this->keyName = 'id';
+                break;
+            case 'user':
+                $this->name = "Downloads by user";
+                $this->columnTitles = ['usrid', 'user', 'fonction', 'societe', 'activite', 'pays', 'min_date', 'max_date', 'nb'];
+                if($this->parms['anonymize']) {
+                    $sql = "SELECT `l`.`usrid`, '-' AS `user`, '-' AS `fonction`, '-' AS `societe`, '-' AS `activite`, '-' AS `pays`,\n"
+                        . "        MIN(`ld`.`date`) AS `dmin`, MAX(`ld`.`date`) AS `dmax`, SUM(1) AS `nb`\n"
+                        . " FROM `log_docs` AS `ld` INNER JOIN `log` AS `l` ON `l`.`id`=`ld`.`log_id`\n"
+                        . " WHERE {{GlobalFilter}}\n"
+                        . " GROUP BY `l`.`usrid`\n"
+                        . " ORDER BY `nb` DESC";
+                }
+                else {
+                    $sql = "SELECT `l`.`usrid`, `l`.`user`, `l`.`fonction`, `l`.`societe`, `l`.`activite`, `l`.`pays`,\n"
+                        . "        MIN(`ld`.`date`) AS `dmin`, MAX(`ld`.`date`) AS `dmax`, SUM(1) AS `nb`\n"
+                        . " FROM `log_docs` AS `ld` INNER JOIN `log` AS `l` ON `l`.`id`=`ld`.`log_id`\n"
+                        . " WHERE {{GlobalFilter}}\n"
+                        . " GROUP BY `l`.`usrid`\n"
+                        . " ORDER BY `nb` DESC";
+                }
+                $this->keyName = 'usrid';
+                break;
+            case 'record':
+                $this->name = "Downloads by record";
+                $this->columnTitles = ['record_id', 'min_date', 'max_date', 'nb'];
+                $sql = "SELECT `ld`.`record_id`,\n"
+                    . "        MIN(`ld`.`date`) AS `dmin`, MAX(`ld`.`date`) AS `dmax`, SUM(1) AS `nb`\n"
+                    . " FROM `log_docs` AS `ld` INNER JOIN `log` AS `l` ON `l`.`id`=`ld`.`log_id`\n"
+                    . " WHERE {{GlobalFilter}}\n"
+                    . " GROUP BY `l`.`usrid`\n"
+                    . " ORDER BY `nb` DESC"
+                ;
+                $this->keyName = 'record_id';
+                break;
+            default:
+                throw new InvalidArgumentException('invalid "group" argument');
+                break;
+        }
+
+        // get acl-filtered coll_id(s) as already sql-quoted
+        $collIds = $this->getCollIds($this->acl, $this->parms['bases']);
+
+        if(!empty($collIds)) {
+
+            // filter subdefs by class
+            $subdefsToReport = ['document' => $this->databox->get_connection()->quote('document')];
+            foreach ($this->getDatabox()->get_subdef_structure() as $subGroup) {
+                foreach ($subGroup->getIterator() as $sub) {
+                    if(in_array($sub->get_class(), ['document', 'preview'])) {
+                        // keep only unique names
+                        $subdefsToReport[$sub->get_name()] = $this->databox->get_connection()->quote($sub->get_name());
+                    }
+                }
+            }
+
+            $subdefsToReport = join(',', $subdefsToReport);
+
+            $filter = "`action`='download' AND `ld`.`coll_id` IN(" . join(',', $collIds) . ")\n"
+                    . "  AND `l`.`usrid`>0\n"
+                    . "  AND `ld`.`final` IN(" . $subdefsToReport . ")";
+
+                // next line : comment to disable "site", to test on an imported dataset from another instance
+            $filter .= "\n  AND `l`.`site` =  " . $this->databox->get_connection()->quote($this->appKey);
+
+            if($this->parms['dmin']) {
+                $filter .= "\n  AND `ld`.`date` >= " . $this->databox->get_connection()->quote($this->parms['dmin']);
+            }
+            if($this->parms['dmax']) {
+                $filter .= "\n  AND `ld`.`date` <= " . $this->databox->get_connection()->quote($this->parms['dmax']);
+            }
+        }
+        else {
+            // no collections report ?
+            // keep the sql intact (to match placeholders/parameters), but enforce empty result
+            $filter = "FALSE";
+        }
+
+        $this->sql = str_replace('{{GlobalFilter}}', $filter, $sql);
+
+        // file_put_contents("/tmp/phraseanet-log.txt", sprintf("%s (%d) %s\n", __FILE__, __LINE__, $this->sql), FILE_APPEND);
+    }
+
+}

lib/Alchemy/Phrasea/Report/ReportFactory.php

@@ -0,0 +1,105 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report;
+
+use Alchemy\Phrasea\Application;
+use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
+
+
+/**
+ * Class ReportFactory
+ *
+ * published as service $app['report.factory']
+ *
+ * @package Alchemy\Phrasea\Report
+ */
+class ReportFactory
+{
+    const CONNECTIONS = 'connections';
+    const DOWNLOADS   = 'downloads';
+    const RECORDS     = 'records';
+
+    protected $appKey;
+    protected $appbox;
+    protected $databox;
+    protected $acl;
+
+    /**
+     * @param string $appKey
+     * @param \appbox $appbox
+     * @param \ACL acl
+     */
+    public function __construct($appKey, \appbox $appbox, \ACL $acl)
+    {
+        $this->appKey = $appKey;
+        $this->appbox = $appbox;
+        $this->acl = $acl;
+    }
+
+    /**
+     * @param $table
+     * @param null $sbasId
+     * @param null $parms
+     *
+     * @return ReportConnections | ReportDownloads
+     */
+    public function createReport($table, $sbasId=null, $parms=null)
+    {
+        switch($table) {
+            case self::CONNECTIONS:
+                return (new ReportConnections(
+                        $this->findDbOr404($sbasId),
+                        $parms
+                    ))
+                    ->setAppKey($this->appKey)
+                    ;
+                break;
+
+            case self::DOWNLOADS:
+                return (new ReportDownloads(
+                        $this->findDbOr404($sbasId),
+                        $parms
+                    ))
+                    ->setAppKey($this->appKey)
+                    ->setACL($this->acl)
+                    ;
+                break;
+
+            case self::RECORDS:
+                return (new ReportRecords(
+                        $this->findDbOr404($sbasId),
+                        $parms
+                    ))
+                    ->setACL($this->acl)
+                    ;
+                break;
+
+            default:
+                throw new \InvalidArgumentException(sprintf("unknown table type \"%s\"", $table));
+                break;
+        }
+    }
+
+    /**
+     * @param int $sbasId
+     * @return \databox
+     */
+    private function findDbOr404($sbasId)
+    {
+        $db = $this->appbox->get_databox(($sbasId));
+        if(!$db) {
+            throw new NotFoundHttpException(sprintf('Databox %s not found', $sbasId));
+        }
+
+        return $db;
+    }
+
+}

lib/Alchemy/Phrasea/Report/ReportRecords.php

@@ -0,0 +1,133 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report;
+
+use Alchemy\Phrasea\Application;
+
+
+class ReportRecords extends Report
+{
+    /** @var  \ACL */
+    private $acl;
+
+    /* those vars will be set once by computeVars() */
+    private $name = null;
+    private $sqlWhere = null;
+    private $sqlColSelect = null;
+    private $columnTitles = null;
+    private $keyName = null;
+
+
+    public function getColumnTitles()
+    {
+        $this->computeVars();
+        return $this->columnTitles;
+    }
+
+    public function getKeyName()
+    {
+        $this->computeVars();
+        return $this->keyName;
+    }
+
+    public function getName()
+    {
+        $this->computeVars();
+        return $this->name;
+    }
+
+    public function setACL($acl)
+    {
+        $this->acl = $acl;
+
+        return $this;
+    }
+
+    public function getAllRows($callback)
+    {
+        $this->computeVars();
+
+        $lastRid = 0;
+        while(true) {
+            $sql = "SELECT MIN(record_id) AS `from`, MAX(record_id) AS `to` FROM (\n"
+                . "SELECT record_id FROM record AS `r`\n"
+                . "WHERE " . $this->sqlWhere . " AND record_id>" . $lastRid . " LIMIT 5000) AS _t";
+            $stmt = $this->databox->get_connection()->executeQuery($sql, []);
+            $row = $stmt->fetch();
+            $stmt->closeCursor();
+
+            if($row && !is_null($row['from']) && !is_null($row['to'])) {
+                $sql = "SELECT r.record_id, c.asciiname, r.moddate, r.mime, r.type, r.originalname,\n"
+                    . $this->sqlColSelect . "\n"
+                    . "FROM (`record` AS `r` LEFT JOIN `coll` AS `c` USING(`coll_id`)) LEFT JOIN `metadatas` AS `m` USING(`record_id`)\n"
+                    . "WHERE " . $this->sqlWhere . "\n"
+                    . "  AND r.record_id >= " . $row['from'] . " AND r.record_id <= " . $row['to'] . "\n"
+                    . "GROUP BY `record_id`\n";
+
+                $stmt = $this->databox->get_connection()->executeQuery($sql, []);
+                $rows = $stmt->fetchAll();
+                $stmt->closeCursor();
+                foreach($rows as $row) {
+                    $callback($row);
+                    $lastRid = $row['record_id'];
+                }
+            }
+            else {
+                break;
+            }
+        }
+    }
+
+    private function computeVars()
+    {
+        if(!is_null($this->name)) {
+            // vars already computed
+            return;
+        }
+
+        // pivot-like query on metadata fields
+        $this->sqlColSelect = [];
+        $this->columnTitles = ['record_id', 'collection', 'moddate', 'mime', 'type', 'originalname'];
+        foreach($this->getDatabox()->get_meta_structure() as $field) {
+            // skip the fields that can't be reported
+            if(!$field->is_report() || ($field->isBusiness() && !$this->acl->can_see_business_fields($this->getDatabox()))) {
+                continue;
+            }
+            // if a list of meta was provided, just keep those
+            if(is_array($this->parms['meta']) && !in_array($field->get_name(), $this->parms['meta'])) {
+                continue;
+            }
+            // column names is not important in the result, simply match the 'title' position
+            $this->columnTitles[] = $field->get_name();
+            $this->sqlColSelect[] = sprintf("GROUP_CONCAT(IF(`m`.`meta_struct_id`=%s, `m`.`value`, NULL)) AS `f%s`", $field->get_id(), $field->get_id());
+        }
+
+        $this->sqlColSelect = join(",\n", $this->sqlColSelect);
+
+        // get acl-filtered coll_id(s) as already sql-quoted
+        $collIds = $this->getCollIds($this->acl, $this->parms['base']);
+        if(!empty($collIds)) {
+            $this->sqlWhere = "`r`.`parent_record_id`=0 AND `r`.`coll_id` IN(" . join(',', $collIds) . ")";
+            if(!is_null($this->parms['dmin'])) {
+                $this->sqlWhere .= " AND r.moddate >= " . $this->databox->get_connection()->quote($this->parms['dmin']);
+            }
+            if(!is_null($this->parms['dmax'])) {
+                $this->sqlWhere .= " AND r.moddate <= " . $this->databox->get_connection()->quote($this->parms['dmax']);
+            }
+        }
+        else {
+            $this->sqlWhere = "FALSE";
+        }
+
+        $this->name = "Databox";
+    }
+
+}

lib/Alchemy/Phrasea/Report/ReportService.php

@@ -0,0 +1,62 @@
+<?php
+/**
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2016 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Report;
+
+use Alchemy\Phrasea\Application;
+
+
+class ReportService
+{
+    protected $appKey;
+    protected $appbox;
+    protected $acl;
+
+    /**
+     * @param string $appKey
+     * @param \appbox $appbox
+     * @param \ACL acl
+     */
+    public function __construct($appKey, \appbox $appbox, \ACL $acl)
+    {
+        $this->appKey = $appKey;
+        $this->appbox = $appbox;
+        $this->acl = $acl;
+    }
+
+    /**
+     * return bases allowed for reporting, grouped by databox
+     * @return array
+     */
+    public function getGranted()
+    {
+        $databoxes = [];
+
+        /** @var \collection $collection */
+        foreach ($this->acl->get_granted_base([\ACL::CANREPORT]) as $collection) {
+            $sbas_id = $collection->get_sbas_id();
+            if (!isset($databoxes[$sbas_id])) {
+                $databoxes[$sbas_id] = [
+                    'id' => $sbas_id,
+                    'name' => $collection->get_databox()->get_viewname(),
+                    'collections' => []
+                ];
+            }
+            $databoxes[$sbas_id]['collections'][$collection->get_base_id()] = [
+                'id' => $collection->get_base_id(),
+                'coll_id' => $collection->get_coll_id(),
+                'name' => $collection->get_name()
+            ];
+        }
+
+        return ['databoxes' => $databoxes];
+    }
+
+}

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/FieldKey.php

@@ -30,6 +30,11 @@ class FieldKey implements Key, QueryPostProcessor
         return $this->getField($context)->getIndexField($raw);
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return $this->getField($context)->getType();
+    }
+
     public function isValueCompatible($value, QueryContext $context)
     {
         return ValueChecker::isValueCompatible($this->getField($context), $value);

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/GeolocationKey.php

@@ -58,6 +58,11 @@ class GeolocationKey implements Key
         $this->key = $key;
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return $this->type;
+    }
+
     public function getIndexField(QueryContext $context, $raw = false)
     {
         return $this->key;

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Key.php

@@ -6,6 +6,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
 
 interface Key
 {
+    public function getFieldType(QueryContext $context);
     public function getIndexField(QueryContext $context, $raw = false);
     public function isValueCompatible($value, QueryContext $context);
     public function __toString();

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MetadataKey.php

@@ -23,6 +23,11 @@ class MetadataKey implements Key
         return $this->getTag($context)->getIndexField($raw);
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return $this->getTag($context)->getType();
+    }
+
     public function isValueCompatible($value, QueryContext $context)
     {
         return ValueChecker::isValueCompatible($this->getTag($context), $value);

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php

@@ -52,6 +52,11 @@ class NativeKey implements Key
         $this->key = $key;
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return $this->type;
+    }
+
     public function getIndexField(QueryContext $context, $raw = false)
     {
         return $this->key;

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/RangeExpression.php

@@ -2,18 +2,20 @@
 
 namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
 
+use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping;
+use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
 use Assert\Assertion;
-use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\FieldKey;
-use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\Key;
 use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node;
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
-use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryPostProcessor;
 
 class RangeExpression extends Node
 {
+    /** @var FieldKey  */
     private $key;
+
     private $lower_bound;
     private $lower_inclusive;
     private $higher_bound;
@@ -55,20 +57,34 @@ class RangeExpression extends Node
     public function buildQuery(QueryContext $context)
     {
         $params = array();
-        if ($this->lower_bound !== null) {
-            $this->assertValueCompatible($this->lower_bound, $context);
-            if ($this->lower_inclusive) {
-                $params['gte'] = $this->lower_bound;
-            } else {
-                $params['gt'] = $this->lower_bound;
+        /** @var StructureField $field */
+        // $field = $this->key->getField($context);
+        $lower_bound = $this->lower_bound;
+        $higher_bound = $this->higher_bound;
+
+        if($this->key->getFieldType($context) === FieldMapping::TYPE_DATE) {
+            if($lower_bound !== null) {
+                $lower_bound = RecordHelper::sanitizeDate($lower_bound);
+            }
+            if($higher_bound !== null) {
+                $higher_bound = RecordHelper::sanitizeDate($higher_bound);
             }
         }
-        if ($this->higher_bound !== null) {
-            $this->assertValueCompatible($this->higher_bound, $context);
-            if ($this->higher_inclusive) {
-                $params['lte'] = $this->higher_bound;
+
+        if ($lower_bound !== null) {
+            $this->assertValueCompatible($lower_bound, $context);
+            if ($this->lower_inclusive) {
+                $params['gte'] = $lower_bound;
             } else {
-                $params['lt'] = $this->higher_bound;
+                $params['gt'] = $lower_bound;
+            }
+        }
+        if ($higher_bound !== null) {
+            $this->assertValueCompatible($higher_bound, $context);
+            if ($this->higher_inclusive) {
+                $params['lte'] = $higher_bound;
+            } else {
+                $params['lt'] = $higher_bound;
             }
         }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/TimestampKey.php

@@ -34,6 +34,11 @@ class TimestampKey implements Key, Typed
         return FieldMapping::TYPE_DATE;
     }
 
+    public function getFieldType(QueryContext $context)
+    {
+        return FieldMapping::TYPE_DATE;
+    }
+
     public function getIndexField(QueryContext $context, $raw = false)
     {
         return $this->index_field;

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php

@@ -43,7 +43,7 @@ class QuotedTextNode extends Node
 
         $private_fields = $context->getPrivateFields();
         $private_fields = ValueChecker::filterByValueCompatibility($private_fields, $this->text);
-        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
+        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $unrestricted_fields, $query_builder) as $private_field_query) {
             $query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
         }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php

@@ -61,7 +61,7 @@ class RawNode extends Node
 
         $private_fields = $context->getPrivateFields();
         $private_fields = ValueChecker::filterByValueCompatibility($private_fields, $this->text);
-        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
+        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $unrestricted_fields, $query_builder) as $private_field_query) {
             $query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
         }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TermNode.php

@@ -19,9 +19,11 @@ class TermNode extends AbstractTermNode
             return $query;
         };
 
-        $query = $query_builder($context->getUnrestrictedFields());
+        $unrestricted_fields = $context->getUnrestrictedFields();
         $private_fields = $context->getPrivateFields();
-        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $concept_query) {
+        $query = $query_builder($unrestricted_fields);
+
+        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $unrestricted_fields, $query_builder) as $concept_query) {
             $query = QueryHelper::applyBooleanClause($query, 'should', $concept_query);
         }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php

@@ -45,6 +45,9 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
                 foreach ($context->localizeField($field) as $f) {
                     $index_fields[] = $f;
                 }
+                foreach ($context->truncationField($field) as $f) {
+                    $index_fields[] = $f;
+                }
             }
             if (!$index_fields) {
                 return null;
@@ -66,12 +69,11 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
             return $query;
         };
 
-        // Unrestricted fields
-        $query = $query_builder($context->getUnrestrictedFields());
-
-        // Private fields
+        $unrestricted_fields = $context->getUnrestrictedFields();
         $private_fields = $context->getPrivateFields();
-        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
+
+        $query = $query_builder($unrestricted_fields);
+        foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $unrestricted_fields, $query_builder) as $private_field_query) {
             $query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
         }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php

@@ -396,10 +396,10 @@ class ElasticSearchEngine implements SearchEngineInterface
         if ($options->getDateFields() && ($options->getMaxDate() || $options->getMinDate())) {
             $range = [];
             if ($options->getMaxDate()) {
-                $range['lte'] = $options->getMaxDate()->format(FieldMapping::DATE_FORMAT_CAPTION_PHP);
+                $range['lte'] = $options->getMaxDate()->format('Y-m-d');
             }
             if ($options->getMinDate()) {
-                $range['gte'] = $options->getMinDate()->format(FieldMapping::DATE_FORMAT_CAPTION_PHP);
+                $range['gte'] = $options->getMinDate()->format('Y-m-d');
             }
 
             foreach ($options->getDateFields() as $dateField) {
@@ -619,6 +619,8 @@ class ElasticSearchEngine implements SearchEngineInterface
         foreach ($context->getHighlightedFields() as $field) {
             switch ($field->getType()) {
                 case FieldMapping::TYPE_STRING:
+                case FieldMapping::TYPE_DOUBLE:
+                case FieldMapping::TYPE_DATE:
                     $index_field = $field->getIndexField();
                     $raw_index_field = $field->getIndexField(true);
                     $highlighted_fields[$index_field . ".light"] = [
@@ -628,13 +630,10 @@ class ElasticSearchEngine implements SearchEngineInterface
                     ];
                     break;
                 case FieldMapping::TYPE_FLOAT:
-                case FieldMapping::TYPE_DOUBLE:
                 case FieldMapping::TYPE_INTEGER:
                 case FieldMapping::TYPE_LONG:
                 case FieldMapping::TYPE_SHORT:
                 case FieldMapping::TYPE_BYTE:
-                    continue;
-                case FieldMapping::TYPE_DATE:
                 default:
                     continue;
             }
@@ -660,7 +659,7 @@ class ElasticSearchEngine implements SearchEngineInterface
                 }
                 $agg = [
                     'terms' => [
-                        'field' => $f['field'],
+                        'field' => $f['esfield'],
                         'size'  => $size
                     ]
                 ];

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php

@@ -256,42 +256,56 @@ class ElasticsearchOptions
     {
         return [
             'base_aggregate' => [
-                'label' => 'prod::facet:base_label',
-                'field' => 'databox_name',
-                'query' => 'database:%s',
+                'type'    => 'string',
+                'label'   => 'prod::facet:base_label',
+                'field'   => "database",
+                'esfield' => 'databox_name',
+                'query'   => 'database:%s',
             ],
             'collection_aggregate' => [
-                'label' => 'prod::facet:collection_label',
-                'field' => 'collection_name',
-                'query' => 'collection:%s',
+                'type'    => 'string',
+                'label'   => 'prod::facet:collection_label',
+                'field'   => "collection",
+                'esfield' => 'collection_name',
+                'query'   => 'collection:%s',
             ],
             'doctype_aggregate' => [
-                'label' => 'prod::facet:doctype_label',
-                'field' => 'type',
-                'query' => 'type:%s',
+                'type'    => 'string',
+                'label'   => 'prod::facet:doctype_label',
+                'field'   => "type",
+                'esfield' => 'type',
+                'query'   => 'type:%s',
             ],
             'camera_model_aggregate' => [
-                'label' => 'Camera Model',
-                'field' => 'metadata_tags.CameraModel',
-                'query' => 'meta.CameraModel:%s',
+                'type'    => 'string',
+                'label'   => 'Camera Model',
+                'field'   => "meta.CameraModel",
+                'esfield' => 'metadata_tags.CameraModel',
+                'query'   => 'meta.CameraModel:%s',
             ],
             'iso_aggregate' => [
-                'label' => 'ISO',
-                'field' => 'metadata_tags.ISO',
-                'query' => 'meta.ISO=%s',
+                'type'    => 'number',
+                'label'   => 'ISO',
+                'field'   => "meta.ISO",
+                'esfield' => 'metadata_tags.ISO',
+                'query'   => 'meta.ISO=%s',
             ],
             'aperture_aggregate' => [
-                'label' => 'Aperture',
-                'field' => 'metadata_tags.Aperture',
-                'query' => 'meta.Aperture=%s',
+                'type'    => 'number',
+                'label'   => 'Aperture',
+                'field'   => "meta.Aperture",
+                'esfield' => 'metadata_tags.Aperture',
+                'query'   => 'meta.Aperture=%s',
                 'output_formatter' => function($value) {
                     return round($value, 1);
                 },
             ],
             'shutterspeed_aggregate' => [
-                'label' => 'Shutter speed',
-                'field' => 'metadata_tags.ShutterSpeed',
-                'query' => 'meta.ShutterSpeed=%s',
+                'type'    => 'number',
+                'label'   => 'Shutter speed',
+                'field'   => "meta.ShutterSpeed",
+                'esfield' => 'metadata_tags.ShutterSpeed',
+                'query'   => 'meta.ShutterSpeed=%s',
                 'output_formatter' => function($value) {
                     if($value < 1.0 && $value != 0) {
                         $value = '1/' . round(1.0 / $value);
@@ -300,51 +314,67 @@ class ElasticsearchOptions
                 },
             ],
             'flashfired_aggregate' => [
-                'label' => 'FlashFired',
-                'field' => 'metadata_tags.FlashFired',
-                'query' => 'meta.FlashFired=%s',
+                'type'    => 'boolean',
+                'label'   => 'FlashFired',
+                'field'   => "meta.FlashFired",
+                'esfield' => 'metadata_tags.FlashFired',
+                'query'   => 'meta.FlashFired=%s',
                 'choices' => [
                     "aggregated (2 values: fired = 0 or 1)" => -1,
                 ],
                 'output_formatter' => function($value) {
-                    static $map = ['0'=>"No flash", '1'=>"Flash"];
+                    static $map = ["false"=>"No flash", "true"=>"Flash", '0'=>"No flash", '1'=>"Flash"];
                     return array_key_exists($value, $map) ? $map[$value] : $value;
                 },
             ],
             'framerate_aggregate' => [
-                'label' => 'FrameRate',
-                'field' => 'metadata_tags.FrameRate',
-                'query' => 'meta.FrameRate=%s',
+                'type'    => 'number',
+                'label'   => 'FrameRate',
+                'field'   => "meta.FrameRate",
+                'esfield' => 'metadata_tags.FrameRate',
+                'query'   => 'meta.FrameRate=%s',
             ],
             'audiosamplerate_aggregate' => [
-                'label' => 'Audio Samplerate',
-                'field' => 'metadata_tags.AudioSamplerate',
-                'query' => 'meta.AudioSamplerate=%s',
+                'type'    => 'number',
+                'label'   => 'Audio Samplerate',
+                'field'   => "meta.AudioSamplerate",
+                'esfield' => 'metadata_tags.AudioSamplerate',
+                'query'   => 'meta.AudioSamplerate=%s',
             ],
             'videocodec_aggregate' => [
-                'label' => 'Video codec',
-                'field' => 'metadata_tags.VideoCodec',
-                'query' => 'meta.VideoCodec:%s',
+                'type'    => 'string',
+                'label'   => 'Video codec',
+                'field'   => "meta.VideoCodec",
+                'esfield' => 'metadata_tags.VideoCodec',
+                'query'   => 'meta.VideoCodec:%s',
             ],
             'audiocodec_aggregate' => [
-                'label' => 'Audio codec',
-                'field' => 'metadata_tags.AudioCodec',
-                'query' => 'meta.AudioCodec:%s',
+                'type'    => 'string',
+                'label'   => 'Audio codec',
+                'field'   => "meta.AudioCodec",
+                'esfield' => 'metadata_tags.AudioCodec',
+                'query'   => 'meta.AudioCodec:%s',
             ],
             'orientation_aggregate' => [
-                'label' => 'Orientation',
-                'field' => 'metadata_tags.Orientation',
-                'query' => 'meta.Orientation=%s',
+                'type'    => 'string',
+                'label'   => 'Orientation',
+                'field'   => "meta.Orientation",
+                'esfield' => 'metadata_tags.Orientation',
+                'query'   => 'meta.Orientation=%s',
             ],
             'colorspace_aggregate' => [
-                'label' => 'Colorspace',
-                'field' => 'metadata_tags.ColorSpace',
-                'query' => 'meta.ColorSpace:%s',
+                'type'    => 'string',
+                'label'   => 'Colorspace',
+                'field'   => "meta.ColorSpace",
+                'esfield' => 'metadata_tags.ColorSpace',
+                'query'   => 'meta.ColorSpace:%s',
             ],
             'mimetype_aggregate' => [
-                'label' => 'MimeType',
-                'field' => 'metadata_tags.MimeType',
-                'query' => 'meta.MimeType:%s',
+                'type'    => 'string',
+                'label'   => 'MimeType',
+                'field'   => "meta.MimeType",
+                'esfield' => 'metadata_tags.MimeType',
+                'query'   => 'meta.MimeType:%s',
             ],
         ];
     }

lib/Alchemy/Phrasea/SearchEngine/Elastic/FieldMapping.php

@@ -16,8 +16,7 @@ class FieldMapping
 
     const DATE_FORMAT_MYSQL = 'yyyy-MM-dd HH:mm:ss';
     const DATE_FORMAT_CAPTION = 'yyyy/MM/dd'; // ES format
-    const DATE_FORMAT_MYSQL_OR_CAPTION = 'yyyy-MM-dd HH:mm:ss||yyyy/MM/dd';
-    const DATE_FORMAT_CAPTION_PHP = 'Y/m/d';  // PHP format
+    const DATE_FORMAT_MYSQL_OR_CAPTION = 'yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||yyyy-MM||yyyy';
 
     // Core types
     const TYPE_STRING     = 'string';

lib/Alchemy/Phrasea/SearchEngine/Elastic/Index.php

@@ -90,6 +90,16 @@ class Index
                     // TODO Maybe replace nfkc_normalizer + asciifolding with icu_folding
                     'filter' => ['nfkc_normalizer', 'asciifolding']
                 ],
+                'truncation_analyzer' => [
+                    'type'      => 'custom',
+                    'tokenizer' => 'truncation_tokenizer',
+                    'filter'    => ['lowercase', 'stop', 'kstem']
+                ],
+                'truncation_analyzer#search' => [
+                    'type'      => 'custom',
+                    'tokenizer' => 'truncation_tokenizer',
+                    'filter'    => ['lowercase', 'stop', 'kstem']
+                ],
                 // Lang specific
                 'fr_full' => [
                     'type' => 'custom',
@@ -145,6 +155,12 @@ class Index
                 ]
             ],
             'tokenizer' => [
+                'truncation_tokenizer' => [
+                    "type" => "edgeNGram",
+                    "min_gram" => "2",
+                    "max_gram" => "15",
+                    "token_chars" => [ "letter", "digit", "punctuation", "symbol" ]
+                ],
                 'thesaurus_path' => [
                     'type' => 'path_hierarchy'
                 ]

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php

@@ -155,15 +155,16 @@ class BulkOperation
         // nb: results (items) are returned IN THE SAME ORDER as commands were pushed in the stack
         // so the items[X] match the operationIdentifiers[X]
         foreach ($response['items'] as $key => $item) {
-            foreach($item as $command=>$result) {   // command may be "index" or "delete"
-                if($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx error
-                    throw new Exception(sprintf('%d: %s', $key, var_export($result, true)));
+            foreach ($item as $command=>$result) {   // command may be "index" or "delete"
+                if ($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx
+                    $err = array_key_exists('error', $result) ? var_export($result['error'], true) : ($command . " error " . $result['status']);
+                    throw new Exception(sprintf('%d: %s', $key, $err));
                 }
             }
 
             $operationIdentifier = $this->operationIdentifiers[$key];
 
-            if(is_string($operationIdentifier) || is_int($operationIdentifier)) {   // dont include null keys
+            if (is_string($operationIdentifier) || is_int($operationIdentifier)) {   // dont include null keys
                 $callbackData[$operationIdentifier] = $response['items'][$key];
             }
         }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php

@@ -127,20 +127,21 @@ class Fetcher
     private function getExecutedStatement()
     {
         if (!$this->statement) {
-            $sql = "SELECT r.record_id"
-                 . ", r.coll_id AS collection_id"
-                 . ", c.asciiname AS collection_name"
-                 . ", r.uuid"
-                 . ", r.status AS flags_bitfield"
-                 . ", r.sha256" // -- TODO rename in "hash"
-                 . ", r.originalname AS original_name"
-                 . ", r.mime, r.type, r.parent_record_id, r.credate AS created_on, r.moddate AS updated_on"
-                 . ", subdef.width, subdef.height, subdef.size"
-                 . " FROM (record r INNER JOIN coll c ON (c.coll_id = r.coll_id))"
-                 . " LEFT JOIN subdef ON subdef.record_id=r.record_id AND subdef.name='document'"
-                 . " -- WHERE"
-                 . " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL()
-                 . " LIMIT :offset, :limit";
+            $sql =  "SELECT r.*, c.asciiname AS collection_name, subdef.width, subdef.height, subdef.size\n"
+                 . " FROM ((\n"
+                 . "     SELECT r.record_id, r.coll_id AS collection_id, r.uuid, r.status AS flags_bitfield, r.sha256,\n"
+                 . "            r.originalname AS original_name, r.mime, r.type, r.parent_record_id,\n"
+                 . "            r.credate AS created_on, r.moddate AS updated_on, r.coll_id\n"
+                 . "     FROM record r\n"
+                 . "     -- WHERE\n"
+                 . "     ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "\n"
+                 . "     LIMIT :offset, :limit\n"
+                 . "   ) AS r\n"
+                 . "   INNER JOIN coll c ON (c.coll_id = r.coll_id)\n"
+                 . " )\n"
+                 . " LEFT JOIN\n"
+                 . " subdef ON subdef.record_id=r.record_id AND subdef.name='document'\n"
+                 . " ORDER BY " . $this->options->getPopulateOrderAsSQL() . " " . $this->options->getPopulateDirectionAsSQL() . "";
 
             $where = $this->delegate->buildWhereClause();
             $sql = str_replace('-- WHERE', $where, $sql);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php

@@ -39,18 +39,13 @@ class MetadataHydrator implements HydratorInterface
 
     public function hydrateRecords(array &$records)
     {
-        $sql = <<<SQL
-            (SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private
-            FROM metadatas AS m
-            INNER JOIN metadatas_structure AS ms ON (ms.id = m.meta_struct_id)
-            WHERE record_id IN (?))
-
-            UNION
-
-            (SELECT record_id, t.name AS `key`, t.value AS value, 'exif' AS type, 0 AS private
-            FROM technical_datas AS t
-            WHERE record_id IN (?))
-SQL;
+        $sql = "(SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private\n"
+            . " FROM metadatas AS m INNER JOIN metadatas_structure AS ms ON (ms.id = m.meta_struct_id)\n"
+            . " WHERE record_id IN (?))\n"
+            . "UNION\n"
+            . "(SELECT record_id, t.name AS `key`, t.value AS value, 'exif' AS type, 0 AS private\n"
+            . " FROM technical_datas AS t\n"
+            . " WHERE record_id IN (?))\n";
 
         $ids = array_keys($records);
         $statement = $this->connection->executeQuery(
@@ -62,7 +57,7 @@ SQL;
         while ($metadata = $statement->fetch()) {
             // Store metadata value
             $key = $metadata['key'];
-            $value = $metadata['value'];
+            $value = trim($metadata['value']);
 
             // Do not keep empty values
             if ($key === '' || $value === '') {
@@ -80,7 +75,7 @@ SQL;
                 case 'caption':
                     // Sanitize fields
                     $value = StringHelper::crlfNormalize($value);
-                    $value = $this->sanitizeValue($value, $this->structure->typeOf($key));
+                    $value = $this->helper->sanitizeValue($value, $this->structure->typeOf($key));
                     // Private caption fields are kept apart
                     $type = $metadata['private'] ? 'private_caption' : 'caption';
                     // Caption are multi-valued
@@ -103,7 +98,7 @@ SQL;
                     }
                     $tag = $this->structure->getMetadataTagByName($key);
                     if ($tag) {
-                        $value = $this->sanitizeValue($value, $tag->getType());
+                        $value = $this->helper->sanitizeValue($value, $tag->getType());
                     }
                     // EXIF data is single-valued
                     $record['metadata_tags'][$key] = $value;
@@ -118,33 +113,6 @@ SQL;
         $this->clearGpsPositionBuffer();
     }
 
-    private function sanitizeValue($value, $type)
-    {
-        switch ($type) {
-            case FieldMapping::TYPE_STRING:
-                return str_replace("\0", "", $value);
-
-            case FieldMapping::TYPE_DATE:
-                return $this->helper->sanitizeDate($value);
-
-            case FieldMapping::TYPE_FLOAT:
-            case FieldMapping::TYPE_DOUBLE:
-                return (float) $value;
-
-            case FieldMapping::TYPE_INTEGER:
-            case FieldMapping::TYPE_LONG:
-            case FieldMapping::TYPE_SHORT:
-            case FieldMapping::TYPE_BYTE:
-                return (int) $value;
-
-            case FieldMapping::TYPE_BOOLEAN:
-                return (bool) $value;
-
-            default:
-                return $value;
-        }
-    }
-
     private function handleGpsPosition(&$records, $id, $tag_name, $value)
     {
         // Get position object

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/ThesaurusHydrator.php

@@ -13,6 +13,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
 use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\CandidateTerms;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept;
@@ -27,7 +28,7 @@ class ThesaurusHydrator implements HydratorInterface
     private $thesaurus;
     private $candidate_terms;
 
-    public function __construct(Structure $structure, Thesaurus $thesaurus, CandidateTerms $candidate_terms)
+    public function __construct(GlobalStructure $structure, Thesaurus $thesaurus, CandidateTerms $candidate_terms)
     {
         $this->structure = $structure;
         $this->thesaurus = $thesaurus;
@@ -42,7 +43,7 @@ class ThesaurusHydrator implements HydratorInterface
         $fields = [];
         $index_fields = [];
         foreach ($structure as $name => $field) {
-            $fields[$name] = $field->getThesaurusRoots();
+            $fields[$name] = $field; // ->getThesaurusRoots();
             $index_fields[$name] = $field->getIndexField();
         }
         // Hydrate records with concepts
@@ -51,7 +52,13 @@ class ThesaurusHydrator implements HydratorInterface
         }
     }
 
-    private function hydrate(array &$record, array $fields, array $index_fields)
+    /**
+     * @param array $record
+     * @param  Field[] $fields
+     * @param array $index_fields
+     * @throws Exception
+     */
+    private function hydrate(array &$record, $fields, array $index_fields)
     {
         if (!isset($record['databox_id'])) {
             throw new Exception('Expected a record with the "databox_id" key set.');
@@ -61,7 +68,14 @@ class ThesaurusHydrator implements HydratorInterface
         $terms = array();
         $filters = array();
         $field_names = array();
-        foreach ($fields as $name => $root_concepts) {
+        /** @var Field[] $dbFields */
+        $dbFields = $this->structure->getAllFieldsByDatabox($record['databox_id']);
+        foreach ($fields as $name => $field) {
+            if(!array_key_exists($name, $dbFields) || !$dbFields[$name]->get_generate_cterms()) {
+                continue;
+            }
+
+            $root_concepts = $field->getThesaurusRoots();
             // Loop through all values to prepare bulk query
             $field_values = \igorw\get_in($record, explode('.', $index_fields[$name]));
             if ($field_values !== null) {
@@ -84,12 +98,13 @@ class ThesaurusHydrator implements HydratorInterface
         $bulk = $this->thesaurus->findConceptsBulk($terms, null, $filters, true);
 
         foreach ($bulk as $offset => $item_concepts) {
+            $name = $field_names[$offset];
             if ($item_concepts && is_array($item_concepts) && count($item_concepts)>0) {
-                $name = $field_names[$offset];
                 foreach ($item_concepts as $concept) {
                     $record['concept_path'][$name][] = $concept->getPath();
                 }
-            } else {
+            }
+            else {
                 $this->candidate_terms->insert($field_names[$offset], $values[$offset]);
             }
         }