Merge branch 'master' into ar-731-status-search-dsl

Conflicts:
	lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php

lib/Alchemy/Phrasea/Application.php

@@ -84,6 +84,8 @@ use MediaVorus\Media\MediaInterface;
 use MediaVorus\MediaVorus;
 use MediaVorus\MediaVorusServiceProvider;
 use Monolog\Handler\NullHandler;
+use Monolog\Handler\RotatingFileHandler;
+use Monolog\Handler\SyslogHandler;
 use Monolog\Logger;
 use Monolog\Processor\IntrospectionProcessor;
 use MP4Box\MP4BoxServiceProvider;
@@ -266,6 +268,7 @@ class Application extends SilexApplication
         $this['phraseanet.exception_handler'] = $this->share(function ($app) {
             $handler =  PhraseaExceptionHandler::register($app['debug']);
             $handler->setTranslator($app['translator']);
+            $handler->setLogger($app['monolog']);
 
             return $handler;
         });
@@ -1050,14 +1053,16 @@ class Application extends SilexApplication
     private function setupMonolog()
     {
         $this['monolog.name'] = 'phraseanet';
-        $this['monolog.handler'] = $this->share(function () {
+        $this['monolog.logfile'] = $this['root.path'] . '/logs/app_error.log';
-            return new NullHandler();
+        $this['monolog.handler'] = $this->share(function (Application $app) {
+            return new RotatingFileHandler(
+                $app['monolog.logfile'],
+                10,
+                Logger::ERROR,
+                $app['monolog.bubble'],
+                $app['monolog.permission']
+            );
         });
-        $this['monolog'] = $this->share($this->extend('monolog', function (Logger $logger) {
-            $logger->pushProcessor(new IntrospectionProcessor());
-
-            return $logger;
-        }));
     }
 
     private function setupEventDispatcher()

lib/Alchemy/Phrasea/Application/Api.php

@@ -41,7 +41,10 @@ return call_user_func(function ($environment = PhraseaApplication::ENV_PROD) {
     $app->loadPlugins();
 
     $app['exception_handler'] = $app->share(function ($app) {
-        return new ApiExceptionHandlerSubscriber($app);
+        $handler = new ApiExceptionHandlerSubscriber($app);
+        $handler->setLogger($app['monolog']);
+
+        return $handler;
     });
     $app['monolog'] = $app->share($app->extend('monolog', function (Logger $monolog) {
         $monolog->pushProcessor(new WebProcessor());
@@ -92,6 +95,7 @@ return call_user_func(function ($environment = PhraseaApplication::ENV_PROD) {
         if ($request->getRequestFormat(Result::FORMAT_JSON) === Result::FORMAT_JSONP && !$response->isOk() && !$response->isServerError()) {
             $response->setStatusCode(200);
         }
+
         // set response content type
         if (!$response->headers->get('Content-Type')) {
             $response->headers->set('Content-Type', $request->getMimeType($request->getRequestFormat(Result::FORMAT_JSON)));

lib/Alchemy/Phrasea/Controller/Admin/DataboxController.php

@@ -577,29 +577,23 @@ class DataboxController extends Controller
 
         $ret = [
             'success'           => false,
-            'msg'               => $this->app->trans('An error occured'),
             'sbas_id'           => null,
+            'msg'               => $this->app->trans('An error occured'),
             'indexable'         => false,
-            'records'           => 0,
-            'xml_indexed'       => 0,
-            'thesaurus_indexed' => 0,
             'viewname'          => null,
             'printLogoURL'      => null,
+            'counts'            => null,
         ];
 
         try {
             $databox = $this->findDataboxById($databox_id);
-            $data = $databox->get_indexed_record_amount();
 
+            $ret['sbas_id'] = $databox_id;
             $ret['indexable'] = $appbox->is_databox_indexable($databox);
             $ret['viewname'] = (($databox->get_dbname() == $databox->get_viewname())
                 ? $this->app->trans('admin::base: aucun alias')
                 : $databox->get_viewname());
-            $ret['records'] = $databox->get_record_amount();
+            $ret['counts'] = $databox->get_counts();
-            $ret['sbas_id'] = $databox_id;
-            $ret['xml_indexed'] = $data['xml_indexed'];
-            $ret['thesaurus_indexed'] = $data['thesaurus_indexed'];
-            $ret['jeton_subdef'] = $data['jeton_subdef'];
             if ($this->app['filesystem']->exists($this->app['root.path'] . '/config/minilogos/logopdf_' . $databox_id . '.jpg')) {
                 $ret['printLogoURL'] = '/custom/minilogos/logopdf_' . $databox_id . '.jpg';
             }

lib/Alchemy/Phrasea/Controller/Api/V1Controller.php

@@ -462,6 +462,13 @@ class V1Controller extends Controller
         ];
     }
 
+    public function getDataboxCollectionAction(Request $request, $base_id)
+    {
+        return Result::create($request, [
+            $this->listCollection($this->app->getApplicationBox()->get_collection($base_id))
+        ])->createResponse();
+    }
+
     /**
      * Get a Response containing the collections of a \databox
      *
@@ -1459,9 +1466,9 @@ class V1Controller extends Controller
         $devices = $request->get('devices', []);
         $mimes = $request->get('mimes', []);
 
-        $ret = array_filter(array_map(function ($media) use ($request, $record) {
+        $ret = array_values(array_filter(array_map(function ($media) use ($request, $record) {
             return $this->listEmbeddableMedia($request, $record, $media);
-        }, $record->get_embedable_medias($devices, $mimes)));
+        }, $record->get_embedable_medias($devices, $mimes))));
 
         return Result::create($request, ["embed" => $ret])->createResponse();
     }
@@ -1978,9 +1985,9 @@ class V1Controller extends Controller
         $devices = $request->get('devices', []);
         $mimes = $request->get('mimes', []);
 
-        $ret = array_filter(array_map(function ($media) use ($request, $record) {
+        $ret = array_values(array_filter(array_map(function ($media) use ($request, $record) {
             return $this->listEmbeddableMedia($request, $record, $media);
-        }, $record->get_embedable_medias($devices, $mimes)));
+        }, $record->get_embedable_medias($devices, $mimes))));
 
         return Result::create($request, ["embed" => $ret])->createResponse();
     }

lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusController.php

@@ -12,7 +12,7 @@ namespace Alchemy\Phrasea\Controller\Thesaurus;
 use Alchemy\Phrasea\Application\Helper\DispatcherAware;
 use Alchemy\Phrasea\Controller\Controller;
 use Alchemy\Phrasea\Core\Event\Thesaurus as ThesaurusEvent;
-use Alchemy\Phrasea\Core\PhraseaEvents;
+use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvents;
 use Doctrine\DBAL\Driver\Connection;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
@@ -699,7 +699,7 @@ class ThesaurusController extends Controller
                 }
 
                 $this->dispatch(
-                    PhraseaEvents::THESAURUS_IMPORTED,
+                    ThesaurusEvents::IMPORTED,
                     new ThesaurusEvent\Imported($databox)
                 );
             }
@@ -987,7 +987,7 @@ class ThesaurusController extends Controller
 
             if ($request->get("reindex")) {
                 $this->dispatch(
-                    PhraseaEvents::THESAURUS_FIELD_LINKED,
+                    ThesaurusEvents::FIELD_LINKED,
                     new ThesaurusEvent\FieldLinked($databox)
                 );
             }
@@ -1360,7 +1360,7 @@ class ThesaurusController extends Controller
                                 $databox->saveThesaurus($domth);
 
                                 $this->dispatch(
-                                    PhraseaEvents::THESAURUS_CANDIDATE_ACCEPTED_AS_CONCEPT,
+                                    ThesaurusEvents::CANDIDATE_ACCEPTED_AS_CONCEPT,
                                     new ThesaurusEvent\CandidateAccepted($databox, $new_te->getAttribute('id'))
                                 );
 
@@ -1404,7 +1404,7 @@ class ThesaurusController extends Controller
                                     $databox->saveThesaurus($domth);
 
                                     $this->dispatch(
-                                        PhraseaEvents::THESAURUS_CANDIDATE_ACCEPTED_AS_SYNONYM,
+                                        ThesaurusEvents::CANDIDATE_ACCEPTED_AS_SYNONYM,
                                         new ThesaurusEvent\CandidateAccepted($databox, $new_te->getAttribute('id'))
                                     );
                                 }
@@ -1493,7 +1493,7 @@ class ThesaurusController extends Controller
                         $databox->saveThesaurus($dom);
 
                         $this->dispatch(
-                            PhraseaEvents::THESAURUS_SYNONYM_LNG_CHANGED,
+                            ThesaurusEvents::SYNONYM_LNG_CHANGED,
                             new ThesaurusEvent\SynonymLngChanged($databox, $sy0->getAttribute('id'))
                         );
                     }
@@ -1575,7 +1575,7 @@ class ThesaurusController extends Controller
                         $databox->saveThesaurus($dom);
 
                         $this->dispatch(
-                            PhraseaEvents::THESAURUS_SYNONYM_POSITION_CHANGED,
+                            ThesaurusEvents::SYNONYM_POSITION_CHANGED,
                             new ThesaurusEvent\SynonymPositionChanged($databox, $sy0->getAttribute('id'))
                         );
 
@@ -1720,7 +1720,7 @@ class ThesaurusController extends Controller
                         $databox->saveThesaurus($dom);
 
                         $this->dispatch(
-                            PhraseaEvents::THESAURUS_SYNONYM_TRASHED,
+                            ThesaurusEvents::SYNONYM_TRASHED,
                             new ThesaurusEvent\ItemTrashed($databox, $te->getAttribute('id'), $delsy->getAttribute('id'))
                         );
 
@@ -1837,7 +1837,7 @@ class ThesaurusController extends Controller
                         $databox->saveThesaurus($domth);
 
                         $this->dispatch(
-                            PhraseaEvents::THESAURUS_CONCEPT_TRASHED,
+                            ThesaurusEvents::CONCEPT_TRASHED,
                             new ThesaurusEvent\ItemTrashed($databox, $thnode_parent->getAttribute('id'), $newte->getAttribute('id'))
                         );
 
@@ -2282,7 +2282,7 @@ class ThesaurusController extends Controller
                         $databox->saveThesaurus($dom);
 
                         $this->dispatch(
-                            PhraseaEvents::THESAURUS_CONCEPT_DELETED,
+                            ThesaurusEvents::CONCEPT_DELETED,
                             new ThesaurusEvent\ConceptDeleted($databox, $refrid, $sy_evt_parm)
                         );
 
@@ -2391,7 +2391,7 @@ class ThesaurusController extends Controller
                     $databox->saveThesaurus($domth);
 
                     $this->dispatch(
-                        PhraseaEvents::THESAURUS_SYNONYM_ADDED,
+                        ThesaurusEvents::SYNONYM_ADDED,
                         new ThesaurusEvent\ItemAdded($databox, $syid)
                     );
 
@@ -2496,7 +2496,7 @@ class ThesaurusController extends Controller
                     $databox->saveThesaurus($domth);
 
                     $this->dispatch(
-                        PhraseaEvents::THESAURUS_CONCEPT_ADDED,
+                        ThesaurusEvents::CONCEPT_ADDED,
                         new ThesaurusEvent\ItemAdded($databox, $syid)
                     );
 

lib/Alchemy/Phrasea/ControllerProvider/Api/V1.php

@@ -20,7 +20,7 @@ use Silex\ServiceProviderInterface;
 
 class V1 implements ControllerProviderInterface, ServiceProviderInterface
 {
-    const VERSION = '1.4.1';
+    const VERSION = '2.0.0';
 
     public static $extendedContentTypes = [
         'json' => ['application/vnd.phraseanet.record-extended+json'],
@@ -72,6 +72,8 @@ class V1 implements ControllerProviderInterface, ServiceProviderInterface
         $controllers->get('/monitor/phraseanet/', 'controller.api.v1:showPhraseanetConfigurationAction')
             ->before('controller.api.v1:ensureAdmin');
 
+        $controllers->get('/collections/{base_id}/', 'controller.api.v1:getDataboxCollectionAction');
+
         $controllers->get('/databoxes/list/', 'controller.api.v1:listDataboxesAction');
 
         $controllers->get('/databoxes/{databox_id}/collections/', 'controller.api.v1:getDataboxCollectionsAction')

lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiExceptionHandlerSubscriber.php

@@ -13,6 +13,8 @@ namespace Alchemy\Phrasea\Core\Event\Subscriber;
 
 use Alchemy\Phrasea\Application;
 use Alchemy\Phrasea\Controller\Api\Result;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
 use Symfony\Component\HttpKernel\Event\GetResponseForExceptionEvent;
 use Symfony\Component\HttpKernel\KernelEvents;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
@@ -27,9 +29,17 @@ class ApiExceptionHandlerSubscriber implements EventSubscriberInterface
 {
     private $app;
 
+    private $logger;
+
     public function __construct(Application $app)
     {
         $this->app = $app;
+        $this->logger = new NullLogger();
+    }
+
+    public function setLogger(LoggerInterface $logger)
+    {
+        $this->logger = $logger;
     }
 
     public static function getSubscribedEvents()
@@ -64,6 +74,13 @@ class ApiExceptionHandlerSubscriber implements EventSubscriberInterface
             $code = 500;
         }
 
+        if ($code == 500) {
+            $this->logger->error($e->getMessage(), [
+                'code' => $e->getCode(),
+                'trace' => $e->getTrace()
+            ]);
+        }
+
         if ($e instanceof HttpExceptionInterface) {
             $headers = $e->getHeaders();
         }

lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiOauth2ErrorsSubscriber.php

@@ -61,7 +61,7 @@ class ApiOauth2ErrorsSubscriber implements EventSubscriberInterface
             $msg = json_encode(['msg'  => $msg, 'code' => $code]);
             $event->setResponse(new Response($msg, $code, $headers));
         } else {
-            $response = $this->handler->createResponseBasedOnRequest($event->getRequest(), $event->getException());
+            $response = $this->handler->createResponse($event->getException());
             $response->headers->set('Content-Type', 'text/html');
             $event->setResponse($response);
         }

lib/Alchemy/Phrasea/Core/Event/Subscriber/PhraseaExceptionHandlerSubscriber.php

@@ -38,7 +38,7 @@ class PhraseaExceptionHandlerSubscriber implements EventSubscriberInterface
             return;
         }
 
-        $event->setResponse($this->handler->createResponseBasedOnRequest($event->getRequest(), $event->getException()));
+        $event->setResponse($this->handler->createResponse($event->getException()));
     }
 
     /**

lib/Alchemy/Phrasea/Core/Event/Thesaurus/ThesaurusEvents.php

@@ -0,0 +1,27 @@
+<?php
+
+/*
+ * This file is part of Phraseanet
+ *
+ * (c) 2005-2014 Alchemy
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Alchemy\Phrasea\Core\Event\Thesaurus;
+
+final class ThesaurusEvents
+{
+    const IMPORTED = 'thesaurus.imported';
+    const FIELD_LINKED = 'thesaurus.field-linked';
+    const CANDIDATE_ACCEPTED_AS_CONCEPT = 'thesaurus.candidate-accepted-as-concept';
+    const CANDIDATE_ACCEPTED_AS_SYNONYM = 'thesaurus.candidate-accepted-as-synonym';
+    const SYNONYM_LNG_CHANGED = 'thesaurus.synonym-lng-changed';
+    const SYNONYM_POSITION_CHANGED = 'thesaurus.synonym-position-changed';
+    const SYNONYM_TRASHED = 'thesaurus.synonym-trashed';
+    const CONCEPT_TRASHED = 'thesaurus.concept-trashed';
+    const CONCEPT_DELETED = 'thesaurus.concept-deleted';
+    const SYNONYM_ADDED = 'thesaurus.synonym-added';
+    const CONCEPT_ADDED = 'thesaurus.concept-added';
+}

lib/Alchemy/Phrasea/Core/PhraseaExceptionHandler.php

@@ -11,6 +11,8 @@
 
 namespace Alchemy\Phrasea\Core;
 
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
 use Symfony\Component\Debug\ExceptionHandler as SymfonyExceptionHandler;
 use Symfony\Component\Debug\Exception\FlattenException;
 use Symfony\Component\HttpFoundation\Request;
@@ -21,6 +23,18 @@ class PhraseaExceptionHandler extends SymfonyExceptionHandler
 {
     private $translator;
 
+    private $logger;
+
+    public function __construct()
+    {
+        $this->logger = new NullLogger();
+    }
+
+    public function setLogger(LoggerInterface $logger)
+    {
+        $this->logger = $logger;
+    }
+
     public function setTranslator(TranslatorInterface $translator)
     {
         $this->translator = $translator;
@@ -33,6 +47,13 @@ class PhraseaExceptionHandler extends SymfonyExceptionHandler
 
     public function getContent(FlattenException $exception)
     {
+        if ($exception->getStatusCode() == '500') {
+            $this->logger->error($exception->getMessage(), [
+                'code' => $exception->getCode(),
+                'trace' => $exception->getTrace()
+            ]);
+        }
+
         switch (true) {
             case 404 === $exception->getStatusCode():
                 if (null !== $this->translator) {

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
-use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
 
 class QuotedTextNode extends Node
 {
@@ -38,11 +38,11 @@ class QuotedTextNode extends Node
         };
 
         $unrestricted_fields = $context->getUnrestrictedFields();
-        $unrestricted_fields = Field::filterByValueCompatibility($unrestricted_fields, $this->text);
+        $unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text);
         $query = $query_builder($unrestricted_fields);
 
         $private_fields = $context->getPrivateFields();
-        $private_fields = Field::filterByValueCompatibility($private_fields, $this->text);
+        $private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text);
         foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
             $query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
         }

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
-use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
 
 class RawNode extends Node
 {
@@ -56,11 +56,11 @@ class RawNode extends Node
         };
 
         $unrestricted_fields = $context->getUnrestrictedFields();
-        $unrestricted_fields = Field::filterByValueCompatibility($unrestricted_fields, $this->text);
+        $unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text);
         $query = $query_builder($unrestricted_fields);
 
         $private_fields = $context->getPrivateFields();
-        $private_fields = Field::filterByValueCompatibility($private_fields, $this->text);
+        $private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text);
         foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
             $query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
         }

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
 use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
-use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Term;
 
 class TextNode extends AbstractTermNode implements ContextAbleInterface
@@ -41,7 +41,7 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
         $query_builder = function (array $fields) use ($context) {
             // Full text
             $index_fields = [];
-            foreach (Field::filterByValueCompatibility($fields, $this->text) as $field) {
+            foreach (StructureField::filterByValueCompatibility($fields, $this->text) as $field) {
                 foreach ($context->localizeField($field) as $f) {
                     $index_fields[] = $f;
                 }
@@ -53,6 +53,7 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
                 'multi_match' => [
                     'fields'   => $index_fields,
                     'query'    => $this->text,
+                    'type'     => 'cross_fields',
                     'operator' => 'and',
                     'lenient'  => true,
                 ]

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php

@@ -41,7 +41,7 @@ class Indexer
     private $recordIndexer;
     private $termIndexer;
 
-    private $indexQueue;
+    private $indexQueue;        // contains RecordInterface(s)
     private $deleteQueue;
 
     private $previousRefreshInterval = self::DEFAULT_REFRESH_INTERVAL;
@@ -155,6 +155,11 @@ class Indexer
         // RecordQueuer::queueRecordsFromDatabox($databox);
     }
 
+    public function scheduleRecordsFromDataboxForIndexing(\databox $databox)
+    {
+        RecordQueuer::queueRecordsFromDatabox($databox);
+    }
+
     public function scheduleRecordsFromCollectionForIndexing(\collection $collection)
     {
         RecordQueuer::queueRecordsFromCollection($collection);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php

@@ -23,10 +23,11 @@ class BulkOperation
     private $logger;
 
     private $stack = array();
-    private $opCount = 0;
+    private $operationIdentifiers = [];
     private $index;
     private $type;
     private $flushLimit = 1000;
+    private $flushCallbacks = [];
 
     public function __construct(Client $client, LoggerInterface $logger)
     {
@@ -52,27 +53,32 @@ class BulkOperation
         $this->flushLimit = (int) $limit;
     }
 
-    public function index(array $params)
+    public function onFlush(\Closure $callback)
+    {
+        $this->flushCallbacks[] = $callback;
+    }
+
+    public function index(array $params, $operationIdentifier)
     {
         $header = $this->buildHeader('index', $params);
         $body = igorw\get_in($params, ['body']);
-        $this->push($header, $body);
+        $this->push($header, $body, $operationIdentifier);
     }
 
-    public function delete(array $params)
+    public function delete(array $params, $operationIdentifier)
     {
-        $this->push($this->buildHeader('delete', $params));
+        $this->push($this->buildHeader('delete', $params), null, $operationIdentifier);
     }
 
-    private function push($header, $body = null)
+    private function push($header, $body, $operationIdentifier)
     {
         $this->stack[] = $header;
         if ($body) {
             $this->stack[] = $body;
         }
-        $this->opCount++;
+        $this->operationIdentifiers[] = $operationIdentifier;
 
-        if ($this->flushLimit === $this->opCount) {
+        if (count($this->operationIdentifiers) === $this->flushLimit) {
             $this->flush();
         }
     }
@@ -93,19 +99,32 @@ class BulkOperation
         }
         $params['body'] = $this->stack;
 
-        $this->logger->debug("ES Bulk query about to be performed\n", ['opCount' => $this->opCount]);
+        $this->logger->debug("ES Bulk query about to be performed\n", ['opCount' => count($this->operationIdentifiers)]);
 
         $response = $this->client->bulk($params);
         $this->stack = array();
-        $this->opCount = 0;
 
-        if (igorw\get_in($response, ['errors'], true)) {
+        $callbackData = [];     // key: operationIdentifier passed when command was pushed on this bulk
-            foreach ($response['items'] as $key => $item) {
+                                // value: json result from es for the command
-                if ($item['index']['status'] >= 400) { // 4xx or 5xx error
+        // nb: results (items) are returned IN THE SAME ORDER as commands were pushed in the stack
-                    throw new Exception(sprintf('%d: %s', $key, $item['index']['error']));
+        // so the items[X] match the operationIdentifiers[X]
+        foreach ($response['items'] as $key => $item) {
+            foreach($item as $command=>$result) {   // command may be "index" or "delete"
+                if($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx error
+                    $err = array_key_exists('error', $result) ? $result['error'] : ($command . " error " . $result['status']);
+                    throw new Exception(sprintf('%d: %s', $key, $err));
                 }
             }
+
+            $operationIdentifier = $this->operationIdentifiers[$key];
+            if(is_string($operationIdentifier) || is_int($operationIdentifier)) {   // dont include null keys
+                $callbackData[$operationIdentifier] = $response['items'][$key];
+            }
         }
+        foreach($this->flushCallbacks as $iCallBack=>$flushCallback) {
+            $flushCallback($callbackData);
+        }
+        $this->operationIdentifiers = [];
     }
 
     private function buildHeader($key, array $params)

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php

@@ -16,12 +16,14 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegate;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegateInterface;
 use Closure;
+use databox;
 use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Driver\Connection as ConnectionInterface;
 use PDO;
 
 class Fetcher
 {
+    private $databox;
     private $connection;
     private $statement;
     private $delegate;
@@ -34,13 +36,19 @@ class Fetcher
     private $postFetch;
     private $onDrain;
 
-    public function __construct(ConnectionInterface $connection, array $hydrators, FetcherDelegateInterface $delegate = null)
+    public function __construct(databox $databox, array $hydrators, FetcherDelegateInterface $delegate = null)
     {
-        $this->connection = $connection;
+        $this->databox = $databox;
+        $this->connection = $databox->get_connection();;
         $this->hydrators  = $hydrators;
         $this->delegate   = $delegate ?: new FetcherDelegate();
     }
 
+    public function getDatabox()
+    {
+        return $this->databox;
+    }
+
     public function fetch()
     {
         if (empty($this->buffer)) {
@@ -64,7 +72,6 @@ class Fetcher
             $records[$record['record_id']] = $record;
             $this->offset++;
         }
-
         if (empty($records)) {
             $this->onDrain->__invoke();
             return;
@@ -87,6 +94,12 @@ class Fetcher
         return $records;
     }
 
+    public function restart()
+    {
+        $this->buffer = array();
+        $this->offset = 0;
+    }
+
     public function setBatchSize($size)
     {
         if ($size < 1) {
@@ -105,28 +118,24 @@ class Fetcher
         $this->onDrain = $onDrain;
     }
 
+    /**
+     * @return \Doctrine\DBAL\Driver\Statement
+     */
     private function getExecutedStatement()
     {
         if (!$this->statement) {
-            $sql = <<<SQL
+            $sql = "SELECT r.record_id"
-            SELECT r.record_id
+                 . ", r.coll_id AS collection_id"
-                 , r.coll_id as collection_id
+                 . ", c.asciiname AS collection_name"
-                 , c.asciiname as collection_name
+                 . ", r.uuid"
-                 , r.uuid
+                 . ", r.status AS flags_bitfield"
-                 , r.status as flags_bitfield
+                 . ", r.sha256" // -- TODO rename in "hash"
-                 , r.sha256 -- TODO rename in "hash"
+                 . ", r.originalname AS original_name"
-                 , r.originalname as original_name
+                 . ", r.mime, r.type, r.parent_record_id, r.credate AS created_on, r.moddate AS updated_on"
-                 , r.mime
+                 . " FROM record r INNER JOIN coll c ON (c.coll_id = r.coll_id)"
-                 , r.type
+                 . " -- WHERE"
-                 , r.parent_record_id
+                 . " ORDER BY r.record_id DESC"
-                 , r.credate as created_on
+                 . " LIMIT :offset, :limit";
-                 , r.moddate as updated_on
-                    FROM record r
-                    INNER JOIN coll c ON (c.coll_id = r.coll_id)
-                    -- WHERE
-                    ORDER BY r.record_id DESC
-                    LIMIT :offset, :limit
-SQL;
 
             $where = $this->delegate->buildWhereClause();
             $sql = str_replace('-- WHERE', $where, $sql);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php

@@ -11,6 +11,7 @@
 
 namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer;
 
+use Alchemy\Phrasea\Model\RecordInterface;
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException;
 use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\BulkOperation;
@@ -60,6 +61,12 @@ class RecordIndexer
 
     private $logger;
 
+    private function getUniqueOperationId($record_key)
+    {
+        $_key = dechex(mt_rand());
+        return $_key . '_' . $record_key;
+    }
+
     public function __construct(Structure $structure, RecordHelper $helper, Thesaurus $thesaurus, \appbox $appbox, array $locales, LoggerInterface $logger)
     {
         $this->structure = $structure;
@@ -70,16 +77,70 @@ class RecordIndexer
         $this->logger = $logger;
     }
 
+    /**
+     * ES made a bulk op, check our (index) operations to drop the "indexing" & "to_index" jetons
+     *
+     * @param databox $databox
+     * @param array $operation_identifiers  key:op_identifier ; value:operation result (json from es)
+     * @param array $submited_records       records indexed, key:op_identifier
+     */
+    private function onBulkFlush(databox $databox, array $operation_identifiers, array &$submited_records)
+    {
+        // nb: because the same bulk could be used by many "clients", this (each) callback may receive
+        // operation_identifiers that does not belong to it.
+        // flag only records that the fetcher worked on
+        $records = array_intersect_key(
+            $submited_records,        // this is OUR records list
+            $operation_identifiers          // reduce to the records indexed by this bulk (should be the same...)
+        );
+        if(count($records) === 0) {
+            return;
+        }
+        // Commit and remove "indexing" flag
+        RecordQueuer::didFinishIndexingRecords(array_values($records), $databox);
+        foreach (array_keys($records) as $id) {
+            unset($submited_records[$id]);
+        }
+    }
+
+    /**
+     * index whole databox(es), don't test actual "jetons"
+     *
+     * @param BulkOperation $bulk
+     * @param databox[] $databoxes
+     */
     public function populateIndex(BulkOperation $bulk, array $databoxes)
     {
         foreach ($databoxes as $databox) {
+            $submited_records = [];
+
             $this->logger->info(sprintf('Indexing database %s...', $databox->get_viewname()));
-            $fetcher = $this->createFetcherForDatabox($databox);
+
-            $this->indexFromFetcher($bulk, $fetcher);
+            $fetcher = $this->createFetcherForDatabox($databox);    // no delegate, scan the whole records
+
+            // post fetch : flag records as "indexing"
+            $fetcher->setPostFetch(function(array $records) use ($databox, $fetcher) {
+                RecordQueuer::didStartIndexingRecords($records, $databox);
+                // do not restart the fetcher since it has no clause on jetons
+            });
+
+            // bulk flush : flag records as "indexed"
+            $bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) {
+                $this->onBulkFlush($databox, $operation_identifiers, $submited_records);
+            });
+
+            // Perform indexing
+            $this->indexFromFetcher($bulk, $fetcher, $submited_records);
+
             $this->logger->info(sprintf('Finished indexing %s', $databox->get_viewname()));
         }
     }
 
+    /**
+     * Index the records flagged as "to_index" on all databoxes
+     *
+     * @param BulkOperation $bulk
+     */
     public function indexScheduled(BulkOperation $bulk)
     {
         foreach ($this->appbox->get_databoxes() as $databox) {
@@ -89,46 +150,82 @@ class RecordIndexer
 
     private function indexScheduledInDatabox(BulkOperation $bulk, databox $databox)
     {
+        $submited_records = [];
+
         // Make fetcher
         $delegate = new ScheduledFetcherDelegate();
         $fetcher = $this->createFetcherForDatabox($databox, $delegate);
-        // Keep track of fetched records, flag them as "indexing"
+
-        $fetched = array();
+        // post fetch : flag records as "indexing"
-        $fetcher->setPostFetch(function(array $records) use ($databox, &$fetched) {
+        $fetcher->setPostFetch(function(array $records) use ($databox, $fetcher) {
-            // TODO Do not keep all indexed records in memory...
-            $fetched += $records;
             RecordQueuer::didStartIndexingRecords($records, $databox);
+            // because changing the flag on the records affects the "where" clause of the fetcher,
+            // restart it each time
+            $fetcher->restart();
         });
+
+        // bulk flush : flag records as "indexed"
+        $bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) {
+            $this->onBulkFlush($databox, $operation_identifiers, $submited_records);
+        });
+
         // Perform indexing
-        $this->indexFromFetcher($bulk, $fetcher);
+        $this->indexFromFetcher($bulk, $fetcher, $submited_records);
-        // Commit and remove "indexing" flag
-        $bulk->flush();
-        RecordQueuer::didFinishIndexingRecords($fetched, $databox);
     }
 
+    /**
+     * Index a list of records
+     *
+     * @param BulkOperation $bulk
+     * @param Iterator $records
+     */
     public function index(BulkOperation $bulk, Iterator $records)
     {
         foreach ($this->createFetchersForRecords($records) as $fetcher) {
-            $this->indexFromFetcher($bulk, $fetcher);
+            $submited_records = [];
+            $databox = $fetcher->getDatabox();
+
+            // post fetch : flag records as "indexing"
+            $fetcher->setPostFetch(function(array $records) use ($fetcher, $databox) {
+                RecordQueuer::didStartIndexingRecords($records, $databox);
+                // do not restart the fetcher since it has no clause on jetons
+            });
+
+            // bulk flush : flag records as "indexed"
+            $bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) {
+                $this->onBulkFlush($databox, $operation_identifiers, $submited_records);
+            });
+
+            // Perform indexing
+            $this->indexFromFetcher($bulk, $fetcher, $submited_records);
         }
     }
 
+    /**
+     * Deleta a list of records
+     *
+     * @param BulkOperation $bulk
+     * @param Iterator $records
+     */
     public function delete(BulkOperation $bulk, Iterator $records)
     {
         foreach ($records as $record) {
             $params = array();
             $params['id'] = $record->getId();
             $params['type'] = self::TYPE_NAME;
-            $bulk->delete($params);
+            $bulk->delete($params, null);       // no operationIdentifier is related to a delete op
         }
     }
 
+    /**
+     * @param Iterator $records
+     * @return Fetcher[]
+     */
     private function createFetchersForRecords(Iterator $records)
     {
         $fetchers = array();
         foreach ($this->groupRecordsByDatabox($records) as $group) {
             $databox = $group['databox'];
-            $connection = $databox->get_connection();
             $delegate = new RecordListFetcherDelegate($group['records']);
             $fetchers[] = $this->createFetcherForDatabox($databox, $delegate);
         }
@@ -140,7 +237,7 @@ class RecordIndexer
     {
         $connection = $databox->get_connection();
         $candidateTerms = new CandidateTerms($databox);
-        $fetcher = new Fetcher($connection, array(
+        $fetcher = new Fetcher($databox, array(
             new CoreHydrator($databox->get_sbas_id(), $databox->get_viewname(), $this->helper),
             new TitleHydrator($connection),
             new MetadataHydrator($connection, $this->structure, $this->helper),
@@ -169,15 +266,21 @@ class RecordIndexer
         return array_values($databoxes);
     }
 
-    private function indexFromFetcher(BulkOperation $bulk, Fetcher $fetcher)
+    private function indexFromFetcher(BulkOperation $bulk, Fetcher $fetcher, array &$submited_records)
     {
+        /** @var RecordInterface $record */
         while ($record = $fetcher->fetch()) {
+            $op_identifier = $this->getUniqueOperationId($record['id']);
+
             $params = array();
             $params['id'] = $record['id'];
             unset($record['id']);
             $params['type'] = self::TYPE_NAME;
             $params['body'] = $record;
-            $bulk->index($params);
+
+            $submited_records[$op_identifier] = $record;
+
+            $bulk->index($params, $op_identifier);
         }
     }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordQueuer.php

@@ -35,41 +35,44 @@ class RecordQueuer
         $connection = $collection->get_connection();
 
         // Set TO_INDEX flag on all records from this collection
-        $sql = <<<SQL
+        $sql = "UPDATE record SET jeton = (jeton | :token) WHERE coll_id = :coll_id";
-            UPDATE record
+
-            SET jeton = (jeton | :token)
-            WHERE coll_id = :coll_id
-SQL;
         $stmt = $connection->prepare($sql);
         $stmt->bindValue(':token', Flag::TO_INDEX, PDO::PARAM_INT);
         $stmt->bindValue(':coll_id', $collection->get_coll_id(), PDO::PARAM_INT);
         $stmt->execute();
     }
 
+    /**
+     * @param array $records
+     * @param $databox
+     *
+     * nb: changing the jeton may affect a fetcher if his "where" clause (delegate) depends on jeton.
+     * in this case the client of the fetcher must set a "postFetch" callback and restart the fetcher
+     */
     public static function didStartIndexingRecords(array $records, $databox)
     {
         $connection = $databox->get_connection();
-        $sql = <<<SQL
+        $sql = "UPDATE record SET jeton = (jeton | :flag) WHERE record_id IN (:record_ids)";
-            UPDATE record
+
-            SET jeton = (jeton | :flag)
-            WHERE record_id IN (:record_ids)
-SQL;
         self::executeFlagQuery($connection, $sql, Flag::INDEXING, $records);
     }
 
-    public static function didFinishIndexingRecords(array $records, $databox)
+    /**
+     * @param array $records
+     * @param $databox
+     *
+     * nb: changing the jeton may affect a fetcher if his "where" clause (delegate) depends on jeton.
+     * in this case the client of the fetcher must set a "postFetch" callback and restart the fetcher
+     */
+    public static function didFinishIndexingRecords(array $records, databox $databox)
     {
         $connection = $databox->get_connection();
-        $sql = <<<SQL
+        $sql = "UPDATE record SET jeton = (jeton & ~ :flag) WHERE record_id IN (:record_ids)";
-            UPDATE record
+        self::executeFlagQuery($connection, $sql, Flag::TO_INDEX | Flag::INDEXING, $records);
-            SET jeton = (jeton & ~ :flag)
-            WHERE record_id IN (:record_ids)
-SQL;
-        $flag = Flag::TO_INDEX | Flag::INDEXING;
-        self::executeFlagQuery($connection, $sql, $flag, $records);
     }
 
-    private static function executeFlagQuery($connection, $sql, $flag, array $records)
+    private static function executeFlagQuery(Connection $connection, $sql, $flag, array $records)
     {
         return $connection->executeQuery($sql, array(
             ':flag'       => $flag,

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/TermIndexer.php

@@ -60,7 +60,7 @@ class TermIndexer
                 $params['type'] = self::TYPE_NAME;
                 $params['body'] = $term;
 
-                $bulk->index($params);
+                $bulk->index($params, null);
             });
 
             $document = Helper::thesaurusFromDatabox($databox);

lib/Alchemy/Phrasea/SearchEngine/Elastic/IndexerSubscriber.php

@@ -19,6 +19,8 @@ use Alchemy\Phrasea\Core\Event\Record\RecordEvents;
 use Alchemy\Phrasea\Core\Event\Record\RecordSubDefinitionCreatedEvent;
 use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvent;
 use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvents;
+use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvent;
+use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvents;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 
 /**
@@ -84,6 +86,17 @@ class IndexerSubscriber implements EventSubscriberInterface
             RecordEvents::STATUS_CHANGED => 'onRecordChange',
             RecordEvents::SUB_DEFINITION_CREATED => 'onRecordChange',
             RecordEvents::MEDIA_SUBSTITUTED => 'onRecordChange',
+            ThesaurusEvents::IMPORTED => 'onThesaurusChange',
+            ThesaurusEvents::FIELD_LINKED => 'onThesaurusChange',
+            ThesaurusEvents::CANDIDATE_ACCEPTED_AS_CONCEPT => 'onThesaurusChange',
+            ThesaurusEvents::CANDIDATE_ACCEPTED_AS_SYNONYM => 'onThesaurusChange',
+            ThesaurusEvents::SYNONYM_LNG_CHANGED => 'onThesaurusChange',
+            ThesaurusEvents::SYNONYM_POSITION_CHANGED => 'onThesaurusChange',
+            ThesaurusEvents::SYNONYM_TRASHED => 'onThesaurusChange',
+            ThesaurusEvents::CONCEPT_TRASHED => 'onThesaurusChange',
+            ThesaurusEvents::CONCEPT_DELETED => 'onThesaurusChange',
+            ThesaurusEvents::SYNONYM_ADDED => 'onThesaurusChange',
+            ThesaurusEvents::CONCEPT_ADDED => 'onThesaurusChange'
         ];
     }
 
@@ -93,6 +106,12 @@ class IndexerSubscriber implements EventSubscriberInterface
         $this->getIndexer()->migrateMappingForDatabox($databox);
     }
 
+    public function onThesaurusChange(ThesaurusEvent $event)
+    {
+        $databox = $event->getDatabox();
+        $this->getIndexer()->scheduleRecordsFromDataboxForIndexing($databox);
+    }
+
     public function onCollectionChange(CollectionEvent $event)
     {
         $collection = $event->getCollection();

lib/Alchemy/Phrasea/TaskManager/Job/SubdefsJob.php

@@ -121,8 +121,7 @@ class SubdefsJob extends AbstractJob
 
                 // rewrite metadata
                 $sql = 'UPDATE record'
-                    . ' SET status=(status & ~0x03),'
+                    . ' SET jeton=(jeton | ' . PhraseaTokens::WRITE_META_SUBDEF | PhraseaTokens::TO_INDEX . ')'
-                    . ' jeton=(jeton | ' . PhraseaTokens::WRITE_META_SUBDEF . ')'
                     . ' WHERE record_id=:record_id';
                 $stmt = $conn->prepare($sql);
                 $stmt->execute([':record_id' => $row['record_id']]);

lib/classes/API/OAuth2/Adapter.php

@@ -16,9 +16,11 @@ use Alchemy\Phrasea\Authentication\Exception\RequireCaptchaException;
 use Alchemy\Phrasea\Exception\RuntimeException;
 use Alchemy\Phrasea\Model\Entities\ApiApplication;
 use Alchemy\Phrasea\Model\Entities\User;
+use Alchemy\Phrasea\Model\Repositories\ApiApplicationRepository;
 use Symfony\Component\HttpFoundation\Session\Session;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
+use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
 
 class API_OAuth2_Adapter extends OAuth2
 {
@@ -601,9 +603,12 @@ class API_OAuth2_Adapter extends OAuth2
             'state' => null,
         ];
 
+        $result = [];
+
         if ($params['state'] !== null) {
             $result["query"]["state"] = $params['state'];
         }
+
         if ($is_authorized === false) {
             $result["query"]["error"] = OAUTH2_ERROR_USER_DENIED;
         } else {
@@ -615,6 +620,7 @@ class API_OAuth2_Adapter extends OAuth2
                 $result["fragment"] = $this->createAccessToken($params['account_id'], $params['scope']);
             }
         }
+
         $this->doRedirectUriCallback($params['redirect_uri'], $result);
     }
 
@@ -684,9 +690,15 @@ class API_OAuth2_Adapter extends OAuth2
                 }
                 break;
             case OAUTH2_GRANT_TYPE_USER_CREDENTIALS:
-                $application = ApiApplication::load_from_client_id($this->app, $client[0]);
+                /** @var ApiApplicationRepository $appRepository */
+                $appRepository = $this->app['repo.api-applications'];
+                $application = $appRepository->findByClientId($client[0]);
 
-                if ( ! $application->is_password_granted()) {
+                if (! $application) {
+                    throw new NotFoundHttpException('Application not found');
+                }
+
+                if ( ! $application->isPasswordGranted()) {
                     $this->errorJsonResponse(OAUTH2_HTTP_BAD_REQUEST, OAUTH2_ERROR_UNSUPPORTED_GRANT_TYPE, 'Password grant type is not enable for your client');
                 }
 
@@ -812,7 +824,7 @@ class API_OAuth2_Adapter extends OAuth2
 
             return [
                 'redirect_uri' => $this->client->getRedirectUri(),
-                'client_id'    => $this->client->getClient(),
+                'client_id'    => $this->client->getClientId(),
                 'account_id'   => $account->getId(),
             ];
         } catch (AccountLockedException $e) {

lib/classes/appbox.php

@@ -268,6 +268,26 @@ class appbox extends base
         return $databoxes[$sbas_id];
     }
 
+    public function get_collection($base_id)
+    {
+        $sbas_id = phrasea::sbasFromBas($this->app, $base_id);
+
+        if ($sbas_id === false) {
+            throw new \RuntimeException('Collection not found.');
+        }
+
+        $collections = $this->get_databox($sbas_id)->get_collections();
+
+        foreach ($collections as $collection) {
+            if ($collection->get_base_id() == $base_id) {
+                return $collection;
+            }
+        }
+
+        // This should not happen, but I'd rather be safe than sorry.
+        throw new \RuntimeException('Collection not found.');
+    }
+
     /**
      * @param string $option
      * @return string

lib/classes/databox.php

@@ -294,42 +294,6 @@ class databox extends base implements ThumbnailedElement
         return $this->id;
     }
 
-    public function get_unique_keywords()
-    {
-        $sql = "SELECT COUNT(kword_id) AS n FROM kword";
-
-        $stmt = $this->get_connection()->prepare($sql);
-        $stmt->execute();
-        $rowbas = $stmt->fetch(PDO::FETCH_ASSOC);
-        $stmt->closeCursor();
-
-        return ($rowbas ? $rowbas['n'] : null);
-    }
-
-    public function get_index_amount()
-    {
-        $sql = "SELECT COUNT(idx_id) AS n FROM idx";
-
-        $stmt = $this->get_connection()->prepare($sql);
-        $stmt->execute();
-        $rowbas = $stmt->fetch(PDO::FETCH_ASSOC);
-        $stmt->closeCursor();
-
-        return ($rowbas ? $rowbas['n'] : null);
-    }
-
-    public function get_thesaurus_hits()
-    {
-        $sql = "SELECT COUNT(thit_id) AS n FROM thit";
-
-        $stmt = $this->get_connection()->prepare($sql);
-        $stmt->execute();
-        $rowbas = $stmt->fetch(PDO::FETCH_ASSOC);
-        $stmt->closeCursor();
-
-        return ($rowbas ? $rowbas['n'] : null);
-    }
-
     public function get_record_details($sort)
     {
         $sql = "SELECT record.coll_id, ISNULL(coll.coll_id) AS lostcoll,
@@ -390,36 +354,46 @@ class databox extends base implements ThumbnailedElement
         return $amount;
     }
 
-    public function get_indexed_record_amount()
+    public function get_counts()
     {
-        $sql = "SELECT status & 3 AS status, SUM(1) AS n FROM record GROUP BY(status & 3)";
+        $mask = PhraseaTokens::MAKE_SUBDEF | PhraseaTokens::TO_INDEX | PhraseaTokens::INDEXING; // we only care about those "jetons"
+        $sql = "SELECT type, jeton & (".$mask.") AS status, SUM(1) AS n FROM record GROUP BY type, (jeton & ".$mask.")";
         $stmt = $this->get_connection()->prepare($sql);
         $stmt->execute();
         $rs = $stmt->fetchAll(PDO::FETCH_ASSOC);
         $stmt->closeCursor();
 
         $ret = array(
-            'xml_indexed'       => 0,
+            'records'             => 0,
-            'thesaurus_indexed' => 0,
+            'records_indexed'     => 0,    // jetons = 0;0
-            'jeton_subdef' => array()
+            'records_to_index'    => 0,    // jetons = 0;1
+            'records_not_indexed' => 0,    // jetons = 1;0
+            'records_indexing'    => 0,    // jetons = 1;1
+            'subdefs_todo'        => array()   // by type "image", "video", ...
         );
-
         foreach ($rs as $row) {
+            $ret['records'] += ($n = (int)($row['n']));
             $status = $row['status'];
-            if ($status & 1)
+            switch($status & (PhraseaTokens::TO_INDEX | PhraseaTokens::INDEXING)) {
-                $ret['xml_indexed'] += $row['n'];
+                case 0:
-            if ($status & 2)
+                    $ret['records_indexed'] += $n;
-                $ret['thesaurus_indexed'] += $row['n'];
+                    break;
-        }
+                case PhraseaTokens::TO_INDEX:
-
+                    $ret['records_to_index'] += $n;
-        $sql = "SELECT type, COUNT(record_id) AS n FROM record WHERE jeton & ".PhraseaTokens::MAKE_SUBDEF." GROUP BY type";
+                    break;
-        $stmt = $this->get_connection()->prepare($sql);
+                case PhraseaTokens::INDEXING:
-        $stmt->execute();
+                    $ret['records_not_indexed'] += $n;
-        $rs = $stmt->fetchAll(PDO::FETCH_ASSOC);
+                    break;
-        $stmt->closeCursor();
+                case PhraseaTokens::INDEXING | PhraseaTokens::TO_INDEX:
-
+                    $ret['records_indexing'] += $n;
-        foreach ($rs as $row) {
+                    break;
-            $ret['jeton_subdef'][$row['type']] = (int)$row['n'];
+            }
+            if($status & PhraseaTokens::MAKE_SUBDEF) {
+                if(!array_key_exists($row['type'], $ret['subdefs_todo'])) {
+                    $ret['subdefs_todo'][$row['type']] = 0;
+                }
+                $ret['subdefs_todo'][$row['type']] += $n;
+            }
         }
 
         return $ret;
@@ -1057,6 +1031,12 @@ class databox extends base implements ThumbnailedElement
     {
         $this->get_connection()->update('pref', ['updated_on' => '0000-00-00 00:00:00'], ['prop' => 'indexes']);
 
+        // Set TO_INDEX flag on all records
+        $sql = "UPDATE record SET jeton = (jeton | :token)";
+        $stmt = $this->connection->prepare($sql);
+        $stmt->bindValue(':token', PhraseaTokens::TO_INDEX, PDO::PARAM_INT);
+        $stmt->execute();
+
         return $this;
     }
 

templates/web/admin/databox/databox.html.twig

@@ -45,7 +45,7 @@
 
     <li>
         {{ 'admin::base: nombre d\'enregistrements sur la base :' | trans }}
-        <span id="nrecords">{{ databox.get_record_amount() }}</span>
+        <span id="records"></span>
 
         (<a href="{{ path('admin_database_display_document_details', {'databox_id': databox.get_sbas_id()}) }}" class="ajax" target="rights">{{ 'phraseanet:: details' | trans }}</a>)
     </li>
@@ -54,41 +54,13 @@
         {{ 'admin::base: subdefs to be created :' | trans }}
         <span id="subdefs_todo"></span>
     </li>
-
-    {% if showDetail %}
-        <li>
-            {{ 'admin::base: nombre de mots uniques sur la base :' | trans }}
-            {{ databox.get_unique_keywords() }}
-        </li>
-        <li>
-            {{ 'admin::base: nombre de mots indexes sur la base' | trans }}
-            {{ databox.get_index_amount() }}
-        </li>
-        {% if app['conf'].get(['registry', 'modules', 'thesaurus']) %}
-            <li>
-            {{ 'admin::base: nombre de termes de Thesaurus indexes :' | trans }}
-            {{ databox.get_thesaurus_hits() }}
-            </li>
-        {% endif %}
-    {% endif %}
 </ul>
 
-<div id="INDEX_P_BAR" style="margin-bottom:20px;">
+<div id="INDEX_P_BAR" style="margin-bottom:20px; width:50%">
-    <div style="height: 35px;">
+    <div class="progress">
-        <p>
+        <div class="bar bar-success records_indexed" style="transition: none; width:0%;">...</div>
-            {{ "admin::base: document indexes en utilisant la fiche xml" | trans }} :
+        <div class="bar bar-warning records_indexing" style="transition:none; width:0%;"></div>
-            <span id="xml_indexed"></span>
+        <div class="bar bar-danger records_not_indexed" style="transition:none; width:0%;"></div>
-        </p>
-        <div id="xml_indexed_bar"></div>
-        <div id="xml_indexed_percent"></div>
-    </div>
-    <div style="height: 35px;">
-        <p>
-            {{ "admin::base: document indexes en utilisant le thesaurus" | trans }} :
-            <span id="thesaurus_indexed"></span>
-        </p>
-        <div id="thesaurus_indexed_bar"></div>
-        <div id="thesaurus_indexed_percent"></div>
     </div>
 </div>
 
@@ -248,65 +220,91 @@
 </div>
 
 <script type="text/javascript">
-    function refreshDatabaseInformations()
-    {
-        // stop the refresh if the page changed
-        if($("#thesaurus_indexed_bar").length == 0) {
-            return;
-        }
 
+    function displayDatabaseInformations(delay)
+    {
+        try {
+            clearTimeout(document.refreshDatabaseInformations_timer);
+        }
+        catch(err) {
+
+        }
+        document.refreshDatabaseInformations_timer = setTimeout("_displayDatabaseInformations();", delay);
+    }
+
+    function _displayDatabaseInformations()
+    {
+        var container = $("#INDEX_P_BAR");
+        if(!container || container.length == 0) {
+            return;     // wrong page ?
+        }
         $.ajax({
             type: "GET",
             url: "/admin/databox/{{ databox.get_sbas_id() }}/informations/documents/",
             dataType: 'json',
             data: {},
-            success: function(data){
+            success: function (data) {
-                if(data.viewname === '') {
+                try {
-                    $("#viewname").html("{{ 'admin::base: aucun alias' | trans }}");
+                    if (data.viewname === '') {
-                } else {
+                        $("#viewname").html("{{ 'admin::base: aucun alias' | trans }}");
-                    $("#viewname").html(data.viewname);
+                    } else {
-                }
+                        $("#viewname").html(data.viewname);
-
-                $("#nrecords").text(data.records);
-                $("#is_indexable").attr('checked', data.indexable);
-                $("#xml_indexed").text(data.xml_indexed);
-                $("#thesaurus_indexed").text(data.thesaurus_indexed);
-
-                if(data.records > 0)
-                {
-                    var p;
-                    p = 100*data.xml_indexed/data.records;
-                    $("#xml_indexed_bar").width(Math.round(2*p)); // 0..200px
-                    $("#xml_indexed_percent").text((Math.round(p*100)/100)+" %");
-                    p = 100*data.thesaurus_indexed/data.records;
-                    $("#thesaurus_indexed_bar").width(Math.round(2*p));
-                    $("#thesaurus_indexed_percent").text((Math.round(p*100)/100)+" %");
-
-                    var t = "";
-                    for(var i in data.jeton_subdef)
-                    {
-                        t += (t==""?"":" ; ") + i + ": " + data.jeton_subdef[i];
                     }
-                    if(t == "") {
+
-                        t = "0";
+                    $("#is_indexable").attr('checked', data.indexable);
+                    $("#records").text(data.counts.records);
+
+                    if (data.counts.records > 0) {
+                        var records_indexed     = data.counts.records_indexed;
+                        var records_not_indexed = data.counts.records_not_indexed;  // flag indexing but NOT to_index ???
+                        var records_indexing    = data.counts.records_indexing;
+                        var p;
+
+                        p = 100 * records_indexed / data.counts.records;
+                        $(".records_indexed", container).width(p + "%").text(records_indexed);
+
+                        if (records_not_indexed > 0) {
+                            p = 100 * records_not_indexed / data.counts.records;
+                            $(".records_not_indexed", container).width(p + "%").text(records_not_indexed);
+                        }
+                        else {
+                            $(".records_not_indexed", container).width(0).text("");
+                        }
+
+                        if (records_indexing > 0) {
+                            p = 100 * records_indexing / data.counts.records;
+                            $(".records_indexing", container).width(p + "%").text(records_indexing);
+                        }
+                        else {
+                            $(".records_indexing", container).width(0).text("");
+                        }
+
+                        var t = "";
+                        for (var i in data.counts.subdefs_todo) {
+                            t += (t == "" ? "" : " ; ") + i + ": " + data.counts.subdefs_todo[i];
+                        }
+                        if (t == "") {
+                            t = "0";
+                        }
+                        $("#subdefs_todo").text(t);
                     }
-                    $("#subdefs_todo").text(t);
-                }
 
-                if(data.printLogoURL)
+                    if (data.printLogoURL) {
-                {
+                        $("#printLogo").attr("src", data.printLogoURL);
-                    $("#printLogo").attr("src", data.printLogoURL);
+                        $("#printLogoDIV_NONE").hide();
-                    $("#printLogoDIV_NONE").hide();
+                        $("#printLogoDIV_OK").show();
-                    $("#printLogoDIV_OK").show();
+                    }
-                }
+                    else {
-                else
+                        $("#printLogoDIV_OK").hide();
-                {
+                        $("#printLogoDIV_NONE").show();
-                    $("#printLogoDIV_OK").hide();
+                    }
-                    $("#printLogoDIV_NONE").show();
-                }
 
-                // refresh every 10 sec.
+                    // refresh every 10 sec.
-                setTimeout("refreshDatabaseInformations();", 10000);
+                    displayDatabaseInformations(10000);
+                }
+                catch(err) {
+                    // wrong page ? don't refresh again
+                }
             }
         });
     }
@@ -407,8 +405,7 @@
             }
         });
 
-        // start the refresh of the page content (progress bar etc...)
+        displayDatabaseInformations(200);   // wait 200ms
-        setTimeout("refreshDatabaseInformations();", 2000);
     });
 
 </script>

tests/Alchemy/Tests/Phrasea/Controller/Admin/DataboxTest.php

@@ -234,9 +234,7 @@ class DataboxTest extends \PhraseanetAuthenticatedWebTestCase
         $this->assertTrue($json->success);
         $this->assertObjectHasAttribute('sbas_id', $json);
         $this->assertObjectHasAttribute('indexable', $json);
-        $this->assertObjectHasAttribute('records', $json);
+        $this->assertObjectHasAttribute('counts', $json);
-        $this->assertObjectHasAttribute('xml_indexed', $json);
-        $this->assertObjectHasAttribute('thesaurus_indexed', $json);
         $this->assertObjectHasAttribute('viewname', $json);
         $this->assertObjectHasAttribute('printLogoURL', $json);
     }

tests/Alchemy/Tests/Phrasea/Controller/Api/ApiTestCase.php

@@ -2288,7 +2288,7 @@ abstract class ApiTestCase extends \PhraseanetWebTestCase
         $this->assertArrayHasKey('response', $content);
         $this->assertTrue(is_array($content['meta']), 'Le bloc meta est un array');
         $this->assertTrue(is_array($content['response']), 'Le bloc reponse est un array');
-        $this->assertEquals('1.4.1', $content['meta']['api_version']);
+        $this->assertEquals('2.0.0', $content['meta']['api_version']);
         $this->assertNotNull($content['meta']['response_time']);
         $this->assertEquals('UTF-8', $content['meta']['charset']);
     }

tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php

@@ -55,6 +55,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
             "multi_match": {
                 "fields": ["foo.fr", "foo.en"],
                 "query": "bar",
+                "type": "cross_fields",
                 "operator": "and",
                 "lenient": true
             }
@@ -94,6 +95,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
                     "multi_match": {
                         "fields": ["foo.fr", "foo.en"],
                         "query": "baz",
+                        "type": "cross_fields",
                         "operator": "and",
                         "lenient": true
                     }
@@ -108,6 +110,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
                             "multi_match": {
                                 "fields": ["private_caption.bar.fr", "private_caption.bar.en"],
                                 "query": "baz",
+                                "type": "cross_fields",
                                 "operator": "and",
                                 "lenient": true
                             }
@@ -140,6 +143,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
                     "multi_match": {
                         "fields": ["foo.fr", "foo.en"],
                         "query": "bar",
+                        "type": "cross_fields",
                         "operator": "and",
                         "lenient": true
                     }
@@ -189,6 +193,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
                     "multi_match": {
                         "fields": ["foo.fr", "foo.en"],
                         "query": "baz",
+                        "type": "cross_fields",
                         "operator": "and",
                         "lenient": true
                     }
@@ -212,6 +217,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
                                     "multi_match": {
                                         "fields": ["private_caption.bar.fr", "private_caption.bar.en"],
                                         "query": "baz",
+                                        "type": "cross_fields",
                                         "operator": "and",
                                         "lenient": true
                                     }