diff --git a/lib/Alchemy/Phrasea/Application.php b/lib/Alchemy/Phrasea/Application.php index af975aea85..09c2814b95 100644 --- a/lib/Alchemy/Phrasea/Application.php +++ b/lib/Alchemy/Phrasea/Application.php @@ -84,6 +84,8 @@ use MediaVorus\Media\MediaInterface; use MediaVorus\MediaVorus; use MediaVorus\MediaVorusServiceProvider; use Monolog\Handler\NullHandler; +use Monolog\Handler\RotatingFileHandler; +use Monolog\Handler\SyslogHandler; use Monolog\Logger; use Monolog\Processor\IntrospectionProcessor; use MP4Box\MP4BoxServiceProvider; @@ -266,6 +268,7 @@ class Application extends SilexApplication $this['phraseanet.exception_handler'] = $this->share(function ($app) { $handler = PhraseaExceptionHandler::register($app['debug']); $handler->setTranslator($app['translator']); + $handler->setLogger($app['monolog']); return $handler; }); @@ -1050,14 +1053,16 @@ class Application extends SilexApplication private function setupMonolog() { $this['monolog.name'] = 'phraseanet'; - $this['monolog.handler'] = $this->share(function () { - return new NullHandler(); + $this['monolog.logfile'] = $this['root.path'] . '/logs/app_error.log'; + $this['monolog.handler'] = $this->share(function (Application $app) { + return new RotatingFileHandler( + $app['monolog.logfile'], + 10, + Logger::ERROR, + $app['monolog.bubble'], + $app['monolog.permission'] + ); }); - $this['monolog'] = $this->share($this->extend('monolog', function (Logger $logger) { - $logger->pushProcessor(new IntrospectionProcessor()); - - return $logger; - })); } private function setupEventDispatcher() diff --git a/lib/Alchemy/Phrasea/Application/Api.php b/lib/Alchemy/Phrasea/Application/Api.php index 751fb1d484..57b0f02ec2 100644 --- a/lib/Alchemy/Phrasea/Application/Api.php +++ b/lib/Alchemy/Phrasea/Application/Api.php @@ -41,7 +41,10 @@ return call_user_func(function ($environment = PhraseaApplication::ENV_PROD) { $app->loadPlugins(); $app['exception_handler'] = $app->share(function ($app) { - return new ApiExceptionHandlerSubscriber($app); + $handler = new ApiExceptionHandlerSubscriber($app); + $handler->setLogger($app['monolog']); + + return $handler; }); $app['monolog'] = $app->share($app->extend('monolog', function (Logger $monolog) { $monolog->pushProcessor(new WebProcessor()); @@ -92,6 +95,7 @@ return call_user_func(function ($environment = PhraseaApplication::ENV_PROD) { if ($request->getRequestFormat(Result::FORMAT_JSON) === Result::FORMAT_JSONP && !$response->isOk() && !$response->isServerError()) { $response->setStatusCode(200); } + // set response content type if (!$response->headers->get('Content-Type')) { $response->headers->set('Content-Type', $request->getMimeType($request->getRequestFormat(Result::FORMAT_JSON))); diff --git a/lib/Alchemy/Phrasea/Controller/Admin/DataboxController.php b/lib/Alchemy/Phrasea/Controller/Admin/DataboxController.php index d1f6e8789b..11c901159e 100644 --- a/lib/Alchemy/Phrasea/Controller/Admin/DataboxController.php +++ b/lib/Alchemy/Phrasea/Controller/Admin/DataboxController.php @@ -577,29 +577,23 @@ class DataboxController extends Controller $ret = [ 'success' => false, - 'msg' => $this->app->trans('An error occured'), 'sbas_id' => null, + 'msg' => $this->app->trans('An error occured'), 'indexable' => false, - 'records' => 0, - 'xml_indexed' => 0, - 'thesaurus_indexed' => 0, 'viewname' => null, 'printLogoURL' => null, + 'counts' => null, ]; try { $databox = $this->findDataboxById($databox_id); - $data = $databox->get_indexed_record_amount(); + $ret['sbas_id'] = $databox_id; $ret['indexable'] = $appbox->is_databox_indexable($databox); $ret['viewname'] = (($databox->get_dbname() == $databox->get_viewname()) ? $this->app->trans('admin::base: aucun alias') : $databox->get_viewname()); - $ret['records'] = $databox->get_record_amount(); - $ret['sbas_id'] = $databox_id; - $ret['xml_indexed'] = $data['xml_indexed']; - $ret['thesaurus_indexed'] = $data['thesaurus_indexed']; - $ret['jeton_subdef'] = $data['jeton_subdef']; + $ret['counts'] = $databox->get_counts(); if ($this->app['filesystem']->exists($this->app['root.path'] . '/config/minilogos/logopdf_' . $databox_id . '.jpg')) { $ret['printLogoURL'] = '/custom/minilogos/logopdf_' . $databox_id . '.jpg'; } diff --git a/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php b/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php index c2eda48729..48496cea46 100644 --- a/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php +++ b/lib/Alchemy/Phrasea/Controller/Api/V1Controller.php @@ -462,6 +462,13 @@ class V1Controller extends Controller ]; } + public function getDataboxCollectionAction(Request $request, $base_id) + { + return Result::create($request, [ + $this->listCollection($this->app->getApplicationBox()->get_collection($base_id)) + ])->createResponse(); + } + /** * Get a Response containing the collections of a \databox * @@ -1459,9 +1466,9 @@ class V1Controller extends Controller $devices = $request->get('devices', []); $mimes = $request->get('mimes', []); - $ret = array_filter(array_map(function ($media) use ($request, $record) { + $ret = array_values(array_filter(array_map(function ($media) use ($request, $record) { return $this->listEmbeddableMedia($request, $record, $media); - }, $record->get_embedable_medias($devices, $mimes))); + }, $record->get_embedable_medias($devices, $mimes)))); return Result::create($request, ["embed" => $ret])->createResponse(); } @@ -1978,9 +1985,9 @@ class V1Controller extends Controller $devices = $request->get('devices', []); $mimes = $request->get('mimes', []); - $ret = array_filter(array_map(function ($media) use ($request, $record) { + $ret = array_values(array_filter(array_map(function ($media) use ($request, $record) { return $this->listEmbeddableMedia($request, $record, $media); - }, $record->get_embedable_medias($devices, $mimes))); + }, $record->get_embedable_medias($devices, $mimes)))); return Result::create($request, ["embed" => $ret])->createResponse(); } diff --git a/lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusController.php b/lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusController.php index eba913764f..76afe86894 100644 --- a/lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusController.php +++ b/lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusController.php @@ -12,7 +12,7 @@ namespace Alchemy\Phrasea\Controller\Thesaurus; use Alchemy\Phrasea\Application\Helper\DispatcherAware; use Alchemy\Phrasea\Controller\Controller; use Alchemy\Phrasea\Core\Event\Thesaurus as ThesaurusEvent; -use Alchemy\Phrasea\Core\PhraseaEvents; +use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvents; use Doctrine\DBAL\Driver\Connection; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; @@ -699,7 +699,7 @@ class ThesaurusController extends Controller } $this->dispatch( - PhraseaEvents::THESAURUS_IMPORTED, + ThesaurusEvents::IMPORTED, new ThesaurusEvent\Imported($databox) ); } @@ -987,7 +987,7 @@ class ThesaurusController extends Controller if ($request->get("reindex")) { $this->dispatch( - PhraseaEvents::THESAURUS_FIELD_LINKED, + ThesaurusEvents::FIELD_LINKED, new ThesaurusEvent\FieldLinked($databox) ); } @@ -1360,7 +1360,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($domth); $this->dispatch( - PhraseaEvents::THESAURUS_CANDIDATE_ACCEPTED_AS_CONCEPT, + ThesaurusEvents::CANDIDATE_ACCEPTED_AS_CONCEPT, new ThesaurusEvent\CandidateAccepted($databox, $new_te->getAttribute('id')) ); @@ -1404,7 +1404,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($domth); $this->dispatch( - PhraseaEvents::THESAURUS_CANDIDATE_ACCEPTED_AS_SYNONYM, + ThesaurusEvents::CANDIDATE_ACCEPTED_AS_SYNONYM, new ThesaurusEvent\CandidateAccepted($databox, $new_te->getAttribute('id')) ); } @@ -1493,7 +1493,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($dom); $this->dispatch( - PhraseaEvents::THESAURUS_SYNONYM_LNG_CHANGED, + ThesaurusEvents::SYNONYM_LNG_CHANGED, new ThesaurusEvent\SynonymLngChanged($databox, $sy0->getAttribute('id')) ); } @@ -1575,7 +1575,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($dom); $this->dispatch( - PhraseaEvents::THESAURUS_SYNONYM_POSITION_CHANGED, + ThesaurusEvents::SYNONYM_POSITION_CHANGED, new ThesaurusEvent\SynonymPositionChanged($databox, $sy0->getAttribute('id')) ); @@ -1720,7 +1720,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($dom); $this->dispatch( - PhraseaEvents::THESAURUS_SYNONYM_TRASHED, + ThesaurusEvents::SYNONYM_TRASHED, new ThesaurusEvent\ItemTrashed($databox, $te->getAttribute('id'), $delsy->getAttribute('id')) ); @@ -1837,7 +1837,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($domth); $this->dispatch( - PhraseaEvents::THESAURUS_CONCEPT_TRASHED, + ThesaurusEvents::CONCEPT_TRASHED, new ThesaurusEvent\ItemTrashed($databox, $thnode_parent->getAttribute('id'), $newte->getAttribute('id')) ); @@ -2282,7 +2282,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($dom); $this->dispatch( - PhraseaEvents::THESAURUS_CONCEPT_DELETED, + ThesaurusEvents::CONCEPT_DELETED, new ThesaurusEvent\ConceptDeleted($databox, $refrid, $sy_evt_parm) ); @@ -2391,7 +2391,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($domth); $this->dispatch( - PhraseaEvents::THESAURUS_SYNONYM_ADDED, + ThesaurusEvents::SYNONYM_ADDED, new ThesaurusEvent\ItemAdded($databox, $syid) ); @@ -2496,7 +2496,7 @@ class ThesaurusController extends Controller $databox->saveThesaurus($domth); $this->dispatch( - PhraseaEvents::THESAURUS_CONCEPT_ADDED, + ThesaurusEvents::CONCEPT_ADDED, new ThesaurusEvent\ItemAdded($databox, $syid) ); diff --git a/lib/Alchemy/Phrasea/ControllerProvider/Api/V1.php b/lib/Alchemy/Phrasea/ControllerProvider/Api/V1.php index 845647ebac..362da070e7 100644 --- a/lib/Alchemy/Phrasea/ControllerProvider/Api/V1.php +++ b/lib/Alchemy/Phrasea/ControllerProvider/Api/V1.php @@ -20,7 +20,7 @@ use Silex\ServiceProviderInterface; class V1 implements ControllerProviderInterface, ServiceProviderInterface { - const VERSION = '1.4.1'; + const VERSION = '2.0.0'; public static $extendedContentTypes = [ 'json' => ['application/vnd.phraseanet.record-extended+json'], @@ -72,6 +72,8 @@ class V1 implements ControllerProviderInterface, ServiceProviderInterface $controllers->get('/monitor/phraseanet/', 'controller.api.v1:showPhraseanetConfigurationAction') ->before('controller.api.v1:ensureAdmin'); + $controllers->get('/collections/{base_id}/', 'controller.api.v1:getDataboxCollectionAction'); + $controllers->get('/databoxes/list/', 'controller.api.v1:listDataboxesAction'); $controllers->get('/databoxes/{databox_id}/collections/', 'controller.api.v1:getDataboxCollectionsAction') diff --git a/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiExceptionHandlerSubscriber.php b/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiExceptionHandlerSubscriber.php index d4b6b98ad1..839cfc60d1 100644 --- a/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiExceptionHandlerSubscriber.php +++ b/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiExceptionHandlerSubscriber.php @@ -13,6 +13,8 @@ namespace Alchemy\Phrasea\Core\Event\Subscriber; use Alchemy\Phrasea\Application; use Alchemy\Phrasea\Controller\Api\Result; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; use Symfony\Component\HttpKernel\Event\GetResponseForExceptionEvent; use Symfony\Component\HttpKernel\KernelEvents; use Symfony\Component\EventDispatcher\EventSubscriberInterface; @@ -27,9 +29,17 @@ class ApiExceptionHandlerSubscriber implements EventSubscriberInterface { private $app; + private $logger; + public function __construct(Application $app) { $this->app = $app; + $this->logger = new NullLogger(); + } + + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; } public static function getSubscribedEvents() @@ -64,6 +74,13 @@ class ApiExceptionHandlerSubscriber implements EventSubscriberInterface $code = 500; } + if ($code == 500) { + $this->logger->error($e->getMessage(), [ + 'code' => $e->getCode(), + 'trace' => $e->getTrace() + ]); + } + if ($e instanceof HttpExceptionInterface) { $headers = $e->getHeaders(); } diff --git a/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiOauth2ErrorsSubscriber.php b/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiOauth2ErrorsSubscriber.php index bff0cad60f..ff6b28523c 100644 --- a/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiOauth2ErrorsSubscriber.php +++ b/lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiOauth2ErrorsSubscriber.php @@ -61,7 +61,7 @@ class ApiOauth2ErrorsSubscriber implements EventSubscriberInterface $msg = json_encode(['msg' => $msg, 'code' => $code]); $event->setResponse(new Response($msg, $code, $headers)); } else { - $response = $this->handler->createResponseBasedOnRequest($event->getRequest(), $event->getException()); + $response = $this->handler->createResponse($event->getException()); $response->headers->set('Content-Type', 'text/html'); $event->setResponse($response); } diff --git a/lib/Alchemy/Phrasea/Core/Event/Subscriber/PhraseaExceptionHandlerSubscriber.php b/lib/Alchemy/Phrasea/Core/Event/Subscriber/PhraseaExceptionHandlerSubscriber.php index dbf2fe2782..bcf78023de 100644 --- a/lib/Alchemy/Phrasea/Core/Event/Subscriber/PhraseaExceptionHandlerSubscriber.php +++ b/lib/Alchemy/Phrasea/Core/Event/Subscriber/PhraseaExceptionHandlerSubscriber.php @@ -38,7 +38,7 @@ class PhraseaExceptionHandlerSubscriber implements EventSubscriberInterface return; } - $event->setResponse($this->handler->createResponseBasedOnRequest($event->getRequest(), $event->getException())); + $event->setResponse($this->handler->createResponse($event->getException())); } /** diff --git a/lib/Alchemy/Phrasea/Core/Event/Thesaurus/ThesaurusEvents.php b/lib/Alchemy/Phrasea/Core/Event/Thesaurus/ThesaurusEvents.php new file mode 100644 index 0000000000..a4c27af2f0 --- /dev/null +++ b/lib/Alchemy/Phrasea/Core/Event/Thesaurus/ThesaurusEvents.php @@ -0,0 +1,27 @@ +logger = new NullLogger(); + } + + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + public function setTranslator(TranslatorInterface $translator) { $this->translator = $translator; @@ -33,6 +47,13 @@ class PhraseaExceptionHandler extends SymfonyExceptionHandler public function getContent(FlattenException $exception) { + if ($exception->getStatusCode() == '500') { + $this->logger->error($exception->getMessage(), [ + 'code' => $exception->getCode(), + 'trace' => $exception->getTrace() + ]); + } + switch (true) { case 404 === $exception->getStatusCode(): if (null !== $this->translator) { diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php index e41381c97f..fb7ac29409 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php @@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper; -use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field; +use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField; class QuotedTextNode extends Node { @@ -38,11 +38,11 @@ class QuotedTextNode extends Node }; $unrestricted_fields = $context->getUnrestrictedFields(); - $unrestricted_fields = Field::filterByValueCompatibility($unrestricted_fields, $this->text); + $unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text); $query = $query_builder($unrestricted_fields); $private_fields = $context->getPrivateFields(); - $private_fields = Field::filterByValueCompatibility($private_fields, $this->text); + $private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text); foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) { $query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query); } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php index f11d2dd6ae..ac387f0371 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php @@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper; -use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field; +use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField; class RawNode extends Node { @@ -56,11 +56,11 @@ class RawNode extends Node }; $unrestricted_fields = $context->getUnrestrictedFields(); - $unrestricted_fields = Field::filterByValueCompatibility($unrestricted_fields, $this->text); + $unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text); $query = $query_builder($unrestricted_fields); $private_fields = $context->getPrivateFields(); - $private_fields = Field::filterByValueCompatibility($private_fields, $this->text); + $private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text); foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) { $query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query); } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php index 2506a9ddf1..8727b6e227 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php @@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext; use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper; -use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field; +use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Term; class TextNode extends AbstractTermNode implements ContextAbleInterface @@ -41,7 +41,7 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface $query_builder = function (array $fields) use ($context) { // Full text $index_fields = []; - foreach (Field::filterByValueCompatibility($fields, $this->text) as $field) { + foreach (StructureField::filterByValueCompatibility($fields, $this->text) as $field) { foreach ($context->localizeField($field) as $f) { $index_fields[] = $f; } @@ -53,6 +53,7 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface 'multi_match' => [ 'fields' => $index_fields, 'query' => $this->text, + 'type' => 'cross_fields', 'operator' => 'and', 'lenient' => true, ] diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php index 32585829e5..22e745a54f 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php @@ -41,7 +41,7 @@ class Indexer private $recordIndexer; private $termIndexer; - private $indexQueue; + private $indexQueue; // contains RecordInterface(s) private $deleteQueue; private $previousRefreshInterval = self::DEFAULT_REFRESH_INTERVAL; @@ -155,6 +155,11 @@ class Indexer // RecordQueuer::queueRecordsFromDatabox($databox); } + public function scheduleRecordsFromDataboxForIndexing(\databox $databox) + { + RecordQueuer::queueRecordsFromDatabox($databox); + } + public function scheduleRecordsFromCollectionForIndexing(\collection $collection) { RecordQueuer::queueRecordsFromCollection($collection); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php index 9d1e85b283..488eb9f9c4 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php @@ -23,10 +23,11 @@ class BulkOperation private $logger; private $stack = array(); - private $opCount = 0; + private $operationIdentifiers = []; private $index; private $type; private $flushLimit = 1000; + private $flushCallbacks = []; public function __construct(Client $client, LoggerInterface $logger) { @@ -52,27 +53,32 @@ class BulkOperation $this->flushLimit = (int) $limit; } - public function index(array $params) + public function onFlush(\Closure $callback) + { + $this->flushCallbacks[] = $callback; + } + + public function index(array $params, $operationIdentifier) { $header = $this->buildHeader('index', $params); $body = igorw\get_in($params, ['body']); - $this->push($header, $body); + $this->push($header, $body, $operationIdentifier); } - public function delete(array $params) + public function delete(array $params, $operationIdentifier) { - $this->push($this->buildHeader('delete', $params)); + $this->push($this->buildHeader('delete', $params), null, $operationIdentifier); } - private function push($header, $body = null) + private function push($header, $body, $operationIdentifier) { $this->stack[] = $header; if ($body) { $this->stack[] = $body; } - $this->opCount++; + $this->operationIdentifiers[] = $operationIdentifier; - if ($this->flushLimit === $this->opCount) { + if (count($this->operationIdentifiers) === $this->flushLimit) { $this->flush(); } } @@ -93,19 +99,32 @@ class BulkOperation } $params['body'] = $this->stack; - $this->logger->debug("ES Bulk query about to be performed\n", ['opCount' => $this->opCount]); + $this->logger->debug("ES Bulk query about to be performed\n", ['opCount' => count($this->operationIdentifiers)]); $response = $this->client->bulk($params); $this->stack = array(); - $this->opCount = 0; - if (igorw\get_in($response, ['errors'], true)) { - foreach ($response['items'] as $key => $item) { - if ($item['index']['status'] >= 400) { // 4xx or 5xx error - throw new Exception(sprintf('%d: %s', $key, $item['index']['error'])); + $callbackData = []; // key: operationIdentifier passed when command was pushed on this bulk + // value: json result from es for the command + // nb: results (items) are returned IN THE SAME ORDER as commands were pushed in the stack + // so the items[X] match the operationIdentifiers[X] + foreach ($response['items'] as $key => $item) { + foreach($item as $command=>$result) { // command may be "index" or "delete" + if($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx error + $err = array_key_exists('error', $result) ? $result['error'] : ($command . " error " . $result['status']); + throw new Exception(sprintf('%d: %s', $key, $err)); } } + + $operationIdentifier = $this->operationIdentifiers[$key]; + if(is_string($operationIdentifier) || is_int($operationIdentifier)) { // dont include null keys + $callbackData[$operationIdentifier] = $response['items'][$key]; + } } + foreach($this->flushCallbacks as $iCallBack=>$flushCallback) { + $flushCallback($callbackData); + } + $this->operationIdentifiers = []; } private function buildHeader($key, array $params) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php index 0b9e57e023..187809c573 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php @@ -16,12 +16,14 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegate; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegateInterface; use Closure; +use databox; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Driver\Connection as ConnectionInterface; use PDO; class Fetcher { + private $databox; private $connection; private $statement; private $delegate; @@ -34,13 +36,19 @@ class Fetcher private $postFetch; private $onDrain; - public function __construct(ConnectionInterface $connection, array $hydrators, FetcherDelegateInterface $delegate = null) + public function __construct(databox $databox, array $hydrators, FetcherDelegateInterface $delegate = null) { - $this->connection = $connection; + $this->databox = $databox; + $this->connection = $databox->get_connection();; $this->hydrators = $hydrators; $this->delegate = $delegate ?: new FetcherDelegate(); } + public function getDatabox() + { + return $this->databox; + } + public function fetch() { if (empty($this->buffer)) { @@ -64,7 +72,6 @@ class Fetcher $records[$record['record_id']] = $record; $this->offset++; } - if (empty($records)) { $this->onDrain->__invoke(); return; @@ -87,6 +94,12 @@ class Fetcher return $records; } + public function restart() + { + $this->buffer = array(); + $this->offset = 0; + } + public function setBatchSize($size) { if ($size < 1) { @@ -105,28 +118,24 @@ class Fetcher $this->onDrain = $onDrain; } + /** + * @return \Doctrine\DBAL\Driver\Statement + */ private function getExecutedStatement() { if (!$this->statement) { - $sql = <<delegate->buildWhereClause(); $sql = str_replace('-- WHERE', $where, $sql); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php index f7bbf8e5aa..50e2830ed7 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php @@ -11,6 +11,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer; +use Alchemy\Phrasea\Model\RecordInterface; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\BulkOperation; @@ -60,6 +61,12 @@ class RecordIndexer private $logger; + private function getUniqueOperationId($record_key) + { + $_key = dechex(mt_rand()); + return $_key . '_' . $record_key; + } + public function __construct(Structure $structure, RecordHelper $helper, Thesaurus $thesaurus, \appbox $appbox, array $locales, LoggerInterface $logger) { $this->structure = $structure; @@ -70,16 +77,70 @@ class RecordIndexer $this->logger = $logger; } + /** + * ES made a bulk op, check our (index) operations to drop the "indexing" & "to_index" jetons + * + * @param databox $databox + * @param array $operation_identifiers key:op_identifier ; value:operation result (json from es) + * @param array $submited_records records indexed, key:op_identifier + */ + private function onBulkFlush(databox $databox, array $operation_identifiers, array &$submited_records) + { + // nb: because the same bulk could be used by many "clients", this (each) callback may receive + // operation_identifiers that does not belong to it. + // flag only records that the fetcher worked on + $records = array_intersect_key( + $submited_records, // this is OUR records list + $operation_identifiers // reduce to the records indexed by this bulk (should be the same...) + ); + if(count($records) === 0) { + return; + } + // Commit and remove "indexing" flag + RecordQueuer::didFinishIndexingRecords(array_values($records), $databox); + foreach (array_keys($records) as $id) { + unset($submited_records[$id]); + } + } + + /** + * index whole databox(es), don't test actual "jetons" + * + * @param BulkOperation $bulk + * @param databox[] $databoxes + */ public function populateIndex(BulkOperation $bulk, array $databoxes) { foreach ($databoxes as $databox) { + $submited_records = []; + $this->logger->info(sprintf('Indexing database %s...', $databox->get_viewname())); - $fetcher = $this->createFetcherForDatabox($databox); - $this->indexFromFetcher($bulk, $fetcher); + + $fetcher = $this->createFetcherForDatabox($databox); // no delegate, scan the whole records + + // post fetch : flag records as "indexing" + $fetcher->setPostFetch(function(array $records) use ($databox, $fetcher) { + RecordQueuer::didStartIndexingRecords($records, $databox); + // do not restart the fetcher since it has no clause on jetons + }); + + // bulk flush : flag records as "indexed" + $bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) { + $this->onBulkFlush($databox, $operation_identifiers, $submited_records); + }); + + // Perform indexing + $this->indexFromFetcher($bulk, $fetcher, $submited_records); + $this->logger->info(sprintf('Finished indexing %s', $databox->get_viewname())); } } + /** + * Index the records flagged as "to_index" on all databoxes + * + * @param BulkOperation $bulk + */ public function indexScheduled(BulkOperation $bulk) { foreach ($this->appbox->get_databoxes() as $databox) { @@ -89,46 +150,82 @@ class RecordIndexer private function indexScheduledInDatabox(BulkOperation $bulk, databox $databox) { + $submited_records = []; + // Make fetcher $delegate = new ScheduledFetcherDelegate(); $fetcher = $this->createFetcherForDatabox($databox, $delegate); - // Keep track of fetched records, flag them as "indexing" - $fetched = array(); - $fetcher->setPostFetch(function(array $records) use ($databox, &$fetched) { - // TODO Do not keep all indexed records in memory... - $fetched += $records; + + // post fetch : flag records as "indexing" + $fetcher->setPostFetch(function(array $records) use ($databox, $fetcher) { RecordQueuer::didStartIndexingRecords($records, $databox); + // because changing the flag on the records affects the "where" clause of the fetcher, + // restart it each time + $fetcher->restart(); }); + + // bulk flush : flag records as "indexed" + $bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) { + $this->onBulkFlush($databox, $operation_identifiers, $submited_records); + }); + // Perform indexing - $this->indexFromFetcher($bulk, $fetcher); - // Commit and remove "indexing" flag - $bulk->flush(); - RecordQueuer::didFinishIndexingRecords($fetched, $databox); + $this->indexFromFetcher($bulk, $fetcher, $submited_records); } + /** + * Index a list of records + * + * @param BulkOperation $bulk + * @param Iterator $records + */ public function index(BulkOperation $bulk, Iterator $records) { foreach ($this->createFetchersForRecords($records) as $fetcher) { - $this->indexFromFetcher($bulk, $fetcher); + $submited_records = []; + $databox = $fetcher->getDatabox(); + + // post fetch : flag records as "indexing" + $fetcher->setPostFetch(function(array $records) use ($fetcher, $databox) { + RecordQueuer::didStartIndexingRecords($records, $databox); + // do not restart the fetcher since it has no clause on jetons + }); + + // bulk flush : flag records as "indexed" + $bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) { + $this->onBulkFlush($databox, $operation_identifiers, $submited_records); + }); + + // Perform indexing + $this->indexFromFetcher($bulk, $fetcher, $submited_records); } } + /** + * Deleta a list of records + * + * @param BulkOperation $bulk + * @param Iterator $records + */ public function delete(BulkOperation $bulk, Iterator $records) { foreach ($records as $record) { $params = array(); $params['id'] = $record->getId(); $params['type'] = self::TYPE_NAME; - $bulk->delete($params); + $bulk->delete($params, null); // no operationIdentifier is related to a delete op } } + /** + * @param Iterator $records + * @return Fetcher[] + */ private function createFetchersForRecords(Iterator $records) { $fetchers = array(); foreach ($this->groupRecordsByDatabox($records) as $group) { $databox = $group['databox']; - $connection = $databox->get_connection(); $delegate = new RecordListFetcherDelegate($group['records']); $fetchers[] = $this->createFetcherForDatabox($databox, $delegate); } @@ -140,7 +237,7 @@ class RecordIndexer { $connection = $databox->get_connection(); $candidateTerms = new CandidateTerms($databox); - $fetcher = new Fetcher($connection, array( + $fetcher = new Fetcher($databox, array( new CoreHydrator($databox->get_sbas_id(), $databox->get_viewname(), $this->helper), new TitleHydrator($connection), new MetadataHydrator($connection, $this->structure, $this->helper), @@ -169,15 +266,21 @@ class RecordIndexer return array_values($databoxes); } - private function indexFromFetcher(BulkOperation $bulk, Fetcher $fetcher) + private function indexFromFetcher(BulkOperation $bulk, Fetcher $fetcher, array &$submited_records) { + /** @var RecordInterface $record */ while ($record = $fetcher->fetch()) { + $op_identifier = $this->getUniqueOperationId($record['id']); + $params = array(); $params['id'] = $record['id']; unset($record['id']); $params['type'] = self::TYPE_NAME; $params['body'] = $record; - $bulk->index($params); + + $submited_records[$op_identifier] = $record; + + $bulk->index($params, $op_identifier); } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordQueuer.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordQueuer.php index 6bed8699f1..7251988fe2 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordQueuer.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordQueuer.php @@ -35,41 +35,44 @@ class RecordQueuer $connection = $collection->get_connection(); // Set TO_INDEX flag on all records from this collection - $sql = <<prepare($sql); $stmt->bindValue(':token', Flag::TO_INDEX, PDO::PARAM_INT); $stmt->bindValue(':coll_id', $collection->get_coll_id(), PDO::PARAM_INT); $stmt->execute(); } + /** + * @param array $records + * @param $databox + * + * nb: changing the jeton may affect a fetcher if his "where" clause (delegate) depends on jeton. + * in this case the client of the fetcher must set a "postFetch" callback and restart the fetcher + */ public static function didStartIndexingRecords(array $records, $databox) { $connection = $databox->get_connection(); - $sql = <<get_connection(); - $sql = <<executeQuery($sql, array( ':flag' => $flag, diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/TermIndexer.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/TermIndexer.php index a9f4fef7d8..76a00e07e0 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/TermIndexer.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/TermIndexer.php @@ -60,7 +60,7 @@ class TermIndexer $params['type'] = self::TYPE_NAME; $params['body'] = $term; - $bulk->index($params); + $bulk->index($params, null); }); $document = Helper::thesaurusFromDatabox($databox); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/IndexerSubscriber.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/IndexerSubscriber.php index e90020d157..9152d455b4 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/IndexerSubscriber.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/IndexerSubscriber.php @@ -19,6 +19,8 @@ use Alchemy\Phrasea\Core\Event\Record\RecordEvents; use Alchemy\Phrasea\Core\Event\Record\RecordSubDefinitionCreatedEvent; use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvent; use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvents; +use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvent; +use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvents; use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** @@ -84,6 +86,17 @@ class IndexerSubscriber implements EventSubscriberInterface RecordEvents::STATUS_CHANGED => 'onRecordChange', RecordEvents::SUB_DEFINITION_CREATED => 'onRecordChange', RecordEvents::MEDIA_SUBSTITUTED => 'onRecordChange', + ThesaurusEvents::IMPORTED => 'onThesaurusChange', + ThesaurusEvents::FIELD_LINKED => 'onThesaurusChange', + ThesaurusEvents::CANDIDATE_ACCEPTED_AS_CONCEPT => 'onThesaurusChange', + ThesaurusEvents::CANDIDATE_ACCEPTED_AS_SYNONYM => 'onThesaurusChange', + ThesaurusEvents::SYNONYM_LNG_CHANGED => 'onThesaurusChange', + ThesaurusEvents::SYNONYM_POSITION_CHANGED => 'onThesaurusChange', + ThesaurusEvents::SYNONYM_TRASHED => 'onThesaurusChange', + ThesaurusEvents::CONCEPT_TRASHED => 'onThesaurusChange', + ThesaurusEvents::CONCEPT_DELETED => 'onThesaurusChange', + ThesaurusEvents::SYNONYM_ADDED => 'onThesaurusChange', + ThesaurusEvents::CONCEPT_ADDED => 'onThesaurusChange' ]; } @@ -93,6 +106,12 @@ class IndexerSubscriber implements EventSubscriberInterface $this->getIndexer()->migrateMappingForDatabox($databox); } + public function onThesaurusChange(ThesaurusEvent $event) + { + $databox = $event->getDatabox(); + $this->getIndexer()->scheduleRecordsFromDataboxForIndexing($databox); + } + public function onCollectionChange(CollectionEvent $event) { $collection = $event->getCollection(); diff --git a/lib/Alchemy/Phrasea/TaskManager/Job/SubdefsJob.php b/lib/Alchemy/Phrasea/TaskManager/Job/SubdefsJob.php index 7d3de8ab27..c38db31ce3 100644 --- a/lib/Alchemy/Phrasea/TaskManager/Job/SubdefsJob.php +++ b/lib/Alchemy/Phrasea/TaskManager/Job/SubdefsJob.php @@ -121,8 +121,7 @@ class SubdefsJob extends AbstractJob // rewrite metadata $sql = 'UPDATE record' - . ' SET status=(status & ~0x03),' - . ' jeton=(jeton | ' . PhraseaTokens::WRITE_META_SUBDEF . ')' + . ' SET jeton=(jeton | ' . PhraseaTokens::WRITE_META_SUBDEF | PhraseaTokens::TO_INDEX . ')' . ' WHERE record_id=:record_id'; $stmt = $conn->prepare($sql); $stmt->execute([':record_id' => $row['record_id']]); diff --git a/lib/classes/API/OAuth2/Adapter.php b/lib/classes/API/OAuth2/Adapter.php index 3b0e73d2ec..80d471909b 100644 --- a/lib/classes/API/OAuth2/Adapter.php +++ b/lib/classes/API/OAuth2/Adapter.php @@ -16,9 +16,11 @@ use Alchemy\Phrasea\Authentication\Exception\RequireCaptchaException; use Alchemy\Phrasea\Exception\RuntimeException; use Alchemy\Phrasea\Model\Entities\ApiApplication; use Alchemy\Phrasea\Model\Entities\User; +use Alchemy\Phrasea\Model\Repositories\ApiApplicationRepository; use Symfony\Component\HttpFoundation\Session\Session; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; +use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; class API_OAuth2_Adapter extends OAuth2 { @@ -601,9 +603,12 @@ class API_OAuth2_Adapter extends OAuth2 'state' => null, ]; + $result = []; + if ($params['state'] !== null) { $result["query"]["state"] = $params['state']; } + if ($is_authorized === false) { $result["query"]["error"] = OAUTH2_ERROR_USER_DENIED; } else { @@ -615,6 +620,7 @@ class API_OAuth2_Adapter extends OAuth2 $result["fragment"] = $this->createAccessToken($params['account_id'], $params['scope']); } } + $this->doRedirectUriCallback($params['redirect_uri'], $result); } @@ -684,9 +690,15 @@ class API_OAuth2_Adapter extends OAuth2 } break; case OAUTH2_GRANT_TYPE_USER_CREDENTIALS: - $application = ApiApplication::load_from_client_id($this->app, $client[0]); + /** @var ApiApplicationRepository $appRepository */ + $appRepository = $this->app['repo.api-applications']; + $application = $appRepository->findByClientId($client[0]); - if ( ! $application->is_password_granted()) { + if (! $application) { + throw new NotFoundHttpException('Application not found'); + } + + if ( ! $application->isPasswordGranted()) { $this->errorJsonResponse(OAUTH2_HTTP_BAD_REQUEST, OAUTH2_ERROR_UNSUPPORTED_GRANT_TYPE, 'Password grant type is not enable for your client'); } @@ -812,7 +824,7 @@ class API_OAuth2_Adapter extends OAuth2 return [ 'redirect_uri' => $this->client->getRedirectUri(), - 'client_id' => $this->client->getClient(), + 'client_id' => $this->client->getClientId(), 'account_id' => $account->getId(), ]; } catch (AccountLockedException $e) { diff --git a/lib/classes/appbox.php b/lib/classes/appbox.php index 5934bfab2e..eaac5de92e 100644 --- a/lib/classes/appbox.php +++ b/lib/classes/appbox.php @@ -268,6 +268,26 @@ class appbox extends base return $databoxes[$sbas_id]; } + public function get_collection($base_id) + { + $sbas_id = phrasea::sbasFromBas($this->app, $base_id); + + if ($sbas_id === false) { + throw new \RuntimeException('Collection not found.'); + } + + $collections = $this->get_databox($sbas_id)->get_collections(); + + foreach ($collections as $collection) { + if ($collection->get_base_id() == $base_id) { + return $collection; + } + } + + // This should not happen, but I'd rather be safe than sorry. + throw new \RuntimeException('Collection not found.'); + } + /** * @param string $option * @return string diff --git a/lib/classes/databox.php b/lib/classes/databox.php index bbffa7dbc7..3cb258dc23 100644 --- a/lib/classes/databox.php +++ b/lib/classes/databox.php @@ -294,42 +294,6 @@ class databox extends base implements ThumbnailedElement return $this->id; } - public function get_unique_keywords() - { - $sql = "SELECT COUNT(kword_id) AS n FROM kword"; - - $stmt = $this->get_connection()->prepare($sql); - $stmt->execute(); - $rowbas = $stmt->fetch(PDO::FETCH_ASSOC); - $stmt->closeCursor(); - - return ($rowbas ? $rowbas['n'] : null); - } - - public function get_index_amount() - { - $sql = "SELECT COUNT(idx_id) AS n FROM idx"; - - $stmt = $this->get_connection()->prepare($sql); - $stmt->execute(); - $rowbas = $stmt->fetch(PDO::FETCH_ASSOC); - $stmt->closeCursor(); - - return ($rowbas ? $rowbas['n'] : null); - } - - public function get_thesaurus_hits() - { - $sql = "SELECT COUNT(thit_id) AS n FROM thit"; - - $stmt = $this->get_connection()->prepare($sql); - $stmt->execute(); - $rowbas = $stmt->fetch(PDO::FETCH_ASSOC); - $stmt->closeCursor(); - - return ($rowbas ? $rowbas['n'] : null); - } - public function get_record_details($sort) { $sql = "SELECT record.coll_id, ISNULL(coll.coll_id) AS lostcoll, @@ -390,36 +354,46 @@ class databox extends base implements ThumbnailedElement return $amount; } - public function get_indexed_record_amount() + public function get_counts() { - $sql = "SELECT status & 3 AS status, SUM(1) AS n FROM record GROUP BY(status & 3)"; + $mask = PhraseaTokens::MAKE_SUBDEF | PhraseaTokens::TO_INDEX | PhraseaTokens::INDEXING; // we only care about those "jetons" + $sql = "SELECT type, jeton & (".$mask.") AS status, SUM(1) AS n FROM record GROUP BY type, (jeton & ".$mask.")"; $stmt = $this->get_connection()->prepare($sql); $stmt->execute(); $rs = $stmt->fetchAll(PDO::FETCH_ASSOC); $stmt->closeCursor(); $ret = array( - 'xml_indexed' => 0, - 'thesaurus_indexed' => 0, - 'jeton_subdef' => array() + 'records' => 0, + 'records_indexed' => 0, // jetons = 0;0 + 'records_to_index' => 0, // jetons = 0;1 + 'records_not_indexed' => 0, // jetons = 1;0 + 'records_indexing' => 0, // jetons = 1;1 + 'subdefs_todo' => array() // by type "image", "video", ... ); - foreach ($rs as $row) { + $ret['records'] += ($n = (int)($row['n'])); $status = $row['status']; - if ($status & 1) - $ret['xml_indexed'] += $row['n']; - if ($status & 2) - $ret['thesaurus_indexed'] += $row['n']; - } - - $sql = "SELECT type, COUNT(record_id) AS n FROM record WHERE jeton & ".PhraseaTokens::MAKE_SUBDEF." GROUP BY type"; - $stmt = $this->get_connection()->prepare($sql); - $stmt->execute(); - $rs = $stmt->fetchAll(PDO::FETCH_ASSOC); - $stmt->closeCursor(); - - foreach ($rs as $row) { - $ret['jeton_subdef'][$row['type']] = (int)$row['n']; + switch($status & (PhraseaTokens::TO_INDEX | PhraseaTokens::INDEXING)) { + case 0: + $ret['records_indexed'] += $n; + break; + case PhraseaTokens::TO_INDEX: + $ret['records_to_index'] += $n; + break; + case PhraseaTokens::INDEXING: + $ret['records_not_indexed'] += $n; + break; + case PhraseaTokens::INDEXING | PhraseaTokens::TO_INDEX: + $ret['records_indexing'] += $n; + break; + } + if($status & PhraseaTokens::MAKE_SUBDEF) { + if(!array_key_exists($row['type'], $ret['subdefs_todo'])) { + $ret['subdefs_todo'][$row['type']] = 0; + } + $ret['subdefs_todo'][$row['type']] += $n; + } } return $ret; @@ -1057,6 +1031,12 @@ class databox extends base implements ThumbnailedElement { $this->get_connection()->update('pref', ['updated_on' => '0000-00-00 00:00:00'], ['prop' => 'indexes']); + // Set TO_INDEX flag on all records + $sql = "UPDATE record SET jeton = (jeton | :token)"; + $stmt = $this->connection->prepare($sql); + $stmt->bindValue(':token', PhraseaTokens::TO_INDEX, PDO::PARAM_INT); + $stmt->execute(); + return $this; } diff --git a/templates/web/admin/databox/databox.html.twig b/templates/web/admin/databox/databox.html.twig index f0437643cc..ed9971fab2 100644 --- a/templates/web/admin/databox/databox.html.twig +++ b/templates/web/admin/databox/databox.html.twig @@ -45,7 +45,7 @@
  • {{ 'admin::base: nombre d\'enregistrements sur la base :' | trans }} - {{ databox.get_record_amount() }} + ({{ 'phraseanet:: details' | trans }})
    • @@ -54,41 +54,13 @@ {{ 'admin::base: subdefs to be created :' | trans }} - - {% if showDetail %} -
  • - {{ 'admin::base: nombre de mots uniques sur la base :' | trans }} - {{ databox.get_unique_keywords() }} -
    • -
  • - {{ 'admin::base: nombre de mots indexes sur la base' | trans }} - {{ databox.get_index_amount() }} -
    • - {% if app['conf'].get(['registry', 'modules', 'thesaurus']) %} -
  • - {{ 'admin::base: nombre de termes de Thesaurus indexes :' | trans }} - {{ databox.get_thesaurus_hits() }} -
    • - {% endif %} - {% endif %} -
    -
    -

    - {{ "admin::base: document indexes en utilisant la fiche xml" | trans }} : - -

    -
    -
    -
    -
    -

    - {{ "admin::base: document indexes en utilisant le thesaurus" | trans }} : - -

    -
    -
    +
    +
    +
    ...
    +
    +
    @@ -248,65 +220,91 @@
    diff --git a/tests/Alchemy/Tests/Phrasea/Controller/Admin/DataboxTest.php b/tests/Alchemy/Tests/Phrasea/Controller/Admin/DataboxTest.php index a074ec40b9..95cbf93d34 100644 --- a/tests/Alchemy/Tests/Phrasea/Controller/Admin/DataboxTest.php +++ b/tests/Alchemy/Tests/Phrasea/Controller/Admin/DataboxTest.php @@ -234,9 +234,7 @@ class DataboxTest extends \PhraseanetAuthenticatedWebTestCase $this->assertTrue($json->success); $this->assertObjectHasAttribute('sbas_id', $json); $this->assertObjectHasAttribute('indexable', $json); - $this->assertObjectHasAttribute('records', $json); - $this->assertObjectHasAttribute('xml_indexed', $json); - $this->assertObjectHasAttribute('thesaurus_indexed', $json); + $this->assertObjectHasAttribute('counts', $json); $this->assertObjectHasAttribute('viewname', $json); $this->assertObjectHasAttribute('printLogoURL', $json); } diff --git a/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiTestCase.php b/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiTestCase.php index 8d0870dc16..c8b2c8980e 100644 --- a/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiTestCase.php +++ b/tests/Alchemy/Tests/Phrasea/Controller/Api/ApiTestCase.php @@ -2288,7 +2288,7 @@ abstract class ApiTestCase extends \PhraseanetWebTestCase $this->assertArrayHasKey('response', $content); $this->assertTrue(is_array($content['meta']), 'Le bloc meta est un array'); $this->assertTrue(is_array($content['response']), 'Le bloc reponse est un array'); - $this->assertEquals('1.4.1', $content['meta']['api_version']); + $this->assertEquals('2.0.0', $content['meta']['api_version']); $this->assertNotNull($content['meta']['response_time']); $this->assertEquals('UTF-8', $content['meta']['charset']); } diff --git a/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php b/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php index db887e0d00..9ff9bcbc43 100644 --- a/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php +++ b/tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php @@ -55,6 +55,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase "multi_match": { "fields": ["foo.fr", "foo.en"], "query": "bar", + "type": "cross_fields", "operator": "and", "lenient": true } @@ -94,6 +95,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase "multi_match": { "fields": ["foo.fr", "foo.en"], "query": "baz", + "type": "cross_fields", "operator": "and", "lenient": true } @@ -108,6 +110,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase "multi_match": { "fields": ["private_caption.bar.fr", "private_caption.bar.en"], "query": "baz", + "type": "cross_fields", "operator": "and", "lenient": true } @@ -140,6 +143,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase "multi_match": { "fields": ["foo.fr", "foo.en"], "query": "bar", + "type": "cross_fields", "operator": "and", "lenient": true } @@ -189,6 +193,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase "multi_match": { "fields": ["foo.fr", "foo.en"], "query": "baz", + "type": "cross_fields", "operator": "and", "lenient": true } @@ -212,6 +217,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase "multi_match": { "fields": ["private_caption.bar.fr", "private_caption.bar.en"], "query": "baz", + "type": "cross_fields", "operator": "and", "lenient": true }