hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-23 18:03:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ar-731-status-search-dsl

Browse Source 
Conflicts:
	lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php
This commit is contained in:
Mathieu Darse
2015-10-06 12:26:03 +02:00
parent 8a5afb1140 b55cb86690
commit d6a82370a9
29 changed files with 517 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
lib/Alchemy/Phrasea/Application.php
View File

@@ -84,6 +84,8 @@ use MediaVorus\Media\MediaInterface;
use MediaVorus\MediaVorus;
use MediaVorus\MediaVorusServiceProvider;
use Monolog\Handler\NullHandler;
use Monolog\Handler\RotatingFileHandler;
use Monolog\Handler\SyslogHandler;
use Monolog\Logger;
use Monolog\Processor\IntrospectionProcessor;
use MP4Box\MP4BoxServiceProvider;
@@ -266,6 +268,7 @@ class Application extends SilexApplication
$this['phraseanet.exception_handler'] = $this->share(function ($app) {
$handler = PhraseaExceptionHandler::register($app['debug']);
$handler->setTranslator($app['translator']);
$handler->setLogger($app['monolog']);
return $handler;
});
@@ -1050,14 +1053,16 @@ class Application extends SilexApplication
private function setupMonolog()
{
$this['monolog.name'] = 'phraseanet';
$this['monolog.handler'] = $this->share(function () {
return new NullHandler();
$this['monolog.logfile'] = $this['root.path'] . '/logs/app_error.log';
$this['monolog.handler'] = $this->share(function (Application $app) {
return new RotatingFileHandler(
$app['monolog.logfile'],
10,
Logger::ERROR,
$app['monolog.bubble'],
$app['monolog.permission']
);
});
$this['monolog'] = $this->share($this->extend('monolog', function (Logger $logger) {
$logger->pushProcessor(new IntrospectionProcessor());
return $logger;
}));
}
private function setupEventDispatcher()

6
lib/Alchemy/Phrasea/Application/Api.php
View File

@@ -41,7 +41,10 @@ return call_user_func(function ($environment = PhraseaApplication::ENV_PROD) {
$app->loadPlugins();
$app['exception_handler'] = $app->share(function ($app) {
return new ApiExceptionHandlerSubscriber($app);
$handler = new ApiExceptionHandlerSubscriber($app);
$handler->setLogger($app['monolog']);
return $handler;
});
$app['monolog'] = $app->share($app->extend('monolog', function (Logger $monolog) {
$monolog->pushProcessor(new WebProcessor());
@@ -92,6 +95,7 @@ return call_user_func(function ($environment = PhraseaApplication::ENV_PROD) {
if ($request->getRequestFormat(Result::FORMAT_JSON) === Result::FORMAT_JSONP && !$response->isOk() && !$response->isServerError()) {
$response->setStatusCode(200);
}
// set response content type
if (!$response->headers->get('Content-Type')) {
$response->headers->set('Content-Type', $request->getMimeType($request->getRequestFormat(Result::FORMAT_JSON)));

14
lib/Alchemy/Phrasea/Controller/Admin/DataboxController.php
View File

@@ -577,29 +577,23 @@ class DataboxController extends Controller
$ret = [
'success' => false,
'msg' => $this->app->trans('An error occured'),
'sbas_id' => null,
'msg' => $this->app->trans('An error occured'),
'indexable' => false,
'records' => 0,
'xml_indexed' => 0,
'thesaurus_indexed' => 0,
'viewname' => null,
'printLogoURL' => null,
'counts' => null,
];
try {
$databox = $this->findDataboxById($databox_id);
$data = $databox->get_indexed_record_amount();
$ret['sbas_id'] = $databox_id;
$ret['indexable'] = $appbox->is_databox_indexable($databox);
$ret['viewname'] = (($databox->get_dbname() == $databox->get_viewname())
? $this->app->trans('admin::base: aucun alias')
: $databox->get_viewname());
$ret['records'] = $databox->get_record_amount();
$ret['sbas_id'] = $databox_id;
$ret['xml_indexed'] = $data['xml_indexed'];
$ret['thesaurus_indexed'] = $data['thesaurus_indexed'];
$ret['jeton_subdef'] = $data['jeton_subdef'];
$ret['counts'] = $databox->get_counts();
if ($this->app['filesystem']->exists($this->app['root.path'] . '/config/minilogos/logopdf_' . $databox_id . '.jpg')) {
$ret['printLogoURL'] = '/custom/minilogos/logopdf_' . $databox_id . '.jpg';
}

15
lib/Alchemy/Phrasea/Controller/Api/V1Controller.php
View File

@@ -462,6 +462,13 @@ class V1Controller extends Controller
];
}
public function getDataboxCollectionAction(Request $request, $base_id)
{
return Result::create($request, [
$this->listCollection($this->app->getApplicationBox()->get_collection($base_id))
])->createResponse();
}
/**
* Get a Response containing the collections of a \databox
*
@@ -1459,9 +1466,9 @@ class V1Controller extends Controller
$devices = $request->get('devices', []);
$mimes = $request->get('mimes', []);
$ret = array_filter(array_map(function ($media) use ($request, $record) {
$ret = array_values(array_filter(array_map(function ($media) use ($request, $record) {
return $this->listEmbeddableMedia($request, $record, $media);
}, $record->get_embedable_medias($devices, $mimes)));
}, $record->get_embedable_medias($devices, $mimes))));
return Result::create($request, ["embed" => $ret])->createResponse();
}
@@ -1978,9 +1985,9 @@ class V1Controller extends Controller
$devices = $request->get('devices', []);
$mimes = $request->get('mimes', []);
$ret = array_filter(array_map(function ($media) use ($request, $record) {
$ret = array_values(array_filter(array_map(function ($media) use ($request, $record) {
return $this->listEmbeddableMedia($request, $record, $media);
}, $record->get_embedable_medias($devices, $mimes)));
}, $record->get_embedable_medias($devices, $mimes))));
return Result::create($request, ["embed" => $ret])->createResponse();
}

24
lib/Alchemy/Phrasea/Controller/Thesaurus/ThesaurusController.php
View File

@@ -12,7 +12,7 @@ namespace Alchemy\Phrasea\Controller\Thesaurus;
use Alchemy\Phrasea\Application\Helper\DispatcherAware;
use Alchemy\Phrasea\Controller\Controller;
use Alchemy\Phrasea\Core\Event\Thesaurus as ThesaurusEvent;
use Alchemy\Phrasea\Core\PhraseaEvents;
use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvents;
use Doctrine\DBAL\Driver\Connection;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
@@ -699,7 +699,7 @@ class ThesaurusController extends Controller
}
$this->dispatch(
PhraseaEvents::THESAURUS_IMPORTED,
ThesaurusEvents::IMPORTED,
new ThesaurusEvent\Imported($databox)
);
}
@@ -987,7 +987,7 @@ class ThesaurusController extends Controller
if ($request->get("reindex")) {
$this->dispatch(
PhraseaEvents::THESAURUS_FIELD_LINKED,
ThesaurusEvents::FIELD_LINKED,
new ThesaurusEvent\FieldLinked($databox)
);
}
@@ -1360,7 +1360,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($domth);
$this->dispatch(
PhraseaEvents::THESAURUS_CANDIDATE_ACCEPTED_AS_CONCEPT,
ThesaurusEvents::CANDIDATE_ACCEPTED_AS_CONCEPT,
new ThesaurusEvent\CandidateAccepted($databox, $new_te->getAttribute('id'))
);
@@ -1404,7 +1404,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($domth);
$this->dispatch(
PhraseaEvents::THESAURUS_CANDIDATE_ACCEPTED_AS_SYNONYM,
ThesaurusEvents::CANDIDATE_ACCEPTED_AS_SYNONYM,
new ThesaurusEvent\CandidateAccepted($databox, $new_te->getAttribute('id'))
);
}
@@ -1493,7 +1493,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($dom);
$this->dispatch(
PhraseaEvents::THESAURUS_SYNONYM_LNG_CHANGED,
ThesaurusEvents::SYNONYM_LNG_CHANGED,
new ThesaurusEvent\SynonymLngChanged($databox, $sy0->getAttribute('id'))
);
}
@@ -1575,7 +1575,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($dom);
$this->dispatch(
PhraseaEvents::THESAURUS_SYNONYM_POSITION_CHANGED,
ThesaurusEvents::SYNONYM_POSITION_CHANGED,
new ThesaurusEvent\SynonymPositionChanged($databox, $sy0->getAttribute('id'))
);
@@ -1720,7 +1720,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($dom);
$this->dispatch(
PhraseaEvents::THESAURUS_SYNONYM_TRASHED,
ThesaurusEvents::SYNONYM_TRASHED,
new ThesaurusEvent\ItemTrashed($databox, $te->getAttribute('id'), $delsy->getAttribute('id'))
);
@@ -1837,7 +1837,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($domth);
$this->dispatch(
PhraseaEvents::THESAURUS_CONCEPT_TRASHED,
ThesaurusEvents::CONCEPT_TRASHED,
new ThesaurusEvent\ItemTrashed($databox, $thnode_parent->getAttribute('id'), $newte->getAttribute('id'))
);
@@ -2282,7 +2282,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($dom);
$this->dispatch(
PhraseaEvents::THESAURUS_CONCEPT_DELETED,
ThesaurusEvents::CONCEPT_DELETED,
new ThesaurusEvent\ConceptDeleted($databox, $refrid, $sy_evt_parm)
);
@@ -2391,7 +2391,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($domth);
$this->dispatch(
PhraseaEvents::THESAURUS_SYNONYM_ADDED,
ThesaurusEvents::SYNONYM_ADDED,
new ThesaurusEvent\ItemAdded($databox, $syid)
);
@@ -2496,7 +2496,7 @@ class ThesaurusController extends Controller
$databox->saveThesaurus($domth);
$this->dispatch(
PhraseaEvents::THESAURUS_CONCEPT_ADDED,
ThesaurusEvents::CONCEPT_ADDED,
new ThesaurusEvent\ItemAdded($databox, $syid)
);

4
lib/Alchemy/Phrasea/ControllerProvider/Api/V1.php
View File

@@ -20,7 +20,7 @@ use Silex\ServiceProviderInterface;
class V1 implements ControllerProviderInterface, ServiceProviderInterface
{
const VERSION = '1.4.1';
const VERSION = '2.0.0';
public static $extendedContentTypes = [
'json' => ['application/vnd.phraseanet.record-extended+json'],
@@ -72,6 +72,8 @@ class V1 implements ControllerProviderInterface, ServiceProviderInterface
$controllers->get('/monitor/phraseanet/', 'controller.api.v1:showPhraseanetConfigurationAction')
->before('controller.api.v1:ensureAdmin');
$controllers->get('/collections/{base_id}/', 'controller.api.v1:getDataboxCollectionAction');
$controllers->get('/databoxes/list/', 'controller.api.v1:listDataboxesAction');
$controllers->get('/databoxes/{databox_id}/collections/', 'controller.api.v1:getDataboxCollectionsAction')

17
lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiExceptionHandlerSubscriber.php
View File

@@ -13,6 +13,8 @@ namespace Alchemy\Phrasea\Core\Event\Subscriber;
use Alchemy\Phrasea\Application;
use Alchemy\Phrasea\Controller\Api\Result;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Symfony\Component\HttpKernel\Event\GetResponseForExceptionEvent;
use Symfony\Component\HttpKernel\KernelEvents;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
@@ -27,9 +29,17 @@ class ApiExceptionHandlerSubscriber implements EventSubscriberInterface
{
private $app;
private $logger;
public function __construct(Application $app)
{
$this->app = $app;
$this->logger = new NullLogger();
}
public function setLogger(LoggerInterface $logger)
{
$this->logger = $logger;
}
public static function getSubscribedEvents()
@@ -64,6 +74,13 @@ class ApiExceptionHandlerSubscriber implements EventSubscriberInterface
$code = 500;
}
if ($code == 500) {
$this->logger->error($e->getMessage(), [
'code' => $e->getCode(),
'trace' => $e->getTrace()
]);
}
if ($e instanceof HttpExceptionInterface) {
$headers = $e->getHeaders();
}

2
lib/Alchemy/Phrasea/Core/Event/Subscriber/ApiOauth2ErrorsSubscriber.php
View File

@@ -61,7 +61,7 @@ class ApiOauth2ErrorsSubscriber implements EventSubscriberInterface
$msg = json_encode(['msg' => $msg, 'code' => $code]);
$event->setResponse(new Response($msg, $code, $headers));
} else {
$response = $this->handler->createResponseBasedOnRequest($event->getRequest(), $event->getException());
$response = $this->handler->createResponse($event->getException());
$response->headers->set('Content-Type', 'text/html');
$event->setResponse($response);
}

2
lib/Alchemy/Phrasea/Core/Event/Subscriber/PhraseaExceptionHandlerSubscriber.php
View File

@@ -38,7 +38,7 @@ class PhraseaExceptionHandlerSubscriber implements EventSubscriberInterface
return;
}
$event->setResponse($this->handler->createResponseBasedOnRequest($event->getRequest(), $event->getException()));
$event->setResponse($this->handler->createResponse($event->getException()));
}
/**

27
lib/Alchemy/Phrasea/Core/Event/Thesaurus/ThesaurusEvents.php Normal file
View File

@@ -0,0 +1,27 @@
<?php
/*
* This file is part of Phraseanet
*
* (c) 2005-2014 Alchemy
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Alchemy\Phrasea\Core\Event\Thesaurus;
final class ThesaurusEvents
{
const IMPORTED = 'thesaurus.imported';
const FIELD_LINKED = 'thesaurus.field-linked';
const CANDIDATE_ACCEPTED_AS_CONCEPT = 'thesaurus.candidate-accepted-as-concept';
const CANDIDATE_ACCEPTED_AS_SYNONYM = 'thesaurus.candidate-accepted-as-synonym';
const SYNONYM_LNG_CHANGED = 'thesaurus.synonym-lng-changed';
const SYNONYM_POSITION_CHANGED = 'thesaurus.synonym-position-changed';
const SYNONYM_TRASHED = 'thesaurus.synonym-trashed';
const CONCEPT_TRASHED = 'thesaurus.concept-trashed';
const CONCEPT_DELETED = 'thesaurus.concept-deleted';
const SYNONYM_ADDED = 'thesaurus.synonym-added';
const CONCEPT_ADDED = 'thesaurus.concept-added';
}

21
lib/Alchemy/Phrasea/Core/PhraseaExceptionHandler.php
View File

@@ -11,6 +11,8 @@
namespace Alchemy\Phrasea\Core;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Symfony\Component\Debug\ExceptionHandler as SymfonyExceptionHandler;
use Symfony\Component\Debug\Exception\FlattenException;
use Symfony\Component\HttpFoundation\Request;
@@ -21,6 +23,18 @@ class PhraseaExceptionHandler extends SymfonyExceptionHandler
{
private $translator;
private $logger;
public function __construct()
{
$this->logger = new NullLogger();
}
public function setLogger(LoggerInterface $logger)
{
$this->logger = $logger;
}
public function setTranslator(TranslatorInterface $translator)
{
$this->translator = $translator;
@@ -33,6 +47,13 @@ class PhraseaExceptionHandler extends SymfonyExceptionHandler
public function getContent(FlattenException $exception)
{
if ($exception->getStatusCode() == '500') {
$this->logger->error($exception->getMessage(), [
'code' => $exception->getCode(),
'trace' => $exception->getTrace()
]);
}
switch (true) {
case 404 === $exception->getStatusCode():
if (null !== $this->translator) {

6
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
class QuotedTextNode extends Node
{
@@ -38,11 +38,11 @@ class QuotedTextNode extends Node
};
$unrestricted_fields = $context->getUnrestrictedFields();
$unrestricted_fields = Field::filterByValueCompatibility($unrestricted_fields, $this->text);
$unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text);
$query = $query_builder($unrestricted_fields);
$private_fields = $context->getPrivateFields();
$private_fields = Field::filterByValueCompatibility($private_fields, $this->text);
$private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text);
foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
$query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
}

6
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
class RawNode extends Node
{
@@ -56,11 +56,11 @@ class RawNode extends Node
};
$unrestricted_fields = $context->getUnrestrictedFields();
$unrestricted_fields = Field::filterByValueCompatibility($unrestricted_fields, $this->text);
$unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text);
$query = $query_builder($unrestricted_fields);
$private_fields = $context->getPrivateFields();
$private_fields = Field::filterByValueCompatibility($private_fields, $this->text);
$private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text);
foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
$query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
}

5
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Term;
class TextNode extends AbstractTermNode implements ContextAbleInterface
@@ -41,7 +41,7 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
$query_builder = function (array $fields) use ($context) {
// Full text
$index_fields = [];
foreach (Field::filterByValueCompatibility($fields, $this->text) as $field) {
foreach (StructureField::filterByValueCompatibility($fields, $this->text) as $field) {
foreach ($context->localizeField($field) as $f) {
$index_fields[] = $f;
}
@@ -53,6 +53,7 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
'multi_match' => [
'fields' => $index_fields,
'query' => $this->text,
'type' => 'cross_fields',
'operator' => 'and',
'lenient' => true,
]

7
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer.php
View File

@@ -41,7 +41,7 @@ class Indexer
private $recordIndexer;
private $termIndexer;
private $indexQueue;
private $indexQueue; // contains RecordInterface(s)
private $deleteQueue;
private $previousRefreshInterval = self::DEFAULT_REFRESH_INTERVAL;
@@ -155,6 +155,11 @@ class Indexer
// RecordQueuer::queueRecordsFromDatabox($databox);
}
public function scheduleRecordsFromDataboxForIndexing(\databox $databox)
{
RecordQueuer::queueRecordsFromDatabox($databox);
}
public function scheduleRecordsFromCollectionForIndexing(\collection $collection)
{
RecordQueuer::queueRecordsFromCollection($collection);

47
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/BulkOperation.php
View File

@@ -23,10 +23,11 @@ class BulkOperation
private $logger;
private $stack = array();
private $opCount = 0;
private $operationIdentifiers = [];
private $index;
private $type;
private $flushLimit = 1000;
private $flushCallbacks = [];
public function __construct(Client $client, LoggerInterface $logger)
{
@@ -52,27 +53,32 @@ class BulkOperation
$this->flushLimit = (int) $limit;
}
public function index(array $params)
public function onFlush(\Closure $callback)
{
$this->flushCallbacks[] = $callback;
}
public function index(array $params, $operationIdentifier)
{
$header = $this->buildHeader('index', $params);
$body = igorw\get_in($params, ['body']);
$this->push($header, $body);
$this->push($header, $body, $operationIdentifier);
}
public function delete(array $params)
public function delete(array $params, $operationIdentifier)
{
$this->push($this->buildHeader('delete', $params));
$this->push($this->buildHeader('delete', $params), null, $operationIdentifier);
}
private function push($header, $body = null)
private function push($header, $body, $operationIdentifier)
{
$this->stack[] = $header;
if ($body) {
$this->stack[] = $body;
}
$this->opCount++;
$this->operationIdentifiers[] = $operationIdentifier;
if ($this->flushLimit === $this->opCount) {
if (count($this->operationIdentifiers) === $this->flushLimit) {
$this->flush();
}
}
@@ -93,19 +99,32 @@ class BulkOperation
}
$params['body'] = $this->stack;
$this->logger->debug("ES Bulk query about to be performed\n", ['opCount' => $this->opCount]);
$this->logger->debug("ES Bulk query about to be performed\n", ['opCount' => count($this->operationIdentifiers)]);
$response = $this->client->bulk($params);
$this->stack = array();
$this->opCount = 0;
if (igorw\get_in($response, ['errors'], true)) {
foreach ($response['items'] as $key => $item) {
if ($item['index']['status'] >= 400) { // 4xx or 5xx error
throw new Exception(sprintf('%d: %s', $key, $item['index']['error']));
$callbackData = []; // key: operationIdentifier passed when command was pushed on this bulk
// value: json result from es for the command
// nb: results (items) are returned IN THE SAME ORDER as commands were pushed in the stack
// so the items[X] match the operationIdentifiers[X]
foreach ($response['items'] as $key => $item) {
foreach($item as $command=>$result) { // command may be "index" or "delete"
if($response['errors'] && $result['status'] >= 400) { // 4xx or 5xx error
$err = array_key_exists('error', $result) ? $result['error'] : ($command . " error " . $result['status']);
throw new Exception(sprintf('%d: %s', $key, $err));
}
}
$operationIdentifier = $this->operationIdentifiers[$key];
if(is_string($operationIdentifier) || is_int($operationIdentifier)) { // dont include null keys
$callbackData[$operationIdentifier] = $response['items'][$key];
}
}
foreach($this->flushCallbacks as $iCallBack=>$flushCallback) {
$flushCallback($callbackData);
}
$this->operationIdentifiers = [];
}
private function buildHeader($key, array $params)

53
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Fetcher.php
View File

@@ -16,12 +16,14 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegate;
use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegateInterface;
use Closure;
use databox;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Driver\Connection as ConnectionInterface;
use PDO;
class Fetcher
{
private $databox;
private $connection;
private $statement;
private $delegate;
@@ -34,13 +36,19 @@ class Fetcher
private $postFetch;
private $onDrain;
public function __construct(ConnectionInterface $connection, array $hydrators, FetcherDelegateInterface $delegate = null)
public function __construct(databox $databox, array $hydrators, FetcherDelegateInterface $delegate = null)
{
$this->connection = $connection;
$this->databox = $databox;
$this->connection = $databox->get_connection();;
$this->hydrators = $hydrators;
$this->delegate = $delegate ?: new FetcherDelegate();
}
public function getDatabox()
{
return $this->databox;
}
public function fetch()
{
if (empty($this->buffer)) {
@@ -64,7 +72,6 @@ class Fetcher
$records[$record['record_id']] = $record;
$this->offset++;
}
if (empty($records)) {
$this->onDrain->__invoke();
return;
@@ -87,6 +94,12 @@ class Fetcher
return $records;
}
public function restart()
{
$this->buffer = array();
$this->offset = 0;
}
public function setBatchSize($size)
{
if ($size < 1) {
@@ -105,28 +118,24 @@ class Fetcher
$this->onDrain = $onDrain;
}
/**
* @return \Doctrine\DBAL\Driver\Statement
*/
private function getExecutedStatement()
{
if (!$this->statement) {
$sql = <<<SQL
SELECT r.record_id
, r.coll_id as collection_id
, c.asciiname as collection_name
, r.uuid
, r.status as flags_bitfield
, r.sha256 -- TODO rename in "hash"
, r.originalname as original_name
, r.mime
, r.type
, r.parent_record_id
, r.credate as created_on
, r.moddate as updated_on
FROM record r
INNER JOIN coll c ON (c.coll_id = r.coll_id)
-- WHERE
ORDER BY r.record_id DESC
LIMIT :offset, :limit
SQL;
$sql = "SELECT r.record_id"
. ", r.coll_id AS collection_id"
. ", c.asciiname AS collection_name"
. ", r.uuid"
. ", r.status AS flags_bitfield"
. ", r.sha256" // -- TODO rename in "hash"
. ", r.originalname AS original_name"
. ", r.mime, r.type, r.parent_record_id, r.credate AS created_on, r.moddate AS updated_on"
. " FROM record r INNER JOIN coll c ON (c.coll_id = r.coll_id)"
. " -- WHERE"
. " ORDER BY r.record_id DESC"
. " LIMIT :offset, :limit";
$where = $this->delegate->buildWhereClause();
$sql = str_replace('-- WHERE', $where, $sql);

137
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php
View File

@@ -11,6 +11,7 @@
namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer;
use Alchemy\Phrasea\Model\RecordInterface;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException;
use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\BulkOperation;
@@ -60,6 +61,12 @@ class RecordIndexer
private $logger;
private function getUniqueOperationId($record_key)
{
$_key = dechex(mt_rand());
return $_key . '_' . $record_key;
}
public function __construct(Structure $structure, RecordHelper $helper, Thesaurus $thesaurus, \appbox $appbox, array $locales, LoggerInterface $logger)
{
$this->structure = $structure;
@@ -70,16 +77,70 @@ class RecordIndexer
$this->logger = $logger;
}
/**
* ES made a bulk op, check our (index) operations to drop the "indexing" & "to_index" jetons
*
* @param databox $databox
* @param array $operation_identifiers key:op_identifier ; value:operation result (json from es)
* @param array $submited_records records indexed, key:op_identifier
*/
private function onBulkFlush(databox $databox, array $operation_identifiers, array &$submited_records)
{
// nb: because the same bulk could be used by many "clients", this (each) callback may receive
// operation_identifiers that does not belong to it.
// flag only records that the fetcher worked on
$records = array_intersect_key(
$submited_records, // this is OUR records list
$operation_identifiers // reduce to the records indexed by this bulk (should be the same...)
);
if(count($records) === 0) {
return;
}
// Commit and remove "indexing" flag
RecordQueuer::didFinishIndexingRecords(array_values($records), $databox);
foreach (array_keys($records) as $id) {
unset($submited_records[$id]);
}
}
/**
* index whole databox(es), don't test actual "jetons"
*
* @param BulkOperation $bulk
* @param databox[] $databoxes
*/
public function populateIndex(BulkOperation $bulk, array $databoxes)
{
foreach ($databoxes as $databox) {
$submited_records = [];
$this->logger->info(sprintf('Indexing database %s...', $databox->get_viewname()));
$fetcher = $this->createFetcherForDatabox($databox);
$this->indexFromFetcher($bulk, $fetcher);
$fetcher = $this->createFetcherForDatabox($databox); // no delegate, scan the whole records
// post fetch : flag records as "indexing"
$fetcher->setPostFetch(function(array $records) use ($databox, $fetcher) {
RecordQueuer::didStartIndexingRecords($records, $databox);
// do not restart the fetcher since it has no clause on jetons
});
// bulk flush : flag records as "indexed"
$bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) {
$this->onBulkFlush($databox, $operation_identifiers, $submited_records);
});
// Perform indexing
$this->indexFromFetcher($bulk, $fetcher, $submited_records);
$this->logger->info(sprintf('Finished indexing %s', $databox->get_viewname()));
}
}
/**
* Index the records flagged as "to_index" on all databoxes
*
* @param BulkOperation $bulk
*/
public function indexScheduled(BulkOperation $bulk)
{
foreach ($this->appbox->get_databoxes() as $databox) {
@@ -89,46 +150,82 @@ class RecordIndexer
private function indexScheduledInDatabox(BulkOperation $bulk, databox $databox)
{
$submited_records = [];
// Make fetcher
$delegate = new ScheduledFetcherDelegate();
$fetcher = $this->createFetcherForDatabox($databox, $delegate);
// Keep track of fetched records, flag them as "indexing"
$fetched = array();
$fetcher->setPostFetch(function(array $records) use ($databox, &$fetched) {
// TODO Do not keep all indexed records in memory...
$fetched += $records;
// post fetch : flag records as "indexing"
$fetcher->setPostFetch(function(array $records) use ($databox, $fetcher) {
RecordQueuer::didStartIndexingRecords($records, $databox);
// because changing the flag on the records affects the "where" clause of the fetcher,
// restart it each time
$fetcher->restart();
});
// bulk flush : flag records as "indexed"
$bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) {
$this->onBulkFlush($databox, $operation_identifiers, $submited_records);
});
// Perform indexing
$this->indexFromFetcher($bulk, $fetcher);
// Commit and remove "indexing" flag
$bulk->flush();
RecordQueuer::didFinishIndexingRecords($fetched, $databox);
$this->indexFromFetcher($bulk, $fetcher, $submited_records);
}
/**
* Index a list of records
*
* @param BulkOperation $bulk
* @param Iterator $records
*/
public function index(BulkOperation $bulk, Iterator $records)
{
foreach ($this->createFetchersForRecords($records) as $fetcher) {
$this->indexFromFetcher($bulk, $fetcher);
$submited_records = [];
$databox = $fetcher->getDatabox();
// post fetch : flag records as "indexing"
$fetcher->setPostFetch(function(array $records) use ($fetcher, $databox) {
RecordQueuer::didStartIndexingRecords($records, $databox);
// do not restart the fetcher since it has no clause on jetons
});
// bulk flush : flag records as "indexed"
$bulk->onFlush(function($operation_identifiers) use ($databox, &$submited_records) {
$this->onBulkFlush($databox, $operation_identifiers, $submited_records);
});
// Perform indexing
$this->indexFromFetcher($bulk, $fetcher, $submited_records);
}
}
/**
* Deleta a list of records
*
* @param BulkOperation $bulk
* @param Iterator $records
*/
public function delete(BulkOperation $bulk, Iterator $records)
{
foreach ($records as $record) {
$params = array();
$params['id'] = $record->getId();
$params['type'] = self::TYPE_NAME;
$bulk->delete($params);
$bulk->delete($params, null); // no operationIdentifier is related to a delete op
}
}
/**
* @param Iterator $records
* @return Fetcher[]
*/
private function createFetchersForRecords(Iterator $records)
{
$fetchers = array();
foreach ($this->groupRecordsByDatabox($records) as $group) {
$databox = $group['databox'];
$connection = $databox->get_connection();
$delegate = new RecordListFetcherDelegate($group['records']);
$fetchers[] = $this->createFetcherForDatabox($databox, $delegate);
}
@@ -140,7 +237,7 @@ class RecordIndexer
{
$connection = $databox->get_connection();
$candidateTerms = new CandidateTerms($databox);
$fetcher = new Fetcher($connection, array(
$fetcher = new Fetcher($databox, array(
new CoreHydrator($databox->get_sbas_id(), $databox->get_viewname(), $this->helper),
new TitleHydrator($connection),
new MetadataHydrator($connection, $this->structure, $this->helper),
@@ -169,15 +266,21 @@ class RecordIndexer
return array_values($databoxes);
}
private function indexFromFetcher(BulkOperation $bulk, Fetcher $fetcher)
private function indexFromFetcher(BulkOperation $bulk, Fetcher $fetcher, array &$submited_records)
{
/** @var RecordInterface $record */
while ($record = $fetcher->fetch()) {
$op_identifier = $this->getUniqueOperationId($record['id']);
$params = array();
$params['id'] = $record['id'];
unset($record['id']);
$params['type'] = self::TYPE_NAME;
$params['body'] = $record;
$bulk->index($params);
$submited_records[$op_identifier] = $record;
$bulk->index($params, $op_identifier);
}
}

41
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordQueuer.php
View File

@@ -35,41 +35,44 @@ class RecordQueuer
$connection = $collection->get_connection();
// Set TO_INDEX flag on all records from this collection
$sql = <<<SQL
UPDATE record
SET jeton = (jeton | :token)
WHERE coll_id = :coll_id
SQL;
$sql = "UPDATE record SET jeton = (jeton | :token) WHERE coll_id = :coll_id";
$stmt = $connection->prepare($sql);
$stmt->bindValue(':token', Flag::TO_INDEX, PDO::PARAM_INT);
$stmt->bindValue(':coll_id', $collection->get_coll_id(), PDO::PARAM_INT);
$stmt->execute();
}
/**
* @param array $records
* @param $databox
*
* nb: changing the jeton may affect a fetcher if his "where" clause (delegate) depends on jeton.
* in this case the client of the fetcher must set a "postFetch" callback and restart the fetcher
*/
public static function didStartIndexingRecords(array $records, $databox)
{
$connection = $databox->get_connection();
$sql = <<<SQL
UPDATE record
SET jeton = (jeton | :flag)
WHERE record_id IN (:record_ids)
SQL;
$sql = "UPDATE record SET jeton = (jeton | :flag) WHERE record_id IN (:record_ids)";
self::executeFlagQuery($connection, $sql, Flag::INDEXING, $records);
}
public static function didFinishIndexingRecords(array $records, $databox)
/**
* @param array $records
* @param $databox
*
* nb: changing the jeton may affect a fetcher if his "where" clause (delegate) depends on jeton.
* in this case the client of the fetcher must set a "postFetch" callback and restart the fetcher
*/
public static function didFinishIndexingRecords(array $records, databox $databox)
{
$connection = $databox->get_connection();
$sql = <<<SQL
UPDATE record
SET jeton = (jeton & ~ :flag)
WHERE record_id IN (:record_ids)
SQL;
$flag = Flag::TO_INDEX | Flag::INDEXING;
self::executeFlagQuery($connection, $sql, $flag, $records);
$sql = "UPDATE record SET jeton = (jeton & ~ :flag) WHERE record_id IN (:record_ids)";
self::executeFlagQuery($connection, $sql, Flag::TO_INDEX | Flag::INDEXING, $records);
}
private static function executeFlagQuery($connection, $sql, $flag, array $records)
private static function executeFlagQuery(Connection $connection, $sql, $flag, array $records)
{
return $connection->executeQuery($sql, array(
':flag' => $flag,

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/TermIndexer.php
View File

@@ -60,7 +60,7 @@ class TermIndexer
$params['type'] = self::TYPE_NAME;
$params['body'] = $term;
$bulk->index($params);
$bulk->index($params, null);
});
$document = Helper::thesaurusFromDatabox($databox);

19
lib/Alchemy/Phrasea/SearchEngine/Elastic/IndexerSubscriber.php
View File

@@ -19,6 +19,8 @@ use Alchemy\Phrasea\Core\Event\Record\RecordEvents;
use Alchemy\Phrasea\Core\Event\Record\RecordSubDefinitionCreatedEvent;
use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvent;
use Alchemy\Phrasea\Core\Event\Record\Structure\RecordStructureEvents;
use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvent;
use Alchemy\Phrasea\Core\Event\Thesaurus\ThesaurusEvents;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
@@ -84,6 +86,17 @@ class IndexerSubscriber implements EventSubscriberInterface
RecordEvents::STATUS_CHANGED => 'onRecordChange',
RecordEvents::SUB_DEFINITION_CREATED => 'onRecordChange',
RecordEvents::MEDIA_SUBSTITUTED => 'onRecordChange',
ThesaurusEvents::IMPORTED => 'onThesaurusChange',
ThesaurusEvents::FIELD_LINKED => 'onThesaurusChange',
ThesaurusEvents::CANDIDATE_ACCEPTED_AS_CONCEPT => 'onThesaurusChange',
ThesaurusEvents::CANDIDATE_ACCEPTED_AS_SYNONYM => 'onThesaurusChange',
ThesaurusEvents::SYNONYM_LNG_CHANGED => 'onThesaurusChange',
ThesaurusEvents::SYNONYM_POSITION_CHANGED => 'onThesaurusChange',
ThesaurusEvents::SYNONYM_TRASHED => 'onThesaurusChange',
ThesaurusEvents::CONCEPT_TRASHED => 'onThesaurusChange',
ThesaurusEvents::CONCEPT_DELETED => 'onThesaurusChange',
ThesaurusEvents::SYNONYM_ADDED => 'onThesaurusChange',
ThesaurusEvents::CONCEPT_ADDED => 'onThesaurusChange'
];
}
@@ -93,6 +106,12 @@ class IndexerSubscriber implements EventSubscriberInterface
$this->getIndexer()->migrateMappingForDatabox($databox);
}
public function onThesaurusChange(ThesaurusEvent $event)
{
$databox = $event->getDatabox();
$this->getIndexer()->scheduleRecordsFromDataboxForIndexing($databox);
}
public function onCollectionChange(CollectionEvent $event)
{
$collection = $event->getCollection();

3
lib/Alchemy/Phrasea/TaskManager/Job/SubdefsJob.php
View File

@@ -121,8 +121,7 @@ class SubdefsJob extends AbstractJob
// rewrite metadata
$sql = 'UPDATE record'
. ' SET status=(status & ~0x03),'
. ' jeton=(jeton | ' . PhraseaTokens::WRITE_META_SUBDEF . ')'
. ' SET jeton=(jeton | ' . PhraseaTokens::WRITE_META_SUBDEF | PhraseaTokens::TO_INDEX . ')'
. ' WHERE record_id=:record_id';
$stmt = $conn->prepare($sql);
$stmt->execute([':record_id' => $row['record_id']]);

18
lib/classes/API/OAuth2/Adapter.php
View File

@@ -16,9 +16,11 @@ use Alchemy\Phrasea\Authentication\Exception\RequireCaptchaException;
use Alchemy\Phrasea\Exception\RuntimeException;
use Alchemy\Phrasea\Model\Entities\ApiApplication;
use Alchemy\Phrasea\Model\Entities\User;
use Alchemy\Phrasea\Model\Repositories\ApiApplicationRepository;
use Symfony\Component\HttpFoundation\Session\Session;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
class API_OAuth2_Adapter extends OAuth2
{
@@ -601,9 +603,12 @@ class API_OAuth2_Adapter extends OAuth2
'state' => null,
];
$result = [];
if ($params['state'] !== null) {
$result["query"]["state"] = $params['state'];
}
if ($is_authorized === false) {
$result["query"]["error"] = OAUTH2_ERROR_USER_DENIED;
} else {
@@ -615,6 +620,7 @@ class API_OAuth2_Adapter extends OAuth2
$result["fragment"] = $this->createAccessToken($params['account_id'], $params['scope']);
}
}
$this->doRedirectUriCallback($params['redirect_uri'], $result);
}
@@ -684,9 +690,15 @@ class API_OAuth2_Adapter extends OAuth2
}
break;
case OAUTH2_GRANT_TYPE_USER_CREDENTIALS:
$application = ApiApplication::load_from_client_id($this->app, $client[0]);
/** @var ApiApplicationRepository $appRepository */
$appRepository = $this->app['repo.api-applications'];
$application = $appRepository->findByClientId($client[0]);
if ( ! $application->is_password_granted()) {
if (! $application) {
throw new NotFoundHttpException('Application not found');
}
if ( ! $application->isPasswordGranted()) {
$this->errorJsonResponse(OAUTH2_HTTP_BAD_REQUEST, OAUTH2_ERROR_UNSUPPORTED_GRANT_TYPE, 'Password grant type is not enable for your client');
}
@@ -812,7 +824,7 @@ class API_OAuth2_Adapter extends OAuth2
return [
'redirect_uri' => $this->client->getRedirectUri(),
'client_id' => $this->client->getClient(),
'client_id' => $this->client->getClientId(),
'account_id' => $account->getId(),
];
} catch (AccountLockedException $e) {

20
lib/classes/appbox.php
View File

@@ -268,6 +268,26 @@ class appbox extends base
return $databoxes[$sbas_id];
}
public function get_collection($base_id)
{
$sbas_id = phrasea::sbasFromBas($this->app, $base_id);
if ($sbas_id === false) {
throw new \RuntimeException('Collection not found.');
}
$collections = $this->get_databox($sbas_id)->get_collections();
foreach ($collections as $collection) {
if ($collection->get_base_id() == $base_id) {
return $collection;
}
}
// This should not happen, but I'd rather be safe than sorry.
throw new \RuntimeException('Collection not found.');
}
/**
* @param string $option
* @return string

92
lib/classes/databox.php
View File

@@ -294,42 +294,6 @@ class databox extends base implements ThumbnailedElement
return $this->id;
}
public function get_unique_keywords()
{
$sql = "SELECT COUNT(kword_id) AS n FROM kword";
$stmt = $this->get_connection()->prepare($sql);
$stmt->execute();
$rowbas = $stmt->fetch(PDO::FETCH_ASSOC);
$stmt->closeCursor();
return ($rowbas ? $rowbas['n'] : null);
}
public function get_index_amount()
{
$sql = "SELECT COUNT(idx_id) AS n FROM idx";
$stmt = $this->get_connection()->prepare($sql);
$stmt->execute();
$rowbas = $stmt->fetch(PDO::FETCH_ASSOC);
$stmt->closeCursor();
return ($rowbas ? $rowbas['n'] : null);
}
public function get_thesaurus_hits()
{
$sql = "SELECT COUNT(thit_id) AS n FROM thit";
$stmt = $this->get_connection()->prepare($sql);
$stmt->execute();
$rowbas = $stmt->fetch(PDO::FETCH_ASSOC);
$stmt->closeCursor();
return ($rowbas ? $rowbas['n'] : null);
}
public function get_record_details($sort)
{
$sql = "SELECT record.coll_id, ISNULL(coll.coll_id) AS lostcoll,
@@ -390,36 +354,46 @@ class databox extends base implements ThumbnailedElement
return $amount;
}
public function get_indexed_record_amount()
public function get_counts()
{
$sql = "SELECT status & 3 AS status, SUM(1) AS n FROM record GROUP BY(status & 3)";
$mask = PhraseaTokens::MAKE_SUBDEF | PhraseaTokens::TO_INDEX | PhraseaTokens::INDEXING; // we only care about those "jetons"
$sql = "SELECT type, jeton & (".$mask.") AS status, SUM(1) AS n FROM record GROUP BY type, (jeton & ".$mask.")";
$stmt = $this->get_connection()->prepare($sql);
$stmt->execute();
$rs = $stmt->fetchAll(PDO::FETCH_ASSOC);
$stmt->closeCursor();
$ret = array(
'xml_indexed' => 0,
'thesaurus_indexed' => 0,
'jeton_subdef' => array()
'records' => 0,
'records_indexed' => 0, // jetons = 0;0
'records_to_index' => 0, // jetons = 0;1
'records_not_indexed' => 0, // jetons = 1;0
'records_indexing' => 0, // jetons = 1;1
'subdefs_todo' => array() // by type "image", "video", ...
);
foreach ($rs as $row) {
$ret['records'] += ($n = (int)($row['n']));
$status = $row['status'];
if ($status & 1)
$ret['xml_indexed'] += $row['n'];
if ($status & 2)
$ret['thesaurus_indexed'] += $row['n'];
}
$sql = "SELECT type, COUNT(record_id) AS n FROM record WHERE jeton & ".PhraseaTokens::MAKE_SUBDEF." GROUP BY type";
$stmt = $this->get_connection()->prepare($sql);
$stmt->execute();
$rs = $stmt->fetchAll(PDO::FETCH_ASSOC);
$stmt->closeCursor();
foreach ($rs as $row) {
$ret['jeton_subdef'][$row['type']] = (int)$row['n'];
switch($status & (PhraseaTokens::TO_INDEX | PhraseaTokens::INDEXING)) {
case 0:
$ret['records_indexed'] += $n;
break;
case PhraseaTokens::TO_INDEX:
$ret['records_to_index'] += $n;
break;
case PhraseaTokens::INDEXING:
$ret['records_not_indexed'] += $n;
break;
case PhraseaTokens::INDEXING | PhraseaTokens::TO_INDEX:
$ret['records_indexing'] += $n;
break;
}
if($status & PhraseaTokens::MAKE_SUBDEF) {
if(!array_key_exists($row['type'], $ret['subdefs_todo'])) {
$ret['subdefs_todo'][$row['type']] = 0;
}
$ret['subdefs_todo'][$row['type']] += $n;
}
}
return $ret;
@@ -1057,6 +1031,12 @@ class databox extends base implements ThumbnailedElement
{
$this->get_connection()->update('pref', ['updated_on' => '0000-00-00 00:00:00'], ['prop' => 'indexes']);
// Set TO_INDEX flag on all records
$sql = "UPDATE record SET jeton = (jeton | :token)";
$stmt = $this->connection->prepare($sql);
$stmt->bindValue(':token', PhraseaTokens::TO_INDEX, PDO::PARAM_INT);
$stmt->execute();
return $this;
}

167
templates/web/admin/databox/databox.html.twig
View File

@@ -45,7 +45,7 @@
<li>
{{ 'admin::base: nombre d\'enregistrements sur la base :' | trans }}
<span id="nrecords">{{ databox.get_record_amount() }}</span>
<span id="records"></span>
(<a href="{{ path('admin_database_display_document_details', {'databox_id': databox.get_sbas_id()}) }}" class="ajax" target="rights">{{ 'phraseanet:: details' | trans }}</a>)
</li>
@@ -54,41 +54,13 @@
{{ 'admin::base: subdefs to be created :' | trans }}
<span id="subdefs_todo"></span>
</li>
{% if showDetail %}
<li>
{{ 'admin::base: nombre de mots uniques sur la base :' | trans }}
{{ databox.get_unique_keywords() }}
</li>
<li>
{{ 'admin::base: nombre de mots indexes sur la base' | trans }}
{{ databox.get_index_amount() }}
</li>
{% if app['conf'].get(['registry', 'modules', 'thesaurus']) %}
<li>
{{ 'admin::base: nombre de termes de Thesaurus indexes :' | trans }}
{{ databox.get_thesaurus_hits() }}
</li>
{% endif %}
{% endif %}
</ul>
<div id="INDEX_P_BAR" style="margin-bottom:20px;">
<div style="height: 35px;">
<p>
{{ "admin::base: document indexes en utilisant la fiche xml" | trans }} :
<span id="xml_indexed"></span>
</p>
<div id="xml_indexed_bar"></div>
<div id="xml_indexed_percent"></div>
</div>
<div style="height: 35px;">
<p>
{{ "admin::base: document indexes en utilisant le thesaurus" | trans }} :
<span id="thesaurus_indexed"></span>
</p>
<div id="thesaurus_indexed_bar"></div>
<div id="thesaurus_indexed_percent"></div>
<div id="INDEX_P_BAR" style="margin-bottom:20px; width:50%">
<div class="progress">
<div class="bar bar-success records_indexed" style="transition: none; width:0%;">...</div>
<div class="bar bar-warning records_indexing" style="transition:none; width:0%;"></div>
<div class="bar bar-danger records_not_indexed" style="transition:none; width:0%;"></div>
</div>
</div>
@@ -248,65 +220,91 @@
</div>
<script type="text/javascript">
function refreshDatabaseInformations()
{
// stop the refresh if the page changed
if($("#thesaurus_indexed_bar").length == 0) {
return;
}
function displayDatabaseInformations(delay)
{
try {
clearTimeout(document.refreshDatabaseInformations_timer);
}
catch(err) {
}
document.refreshDatabaseInformations_timer = setTimeout("_displayDatabaseInformations();", delay);
}
function _displayDatabaseInformations()
{
var container = $("#INDEX_P_BAR");
if(!container || container.length == 0) {
return; // wrong page ?
}
$.ajax({
type: "GET",
url: "/admin/databox/{{ databox.get_sbas_id() }}/informations/documents/",
dataType: 'json',
data: {},
success: function(data){
if(data.viewname === '') {
$("#viewname").html("{{ 'admin::base: aucun alias' | trans }}");
} else {
$("#viewname").html(data.viewname);
}
$("#nrecords").text(data.records);
$("#is_indexable").attr('checked', data.indexable);
$("#xml_indexed").text(data.xml_indexed);
$("#thesaurus_indexed").text(data.thesaurus_indexed);
if(data.records > 0)
{
var p;
p = 100*data.xml_indexed/data.records;
$("#xml_indexed_bar").width(Math.round(2*p)); // 0..200px
$("#xml_indexed_percent").text((Math.round(p*100)/100)+" %");
p = 100*data.thesaurus_indexed/data.records;
$("#thesaurus_indexed_bar").width(Math.round(2*p));
$("#thesaurus_indexed_percent").text((Math.round(p*100)/100)+" %");
var t = "";
for(var i in data.jeton_subdef)
{
t += (t==""?"":" ; ") + i + ": " + data.jeton_subdef[i];
success: function (data) {
try {
if (data.viewname === '') {
$("#viewname").html("{{ 'admin::base: aucun alias' | trans }}");
} else {
$("#viewname").html(data.viewname);
}
if(t == "") {
t = "0";
$("#is_indexable").attr('checked', data.indexable);
$("#records").text(data.counts.records);
if (data.counts.records > 0) {
var records_indexed = data.counts.records_indexed;
var records_not_indexed = data.counts.records_not_indexed; // flag indexing but NOT to_index ???
var records_indexing = data.counts.records_indexing;
var p;
p = 100 * records_indexed / data.counts.records;
$(".records_indexed", container).width(p + "%").text(records_indexed);
if (records_not_indexed > 0) {
p = 100 * records_not_indexed / data.counts.records;
$(".records_not_indexed", container).width(p + "%").text(records_not_indexed);
}
else {
$(".records_not_indexed", container).width(0).text("");
}
if (records_indexing > 0) {
p = 100 * records_indexing / data.counts.records;
$(".records_indexing", container).width(p + "%").text(records_indexing);
}
else {
$(".records_indexing", container).width(0).text("");
}
var t = "";
for (var i in data.counts.subdefs_todo) {
t += (t == "" ? "" : " ; ") + i + ": " + data.counts.subdefs_todo[i];
}
if (t == "") {
t = "0";
}
$("#subdefs_todo").text(t);
}
$("#subdefs_todo").text(t);
}
if(data.printLogoURL)
{
$("#printLogo").attr("src", data.printLogoURL);
$("#printLogoDIV_NONE").hide();
$("#printLogoDIV_OK").show();
}
else
{
$("#printLogoDIV_OK").hide();
$("#printLogoDIV_NONE").show();
}
if (data.printLogoURL) {
$("#printLogo").attr("src", data.printLogoURL);
$("#printLogoDIV_NONE").hide();
$("#printLogoDIV_OK").show();
}
else {
$("#printLogoDIV_OK").hide();
$("#printLogoDIV_NONE").show();
}
// refresh every 10 sec.
setTimeout("refreshDatabaseInformations();", 10000);
// refresh every 10 sec.
displayDatabaseInformations(10000);
}
catch(err) {
// wrong page ? don't refresh again
}
}
});
}
@@ -407,8 +405,7 @@
}
});
// start the refresh of the page content (progress bar etc...)
setTimeout("refreshDatabaseInformations();", 2000);
displayDatabaseInformations(200); // wait 200ms
});
</script>

4
tests/Alchemy/Tests/Phrasea/Controller/Admin/DataboxTest.php
View File

@@ -234,9 +234,7 @@ class DataboxTest extends \PhraseanetAuthenticatedWebTestCase
$this->assertTrue($json->success);
$this->assertObjectHasAttribute('sbas_id', $json);
$this->assertObjectHasAttribute('indexable', $json);
$this->assertObjectHasAttribute('records', $json);
$this->assertObjectHasAttribute('xml_indexed', $json);
$this->assertObjectHasAttribute('thesaurus_indexed', $json);
$this->assertObjectHasAttribute('counts', $json);
$this->assertObjectHasAttribute('viewname', $json);
$this->assertObjectHasAttribute('printLogoURL', $json);
}

2
tests/Alchemy/Tests/Phrasea/Controller/Api/ApiTestCase.php
View File

@@ -2288,7 +2288,7 @@ abstract class ApiTestCase extends \PhraseanetWebTestCase
$this->assertArrayHasKey('response', $content);
$this->assertTrue(is_array($content['meta']), 'Le bloc meta est un array');
$this->assertTrue(is_array($content['response']), 'Le bloc reponse est un array');
$this->assertEquals('1.4.1', $content['meta']['api_version']);
$this->assertEquals('2.0.0', $content['meta']['api_version']);
$this->assertNotNull($content['meta']['response_time']);
$this->assertEquals('UTF-8', $content['meta']['charset']);
}

6
tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php
View File

@@ -55,6 +55,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
"multi_match": {
"fields": ["foo.fr", "foo.en"],
"query": "bar",
"type": "cross_fields",
"operator": "and",
"lenient": true
}
@@ -94,6 +95,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
"multi_match": {
"fields": ["foo.fr", "foo.en"],
"query": "baz",
"type": "cross_fields",
"operator": "and",
"lenient": true
}
@@ -108,6 +110,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
"multi_match": {
"fields": ["private_caption.bar.fr", "private_caption.bar.en"],
"query": "baz",
"type": "cross_fields",
"operator": "and",
"lenient": true
}
@@ -140,6 +143,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
"multi_match": {
"fields": ["foo.fr", "foo.en"],
"query": "bar",
"type": "cross_fields",
"operator": "and",
"lenient": true
}
@@ -189,6 +193,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
"multi_match": {
"fields": ["foo.fr", "foo.en"],
"query": "baz",
"type": "cross_fields",
"operator": "and",
"lenient": true
}
@@ -212,6 +217,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
"multi_match": {
"fields": ["private_caption.bar.fr", "private_caption.bar.en"],
"query": "baz",
"type": "cross_fields",
"operator": "and",
"lenient": true
}