PHRAS-1895_PORT_PHRAS-1401_include_json_format_return_record (#2494)

* PHRAS-1895_PORT_PHRAS-1401 include json format return record

* PHRAS-1895_rollback deleted action ref PHRAS-1688
This commit is contained in:
KallooaSan
2018-02-28 20:31:45 +04:00
committed by jygaulier
parent 71f6e5d964
commit 00be9e51b0
11 changed files with 225 additions and 95 deletions

View File

@@ -59,14 +59,14 @@ class SearchController extends Controller
$result = $this->getSearchEngine()->query($query, $options);
$this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $result->getUserQuery());
$this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $result->getQueryText());
// log array of collectionIds (from $options) for each databox
$collectionsReferencesByDatabox = $options->getCollectionsReferencesByDatabox();
foreach ($collectionsReferencesByDatabox as $sbid => $references) {
$databox = $this->findDataboxById($sbid);
$collectionsIds = array_map(function(CollectionReference $ref){return $ref->getCollectionId();}, $references);
$this->getSearchEngineLogger()->log($databox, $result->getUserQuery(), $result->getTotal(), $collectionsIds);
$this->getSearchEngineLogger()->log($databox, $result->getQueryText(), $result->getTotal(), $collectionsIds);
}
$this->getSearchEngine()->clearCache();

View File

@@ -1509,14 +1509,14 @@ class V1Controller extends Controller
$search_result = $this->getSearchEngine()->query((string)$request->get('query'), $options);
$this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $search_result->getUserQuery());
$this->getUserManipulator()->logQuery($this->getAuthenticatedUser(), $search_result->getQueryText());
// log array of collectionIds (from $options) for each databox
$collectionsReferencesByDatabox = $options->getCollectionsReferencesByDatabox();
foreach ($collectionsReferencesByDatabox as $sbid => $references) {
$databox = $this->findDataboxById($sbid);
$collectionsIds = array_map(function(CollectionReference $ref){return $ref->getCollectionId();}, $references);
$this->getSearchEngineLogger()->log($databox, $search_result->getUserQuery(), $search_result->getTotal(), $collectionsIds);
$this->getSearchEngineLogger()->log($databox, $search_result->getQueryText(), $search_result->getTotal(), $collectionsIds);
}
$this->getSearchEngine()->clearCache();

View File

@@ -15,6 +15,7 @@ use Alchemy\Phrasea\Cache\Exception;
use Alchemy\Phrasea\Collection\Reference\CollectionReference;
use Alchemy\Phrasea\Controller\Controller;
use Alchemy\Phrasea\Core\Configuration\DisplaySettingService;
use Alchemy\Phrasea\Model\Entities\ElasticsearchRecord;
use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContextFactory;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
@@ -132,9 +133,9 @@ class QueryController extends Controller
// since the query comes from a submited form, normalize crlf,cr,lf ...
$query = StringHelper::crlfNormalize($query);
$json = array(
$json = [
'query' => $query
);
];
$options = SearchEngineOptions::fromRequest($this->app, $request);
@@ -168,7 +169,7 @@ class QueryController extends Controller
foreach ($collectionsReferencesByDatabox as $sbid => $references) {
$databox = $this->findDataboxById($sbid);
$collectionsIds = array_map(function(CollectionReference $ref){return $ref->getCollectionId();}, $references);
$this->getSearchEngineLogger()->log($databox, $result->getUserQuery(), $result->getTotal(), $collectionsIds);
$this->getSearchEngineLogger()->log($databox, $result->getQueryText(), $result->getTotal(), $collectionsIds);
}
$proposals = $firstPage ? $result->getProposals() : false;
@@ -274,29 +275,69 @@ class QueryController extends Controller
} else {
$template = 'prod/results/records.html.twig';
}
$json['results'] = $this->render($template, ['results'=> $result]);
/** Debug */
$json['parsed_query'] = $result->getEngineQuery();
/** End debug */
$fieldLabels = [];
// add technical fields
$fieldLabels = [];
foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) {
$fieldLabels[$k] = $this->app->trans($f['label']);
}
// add databox fields
// get infos about fields, fusionned and by databox
$fieldsInfos = []; // by databox
foreach ($this->app->getDataboxes() as $databox) {
$sbasId = $databox->get_sbas_id();
$fieldsInfos[$sbasId] = [];
foreach ($databox->get_meta_structure() as $field) {
if (!isset($fieldLabels[$field->get_name()])) {
$fieldLabels[$field->get_name()] = $field->get_label($this->app['locale']);
$name = $field->get_name();
$fieldsInfos[$sbasId][$name] = [
'label' => $field->get_label($this->app['locale']),
'type' => $field->get_type(),
'business' => $field->isBusiness(),
'multi' => $field->is_multi(),
];
if (!isset($fieldLabels[$name])) {
$fieldLabels[$name] = $field->get_label($this->app['locale']);
}
}
}
$facets = [];
// populates fileds infos
$json['fields'] = $fieldsInfos;
// populates rawresults
// need acl so the result will not include business fields where not allowed
$acl = $this->getAclForUser();
$json['rawResults'] = [];
/** @var ElasticsearchRecord $record */
foreach($result->getResults() as $record) {
$rawRecord = $record->asArray();
$sbasId = $record->getDataboxId();
$baseId = $record->getBaseId();
$caption = $rawRecord['caption'];
if($acl && $acl->has_right_on_base($baseId, \ACL::CANMODIFRECORD)) {
$caption = array_merge($caption, $rawRecord['privateCaption']);
}
// read the fields following the structure order
$rawCaption = [];
foreach($fieldsInfos[$sbasId] as $fieldName=>$fieldInfos) {
if(array_key_exists($fieldName, $caption)) {
$rawCaption[$fieldName] = $caption[$fieldName];
}
}
$rawRecord['caption'] = $rawCaption;
unset($rawRecord['privateCaption']);
$json['rawResults'][$record->getId()] = $rawRecord;
}
// populates facets (aggregates)
$facets = [];
foreach ($result->getFacets() as $facet) {
$facetName = $facet['name'];
@@ -311,6 +352,9 @@ class QueryController extends Controller
$json['next_page'] = ($page < $npages && $result->getAvailable() > 0) ? ($page + 1) : false;
$json['prev_page'] = ($page > 1 && $result->getAvailable() > 0) ? ($page - 1) : false;
$json['form'] = $options->serialize();
$json['queryCompiled'] = $result->getQueryCompiled();
$json['queryAST'] = $result->getQueryAST();
$json['queryESLib'] = $result->getQueryESLib();
}
catch(\Exception $e) {
// we'd like a message from the parser so get all the exceptions messages
@@ -319,9 +363,9 @@ class QueryController extends Controller
$msg .= ($msg ? "\n":"") . $e->getMessage();
}
$template = 'prod/results/help.html.twig';
$result = array(
$result = [
'error' => $msg
);
];
$json['results'] = $this->render($template, ['results'=> $result]);
}

View File

@@ -53,6 +53,42 @@ class ElasticsearchRecord implements RecordInterface, MutableRecordInterface
private $flags = [];
private $highlight = [];
public function asArray()
{
return [
'_index' => $this->_index,
'_type' => $this->_type,
'_id' => $this->_id,
'_version' => $this->_version,
'_score' => $this->_score,
'databoxId' => $this->databoxId,
'recordId' => $this->recordId,
'collectionId' => $this->collectionId,
'baseId' => $this->baseId,
'collectionName' => $this->collectionName,
'mimeType' => $this->mimeType,
'title' => $this->title,
'originalName' => $this->originalName,
'updated' => $this->updated,
'created' => $this->created,
'sha256' => $this->sha256,
'width' => $this->width,
'height' => $this->height,
'size' => $this->size,
'uuid' => $this->uuid,
'position' => $this->position,
'type' => $this->type,
'status' => $this->status,
'isStory' => $this->isStory,
'caption' => $this->caption,
'privateCaption' => $this->privateCaption,
'exif' => $this->exif,
'subdefs' => $this->subdefs,
'flags' => $this->flags,
'highlight' => $this->highlight,
];
}
/**
* @param string $index
* @param string $type

View File

@@ -36,7 +36,7 @@ abstract class V1SearchTransformer extends TransformerAbstract
return $suggestion->toArray();
}, $result->getSuggestions()->toArray()),
'facets' => $result->getFacets(),
'query' => $result->getEngineQuery(),
'query' => $result->getQueryText(),
];
}

View File

@@ -273,35 +273,35 @@ class ElasticSearchEngine implements SearchEngineInterface
/**
* {@inheritdoc}
*/
public function query($string, SearchEngineOptions $options = null)
public function query($queryText, SearchEngineOptions $options = null)
{
$options = $options ?: new SearchEngineOptions();
$context = $this->context_factory->createContext($options);
/** @var QueryCompiler $query_compiler */
$query_compiler = $this->app['query_compiler'];
$recordQuery = $query_compiler->compile($string, $context);
$queryAST = $query_compiler->parse($queryText)->dump();
$queryCompiled = $query_compiler->compile($queryText, $context);
$params = $this->createRecordQueryParams($recordQuery, $options, null);
$queryESLib = $this->createRecordQueryParams($queryCompiled, $options, null);
// ask ES to return field _version (incremental version number of document)
$params['body']['version'] = true;
$queryESLib['body']['version'] = true;
$params['body']['from'] = $options->getFirstResult();
$params['body']['size'] = $options->getMaxResults();
$queryESLib['body']['from'] = $options->getFirstResult();
$queryESLib['body']['size'] = $options->getMaxResults();
if($this->options->getHighlight()) {
$params['body']['highlight'] = $this->buildHighlightRules($context);
$queryESLib['body']['highlight'] = $this->buildHighlightRules($context);
}
$aggs = $this->getAggregationQueryParams($options);
if ($aggs) {
$params['body']['aggs'] = $aggs;
$queryESLib['body']['aggs'] = $aggs;
}
$res = $this->client->search($params);
$res = $this->client->search($queryESLib);
$results = new ArrayCollection();
$n = 0;
foreach ($res['hits']['hits'] as $hit) {
$results[] = ElasticsearchRecordHydrator::hydrate($hit, $n++);
@@ -310,16 +310,13 @@ class ElasticSearchEngine implements SearchEngineInterface
/** @var FacetsResponse $facets */
$facets = $this->facetsResponseFactory->__invoke($res);
$query['ast'] = $query_compiler->parse($string)->dump();
$query['query_main'] = $recordQuery;
$query['query'] = $params['body'];
$query['query_string'] = json_encode($params['body'], JSON_PRETTY_PRINT);
return new SearchEngineResult(
$options,
$results, // ArrayCollection of results
$string, // the query as typed by the user
json_encode($query),
$results, // ArrayCollection of results
$queryText, // the query as typed by the user
$queryAST,
$queryCompiled,
$queryESLib,
$res['took'], // duration
$options->getFirstResult(),
$res['hits']['total'], // available

View File

@@ -17,8 +17,10 @@ use Doctrine\Common\Collections\ArrayCollection;
class SearchEngineResult
{
protected $results;
protected $user_query;
protected $engine_query;
protected $queryText;
protected $queryAST;
protected $queryCompiled;
protected $queryESLib;
protected $duration;
protected $offsetStart;
protected $available;
@@ -37,11 +39,12 @@ class SearchEngineResult
private $options;
/**
* SearchEngineResult constructor.
* @param SearchEngineOptions $options
* @param ArrayCollection $results
* @param string $user_query query as user typed, "dog"
* @param string $engine_query query parsed for engine, "{"ast":"<text:\"dog\">","query_main" ....
* @param string $queryText
* @param string $queryAST
* @param string $queryCompiled
* @param string $queryESLib
* @param float $duration
* @param int $offsetStart
* @param int $available
@@ -49,15 +52,17 @@ class SearchEngineResult
* @param mixed $error
* @param mixed $warning
* @param ArrayCollection $suggestions
* @param array $propositions
* @param array $indexes
* @param FacetsResponse|null $facets
* @param Array $propositions
* @param Array $indexes
* @param FacetsResponse $facets
*/
public function __construct(
SearchEngineOptions $options,
ArrayCollection $results,
$user_query,
$engine_query,
$queryText,
$queryAST,
$queryCompiled,
$queryESLib,
$duration,
$offsetStart,
$available,
@@ -70,10 +75,11 @@ class SearchEngineResult
FacetsResponse $facets = null
) {
$this->options = $options;
$this->results = $results;
$this->user_query = $user_query;
$this->engine_query = $engine_query;
$this->queryText = $queryText;
$this->queryAST = $queryAST;
$this->queryCompiled = $queryCompiled;
$this->queryESLib = $queryESLib;
$this->duration = (float) $duration;
$this->offsetStart = (int) $offsetStart;
$this->available = (int) $available;
@@ -104,35 +110,30 @@ class SearchEngineResult
return $this->results;
}
/**
* The query related to these results
* @obsolete use getUserQuery (unparsed query) or getEngineQuery (parsed)
*
* @return string
*/
public function getQuery()
{
return $this->getEngineQuery();
}
/**
* The unparsed query related to these results
*
* @return string
*/
public function getUserQuery()
public function getQueryText()
{
return $this->user_query;
return $this->queryText;
}
/**
* The parsed query related to these results
*
* @return string
*/
public function getEngineQuery()
public function getQueryAST()
{
return $this->engine_query;
return $this->queryAST;
}
public function getQueryCompiled()
{
return $this->queryCompiled;
}
public function getQueryESLib()
{
return $this->queryESLib;
}
/**

View File

@@ -59,10 +59,10 @@ class ApiJsonTest extends ApiTestCase
);
$record = \record_adapter::createFromFile($file, $app);
$story['story_records'] = array(array(
$story['story_records'] = [[
'databox_id' => $record->getDataboxId(),
'record_id' => $record->getRecordId()
));
]];
$client = $this->getClient();
$client->request(
@@ -74,7 +74,7 @@ class ApiJsonTest extends ApiTestCase
'HTTP_ACCEPT' => $this->getAcceptMimeType(),
'CONTENT_TYPE' => 'application/json',
],
json_encode(array('stories' => array($story)))
json_encode(['stories' => [$story]])
);
$content = $this->unserialize($client->getResponse()->getContent());
@@ -105,10 +105,10 @@ class ApiJsonTest extends ApiTestCase
);
$record = \record_adapter::createFromFile($file, self::$DI['app']);
$records = array(
$records = [
'databox_id' => $record->getDataboxId(),
'record_id' => $record->getRecordId()
);
];
self::$DI['client']->request(
'POST',
@@ -119,7 +119,7 @@ class ApiJsonTest extends ApiTestCase
'HTTP_ACCEPT' => $this->getAcceptMimeType(),
'CONTENT_TYPE' => 'application/json',
],
json_encode(array('story_records' => array($records)))
json_encode(['story_records' => [$records]])
);
$content = $this->unserialize(self::$DI['client']->getResponse()->getContent());
@@ -147,10 +147,10 @@ class ApiJsonTest extends ApiTestCase
$story->appendChild($record);
$route = sprintf('/api/v1/stories/%s/%s/delrecords', $story->getDataboxId(), $story->getRecordId());
$records = array(
$records = [
'databox_id' => $record->getDataboxId(),
'record_id' => $record->getRecordId()
);
];
self::$DI['client']->request(
'DELETE',
@@ -161,7 +161,7 @@ class ApiJsonTest extends ApiTestCase
'HTTP_ACCEPT' => $this->getAcceptMimeType(),
'CONTENT_TYPE' => 'application/json',
],
json_encode(array('story_records' => array($records)))
json_encode(['story_records' => [$records]])
);
$content = $this->unserialize(self::$DI['client']->getResponse()->getContent());
@@ -877,7 +877,6 @@ class ApiJsonTest extends ApiTestCase
$this->setToken($this->userAccessToken);
$response = $this->request('POST', '/api/v1/records/search/', $this->getParameters(), ['HTTP_Accept' => $this->getAcceptMimeType()]);
$content = $this->unserialize($response->getContent());
$this->evaluateResponse200($response);
$this->evaluateMeta200($content);
@@ -909,7 +908,7 @@ class ApiJsonTest extends ApiTestCase
$mock = $this->getMock('Alchemy\Phrasea\SearchEngine\SearchEngineInterface');
$app['phraseanet.SE'] = $mock;
$mock
$mock
->expects($this->once())
->method('query')
->withAnyParameters()
@@ -1065,7 +1064,7 @@ class ApiJsonTest extends ApiTestCase
$route = '/api/v1/records/' . self::$DI['record_1']->get_sbas_id() . '/' . self::$DI['record_1']->get_record_id() . '/embed/';
self::$DI['client']->request('GET', $route, $this->getParameters(), array(), array('HTTP_Accept' => $this->getAcceptMimeType()));
self::$DI['client']->request('GET', $route, $this->getParameters(), [], array('HTTP_Accept' => $this->getAcceptMimeType()));
$content = $this->unserialize(self::$DI['client']->getResponse()->getContent());
$this->evaluateResponse200(self::$DI['client']->getResponse());
@@ -1093,7 +1092,7 @@ class ApiJsonTest extends ApiTestCase
$route = '/api/v1/records/' . self::$DI['record_1']->get_sbas_id() . '/' . self::$DI['record_1']->get_record_id() . '/embed/';
self::$DI['client']->request('GET', $route, $this->getParameters(), array(), array('HTTP_Accept' => $this->getAcceptMimeType()));
self::$DI['client']->request('GET', $route, $this->getParameters(), [], array('HTTP_Accept' => $this->getAcceptMimeType()));
$content = $this->unserialize(self::$DI['client']->getResponse()->getContent());
$this->evaluateResponse200(self::$DI['client']->getResponse());

View File

@@ -23,8 +23,11 @@ class SearchEngineResultTest extends \PhraseanetTestCase
self::$DI['record_2']
]);
$user_query = 'Gotainer';
$engine_query = '{text:"Gotainer"}'; // fake, real is really more complex
$queryText = 'azerty';
$queryAST = '<text:"azerty">'; // fake, real is really more complex
$queryCompiled = '{match:"azerty"}'; // fake, real is really more complex
$queryESLib = '{index:"test", match:{"azerty"}}'; // fake, real is really more complex
$duration = 1 / 3;
$offsetStart = 23;
$available = 25;
@@ -32,12 +35,30 @@ class SearchEngineResultTest extends \PhraseanetTestCase
$error = 'this is an error message';
$warning = 'this is a warning message';
$suggestions = new ArrayCollection([
new SearchEngineSuggestion($user_query, 'Richard', 22)
new SearchEngineSuggestion($queryText, 'Richard', 22)
]);
$propositions = new ArrayCollection();
$indexes = 'new-index';
$result = new SearchEngineResult($options, $results, $user_query, $engine_query, $duration, $offsetStart, $available, $total, $error, $warning, $suggestions, $propositions, $indexes);
$result = new SearchEngineResult(
$options,
$results,
$queryText, // the query as typed by the user
$queryAST,
$queryCompiled,
$queryESLib,
$duration,
$offsetStart,
$available,
$total,
$error,
$warning,
$suggestions,
$propositions,
$indexes
);
$this->assertEquals($warning, $result->getWarning());
$this->assertEquals(2, $result->getTotalPages(23));
@@ -45,8 +66,12 @@ class SearchEngineResultTest extends \PhraseanetTestCase
$this->assertEquals($total, $result->getTotal());
$this->assertEquals($suggestions, $result->getSuggestions());
$this->assertEquals($results, $result->getResults());
$this->assertEquals($user_query, $result->getUserQuery());
$this->assertEquals($engine_query, $result->getEngineQuery());
$this->assertEquals($queryText, $result->getQueryText());
$this->assertEquals($queryAST, $result->getQueryAST());
$this->assertEquals($queryCompiled, $result->getQueryCompiled());
$this->assertEquals($queryESLib, $result->getQueryESLib());
$this->assertEquals($propositions, $result->getProposals());
$this->assertEquals($indexes, $result->getIndexes());
$this->assertEquals($error, $result->getError());
@@ -62,8 +87,11 @@ class SearchEngineResultTest extends \PhraseanetTestCase
self::$DI['record_2']
]);
$user_query = 'Gotainer';
$engine_query = '{text:"Gotainer"}'; // fake, real is really more complex
$queryText = 'azerty';
$queryAST = '<text:"azerty">'; // fake, real is really more complex
$queryCompiled = '{match:"azerty"}'; // fake, real is really more complex
$queryESLib = '{index:"test", match:{"azerty"}}'; // fake, real is really more complex
$duration = 1 / 3;
$offsetStart = 0;
$available = 25;
@@ -71,12 +99,30 @@ class SearchEngineResultTest extends \PhraseanetTestCase
$error = 'this is an error message';
$warning = 'this is a warning message';
$suggestions = new ArrayCollection([
new SearchEngineSuggestion($user_query, 'Richard', 22)
new SearchEngineSuggestion($queryText, 'Richard', 22)
]);
$propositions = new ArrayCollection();
$indexes = 'new-index';
$result = new SearchEngineResult($options, $results, $user_query, $engine_query, $duration, $offsetStart, $available, $total, $error, $warning, $suggestions, $propositions, $indexes);
$result = new SearchEngineResult(
$options,
$results,
$queryText, // the query as typed by the user
$queryAST,
$queryCompiled,
$queryESLib,
$duration,
$offsetStart,
$available,
$total,
$error,
$warning,
$suggestions,
$propositions,
$indexes
);
$this->assertEquals(1, $result->getCurrentPage(10));
$this->assertEquals(1, $result->getCurrentPage(25));

View File

@@ -206,11 +206,18 @@ abstract class PhraseanetAuthenticatedWebTestCase extends \PhraseanetAuthenticat
$elasticsearchRecord->setDataboxId($record->getDataboxId());
$elasticsearchRecord->setRecordId($record->getRecordId());
$queryText = '';
$queryAST = '<NULL>'; // fake, real is really more complex
$queryCompiled = '{match_all:[]}'; // fake, real is really more complex
$queryESLib = '{index:"test", match:{""}}'; // fake, real is really more complex
$result = new SearchEngineResult(
new SearchEngineOptions(),
new ArrayCollection([$elasticsearchRecord]), // Records
'', // Query as user typed
'{}', // Query as engine parsed
$queryText, // the query as typed by the user
$queryAST,
$queryCompiled,
$queryESLib,
0, // Duration
0, // offsetStart
1, // available