hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-15 14:03:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1564 from mdarse/query-on-exif

Browse Source 
Query support for metadata tags
This commit is contained in:
Benoît Burnichon
2015-11-12 22:47:24 +01:00
parent dd683187da b8c3ea91c8
commit 5dc1fbe2cb
39 changed files with 927 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/AndOperator.php → lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/AndExpression.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\Boolean;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
class AndOperator extends BinaryOperator
class AndExpression extends BinaryExpression
{
protected $operator = 'AND';

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/BinaryOperator.php → lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/BinaryExpression.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\Boolean;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node;
abstract class BinaryOperator extends Node
abstract class BinaryExpression extends Node
{
protected $left;
protected $right;

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/ExceptOperator.php → lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/ExceptExpression.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\Boolean;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
class ExceptOperator extends BinaryOperator
class ExceptExpression extends BinaryExpression
{
protected $operator = 'EXCEPT';

2
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/OrOperator.php → lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/Boolean/OrExpression.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\Boolean;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
class OrOperator extends BinaryOperator
class OrExpression extends BinaryExpression
{
protected $operator = 'OR';

48
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/FieldEqualsExpression.php
View File

@@ -1,48 +0,0 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
class FieldEqualsExpression extends Node
{
private $field;
private $value;
public function __construct(Field $field, $value)
{
$this->field = $field;
$this->value = $value;
}
public function buildQuery(QueryContext $context)
{
$structure_field = $context->get($this->field);
if (!$structure_field) {
throw new QueryException(sprintf('Field "%s" does not exist', $this->field->getValue()));
}
if (!$structure_field->isValueCompatible($this->value)) {
return null;
}
$query = [
'term' => [
$structure_field->getIndexField(true) => $this->value
]
];
return QueryHelper::wrapPrivateFieldQuery($structure_field, $query);
}
public function getTermNodes()
{
return array();
}
public function __toString()
{
return sprintf('(%s == <value:"%s">)', $this->field, $this->value);
}
}

52
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/EqualExpression.php Normal file
View File

@@ -0,0 +1,52 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\FieldKey;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\Key;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryPostProcessor;
class EqualExpression extends Node
{
private $key;
private $value;
public function __construct(Key $key, $value)
{
$this->key = $key;
$this->value = $value;
}
public function buildQuery(QueryContext $context)
{
if (!$this->key->isValueCompatible($this->value, $context)) {
throw new QueryException(sprintf('Value "%s" for metadata tag "%s" is not valid.', $this->value, $this->key));
}
$query = [
'term' => [
$this->key->getIndexField($context, true) => $this->value
]
];
if ($this->key instanceof QueryPostProcessor) {
return $this->key->postProcessQuery($query, $context);
}
return $query;
}
public function getTermNodes()
{
return array();
}
public function __toString()
{
return sprintf('(<%s> == <value:"%s">)', $this->key, $this->value);
}
}

61
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/FieldKey.php Normal file
View File

@@ -0,0 +1,61 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryPostProcessor;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\ValueChecker;
use Assert\Assertion;
class FieldKey implements Key, QueryPostProcessor
{
private $name;
private $field_cache = [];
public function __construct($name)
{
Assertion::string($name);
$this->name = $name;
}
public function getIndexField(QueryContext $context, $raw = false)
{
return $this->getField($context)->getIndexField($raw);
}
public function isValueCompatible($value, QueryContext $context)
{
return ValueChecker::isValueCompatible($this->getField($context), $value);
}
public function postProcessQuery($query, QueryContext $context)
{
$field = $this->getField($context);
return QueryHelper::wrapPrivateFieldQuery($field, $query);
}
private function getField(QueryContext $context)
{
$hash = spl_object_hash($context);
if (!isset($this->field_cache[$hash])) {
$this->field_cache[$hash] = $context->get($this->name);
}
$field = $this->field_cache[$hash];
if ($field === null) {
throw new QueryException(sprintf('Field "%s" does not exist', $this->name));
}
return $field;
}
public function clearCache()
{
$this->field_cache = [];
}
public function __toString()
{
return sprintf('field.%s', $this->name);
}
}

3
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Key.php
View File

@@ -6,6 +6,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
interface Key
{
public function buildQueryForValue($value, QueryContext $context);
public function getIndexField(QueryContext $context, $raw = false);
public function isValueCompatible($value, QueryContext $context);
public function __toString();
}

19
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/Expression.php → lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MatchExpression.php
View File

@@ -3,13 +3,14 @@
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Assert\Assertion;
class Expression extends Node
class MatchExpression extends Node
{
protected $key;
protected $value;
private $key;
private $value;
public function __construct(Key $key, $value)
{
@@ -20,7 +21,15 @@ class Expression extends Node
public function buildQuery(QueryContext $context)
{
return $this->key->buildQueryForValue($this->value, $context);
if (!$this->key->isValueCompatible($this->value, $context)) {
throw new QueryException(sprintf('Value "%s" for metadata tag "%s" is not valid.', $this->value, $this->key));
}
return [
'match' => [
$this->key->getIndexField($context) => $this->value
]
];
}
public function getTermNodes()
@@ -30,6 +39,6 @@ class Expression extends Node
public function __toString()
{
return sprintf('<%s:%s>', $this->key, $this->value);
return sprintf('<%s:"%s">', $this->key, $this->value);
}
}

54
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/MetadataKey.php Normal file
View File

@@ -0,0 +1,54 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\ValueChecker;
use Assert\Assertion;
class MetadataKey implements Key
{
private $name;
private $tag_cache = [];
public function __construct($name)
{
Assertion::string($name);
$this->name = $name;
}
public function getIndexField(QueryContext $context, $raw = false)
{
return $this->getTag($context)->getIndexField($raw);
}
public function isValueCompatible($value, QueryContext $context)
{
return ValueChecker::isValueCompatible($this->getTag($context), $value);
}
private function getTag(QueryContext $context)
{
$hash = spl_object_hash($context);
if (!isset($this->tag_cache[$hash])) {
$this->tag_cache[$hash] = $context->getMetadataTag($this->name);
}
$tag = $this->tag_cache[$hash];
if ($tag === null) {
throw new QueryException(sprintf('Metadata tag "%s" does not exist', $this->name));
}
return $tag;
}
public function clearCache()
{
$this->tag_cache = [];
}
public function __toString()
{
return sprintf('metadata.%s', $this->name);
}
}

15
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/NativeKey.php
View File

@@ -3,7 +3,6 @@
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Assert\Assertion;
class NativeKey implements Key
{
@@ -41,14 +40,14 @@ class NativeKey implements Key
$this->key = $key;
}
public function buildQueryForValue($value, QueryContext $context)
public function getIndexField(QueryContext $context, $raw = false)
{
Assertion::string($value);
return [
'term' => [
$this->key => $value
]
];
return $this->key;
}
public function isValueCompatible($value, QueryContext $context)
{
return true;
}
public function __toString()

62
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RangeExpression.php → lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeyValue/RangeExpression.php
View File

@@ -1,42 +1,50 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
namespace Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue;
use Assert\Assertion;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\FieldKey;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\KeyValue\Key;
use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\QueryException;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
class RangeExpression extends Node
{
private $field;
private $key;
private $lower_bound;
private $lower_inclusive;
private $higher_bound;
private $higher_inclusive;
public static function lessThan(Field $field, $bound)
public static function lessThan(Key $key, $bound)
{
return new self($field, $bound, false);
return new self($key, $bound, false);
}
public static function lessThanOrEqual(Field $field, $bound)
public static function lessThanOrEqual(Key $key, $bound)
{
return new self($field, $bound, true);
return new self($key, $bound, true);
}
public static function greaterThan(Field $field, $bound)
public static function greaterThan(Key $key, $bound)
{
return new self($field, null, null, $bound, false);
return new self($key, null, false, $bound, false);
}
public static function greaterThanOrEqual(Field $field, $bound)
public static function greaterThanOrEqual(Key $key, $bound)
{
return new self($field, null, null, $bound, true);
return new self($key, null, false, $bound, true);
}
public function __construct(Field $field, $lb, $li = false, $hb = null, $hi = false)
public function __construct(Key $key, $lb, $li = false, $hb = null, $hi = false)
{
$this->field = $field;
Assertion::nullOrScalar($lb);
Assertion::boolean($li);
Assertion::nullOrScalar($hb);
Assertion::boolean($hi);
$this->key = $key;
$this->lower_bound = $lb;
$this->lower_inclusive = $li;
$this->higher_bound = $hb;
@@ -45,16 +53,9 @@ class RangeExpression extends Node
public function buildQuery(QueryContext $context)
{
$structure_field = $context->get($this->field);
if (!$structure_field) {
throw new QueryException(sprintf('Field "%s" does not exist', $this->field->getValue()));
}
$params = array();
if ($this->lower_bound !== null) {
if (!$structure_field->isValueCompatible($this->lower_bound)) {
return;
}
$this->assertValueCompatible($this->lower_bound, $context);
if ($this->lower_inclusive) {
$params['lte'] = $this->lower_bound;
} else {
@@ -62,9 +63,7 @@ class RangeExpression extends Node
}
}
if ($this->higher_bound !== null) {
if (!$structure_field->isValueCompatible($this->higher_bound)) {
return;
}
$this->assertValueCompatible($this->higher_bound, $context);
if ($this->higher_inclusive) {
$params['gte'] = $this->higher_bound;
} else {
@@ -73,9 +72,20 @@ class RangeExpression extends Node
}
$query = [];
$query['range'][$structure_field->getIndexField()] = $params;
$query['range'][$this->key->getIndexField($context)] = $params;
return QueryHelper::wrapPrivateFieldQuery($structure_field, $query);
if ($this->key instanceof QueryPostProcessor) {
return $this->key->postProcessQuery($query, $context);
}
return $query;
}
private function assertValueCompatible($value, QueryContext $context)
{
if (!$this->key->isValueCompatible($value, $context)) {
throw new QueryException(sprintf('Value "%s" for metadata tag "%s" is not valid.', $value, $this->key));
}
}
public function getTermNodes()
@@ -101,6 +111,6 @@ class RangeExpression extends Node
}
}
return sprintf('<range:%s%s>', $this->field->getValue(), $string);
return sprintf('<range:%s%s>', $this->key, $string);
}
}

6
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\ValueChecker;
class QuotedTextNode extends Node
{
@@ -38,11 +38,11 @@ class QuotedTextNode extends Node
};
$unrestricted_fields = $context->getUnrestrictedFields();
$unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text);
$unrestricted_fields = ValueChecker::filterByValueCompatibility($unrestricted_fields, $this->text);
$query = $query_builder($unrestricted_fields);
$private_fields = $context->getPrivateFields();
$private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text);
$private_fields = ValueChecker::filterByValueCompatibility($private_fields, $this->text);
foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
$query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
}

6
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/RawNode.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\ValueChecker;
class RawNode extends Node
{
@@ -56,11 +56,11 @@ class RawNode extends Node
};
$unrestricted_fields = $context->getUnrestrictedFields();
$unrestricted_fields = StructureField::filterByValueCompatibility($unrestricted_fields, $this->text);
$unrestricted_fields = ValueChecker::filterByValueCompatibility($unrestricted_fields, $this->text);
$query = $query_builder($unrestricted_fields);
$private_fields = $context->getPrivateFields();
$private_fields = StructureField::filterByValueCompatibility($private_fields, $this->text);
$private_fields = ValueChecker::filterByValueCompatibility($private_fields, $this->text);
foreach (QueryHelper::wrapPrivateFieldQueries($private_fields, $query_builder) as $private_field_query) {
$query = QueryHelper::applyBooleanClause($query, 'should', $private_field_query);
}

4
lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php
View File

@@ -4,7 +4,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field as StructureField;
use Alchemy\Phrasea\SearchEngine\Elastic\Structure\ValueChecker;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Term;
class TextNode extends AbstractTermNode implements ContextAbleInterface
@@ -41,7 +41,7 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
$query_builder = function (array $fields) use ($context) {
// Full text
$index_fields = [];
foreach (StructureField::filterByValueCompatibility($fields, $this->text) as $field) {
foreach (ValueChecker::filterByValueCompatibility($fields, $this->text) as $field) {
foreach ($context->localizeField($field) as $f) {
$index_fields[] = $f;
}

48
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php
View File

@@ -61,7 +61,7 @@ SQL;
$value = $metadata['value'];
// Do not keep empty values
if (empty($key) || empty($value)) {
if ($key === '' || $value === '') {
continue;
}
@@ -76,23 +76,7 @@ SQL;
case 'caption':
// Sanitize fields
$value = StringHelper::crlfNormalize($value);
switch ($this->structure->typeOf($key)) {
case Mapping::TYPE_DATE:
$value = $this->helper->sanitizeDate($value);
break;
case Mapping::TYPE_FLOAT:
case Mapping::TYPE_DOUBLE:
$value = (float) $value;
break;
case Mapping::TYPE_INTEGER:
case Mapping::TYPE_LONG:
case Mapping::TYPE_SHORT:
case Mapping::TYPE_BYTE:
$value = (int) $value;
break;
}
$value = $this->sanitizeValue($value, $this->structure->typeOf($key));
// Private caption fields are kept apart
$type = $metadata['private'] ? 'private_caption' : 'caption';
// Caption are multi-valued
@@ -110,6 +94,10 @@ SQL;
case 'exif':
// EXIF data is single-valued
$tag = $this->structure->getMetadataTagByName($key);
if ($tag) {
$value = $this->sanitizeValue($value, $tag->getType());
}
$record['exif'][$key] = $value;
break;
@@ -119,4 +107,28 @@ SQL;
}
}
}
private function sanitizeValue($value, $type)
{
switch ($type) {
case Mapping::TYPE_DATE:
return $this->helper->sanitizeDate($value);
case Mapping::TYPE_FLOAT:
case Mapping::TYPE_DOUBLE:
return (float) $value;
case Mapping::TYPE_INTEGER:
case Mapping::TYPE_LONG:
case Mapping::TYPE_SHORT:
case Mapping::TYPE_BYTE:
return (int) $value;
case Mapping::TYPE_BOOLEAN:
return (bool) $value;
default:
return $value;
}
}
}

43
lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/RecordIndexer.php
View File

@@ -36,7 +36,6 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\CandidateTerms;
use databox;
use Iterator;
use media_subdef;
use Psr\Log\LoggerInterface;
class RecordIndexer
@@ -308,7 +307,7 @@ class RecordIndexer
// Thesaurus
->add('concept_path', $this->getThesaurusPathMapping())
// EXIF
->add('exif', $this->getExifMapping())
->add('exif', $this->getMetadataTagMapping())
// Status
->add('flags', $this->getFlagsMapping())
->add('flags_bitfield', 'integer')->notIndexed()
@@ -375,36 +374,20 @@ class RecordIndexer
return $mapping;
}
// @todo Add call to addAnalyzedVersion ?
private function getExifMapping()
private function getMetadataTagMapping()
{
$mapping = new Mapping();
$mapping
->add(media_subdef::TC_DATA_WIDTH, 'integer')
->add(media_subdef::TC_DATA_HEIGHT, 'integer')
->add(media_subdef::TC_DATA_COLORSPACE, 'string')->notAnalyzed()
->add(media_subdef::TC_DATA_CHANNELS, 'integer')
->add(media_subdef::TC_DATA_ORIENTATION, 'integer')
->add(media_subdef::TC_DATA_COLORDEPTH, 'integer')
->add(media_subdef::TC_DATA_DURATION, 'float')
->add(media_subdef::TC_DATA_AUDIOCODEC, 'string')->notAnalyzed()
->add(media_subdef::TC_DATA_AUDIOSAMPLERATE, 'float')
->add(media_subdef::TC_DATA_VIDEOCODEC, 'string')->notAnalyzed()
->add(media_subdef::TC_DATA_FRAMERATE, 'float')
->add(media_subdef::TC_DATA_MIMETYPE, 'string')->notAnalyzed()
->add(media_subdef::TC_DATA_FILESIZE, 'long')
// TODO use geo point type for lat/long
->add(media_subdef::TC_DATA_LONGITUDE, 'float')
->add(media_subdef::TC_DATA_LATITUDE, 'float')
->add(media_subdef::TC_DATA_FOCALLENGTH, 'float')
->add(media_subdef::TC_DATA_CAMERAMODEL, 'string')
->add(media_subdef::TC_DATA_FLASHFIRED, 'boolean')
->add(media_subdef::TC_DATA_APERTURE, 'float')
->add(media_subdef::TC_DATA_SHUTTERSPEED, 'float')
->add(media_subdef::TC_DATA_HYPERFOCALDISTANCE, 'float')
->add(media_subdef::TC_DATA_ISO, 'integer')
->add(media_subdef::TC_DATA_LIGHTVALUE, 'float')
;
foreach ($this->structure->getMetadataTags() as $tag) {
$type = $tag->getType();
$mapping->add($tag->getName(), $type);
if ($type === Mapping::TYPE_STRING) {
if ($tag->isAnalyzable()) {
$mapping->addRawVersion();
} else {
$mapping->notAnalyzed();
}
}
}
return $mapping;
}

4
lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/NodeTypes.php
View File

@@ -15,15 +15,17 @@ class NodeTypes
const LTE_EXPR = '#less_than_or_equal_to';
const GTE_EXPR = '#greater_than_or_equal_to';
const EQUAL_EXPR = '#equal_to';
const MATCH_EXPR = '#match_expression';
const FIELD_STATEMENT = '#field_statement';
const FIELD = '#field';
const FIELD_KEY = '#field_key';
const VALUE = '#value';
const TERM = '#thesaurus_term';
const TEXT = '#text';
const CONTEXT = '#context';
const METADATA_KEY = '#meta_key';
const FLAG_STATEMENT = '#flag_statement';
const FLAG = '#flag';
const NATIVE_KEY_VALUE = '#native_key_value';
const NATIVE_KEY = '#native_key';
// Token types for leaf nodes
const TOKEN_WORD = 'word';

5
lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php
View File

@@ -85,6 +85,11 @@ class QueryContext
return $this->structure->getFlagByName($name);
}
public function getMetadataTag($name)
{
return $this->structure->getMetadataTagByName($name);
}
/**
* @todo Maybe we should put this logic in Field class?
*/

10
lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryPostProcessor.php Normal file
View File

@@ -0,0 +1,10 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\Search;
use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryContext;
interface QueryPostProcessor
{
public function postProcessQuery($query, QueryContext $context);
}

92
lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php
View File

@@ -89,12 +89,18 @@ class QueryVisitor implements Visit
case NodeTypes::FLAG:
return new AST\Flag($this->visitString($element));
case NodeTypes::NATIVE_KEY_VALUE:
return $this->visitNativeKeyValueNode($element);
case NodeTypes::MATCH_EXPR:
return $this->visitMatchExpressionNode($element);
case NodeTypes::NATIVE_KEY:
return $this->visitNativeKeyNode($element);
case NodeTypes::METADATA_KEY:
return new AST\KeyValue\MetadataKey($this->visitString($element));
case NodeTypes::FIELD_KEY:
return new AST\KeyValue\FieldKey($this->visitString($element));
default:
throw new Exception(sprintf('Unknown node type "%s".', $element->getId()));
}
@@ -111,60 +117,53 @@ class QueryVisitor implements Visit
private function visitFieldStatementNode(TreeNode $node)
{
if ($node->getChildrenNumber() !== 2) {
throw new Exception('Field statement must have 2 childs.');
}
$field = $this->visit($node->getChild(0));
$value = $this->visit($node->getChild(1));
return new AST\FieldMatchExpression($field, $value);
return $this->handleBinaryExpression($node, function($left, $right) {
return new AST\FieldMatchExpression($left, $right);
});
}
private function visitAndNode(Element $element)
{
return $this->handleBinaryOperator($element, function($left, $right) {
return new AST\Boolean\AndOperator($left, $right);
return $this->handleBinaryExpression($element, function($left, $right) {
return new AST\Boolean\AndExpression($left, $right);
});
}
private function visitOrNode(Element $element)
{
return $this->handleBinaryOperator($element, function($left, $right) {
return new AST\Boolean\OrOperator($left, $right);
return $this->handleBinaryExpression($element, function($left, $right) {
return new AST\Boolean\OrExpression($left, $right);
});
}
private function visitExceptNode(Element $element)
{
return $this->handleBinaryOperator($element, function($left, $right) {
return new AST\Boolean\ExceptOperator($left, $right);
return $this->handleBinaryExpression($element, function($left, $right) {
return new AST\Boolean\ExceptExpression($left, $right);
});
}
private function visitRangeNode(TreeNode $node)
{
if ($node->getChildrenNumber() !== 2) {
throw new Exception('Comparison operator can only have 2 childs.');
}
$field = $node->getChild(0)->accept($this);
$expression = $node->getChild(1)->accept($this);
$this->assertChildrenCount($node, 2);
$key = $node->getChild(0)->accept($this);
$boundary = $node->getChild(1)->accept($this);
switch ($node->getId()) {
case NodeTypes::LT_EXPR:
return AST\RangeExpression::lessThan($field, $expression);
return AST\KeyValue\RangeExpression::lessThan($key, $boundary);
case NodeTypes::LTE_EXPR:
return AST\RangeExpression::lessThanOrEqual($field, $expression);
return AST\KeyValue\RangeExpression::lessThanOrEqual($key, $boundary);
case NodeTypes::GT_EXPR:
return AST\RangeExpression::greaterThan($field, $expression);
return AST\KeyValue\RangeExpression::greaterThan($key, $boundary);
case NodeTypes::GTE_EXPR:
return AST\RangeExpression::greaterThanOrEqual($field, $expression);
return AST\KeyValue\RangeExpression::greaterThanOrEqual($key, $boundary);
}
}
private function handleBinaryOperator(Element $element, \Closure $factory)
private function handleBinaryExpression(Element $element, \Closure $factory)
{
if ($element->getChildrenNumber() !== 2) {
throw new Exception('Binary expression can only have 2 childs.');
}
$this->assertChildrenCount($element, 2);
$left = $element->getChild(0)->accept($this);
$right = $element->getChild(1)->accept($this);
@@ -173,14 +172,9 @@ class QueryVisitor implements Visit
private function visitEqualNode(TreeNode $node)
{
if ($node->getChildrenNumber() !== 2) {
throw new Exception('Equality operator can only have 2 childs.');
}
return new AST\FieldEqualsExpression(
$node->getChild(0)->accept($this),
$node->getChild(1)->accept($this)
);
return $this->handleBinaryExpression($node, function($left, $right) {
return new AST\KeyValue\EqualExpression($left, $right);
});
}
private function visitTerm(Element $element)
@@ -242,7 +236,7 @@ class QueryVisitor implements Visit
throw new Exception('Unexpected context after non-contextualizable node');
}
} elseif ($node instanceof AST\Node) {
$root = new AST\Boolean\AndOperator($root, $node);
$root = new AST\Boolean\AndExpression($root, $node);
} else {
throw new Exception('Unexpected node type inside text node.');
}
@@ -267,9 +261,7 @@ class QueryVisitor implements Visit
private function visitFlagStatementNode(TreeNode $node)
{
if ($node->getChildrenNumber() !== 2) {
throw new Exception('Flag statement can only have 2 childs.');
}
$this->assertChildrenCount($node, 2);
$flag = $node->getChild(0)->accept($this);
if (!$flag instanceof AST\Flag) {
throw new \Exception('Flag statement key must be a flag node.');
@@ -298,21 +290,16 @@ class QueryVisitor implements Visit
}
}
private function visitNativeKeyValueNode(TreeNode $node)
private function visitMatchExpressionNode(TreeNode $node)
{
if ($node->getChildrenNumber() !== 2) {
throw new Exception('Key value expression can only have 2 childs.');
}
$key = $this->visit($node->getChild(0));
$value = $this->visit($node->getChild(1));
return new AST\KeyValue\Expression($key, $value);
return $this->handleBinaryExpression($node, function($left, $right) {
return new AST\KeyValue\MatchExpression($left, $right);
});
}
private function visitNativeKeyNode(Element $element)
{
if ($element->getChildrenNumber() !== 1) {
throw new Exception('Native key node can only have a single child.');
}
$this->assertChildrenCount($element, 1);
$type = $element->getChild(0)->getValue()['token'];
switch ($type) {
case NodeTypes::TOKEN_DATABASE:
@@ -327,4 +314,11 @@ class QueryVisitor implements Visit
throw new InvalidArgumentException(sprintf('Unexpected token type "%s" for native key.', $type));
}
}
private function assertChildrenCount(TreeNode $node, $count)
{
if ($node->getChildrenNumber() !== $count) {
throw new Exception(sprintf('Node was expected to have only %s children.', $count));
}
}
}

38
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php
View File

@@ -4,7 +4,6 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Alchemy\Phrasea\SearchEngine\Elastic\Exception\MergeException;
use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept;
use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Helper as ThesaurusHelper;
use Assert\Assertion;
@@ -13,7 +12,7 @@ use databox_field;
/**
* @todo Field labels
*/
class Field
class Field implements Typed
{
private $name;
private $type;
@@ -119,41 +118,6 @@ class Field
);
}
public function isValueCompatible($value)
{
return count(self::filterByValueCompatibility([$this], $value)) > 0;
}
public static function filterByValueCompatibility(array $fields, $value)
{
$is_numeric = is_numeric($value);
$is_valid_date = RecordHelper::validateDate($value);
$filtered = [];
foreach ($fields as $field) {
switch ($field->type) {
case Mapping::TYPE_FLOAT:
case Mapping::TYPE_DOUBLE:
case Mapping::TYPE_INTEGER:
case Mapping::TYPE_LONG:
case Mapping::TYPE_SHORT:
case Mapping::TYPE_BYTE:
if ($is_numeric) {
$filtered[] = $field;
}
break;
case Mapping::TYPE_DATE:
if ($is_valid_date) {
$filtered[] = $field;
}
break;
case Mapping::TYPE_STRING:
default:
$filtered[] = $field;
}
}
return $filtered;
}
public function getConceptPathIndexField()
{
return sprintf('concept_path.%s', $this->name);

20
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php
View File

@@ -19,6 +19,7 @@ final class GlobalStructure implements Structure
/** @var Field[] */
private $facets = array();
private $flags = array();
private $metadata_tags = array();
/**
* @param \databox[] $databoxes
@@ -36,19 +37,23 @@ final class GlobalStructure implements Structure
$flags[] = Flag::createFromLegacyStatus($status);
}
}
return new self($fields, $flags);
return new self($fields, $flags, MetadataHelper::createTags());
}
public function __construct(array $fields = [], array $flags = [])
public function __construct(array $fields = [], array $flags = [], array $metadata_tags = [])
{
Assertion::allIsInstanceOf($fields, Field::class);
Assertion::allIsInstanceOf($flags, Flag::class);
Assertion::allIsInstanceOf($metadata_tags, Tag::class);
foreach ($fields as $field) {
$this->add($field);
}
foreach ($flags as $flag) {
$this->flags[$flag->getName()] = $flag;
}
foreach ($metadata_tags as $tag) {
$this->metadata_tags[$tag->getName()] = $tag;
}
}
public function add(Field $field)
@@ -148,6 +153,17 @@ final class GlobalStructure implements Structure
$this->flags[$name] : null;
}
public function getMetadataTags()
{
return $this->metadata_tags;
}
public function getMetadataTagByName($name)
{
return isset($this->metadata_tags[$name]) ?
$this->metadata_tags[$name] : null;
}
/**
* Returns an array of collections indexed by field name.
*

10
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/LimitedStructure.php
View File

@@ -85,6 +85,16 @@ final class LimitedStructure implements Structure
return $this->structure->getFlagByName($name);
}
public function getMetadataTags()
{
return $this->structure->getMetadataTags();
}
public function getMetadataTagByName($name)
{
return $this->structure->getMetadataTagByName($name);
}
private function limit(array $fields)
{
$allowed_collections = $this->allowedCollections();

49
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/MetadataHelper.php Normal file
View File

@@ -0,0 +1,49 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Assert\Assertion;
use InvalidArgumentException;
use media_subdef;
class MetadataHelper
{
private function __construct() {}
public static function createTags()
{
static $tag_descriptors = [
[media_subdef::TC_DATA_WIDTH , 'integer', false],
[media_subdef::TC_DATA_HEIGHT , 'integer', false],
[media_subdef::TC_DATA_COLORSPACE , 'string' , false],
[media_subdef::TC_DATA_CHANNELS , 'integer', false],
[media_subdef::TC_DATA_ORIENTATION , 'integer', false],
[media_subdef::TC_DATA_COLORDEPTH , 'integer', false],
[media_subdef::TC_DATA_DURATION , 'float' , false],
[media_subdef::TC_DATA_AUDIOCODEC , 'string' , false],
[media_subdef::TC_DATA_AUDIOSAMPLERATE , 'float' , false],
[media_subdef::TC_DATA_VIDEOCODEC , 'string' , false],
[media_subdef::TC_DATA_FRAMERATE , 'float' , false],
[media_subdef::TC_DATA_MIMETYPE , 'string' , false],
[media_subdef::TC_DATA_FILESIZE , 'long' , false],
// TODO use geo point type for lat/long
[media_subdef::TC_DATA_LONGITUDE , 'float' , false],
[media_subdef::TC_DATA_LATITUDE , 'float' , false],
[media_subdef::TC_DATA_FOCALLENGTH , 'float' , false],
[media_subdef::TC_DATA_CAMERAMODEL , 'string' , true ],
[media_subdef::TC_DATA_FLASHFIRED , 'boolean', false],
[media_subdef::TC_DATA_APERTURE , 'float' , false],
[media_subdef::TC_DATA_SHUTTERSPEED , 'float' , false],
[media_subdef::TC_DATA_HYPERFOCALDISTANCE, 'float' , false],
[media_subdef::TC_DATA_ISO , 'integer', false],
[media_subdef::TC_DATA_LIGHTVALUE , 'float' , false]
];
$tags = [];
foreach ($tag_descriptors as $descriptor) {
$tags[] = new Tag($descriptor[0], $descriptor[1], $descriptor[2]);
}
return $tags;
}
}

3
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php
View File

@@ -27,4 +27,7 @@ interface Structure
public function getAllFlags();
public function getFlagByName($name);
public function getMetadataTags();
public function getMetadataTagByName($name);
}

47
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Tag.php Normal file
View File

@@ -0,0 +1,47 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
use Assert\Assertion;
class Tag implements Typed
{
private $name;
private $type;
private $analyzable;
public function __construct($name, $type, $analyzable = false)
{
Assertion::string($name);
Assertion::string($type);
Assertion::boolean($analyzable);
$this->name = $name;
$this->type = $type;
$this->analyzable = $analyzable;
}
public function getName()
{
return $this->name;
}
public function getType()
{
return $this->type;
}
public function isAnalyzable()
{
return $this->analyzable;
}
public function getIndexField($raw = false)
{
return sprintf(
'exif.%s%s',
$this->name,
$raw && $this->type === Mapping::TYPE_STRING ? '.raw' : ''
);
}
}

13
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Typed.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
interface Typed
{
/**
* Get the type of the object
*
* @return string One of Mapping::TYPE_* constants
*/
public function getType();
}

48
lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/ValueChecker.php Normal file
View File

@@ -0,0 +1,48 @@
<?php
namespace Alchemy\Phrasea\SearchEngine\Elastic\Structure;
use Alchemy\Phrasea\SearchEngine\Elastic\Mapping;
use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
use Assert\Assertion;
class ValueChecker
{
private function __construct() {}
public static function isValueCompatible(Typed $typed, $value)
{
return count(self::filterByValueCompatibility([$typed], $value)) > 0;
}
public static function filterByValueCompatibility(array $list, $value)
{
Assertion::allIsInstanceOf($list, Typed::class);
$is_numeric = is_numeric($value);
$is_valid_date = RecordHelper::validateDate($value);
$filtered = [];
foreach ($list as $item) {
switch ($item->getType()) {
case Mapping::TYPE_FLOAT:
case Mapping::TYPE_DOUBLE:
case Mapping::TYPE_INTEGER:
case Mapping::TYPE_LONG:
case Mapping::TYPE_SHORT:
case Mapping::TYPE_BYTE:
if ($is_numeric) {
$filtered[] = $item;
}
break;
case Mapping::TYPE_DATE:
if ($is_valid_date) {
$filtered[] = $item;
}
break;
case Mapping::TYPE_STRING:
default:
$filtered[] = $item;
}
}
return $filtered;
}
}