hazza/Phraseanet
1
0
Fork 0
mirror of https://github.com/alchemy-fr/Phraseanet.git synced 2025-10-14 13:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

[SearchEngine] Rename directories

Browse Source
This commit is contained in:
Romain Neutron
2012-08-27 12:32:36 +02:00
parent 82d52da228
commit a104cd879f
7 changed files with 259 additions and 3546 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
lib/Alchemy/Phrasea/SearchEngine/Phrasea/PhraseaEngine.php
View File

@@ -11,86 +11,37 @@
namespace Alchemy\Phrasea\SearchEngine\Phrasea;
use Alchemy\Phrasea\Application;
use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
use Alchemy\Phrasea\SearchEngine\SearchEngineResult;
use Alchemy\Phrasea\Exception\RuntimeException;
use Doctrine\Common\Collections\ArrayCollection;
use Silex\Application;
use Symfony\Component\HttpFoundation\Request;
class PhraseaEngine implements SearchEngineInterface
{
private $initialized;
/**
*
* @var SearchEngineOptions
*/
private $options;
private $app;
private $queries = array();
private $arrayq = array();
private $colls = array();
private $qp = array();
private $needthesaurus = array();
private $configurationPanel;
private $resetCacheNextQuery = false;
protected $options;
protected $queries = array();
protected $arrayq = array();
protected $colls = array();
protected $qp = array();
protected $needthesaurus = array();
protected $configurationPanel;
protected $resetCacheNextQuery = false;
/**
* {@inheritdoc}
*/
public function __construct(Application $app)
public function __construct()
{
$this->app = $app;
$this->options = new SearchEngineOptions();
}
public function initialize()
{
if ($this->initialized) {
return $this;
}
$choosenConnexion = $this->app['phraseanet.configuration']->getPhraseanet()->get('database');
$connexion = $this->app['phraseanet.configuration']->getConnexion($choosenConnexion);
$hostname = $connexion->get('host');
$port = (int) $connexion->get('port');
$user = $connexion->get('user');
$password = $connexion->get('password');
$dbname = $connexion->get('dbname');
if (!extension_loaded('phrasea2')) {
throw new RuntimeException('Phrasea extension is required');
}
if (!function_exists('phrasea_conn')) {
throw new RuntimeException('Phrasea extension requires upgrade');
}
if (phrasea_conn($hostname, $port, $user, $password, $dbname) !== true) {
throw new RuntimeException('Unable to initialize Phrasea connection');
}
$this->initialized = true;
return $this;
}
private function checkSession()
{
if (!$this->app['phraseanet.user']) {
throw new \RuntimeException('Phrasea currently support only authenticated queries');
}
if (!phrasea_open_session($this->app['session']->get('phrasea_session_id'), $this->app['phraseanet.user']->get_id())) {
if (!$ses_id = phrasea_create_session((string) $this->app['phraseanet.user']->get_id())) {
throw new \Exception_InternalServerError('Unable to create phrasea session');
}
$this->app['session']->set('phrasea_session_id', $ses_id);
}
}
/**
* {@inheritdoc}
*/
@@ -104,7 +55,17 @@ class PhraseaEngine implements SearchEngineInterface
return $status;
}
public function configurationPanel()
public function getConfigurationPanel(Application $app, Request $request)
{
}
public function postConfigurationPanel(Application $app, Request $request)
{
}
private function configurationPanel()
{
if ( ! $this->configurationPanel) {
$this->configurationPanel = new ConfigurationPanel($this);
@@ -113,6 +74,7 @@ class PhraseaEngine implements SearchEngineInterface
return $this->configurationPanel;
}
/**
* {@inheritdoc}
*/
@@ -161,7 +123,7 @@ class PhraseaEngine implements SearchEngineInterface
*/
public function updateRecord(\record_adapter $record)
{
$record->set_binary_status(\databox_status::dec2bin($this->app, bindec($record->get_status()) & ~7 | 4));
$record->set_binary_status(\databox_status::dec2bin(bindec($record->get_status()) & ~7 | 4));
return $this;
}
@@ -235,9 +197,6 @@ class PhraseaEngine implements SearchEngineInterface
*/
public function query($query, $offset, $perPage)
{
$this->initialize();
$this->checkSession();
assert(is_int($offset));
assert($offset >= 0);
assert(is_int($perPage));
@@ -250,9 +209,12 @@ class PhraseaEngine implements SearchEngineInterface
$query .= ' AND recordtype=' . $this->options->getRecordType();
}
$appbox = \appbox::get_instance(\bootstrap::getCore());
$session = $appbox->get_session();
$sql = 'SELECT query, query_time, duration, total FROM cache WHERE session_id = :ses_id';
$stmt = $this->app['phraseanet.appbox']->get_connection()->prepare($sql);
$stmt->execute(array(':ses_id' => $this->app['session']->get('phrasea_session_id')));
$stmt = $appbox->get_connection()->prepare($sql);
$stmt->execute(array(':ses_id' => $session->get_ses_id()));
$row = $stmt->fetch(\PDO::FETCH_ASSOC);
$stmt->closeCursor();
@@ -267,24 +229,24 @@ class PhraseaEngine implements SearchEngineInterface
}
if ($this->resetCacheNextQuery === true) {
phrasea_clear_cache($this->app['session']->get('phrasea_session_id'));
phrasea_clear_cache($session->get_ses_id());
$this->addQuery($query);
$this->executeQuery($query);
$sql = 'SELECT query, query_time, duration, total FROM cache WHERE session_id = :ses_id';
$stmt = $this->app['phraseanet.appbox']->get_connection()->prepare($sql);
$stmt->execute(array(':ses_id' => $this->app['session']->get('phrasea_session_id')));
$stmt = $appbox->get_connection()->prepare($sql);
$stmt->execute(array(':ses_id' => $session->get_ses_id()));
$row = $stmt->fetch(\PDO::FETCH_ASSOC);
$stmt->closeCursor();
} else {
/**
* @todo clean this in DB
*/
$this->total_available = $this->total_results = $this->app['session']->get('phrasea_engine_n_results');
$this->total_available = $this->total_results = $session->get_session_prefs('phrasea_engine_n_results');
}
$res = phrasea_fetch_results(
$this->app['session']->get('phrasea_session_id'), $offset + 1, $perPage, false
$session->get_ses_id(), $offset + 1, $perPage, false
);
$rs = array();
@@ -301,8 +263,7 @@ class PhraseaEngine implements SearchEngineInterface
foreach ($rs as $data) {
try {
$records->add(new \record_adapter(
$this->app,
\phrasea::sbasFromBas($this->app, $data['base_id']),
\phrasea::sbasFromBas($data['base_id']),
$data['record_id'],
$resultNumber
));
@@ -316,16 +277,15 @@ class PhraseaEngine implements SearchEngineInterface
return new SearchEngineResult($records, $query, $row['duration'], $offset, $row['total'], $row['total'], $error, '', new ArrayCollection(), new ArrayCollection(), '');
}
public static function create(Application $app)
{
return new static($app);
}
/**
* {@inheritdoc}
*/
private function executeQuery($query)
{
$appbox = \appbox::get_instance(\bootstrap::getCore());
$session = $appbox->get_session();
$registry = $appbox->get_registry();
$dateLog = date("Y-m-d H:i:s");
$nbanswers = $total_time = 0;
$sort = '';
@@ -351,12 +311,12 @@ class PhraseaEngine implements SearchEngineInterface
}
$results = phrasea_query2(
$this->app['session']->get('phrasea_session_id')
$session->get_ses_id()
, $sbas_id
, $this->colls[$sbas_id]
, $this->arrayq[$sbas_id]
, $this->app['phraseanet.registry']->get('GV_sit')
, $this->app['session']->get('usr_id')
, $registry->get('GV_sit')
, (string) $session->get_usr_id()
, false
, $this->options->searchType() == SearchEngineOptions::RECORD_GROUPING ? PHRASEA_MULTIDOC_REGONLY : PHRASEA_MULTIDOC_DOCONLY
, $sort
@@ -368,26 +328,26 @@ class PhraseaEngine implements SearchEngineInterface
$nbanswers += $results["nbanswers"];
}
// $logger = $session->get_logger($this->appbox->get_databox($sbas_id));
//
// $conn2 = \connection::getPDOConnection($sbas_id);
//
// $sql3 = "INSERT INTO log_search
// (id, log_id, date, search, results, coll_id )
// VALUES
// (null, :log_id, :date, :query, :nbresults, :colls)";
//
// $params = array(
// ':log_id' => $logger->get_id()
// , ':date' => $dateLog
// , ':query' => $query
// , ':nbresults' => $results["nbanswers"]
// , ':colls' => implode(',', $this->colls[$sbas_id])
// );
//
// $stmt = $conn2->prepare($sql3);
// $stmt->execute($params);
// $stmt->closeCursor();
$logger = $session->get_logger($appbox->get_databox($sbas_id));
$conn2 = \connection::getPDOConnection($sbas_id);
$sql3 = "INSERT INTO log_search
(id, log_id, date, search, results, coll_id )
VALUES
(null, :log_id, :date, :query, :nbresults, :colls)";
$params = array(
':log_id' => $logger->get_id()
, ':date' => $dateLog
, ':query' => $query
, ':nbresults' => $results["nbanswers"]
, ':colls' => implode(',', $this->colls[$sbas_id])
);
$stmt = $conn2->prepare($sql3);
$stmt->execute($params);
$stmt->closeCursor();
}
$sql = 'UPDATE cache
@@ -396,18 +356,16 @@ class PhraseaEngine implements SearchEngineInterface
$params = array(
'query' => $query,
':ses_id' => $this->app['session']->get('phrasea_session_id'),
':ses_id' => $session->get_ses_id(),
':duration' => $total_time,
':total' => $nbanswers,
);
$stmt = $this->app['phraseanet.appbox']->get_connection()->prepare($sql);
$stmt = $appbox->get_connection()->prepare($sql);
$stmt->execute($params);
$stmt->closeCursor();
if ($this->app['phraseanet.user']) {
\User_Adapter::saveQuery($this->app, $query);
}
\User_Adapter::saveQuery($query);
return $this;
}
@@ -427,8 +385,10 @@ class PhraseaEngine implements SearchEngineInterface
{
$ret = array();
$appbox = \appbox::get_instance(\bootstrap::getCore());
$session = $appbox->get_session();
$res = phrasea_fetch_results(
$this->app['session']->get('phrasea_session_id'), ($record->get_number() + 1), 1, true, "[[em]]", "[[/em]]"
$session->get_ses_id(), ($record->get_number() + 1), 1, true, "[[em]]", "[[/em]]"
);
if ( ! isset($res['results']) || ! is_array($res['results'])) {
@@ -484,7 +444,7 @@ class PhraseaEngine implements SearchEngineInterface
if ($status) {
$requestStat = 'xxxx';
for ($i = 4; ($i <= 32); $i++) {
for ($i = 4; ($i <= 64); $i ++ ) {
if ( ! isset($status[$i])) {
$requestStat = 'x' . $requestStat;
continue;
@@ -507,9 +467,7 @@ class PhraseaEngine implements SearchEngineInterface
}
}
if ($this->options->fields()) {
$this->queries[$sbas] .= ' IN (' . implode(' OR ', array_map(function(\databox_field $field) {
return $field->get_name();
}, $this->options->fields())) . ')';
$this->queries[$sbas] .= ' IN (' . implode(' OR ', $this->options->fields()) . ')';
}
if (($this->options->getMinDate() || $this->options->getMaxDate()) && $this->options->getDateFields()) {
if ($this->options->getMinDate()) {
@@ -568,7 +526,7 @@ class PhraseaEngine implements SearchEngineInterface
private function singleParse($sbas, $query)
{
$this->qp[$sbas] = new PhraseaEngineQueryParser($this->app, $this->options->getLocale());
$this->qp[$sbas] = new PhraseaEngineQueryParser($this->options->getLocale());
$this->qp[$sbas]->debug = false;
if ($sbas == 'main') {
@@ -586,39 +544,5 @@ class PhraseaEngine implements SearchEngineInterface
return $this;
}
/**
* @inheritdoc
*/
public function clearCache()
{
if ($this->app['session']->has('phrasea_session_id')) {
$this->initialize();
phrasea_close_session($this->app['session']->get('phrasea_session_id'));
$this->app['session']->remove('phrasea_session_id');
}
}
/**
* @inheritdoc
*/
public function clearAllCache(\DateTime $date = null)
{
if (!$date) {
$date = new \DateTime();
}
$sql = "SELECT session_id FROM cache WHERE lastaccess <= :date";
$stmt = $this->app['phraseanet.appbox']->get_connection()->prepare($sql);
$stmt->execute(array(':date' => $date->format(DATE_ISO8601)));
$rs = $stmt->fetchAll(\PDO::FETCH_ASSOC);
$stmt->closeCursor();
foreach ($rs as $row) {
phrasea_close_session($row['session_id']);
}
return $this;
}
}

82
lib/Alchemy/Phrasea/SearchEngine/Phrasea/PhraseaEngineQueryParser.php
View File

@@ -11,8 +11,6 @@
namespace Alchemy\Phrasea\SearchEngine\Phrasea;
use Alchemy\Phrasea\Application;
/**
*
* @package searchEngine
@@ -86,17 +84,15 @@ class PhraseaEngineQueryParser
*/
public $proposals = Array("QRY" => "", "BASES" => array(), "QUERIES" => array());
public $app;
/**
* Current language for thesaurus
* @var <type>
*/
public $lng;
public $lng = null;
protected $unicode;
public function __construct(Application $app, $lng = "???")
public function __construct($lng = "???")
{
$this->app = $app;
$this->lng = $lng;
$this->unicode = new \unicode();
@@ -625,15 +621,15 @@ class PhraseaEngineQueryParser
foreach ($tsy as $lng => $tsy2) {
foreach ($tsy2 as $sy) {
$alt .= $alt ? "\n" : "";
$alt .= "" . $lng . ": " . \p4string::MakeString($sy["v"], "js");
$alt .= "" . $lng . ": " . p4string::MakeString($sy["v"], "js");
}
}
$this->proposals['QUERIES'][$syfound["w"]] = $syfound["w"];
$thtml = $syfound["v"];
$kjs = $syfound["k"] ? ("'" . \p4string::MakeString($syfound["k"], "js") . "'") : "null";
$wjs = "'" . \p4string::MakeString($syfound["w"], "js") . "'";
$kjs = $syfound["k"] ? ("'" . p4string::MakeString($syfound["k"], "js") . "'") : "null";
$wjs = "'" . p4string::MakeString($syfound["w"], "js") . "'";
if ($node->getAttribute("term")) {
$thtml = "<b>" . $thtml . "</b>";
@@ -752,7 +748,7 @@ class PhraseaEngineQueryParser
if ($this->debug)
printf("searching thesaurus with xpath='%s'<br/>\n", $x);
$dxp = new \DOMXPath($domthe);
$dxp = new DOMXPath($domthe);
$nodes = $dxp->query($x);
if ( ! isset($tree["RB"]["SREF"]["TIDS"]))
@@ -777,11 +773,70 @@ class PhraseaEngineQueryParser
$this->proposals["BASES"]["b$bid"]["TERMS"][$path]["HTML"] = $prophtml;
} else {
// le mot n'est pas dans le thesaurus
$tree = null;
}
return($ambigus);
}
/*
function dead_setTids(&$tree, &$simple, $bid, &$domthe, $searchsynonyms)
{
// if($this->debug)
print("setTids:\n\$tree=" . var_export($tree, true) . "\n");
$ambigus = 0;
if(is_array($w = $simple["VALUE"]))
$t = $w = implode(" ", $w);
if (isset($tree["CONTEXT"])) {
if (!$tree["CONTEXT"]) {
$x0 = "@w=\"" . $w ."\" and not(@k)";
} else {
if ($tree["CONTEXT"]=="*") {
$x0 = "@w=\"" . $w ."\"";
} else {
$x0 = "@w=\"" . $w ."\" and @k=\"" . $tree["CONTEXT"] . "\"";
$t .= " (" . $tree["CONTEXT"] . ")";
}
}
} else {
$x0 = "@w=\"" . $w ."\"";
}
$x = "/thesaurus//sy[" . $x0 ."]";
if($this->debug)
printf("searching thesaurus with xpath='%s'<br/>\n", $x);
$dxp = new DOMXPath($domthe);
$nodes = $dxp->query($x);
if(!isset($tree["RB"]["SREF"]["TIDS"]))
$tree["RB"]["SREF"]["TIDS"] = array();
if ($nodes->length >= 1) {
if ($nodes->length == 1) {
// on cherche un id simple, on utilisera la syntaxe sql 'like' (l'extension repérera elle méme la syntaxe car la value finira par '%')
$this->addtoTIDS($tree["RB"], $bid, $nodes->item(0));
// $this->thesaurusDOMNodes[] = $nodes->item(0);
} else {
// on cherche plusieurs id's, on utilisera la syntaxe 'regexp' (l'extension repérera elle meme la syntaxe car la value finira par '$')
$val = "";
foreach ($nodes as $node) {
if(!isset($tree["CONTEXT"]))
$ambigus++;
$this->addtoTIDS($tree["RB"], $bid, $node);
}
}
$path = $tree["RB"]["SREF"]["PATH"];
$prophtml = "";
$this->propAsHTML($domthe->documentElement, $prophtml, $path);
$this->proposals["TERMS"][$path]["HTML"] = $prophtml;
} else {
// le mot n'est pas dans le thesaurus
}
return($ambigus);
}
*/
public function containsColonOperator(&$tree)
{
@@ -1668,15 +1723,16 @@ class PhraseaEngineQueryParser
public function addsimple($t, $type, $nodetype, $pnum, $tree, $depth)
{
$nok = 0;
$registry = \registry::get_instance();
$w = $t["VALUE"];
if ($w != "?" && $w != "*") { // on laisse passer les 'isolés' pour les traiter plus tard comme des mots vides
for ($i = 0; $i < strlen($w); $i ++ ) {
$c = substr($w, $i, 1);
if ($c == "?" || $c == "*") {
if ($nok < $this->app['phraseanet.registry']->get('GV_min_letters_truncation')) {
if ($nok < $registry->get('GV_min_letters_truncation')) {
if ($this->errmsg != "")
$this->errmsg .= sprintf("\\n");
$this->errmsg .= _('qparser:: Formulation incorrecte, necessite plus de caractere : ') . "<br>" . $this->app['phraseanet.registry']->get('GV_min_letters_truncation');
$this->errmsg .= _('qparser:: Formulation incorrecte, necessite plus de caractere : ') . "<br>" . $registry->get('GV_min_letters_truncation');
return(null);
}

517
lib/Alchemy/Phrasea/SearchEngine/PhraseaEngine.php
View File

@@ -1,517 +0,0 @@
<?php
/*
* This file is part of Phraseanet
*
* (c) 2005-2012 Alchemy
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Alchemy\Phrasea\SearchEngine;
use Alchemy\Phrasea\Exception\RuntimeException;
use Doctrine\Common\Collections\ArrayCollection;
class PhraseaEngine implements SearchEngineInterface
{
/**
*
* @var SearchEngineOptions
*/
protected $options;
protected $queries = array();
protected $arrayq = array();
protected $colls = array();
protected $qp = array();
protected $needthesaurus = array();
protected $resetCacheNextQuery = false;
/**
* {@inheritdoc}
*/
public function __construct()
{
$this->options = new SearchEngineOptions();
}
/**
* {@inheritdoc}
*/
public function status()
{
return true;
}
/**
* {@inheritdoc}
*/
public function availableTypes()
{
return array(self::GEM_TYPE_RECORD, self::GEM_TYPE_STORY);
}
/**
* {@inheritdoc}
*/
public function addRecord(\record_adapter $record)
{
return $this->updateRecord($record);
}
/**
* {@inheritdoc}
*/
public function removeRecord(\record_adapter $record)
{
$connbas = $record->get_databox()->get_connection();
$sql = "DELETE FROM prop WHERE record_id = :record_id";
$stmt = $connbas->prepare($sql);
$stmt->execute(array(':record_id' => $record->get_record_id()));
$stmt->closeCursor();
$sql = "DELETE FROM idx WHERE record_id = :record_id";
$stmt = $connbas->prepare($sql);
$stmt->execute(array(':record_id' => $record->get_record_id()));
$stmt->closeCursor();
$sql = "DELETE FROM thit WHERE record_id = :record_id";
$stmt = $connbas->prepare($sql);
$stmt->execute(array(':record_id' => $record->get_record_id()));
$stmt->closeCursor();
unset($stmt, $connbas);
return $this;
}
/**
* {@inheritdoc}
*/
public function updateRecord(\record_adapter $record)
{
$record->set_binary_status(\databox_status::dec2bin(bindec($record->get_status()) & ~7 | 4));
return $this;
}
/**
* {@inheritdoc}
*/
public function addStory(\record_adapter $record)
{
return $this->updateRecord($record);
}
/**
* {@inheritdoc}
*/
public function removeStory(\record_adapter $record)
{
return $this->removeRecord($record);
}
/**
* {@inheritdoc}
*/
public function updateStory(\record_adapter $record)
{
return $this->updateRecord($record);
}
/**
* {@inheritdoc}
*/
public function addFeedEntry(\Feed_Entry_Adapter $entry)
{
throw new RuntimeException('Feed Entry indexing not supported by Phrasea Engine');
}
/**
* {@inheritdoc}
*/
public function removeFeedEntry(\Feed_Entry_Adapter $entry)
{
throw new RuntimeException('Feed Entry indexing not supported by Phrasea Engine');
}
/**
* {@inheritdoc}
*/
public function updateFeedEntry(\Feed_Entry_Adapter $entry)
{
throw new RuntimeException('Feed Entry indexing not supported by Phrasea Engine');
}
/**
* {@inheritdoc}
*/
public function setOptions(SearchEngineOptions $options)
{
$this->options = $options;
}
/**
* {@inheritdoc}
*/
public function resetOptions()
{
$this->options = new SearchEngineOptions();
}
/**
* {@inheritdoc}
*/
public function query($query, $offset, $perPage)
{
assert(is_int($offset));
assert($offset >= 0);
assert(is_int($perPage));
if (trim($query) === '') {
$query = "all";
}
if ($this->options->getRecordType()) {
$query .= ' AND recordtype=' . $this->options->getRecordType();
}
$appbox = \appbox::get_instance(\bootstrap::getCore());
$session = $appbox->get_session();
$sql = 'SELECT query, query_time, duration, total FROM cache WHERE session_id = :ses_id';
$stmt = $appbox->get_connection()->prepare($sql);
$stmt->execute(array(':ses_id' => $session->get_ses_id()));
$row = $stmt->fetch(\PDO::FETCH_ASSOC);
$stmt->closeCursor();
$date_obj = new \DateTime('-10 min');
$date_quest = new \DateTime($row['query_time']);
if ($query != $row['query']) {
$this->resetCacheNextQuery = true;
}
if ($date_obj > $date_quest) {
$this->resetCacheNextQuery = true;
}
if ($this->resetCacheNextQuery === true) {
phrasea_clear_cache($session->get_ses_id());
$this->addQuery($query);
$this->executeQuery($query);
$sql = 'SELECT query, query_time, duration, total FROM cache WHERE session_id = :ses_id';
$stmt = $appbox->get_connection()->prepare($sql);
$stmt->execute(array(':ses_id' => $session->get_ses_id()));
$row = $stmt->fetch(\PDO::FETCH_ASSOC);
$stmt->closeCursor();
} else {
/**
* @todo clean this in DB
*/
$this->total_available = $this->total_results = $session->get_session_prefs('phrasea_engine_n_results');
}
$res = phrasea_fetch_results(
$session->get_ses_id(), $offset + 1, $perPage, false
);
$rs = array();
$error = _('Unable to execute query');
if (isset($res['results']) && is_array($res['results'])) {
$rs = $res['results'];
$error = '';
}
$resultNumber = $offset;
$records = new ArrayCollection();
foreach ($rs as $data) {
try {
$records->add(new \record_adapter(
\phrasea::sbasFromBas($data['base_id']),
$data['record_id'],
$resultNumber
));
} catch (Exception $e) {
}
$resultNumber ++;
}
return new SearchEngineResult($records, $query, $row['duration'], $offset, $row['total'], $row['total'], $error, '', new ArrayCollection(), new ArrayCollection(), '');
}
/**
* {@inheritdoc}
*/
private function executeQuery($query)
{
$appbox = \appbox::get_instance(\bootstrap::getCore());
$session = $appbox->get_session();
$registry = $appbox->get_registry();
$dateLog = date("Y-m-d H:i:s");
$nbanswers = $total_time = 0;
$sort = '';
if ($this->options->sortBy()) {
switch ($this->options->sortOrder()) {
case SearchEngineOptions::SORT_MODE_ASC:
$sort = '+';
break;
case SearchEngineOptions::SORT_MODE_DESC:
default:
$sort = '-';
break;
}
$sort .= '0' . $this->options->sortBy();
}
foreach ($this->queries as $sbas_id => $qry) {
$BF = array();
foreach ($this->options->businessFieldsOn() as $collection) {
$BF[] = $collection->get_base_id();
}
$results = phrasea_query2(
$session->get_ses_id()
, $sbas_id
, $this->colls[$sbas_id]
, $this->arrayq[$sbas_id]
, $registry->get('GV_sit')
, (string) $session->get_usr_id()
, false
, $this->options->searchType() == SearchEngineOptions::RECORD_GROUPING ? PHRASEA_MULTIDOC_REGONLY : PHRASEA_MULTIDOC_DOCONLY
, $sort
, $BF
);
if ($results) {
$total_time += $results['time_all'];
$nbanswers += $results["nbanswers"];
}
$logger = $session->get_logger($appbox->get_databox($sbas_id));
$conn2 = \connection::getPDOConnection($sbas_id);
$sql3 = "INSERT INTO log_search
(id, log_id, date, search, results, coll_id )
VALUES
(null, :log_id, :date, :query, :nbresults, :colls)";
$params = array(
':log_id' => $logger->get_id()
, ':date' => $dateLog
, ':query' => $query
, ':nbresults' => $results["nbanswers"]
, ':colls' => implode(',', $this->colls[$sbas_id])
);
$stmt = $conn2->prepare($sql3);
$stmt->execute($params);
$stmt->closeCursor();
}
$sql = 'UPDATE cache
SET query = :query, query_time = NOW(), duration = :duration, total = :total
WHERE session_id = :ses_id';
$params = array(
'query' => $query,
':ses_id' => $session->get_ses_id(),
':duration' => $total_time,
':total' => $nbanswers,
);
$stmt = $appbox->get_connection()->prepare($sql);
$stmt->execute($params);
$stmt->closeCursor();
\User_Adapter::saveQuery($query);
return $this;
}
/**
* {@inheritdoc}
*/
public function autocomplete($query)
{
return new ArrayCollection();
}
/**
* {@inheritdoc}
*/
public function excerpt($query, $fields, \record_adapter $record)
{
$ret = array();
$appbox = \appbox::get_instance(\bootstrap::getCore());
$session = $appbox->get_session();
$res = phrasea_fetch_results(
$session->get_ses_id(), ($record->get_number() + 1), 1, true, "[[em]]", "[[/em]]"
);
if ( ! isset($res['results']) || ! is_array($res['results'])) {
return array();
}
$rs = $res['results'];
$res = array_shift($rs);
if ( ! isset($res['xml'])) {
return array();
}
$sxe = simplexml_load_string($res['xml']);
foreach ($fields as $name => $field) {
if ($sxe && $sxe->description && $sxe->description->$name) {
$val = array();
foreach ($sxe->description->$name as $value) {
$val[] = str_replace(array('[[em]]', '[[/em]]'), array('<em>', '</em>'), (string) $value);
}
$separator = $field['separator'] ? $field['separator'][0] : '';
$val = implode(' ' . $separator . ' ', $val);
} else {
$val = $field['value'];
}
$ret[] = $val;
}
return $ret;
}
/**
* {@inheritdoc}
*/
public function resetCache()
{
$this->resetCacheNextQuery = true;
$this->queries = $this->arrayq = $this->colls = $this->qp = $this->needthesaurus = array();
return $this;
}
private function addQuery($query)
{
foreach ($this->options->databoxes() as $databox) {
$this->queries[$databox->get_sbas_id()] = $query;
}
$status = $this->options->getStatus();
foreach ($this->queries as $sbas => $qs) {
if ($status) {
$requestStat = 'xxxx';
for ($i = 4; ($i <= 64); $i ++ ) {
if ( ! isset($status[$i])) {
$requestStat = 'x' . $requestStat;
continue;
}
$val = 'x';
if (isset($status[$i][$sbas])) {
if ($status[$i][$sbas] == '0') {
$val = '0';
} elseif ($status[$i][$sbas] == '1') {
$val = '1';
}
}
$requestStat = $val . $requestStat;
}
$requestStat = ltrim($requestStat, 'x');
if ($requestStat !== '') {
$this->queries[$sbas] .= ' AND (recordstatus=' . $requestStat . ')';
}
}
if ($this->options->fields()) {
$this->queries[$sbas] .= ' IN (' . implode(' OR ', $this->options->fields()) . ')';
}
if (($this->options->getMinDate() || $this->options->getMaxDate()) && $this->options->getDateFields()) {
if ($this->options->getMinDate()) {
$this->queries[$sbas] .= ' AND ( ' . implode(' >= ' . $this->options->getMinDate()->format('Y-m-d') . ' OR ', $this->options->getDateFields()) . ' >= ' . $this->options->getMinDate()->format('Y-m-d') . ' ) ';
}
if ($this->options->getMaxDate()) {
$this->queries[$sbas] .= ' AND ( ' . implode(' <= ' . $this->options->getMaxDate()->format('Y-m-d') . ' OR ', $this->options->getDateFields()) . ' <= ' . $this->options->getMaxDate()->format('Y-m-d') . ' ) ';
}
}
}
$this->singleParse('main', $query);
foreach ($this->queries as $sbas => $db_query) {
$this->singleParse($sbas, $query);
}
$base_ids = array_map(function(\collection $collection) {
return $collection->get_base_id();
}, $this->options->collections());
foreach ($this->options->databoxes() as $databox) {
$sbas_id = $databox->get_sbas_id();
$this->colls[$sbas_id] = array();
foreach ($databox->get_collections() as $collection) {
if (in_array($collection->get_base_id(), $base_ids)) {
$this->colls[$sbas_id][] = $collection->get_base_id();
}
}
if (sizeof($this->colls[$sbas_id]) <= 0) {
continue;
}
if ($this->needthesaurus[$sbas_id]) {
if ($databox->get_dom_thesaurus()) {
$this->qp[$sbas_id]->thesaurus2($this->indep_treeq[$sbas_id], $sbas_id, $databox->get_dbname(), $databox->get_dom_thesaurus(), true);
$this->qp['main']->thesaurus2($this->indep_treeq['main'], $sbas_id, $databox->get_dbname(), $databox->get_dom_thesaurus(), true);
}
}
$emptyw = false;
$this->qp[$sbas_id]->set_default($this->indep_treeq[$sbas_id], $emptyw);
$this->qp[$sbas_id]->distrib_in($this->indep_treeq[$sbas_id]);
$this->qp[$sbas_id]->factor_or($this->indep_treeq[$sbas_id]);
$this->qp[$sbas_id]->setNumValue($this->indep_treeq[$sbas_id], $databox->get_sxml_structure());
$this->qp[$sbas_id]->thesaurus2_apply($this->indep_treeq[$sbas_id], $sbas_id);
$this->arrayq[$sbas_id] = $this->qp[$sbas_id]->makequery($this->indep_treeq[$sbas_id]);
}
return $this;
}
private function singleParse($sbas, $query)
{
$this->qp[$sbas] = new PhraseaEngineQueryParser($this->options->getLocale());
$this->qp[$sbas]->debug = false;
if ($sbas == 'main') {
$simple_treeq = $this->qp[$sbas]->parsequery($query);
} else {
$simple_treeq = $this->qp[$sbas]->parsequery($this->queries[$sbas]);
}
$this->qp[$sbas]->priority_opk($simple_treeq);
$this->qp[$sbas]->distrib_opk($simple_treeq);
$this->needthesaurus[$sbas] = false;
$this->indep_treeq[$sbas] = $this->qp[$sbas]->extendThesaurusOnTerms($simple_treeq, true, true, false);
$this->needthesaurus[$sbas] = $this->qp[$sbas]->containsColonOperator($this->indep_treeq[$sbas]);
return $this;
}
}

1954
lib/Alchemy/Phrasea/SearchEngine/PhraseaEngineQueryParser.php
View File

File diff suppressed because it is too large Load Diff

2
lib/Alchemy/Phrasea/SearchEngine/SearchEngineInterface.php
View File

@@ -15,6 +15,8 @@ use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
use Alchemy\Phrasea\SearchEngine\SearchEngineResult;
use Alchemy\Phrasea\Exception\RuntimeException;
use Doctrine\Common\Collections\ArrayCollection;
use Silex\Application;
use Symfony\Component\HttpFoundation\Request;
interface SearchEngineInterface
{

689
lib/Alchemy/Phrasea/SearchEngine/SphinxSearch.php
View File

@@ -1,689 +0,0 @@
<?php
/*
* This file is part of Phraseanet
*
* (c) 2005-2012 Alchemy
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Alchemy\Phrasea\SearchEngine;
use Alchemy\Phrasea\Exception\RuntimeException;
use Doctrine\Common\Collections\ArrayCollection;
require_once __DIR__ . '/../../../vendor/sphinx/sphinxapi.php';
class SphinxSearch implements SearchEngineInterface
{
/**
*
* @var \SphinxClient
*/
protected $sphinx;
/**
*
* @var \PDO
*/
protected $rt_conn;
protected $options;
public function __construct($host, $port, $rt_host, $rt_port)
{
$this->options = new SearchEngineOptions();
$this->sphinx = new \SphinxClient();
$this->sphinx->SetServer($host, $port);
$this->sphinx->SetArrayResult(true);
$this->sphinx->SetConnectTimeout(1);
try {
$this->rt_conn = @new \PDO(sprintf('mysql:host=%s;port=%s;', $rt_host, $rt_port));
} catch (\PDOException $e) {
$this->rt_conn = null;
}
return $this;
}
public function status()
{
$status = $this->sphinx->Status();
if (false === $status) {
throw new Exception(_('Sphinx server is offline'));
}
if (null === $this->rt_conn) {
throw new RuntimeException('Unable to connect to sphinx rt');
}
return $status;
}
public function availableTypes()
{
return array(self::GEM_TYPE_RECORD, self::GEM_TYPE_STORY);
}
public function addRecord(\record_adapter $record)
{
$all_datas = array();
foreach ($record->get_caption()->get_fields(null, true) as $field) {
if ( ! $field->is_indexable()) {
continue;
}
$all_datas[] = $field->get_serialized_values();
foreach ($field->get_values() as $value) {
$this->rt_conn->exec("REPLACE INTO "
. "metas_realtime" . $this->CRCdatabox($record->get_databox()) . " VALUES (
'" . $value->getId() . "'
,'" . str_replace("'", "\'", $value->getValue()) . "'
,'" . $value->getDatabox_field()->get_id() . "'
," . $record->get_record_id() . "
," . $record->get_sbas_id() . "
," . $record->get_collection()->get_coll_id() . "
," . (int) $record->is_grouping() . "
," . crc32($record->get_sbas_id() . '_' . $value->getDatabox_field()->get_id()) . "
," . crc32($record->get_sbas_id() . '_' . $record->get_collection()->get_coll_id()) . "
," . crc32($record->get_sbas_id() . '_' . $record->get_record_id()) . "
," . crc32($record->get_type()) . "
,0
," . (int) $value->getDatabox_field()->isBusiness() . "
," . crc32($record->get_collection()->get_coll_id() . '_' . (int) $value->getDatabox_field()->isBusiness()) . "
," . $record->get_creation_date()->format('U') . " )");
}
}
$this->rt_conn->exec("REPLACE INTO "
. "docs_realtime" . $this->CRCdatabox($record->get_databox()) . " VALUES (
'" . $record->get_record_id() . "'
,'" . str_replace("'", "\'", implode(' ', $all_datas)) . "'
," . $record->get_record_id() . "
," . $record->get_sbas_id() . "
," . $record->get_collection()->get_coll_id() . "
," . (int) $record->is_grouping() . "
," . crc32($record->get_sbas_id() . '_' . $record->get_collection()->get_coll_id()) . "
," . crc32($record->get_sbas_id() . '_' . $record->get_record_id()) . "
," . crc32($record->get_type()) . "
,0
," . $record->get_creation_date()->format('U') . " )");
}
public function removeRecord(\record_adapter $record)
{
$CRCdatabox = $this->CRCdatabox($record->get_databox());
$indexes = array(
"metadatas" . $CRCdatabox,
"metadatas" . $CRCdatabox . "_stemmed_en",
"metadatas" . $CRCdatabox . "_stemmed_fr",
);
foreach ($record->get_caption()->get_fields(null, true) as $field) {
foreach ($field->get_values() as $value) {
foreach ($indexes as $index) {
$this->sphinx->UpdateAttributes($index, array("deleted"), array($value->getId() => array(1)));
}
$this->rt_conn->exec("DELETE FROM metas_realtime" . $CRCdatabox . " WHERE id = " . $value->getId());
}
}
$indexes = array(
"documents" . $CRCdatabox,
"documents" . $CRCdatabox . "_stemmed_fr",
"documents" . $CRCdatabox . "_stemmed_en"
);
foreach ($indexes as $index) {
$this->sphinx->UpdateAttributes($index, array("deleted"), array($record->get_record_id() => array(1)));
}
$this->rt_conn->exec("DELETE FROM docs_realtime" . $CRCdatabox . " WHERE id = " . $record->get_record_id());
}
public function updateRecord(\record_adapter $record)
{
$this->removeRecord($record);
$this->addRecord($record);
}
public function addStory(\record_adapter $record)
{
return $this->addRecord($record);
}
public function removeStory(\record_adapter $record)
{
return $this->removeRecord($record);
}
public function updateStory(\record_adapter $record)
{
return $this->updateRecord($record);
}
public function addFeedEntry(\Feed_Entry_Adapter $entry)
{
throw new RuntimeException('Feed Entry indexing not supported by Sphinx Search Engine');
}
public function removeFeedEntry(\Feed_Entry_Adapter $entry)
{
throw new RuntimeException('Feed Entry indexing not supported by Sphinx Search Engine');
}
public function updateFeedEntry(\Feed_Entry_Adapter $entry)
{
throw new RuntimeException('Feed Entry indexing not supported by Sphinx Search Engine');
}
public function setOptions(SearchEngineOptions $options)
{
$this->options = $options;
$this->applyOptions($options);
}
public function resetOptions()
{
$this->options = new SearchEngineOptions();
$this->resetSphinx();
}
private function resetSphinx()
{
$this->sphinx->ResetGroupBy();
$this->sphinx->ResetFilters();
$this->sphinx->ResetOverrides();
}
public function query($query, $offset, $perPage)
{
assert(is_int($offset));
assert($offset >= 0);
assert(is_int($perPage));
$query = $this->parseQuery($query);
$preg = preg_match('/\s?recordid\s?=\s?([0-9]+)/i', $query, $matches, 0, 0);
if ($preg > 0) {
$this->sphinx->SetFilter('record_id', array($matches[1]));
$query = '';
}
$this->sphinx->SetLimits($offset, $perPage);
$this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED2);
$index = $this->getQueryIndex($query);
$res = $this->sphinx->Query($query, $index);
$results = new ArrayCollection();
if ($res === false) {
if ($this->sphinx->IsConnectError() === true) {
$error = _('Sphinx server is offline');
} else {
$error = $this->sphinx->GetLastError();
}
$warning = $this->sphinx->GetLastWarning();
$total = $available = $duration = 0;
$suggestions = $propositions = array();
} else {
$error = $res['error'];
$warning = $res['warning'];
$duration = $res['time'];
$total = $res['total_found'];
$available = $res['total'];
$resultOffset = $offset;
if (isset($res['matches'])) {
foreach ($res['matches'] as $record_id => $match) {
try {
$record =
new \record_adapter(
$match['attrs']['sbas_id']
, $match['attrs']['record_id']
, $resultOffset
);
$results->add($record);
} catch (Exception $e) {
}
$resultOffset ++;
}
}
$suggestions = $this->getSuggestions($query);
$propositions = array();
}
return new SearchEngineResult($results, $query, $duration, $offset, $available, $total, $error, $warning, $suggestions, $propositions, $index);
}
public function autocomplete($query)
{
$words = explode(" ", $this->cleanupQuery($query));
return $this->getSuggestions(array_pop($words));
}
public function excerpt($query, $fields, \record_adapter $record)
{
$index = '';
// in this case search is done on metas
if ($this->options->fields() || $this->options->businessFieldsOn()) {
if ($this->options->stemmed() && $this->options->getLocale()) {
$index = 'metadatas' . $this->CRCdatabox($record->get_databox()) . '_stemmed_' . $this->options->getLocale();
} else {
$index = 'metadatas' . $this->CRCdatabox($record->get_databox());
}
} else {
if ($this->options->stemmed()) {
$index = 'documents' . $this->CRCdatabox($record->get_databox()) . '_stemmed_' . $this->options->getLocale();
} else {
$index = 'documents' . $this->CRCdatabox($record->get_databox());
}
}
$opts = array(
'before_match' => "<em>",
'after_match' => "</em>",
);
$fields_to_send = array();
foreach ($fields as $k => $f) {
$fields_to_send[$k] = $f['value'];
}
return $this->sphinx->BuildExcerpts($fields_to_send, $index, $query, $opts);
}
public function resetCache()
{
return $this;
}
/**
* Reset sphinx client and apply the options
*
* Only apply filters and group by
*
* @param SearchEngineOptions $options
* @return SphinxSearch
*/
protected function applyOptions(SearchEngineOptions $options)
{
$this->resetSphinx();
$filters = array();
foreach ($options->collections() as $collection) {
$filters[] = crc32($collection->get_databox()->get_sbas_id() . '_' . $collection->get_coll_id());
}
$this->sphinx->SetFilter('crc_sbas_coll', $filters);
$this->sphinx->SetFilter('deleted', array(0));
$this->sphinx->SetFilter('parent_record_id', array($options->searchType()));
if ($options->fields()) {
$filters = array();
foreach ($options->fields() as $field) {
$filters[] = crc32($field->get_databox()->get_sbas_id() . '_' . $field->get_id());
}
$this->sphinx->SetFilter('crc_struct_id', $filters);
}
if ($options->businessFieldsOn()) {
$crc_coll_business = array();
foreach ($options->businessFieldsOn() as $collection) {
$crc_coll_business[] = crc32($collection->get_coll_id() . '_1');
$crc_coll_business[] = crc32($collection->get_coll_id() . '_0');
}
$non_business = array();
foreach ($options->collections() as $collection) {
foreach ($options->businessFieldsOn() as $BFcollection) {
if ($collection->get_base_id() == $BFcollection->get_base_id()) {
continue 2;
}
}
$non_business[] = $collection;
}
foreach ($non_business as $collection) {
$crc_coll_business[] = crc32($collection->get_coll_id() . '_0');
}
$this->sphinx->SetFilter('crc_coll_business', $crc_coll_business);
} elseif ($options->fields()) {
$this->sphinx->SetFilter('business', array(0));
}
/**
* @todo : enhance : check status in a better way
*/
foreach ($options->databoxes() as $databox) {
$status_opts = $options->getStatus();
foreach ($databox->get_statusbits() as $n => $status) {
if ( ! array_key_exists($n, $status_opts))
continue;
if ( ! array_key_exists($databox->get_sbas_id(), $status_opts[$n]))
continue;
$crc = crc32($databox->get_sbas_id() . '_' . $n);
$this->sphinx->SetFilter('status', array($crc), ($status_opts[$n][$databox->get_sbas_id()] == '0'));
}
}
if ($options->getRecordType()) {
$this->sphinx->SetFilter('crc_type', array(crc32($options->getRecordType())));
}
$order = '';
switch ($options->sortOrder()) {
case SearchEngineOptions::SORT_MODE_ASC:
$order = 'ASC';
break;
case SearchEngineOptions::SORT_MODE_DESC:
default:
$order = 'DESC';
break;
}
switch ($options->sortBy()) {
case SearchEngineOptions::SORT_RANDOM:
$sort = '@random';
break;
case SearchEngineOptions::SORT_RELEVANCE:
default:
$sort = '@relevance ' . $order . ', created_on ' . $order;
break;
case SearchEngineOptions::SORT_CREATED_ON:
$sort = 'created_on ' . $order;
break;
}
$this->sphinx->SetGroupBy('crc_sbas_record', SPH_GROUPBY_ATTR, $sort);
return $this;
}
/**
* Return unique integer key for a databox
*
* @param \databox $databox
* @return int
*/
private function CRCdatabox(\databox $databox)
{
return crc32(
str_replace(
array('.', '%')
, '_'
, sprintf('%s_%s_%s_%s', $databox->get_host(), $databox->get_port(), $databox->get_user(), $databox->get_dbname())
)
);
}
/**
* Remove all keywords, operators, quotes from a query string
*
* @param string $query
* @return string
*/
private function cleanupQuery($query)
{
return str_replace(array("all", "last", "et", "ou", "sauf", "and", "or", "except", "in", "dans", "'", '"', "(", ")", "_", "-", "+"), ' ', $query);
}
/**
* Return a collection of suggestion corresponding a query
*
* @param string $query
* @return ArrayCollection An array collection of SearchEngineSuggestion
*/
private function getSuggestions($query)
{
// First we split the query into simple words
$words = explode(" ", $this->cleanupQuery(mb_strtolower($query)));
$tmpWords = array();
foreach ($words as $word) {
if (trim($word) === '') {
continue;
}
$tmpWords[] = $word;
}
$words = array_unique($tmpWords);
$altVersions = array();
// As we got words, we look for alternate word for each of them
if (function_exists('enchant_broker_init') && $this->options->getLocale()) {
$broker = enchant_broker_init();
if (enchant_broker_dict_exists($broker, $this->options->getLocale())) {
$dictionnary = enchant_broker_request_dict($broker, $this->options->getLocale());
foreach ($words as $word) {
if (enchant_dict_check($dictionnary, $word) == false) {
$suggs = array_merge(array($word), enchant_dict_suggest($dictionnary, $word));
}
$altVersions[$word] = array_unique($suggs);
}
enchant_broker_free_dict($dictionnary);
}
enchant_broker_free($broker);
}
/**
* @todo enhance the trigramm query, as it could be sent in one batch
*/
foreach ($altVersions as $word => $versions) {
$altVersions[$word] = array_unique(array_merge($versions, $this->get_sugg_trigrams($word)));
}
// We now build an array of all possibilities based on the original query
$queries = array($query);
foreach ($altVersions as $word => $versions) {
$tmp_queries = array();
foreach ($versions as $version) {
foreach ($queries as $alt_query) {
$tmp_queries[] = $alt_query;
$tmp_queries[] = str_replace($word, $version, $alt_query);
}
$tmp_queries[] = str_replace($word, $version, $query);
}
$queries = array_unique(array_merge($queries, $tmp_queries));
}
$suggestions = array();
$max_results = 0;
foreach ($queries as $alt_query) {
$results = $this->sphinx->Query($alt_query, $this->getQueryIndex($alt_query));
if ($results !== false && isset($results['total_found'])) {
if ($results['total_found'] > 0) {
$max_results = max($max_results, (int) $results['total_found']);
$suggestions[] = new SearchEngineSuggestion($query, $alt_query, (int) $results['total_found']);
}
}
}
usort($suggestions, array('self', 'suggestionsHitSorter'));
$tmpSuggestions = new ArrayCollection();
foreach ($suggestions as $key => $suggestion) {
if ($suggestion->hits() < ($max_results / 100)) {
continue;
}
$tmpSuggestions->add($suggestion);
}
return $tmpSuggestions;
}
private static function suggestionsHitSorter(SearchEngineSuggestion $a, SearchEngineSuggestion $b)
{
if ($a->hits() == $b->hits()) {
return 0;
}
return ($a->hits() > $b->hits()) ? -1 : 1;
}
private function BuildTrigrams($keyword)
{
$t = "__" . $keyword . "__";
$trigrams = "";
for ($i = 0; $i < strlen($t) - 2; $i ++ ) {
$trigrams .= substr($t, $i, 3) . " ";
}
return $trigrams;
}
private function get_sugg_trigrams($word)
{
$trigrams = $this->BuildTrigrams($word);
$query = "\"$trigrams\"/1";
$len = strlen($word);
$this->resetSphinx();
$this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED2);
$this->sphinx->SetRankingMode(SPH_RANK_WORDCOUNT);
$this->sphinx->SetFilterRange("len", $len - 2, $len + 4);
$this->sphinx->SetSortMode(SPH_SORT_EXTENDED, "@weight DESC");
$this->sphinx->SetLimits(0, 10);
$indexes = array();
foreach ($this->options->databoxes() as $databox) {
$indexes[] = 'suggest' . $this->CRCdatabox($databox);
}
$index = implode(',', $indexes);
$res = $this->sphinx->Query($query, $index);
if ($this->sphinx->Status() === false) {
return array();
}
if ( ! $res || ! isset($res["matches"])) {
return array();
}
$this->sphinx->ResetGroupBy();
$this->sphinx->ResetFilters();
$words = array();
foreach ($res["matches"] as $match) {
$words[] = $match['attrs']['keyword'];
}
$this->applyOptions($this->options);
return $words;
}
private function getQueryIndex($query)
{
$index = '*';
$index_keys = array();
foreach ($this->options->databoxes() as $databox) {
$index_keys[] = $this->CRCdatabox($databox);
}
if (count($index_keys) > 0) {
if ($this->options->fields() || $this->options->businessFieldsOn()) {
if ($query !== '' && $this->options->stemmed() && $this->options->getLocale()) {
$index = ', metadatas' . implode('_stemmed_' . $this->options->getLocale() . ', metadatas', $index_keys) . '_stemmed_' . $this->options->getLocale();
} else {
$index = 'metadatas' . implode(',metadatas', $index_keys);
}
$index .= ', metas_realtime' . implode(', metas_realtime', $index_keys);
} else {
if ($query !== '' && $this->options->stemmed() && $this->options->getLocale()) {
$index .= ', documents' . implode('_stemmed_' . $this->options->getLocale() . ', documents', $index_keys) . '_stemmed_' . $this->options->getLocale();
} else {
$index .= 'documents' . implode(', documents', $index_keys);
}
$index .= ', docs_realtime' . implode(', docs_realtime', $index_keys);
}
}
return $index;
}
private function parseQuery($query)
{
$query = trim($query);
while (substr($query, 0, 1) === '(' && substr($query, -1) === ')') {
$query = substr($query, 1, (mb_strlen($query) - 2));
}
if ($query == 'all') {
$query = '';
}
while (mb_strpos($query, ' ') !== false) {
$query = str_replace(' ', ' ', $query);
}
$offset = 0;
while (false !== $pos = mb_strpos($query, '-', $offset)) {
$offset = $pos + 1;
if ($pos === 0) {
continue;
}
if (mb_substr($query, ($pos - 1), 1) !== ' ') {
$query = mb_substr($query, 0, ($pos)) . ' ' . mb_substr($query, $pos + 1);
}
}
$query = str_ireplace(array(' ou ', ' or '), '|', $query);
$query = str_ireplace(array(' sauf ', ' except '), ' -', $query);
$query = str_ireplace(array(' and ', ' et '), ' +', $query);
return $query;
}
}

209
lib/Alchemy/Phrasea/SearchEngine/SphinxSearch/SphinxSearchEngine.php
View File

@@ -11,15 +11,16 @@
namespace Alchemy\Phrasea\SearchEngine\SphinxSearch;
use Alchemy\Phrasea\Application;
use Alchemy\Phrasea\SearchEngine\SearchEngineInterface;
use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
use Alchemy\Phrasea\SearchEngine\SearchEngineResult;
use Alchemy\Phrasea\SearchEngine\SearchEngineSuggestion;
use Alchemy\Phrasea\Exception\RuntimeException;
use Doctrine\Common\Collections\ArrayCollection;
use Symfony\Component\Process\ExecutableFinder;
use Symfony\Component\Process\Process;
use Silex\Application;
use Symfony\Component\HttpFoundation\Request;
require_once __DIR__ . '/../../../../vendor/sphinx/sphinxapi.php';
class SphinxSearchEngine implements SearchEngineInterface
{
@@ -29,12 +30,6 @@ class SphinxSearchEngine implements SearchEngineInterface
*/
protected $sphinx;
/**
*
* @var \SphinxClient
*/
protected $suggestionClient;
/**
*
* @var \PDO
@@ -42,26 +37,19 @@ class SphinxSearchEngine implements SearchEngineInterface
protected $rt_conn;
protected $configurationPanel;
protected $options;
protected $app;
public function __construct(Application $app, $host, $port, $rt_host, $rt_port)
public function __construct($host, $port, $rt_host, $rt_port)
{
$this->app = $app;
$this->options = new SearchEngineOptions();
$this->sphinx = new \SphinxClient();
$this->sphinx->SetServer($host, $port);
$this->sphinx->SetArrayResult(true);
$this->sphinx->SetConnectTimeout(1);
$this->suggestionClient = new \SphinxClient();
$this->suggestionClient->SetServer($host, $port);
$this->suggestionClient->SetArrayResult(true);
$this->suggestionClient->SetConnectTimeout(1);
try {
$this->rt_conn = @new \PDO(sprintf('mysql:host=%s;port=%s;', $rt_host, $rt_port));
$this->rt_conn->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
} catch (\PDOException $e) {
$this->rt_conn = null;
}
@@ -71,11 +59,9 @@ class SphinxSearchEngine implements SearchEngineInterface
public function status()
{
if (false === $this->sphinx->Status()) {
throw new RuntimeException(_('Sphinx server is offline'));
}
$status = $this->sphinx->Status();
if (false === $this->suggestionClient->Status()) {
if (false === $status) {
throw new RuntimeException(_('Sphinx server is offline'));
}
@@ -83,7 +69,17 @@ class SphinxSearchEngine implements SearchEngineInterface
throw new RuntimeException('Unable to connect to sphinx rt');
}
return $this->sphinx->Status();
return $status;
}
public function getConfigurationPanel(Application $app, Request $request)
{
return $this->configurationPanel()->get($app, $request);
}
public function postConfigurationPanel(Application $app, Request $request)
{
return $this->configurationPanel()->post($app, $request);
}
/**
@@ -106,33 +102,18 @@ class SphinxSearchEngine implements SearchEngineInterface
public function addRecord(\record_adapter $record)
{
if (!$this->rt_conn) {
throw new RuntimeException('Unable to connect to sphinx real-time index');
}
$all_datas = array();
$status = array();
$binStatus = strrev($record->get_status());
for ($i = 4; $i < 32; $i++) {
if ($binStatus[$i]) {
$status[] = crc32($record->get_databox()->get_sbas_id() . '_' . $i);
}
}
foreach ($record->get_caption()->get_fields(null, true) as $field) {
if ( ! $field->is_indexable()) {
continue;
}
if (!$field->get_databox_field()->isBusiness()) {
$all_datas[] = $field->get_serialized_values();
}
foreach ($field->get_values() as $value) {
$sql = "REPLACE INTO "
$this->rt_conn->exec("REPLACE INTO "
. "metas_realtime" . $this->CRCdatabox($record->get_databox()) . " VALUES (
'" . $value->getId() . "'
,'" . str_replace("'", "\'", $value->getValue()) . "'
@@ -148,10 +129,7 @@ class SphinxSearchEngine implements SearchEngineInterface
,0
," . (int) $value->getDatabox_field()->isBusiness() . "
," . crc32($record->get_collection()->get_coll_id() . '_' . (int) $value->getDatabox_field()->isBusiness()) . "
," . $record->get_creation_date()->format('U') . "
,(" . implode(',', $status) . ") )";
$this->rt_conn->exec($sql);
," . $record->get_creation_date()->format('U') . " )");
}
}
@@ -167,18 +145,13 @@ class SphinxSearchEngine implements SearchEngineInterface
," . crc32($record->get_sbas_id() . '_' . $record->get_record_id()) . "
," . crc32($record->get_type()) . "
,0
," . $record->get_creation_date()->format('U') . "
,(" . implode(',', $status) . ") )");
," . $record->get_creation_date()->format('U') . " )");
return $this;
}
public function removeRecord(\record_adapter $record)
{
if (!$this->rt_conn) {
throw new RuntimeException('Unable to connect to sphinx real-time index');
}
$CRCdatabox = $this->CRCdatabox($record->get_databox());
$indexes = array(
"metadatas" . $CRCdatabox,
@@ -194,7 +167,9 @@ class SphinxSearchEngine implements SearchEngineInterface
$this->sphinx->UpdateAttributes($index, array("deleted"), array($value->getId() => array(1)));
}
$this->rt_conn->exec("DELETE FROM metas_realtime" . $CRCdatabox . " WHERE id = " . $value->getId());
$stmt = $this->rt_conn->exec("DELETE FROM metas_realtime" . $CRCdatabox . " WHERE id = " . $value->getId());
$stmt->execute();
$stmt->closeCursor();
}
}
@@ -265,6 +240,7 @@ class SphinxSearchEngine implements SearchEngineInterface
{
$this->sphinx->ResetGroupBy();
$this->sphinx->ResetFilters();
$this->sphinx->ResetOverrides();
}
public function query($query, $offset, $perPage)
@@ -275,10 +251,10 @@ class SphinxSearchEngine implements SearchEngineInterface
$query = $this->parseQuery($query);
$preg = preg_match('/\s?(recordid|storyid)\s?=\s?([0-9]+)/i', $query, $matches, 0, 0);
$preg = preg_match('/\s?recordid\s?=\s?([0-9]+)/i', $query, $matches, 0, 0);
if ($preg > 0) {
$this->sphinx->SetFilter('record_id', array($matches[2]));
$this->sphinx->SetFilter('record_id', array($matches[1]));
$query = '';
}
@@ -315,10 +291,9 @@ class SphinxSearchEngine implements SearchEngineInterface
try {
$record =
new \record_adapter(
$this->app,
$match['attrs']['sbas_id'],
$match['attrs']['record_id'],
$resultOffset
$match['attrs']['sbas_id']
, $match['attrs']['record_id']
, $resultOffset
);
$results->add($record);
@@ -354,7 +329,7 @@ class SphinxSearchEngine implements SearchEngineInterface
$index = 'metadatas' . $this->CRCdatabox($record->get_databox());
}
} else {
if ($this->options->stemmed() && $this->options->getLocale()) {
if ($this->options->stemmed()) {
$index = 'documents' . $this->CRCdatabox($record->get_databox()) . '_stemmed_' . $this->options->getLocale();
} else {
$index = 'documents' . $this->CRCdatabox($record->get_databox());
@@ -463,8 +438,8 @@ class SphinxSearchEngine implements SearchEngineInterface
/**
* @todo : enhance : check status in a better way
*/
$status_opts = $options->getStatus();
foreach ($options->databoxes() as $databox) {
$status_opts = $options->getStatus();
foreach ($databox->get_statusbits() as $n => $status) {
if ( ! array_key_exists($n, $status_opts))
continue;
@@ -545,10 +520,6 @@ class SphinxSearchEngine implements SearchEngineInterface
$altVersions = array();
foreach ($words as $word) {
$altVersions[$word] = array($word);
}
// As we got words, we look for alternate word for each of them
if (function_exists('enchant_broker_init') && $this->options->getLocale()) {
$broker = enchant_broker_init();
@@ -558,7 +529,7 @@ class SphinxSearchEngine implements SearchEngineInterface
foreach ($words as $word) {
if (enchant_dict_check($dictionnary, $word) == false) {
$suggs = array_merge(enchant_dict_suggest($dictionnary, $word));
$suggs = array_merge(array($word), enchant_dict_suggest($dictionnary, $word));
}
$altVersions[$word] = array_unique($suggs);
@@ -595,6 +566,7 @@ class SphinxSearchEngine implements SearchEngineInterface
foreach ($queries as $alt_query) {
$results = $this->sphinx->Query($alt_query, $this->getQueryIndex($alt_query));
if ($results !== false && isset($results['total_found'])) {
if ($results['total_found'] > 0) {
@@ -646,12 +618,12 @@ class SphinxSearchEngine implements SearchEngineInterface
$this->resetSphinx();
$this->suggestionClient->SetMatchMode(SPH_MATCH_EXTENDED2);
$this->suggestionClient->SetRankingMode(SPH_RANK_WORDCOUNT);
$this->suggestionClient->SetFilterRange("len", $len - 2, $len + 4);
$this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED2);
$this->sphinx->SetRankingMode(SPH_RANK_WORDCOUNT);
$this->sphinx->SetFilterRange("len", $len - 2, $len + 4);
$this->suggestionClient->SetSortMode(SPH_SORT_EXTENDED, "@weight DESC");
$this->suggestionClient->SetLimits(0, 10);
$this->sphinx->SetSortMode(SPH_SORT_EXTENDED, "@weight DESC");
$this->sphinx->SetLimits(0, 10);
$indexes = array();
@@ -660,9 +632,10 @@ class SphinxSearchEngine implements SearchEngineInterface
}
$index = implode(',', $indexes);
$res = $this->suggestionClient->Query($query, $index);
if ($this->suggestionClient->Status() === false) {
$res = $this->sphinx->Query($query, $index);
if ($this->sphinx->Status() === false) {
return array();
}
@@ -670,11 +643,16 @@ class SphinxSearchEngine implements SearchEngineInterface
return array();
}
$this->sphinx->ResetGroupBy();
$this->sphinx->ResetFilters();
$words = array();
foreach ($res["matches"] as $match) {
$words[] = $match['attrs']['keyword'];
}
$this->applyOptions($this->options);
return $words;
}
@@ -691,14 +669,14 @@ class SphinxSearchEngine implements SearchEngineInterface
if (count($index_keys) > 0) {
if ($this->options->fields() || $this->options->businessFieldsOn()) {
if ($query !== '' && $this->options->stemmed() && $this->options->getLocale()) {
$index = 'metadatas' . implode('_stemmed_' . $this->options->getLocale() . ', metadatas', $index_keys) . '_stemmed_' . $this->options->getLocale();
$index = ', metadatas' . implode('_stemmed_' . $this->options->getLocale() . ', metadatas', $index_keys) . '_stemmed_' . $this->options->getLocale();
} else {
$index = 'metadatas' . implode(',metadatas', $index_keys);
}
$index .= ', metas_realtime' . implode(', metas_realtime', $index_keys);
} else {
if ($query !== '' && $this->options->stemmed() && $this->options->getLocale()) {
$index = 'documents' . implode('_stemmed_' . $this->options->getLocale() . ', documents', $index_keys) . '_stemmed_' . $this->options->getLocale();
$index = ', documents' . implode('_stemmed_' . $this->options->getLocale() . ', documents', $index_keys) . '_stemmed_' . $this->options->getLocale();
} else {
$index = 'documents' . implode(', documents', $index_keys);
}
@@ -743,92 +721,5 @@ class SphinxSearchEngine implements SearchEngineInterface
return $query;
}
public function buildSuggestions(array $databoxes, $configuration, $threshold = 10)
{
$executableFinder = new ExecutableFinder();
$indexer = $executableFinder->find('indexer');
if (!is_executable($indexer)) {
throw new RuntimeException('Indexer does not seem to be executable');
}
foreach ($databoxes as $databox) {
$tmp_file = tempnam(sys_get_temp_dir(), 'sphinx_sugg');
$cmd = $indexer . ' --config ' . $configuration . ' metadatas' . $this->CRCdatabox($databox)
. ' --buildstops ' . $tmp_file . ' 1000000 --buildfreqs';
$process = new Process($cmd);
$process->run();
$sql = 'TRUNCATE suggest';
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
$stmt->closeCursor();
if (null !== $sql = $this->BuildDictionarySQL(file_get_contents($tmp_file), $threshold)) {
$stmt = $databox->get_connection()->prepare($sql);
$stmt->execute();
$stmt->closeCursor();
}
unlink($tmp_file);
}
return $this;
}
protected function BuildDictionarySQL($dictionnary, $threshold)
{
$out = array();
$n = 1;
$lines = explode("\n", $dictionnary);
foreach ($lines as $line) {
if (trim($line) === '') {
continue;
}
list ( $keyword, $freq ) = explode(" ", trim($line));
if ($freq < $threshold || strstr($keyword, "_") !== false || strstr($keyword, "'") !== false) {
continue;
}
if (ctype_digit($keyword)) {
continue;
}
if (mb_strlen($keyword) < 3) {
continue;
}
$trigrams = $this->BuildTrigrams($keyword);
$out[] = "( $n, '$keyword', '$trigrams', $freq )";
$n++;
}
if ($out) {
return "INSERT INTO suggest VALUES " . implode(",\n", $out) . ";";
}
return null;
}
/**
* @inheritdoc
*/
public function clearCache()
{
return $this;
}
/**
* @inheritdoc
*/
public function clearAllCache(\DateTime $date = null)
{
return $this;
}
}