From e20bb42f9850163b3129ed5c53308933e3b34245 Mon Sep 17 00:00:00 2001 From: aynsix Date: Wed, 5 Feb 2020 11:21:45 +0300 Subject: [PATCH 01/25] hide cterms ckbox if not relevant --- templates/web/admin/fields/index.html.twig | 14 ++++++++++++++ templates/web/admin/fields/templates.html.twig | 14 ++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/templates/web/admin/fields/index.html.twig b/templates/web/admin/fields/index.html.twig index e8198a324f..531ef168eb 100644 --- a/templates/web/admin/fields/index.html.twig +++ b/templates/web/admin/fields/index.html.twig @@ -27,3 +27,17 @@ {# bootstrap admin field backbone application #} + + diff --git a/templates/web/admin/fields/templates.html.twig b/templates/web/admin/fields/templates.html.twig index 0dc661fc5c..a021a296c2 100644 --- a/templates/web/admin/fields/templates.html.twig +++ b/templates/web/admin/fields/templates.html.twig @@ -261,12 +261,14 @@ - - - - - /> - + + +
> + +
+ From fc53e158da3cd06b3d12bca0e2cd9ad059a24935 Mon Sep 17 00:00:00 2001 From: aynsix Date: Wed, 5 Feb 2020 18:06:04 +0300 Subject: [PATCH 02/25] fix visible ckbox in the right side --- templates/web/admin/fields/index.html.twig | 14 -------------- templates/web/admin/fields/templates.html.twig | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/templates/web/admin/fields/index.html.twig b/templates/web/admin/fields/index.html.twig index 531ef168eb..e8198a324f 100644 --- a/templates/web/admin/fields/index.html.twig +++ b/templates/web/admin/fields/index.html.twig @@ -27,17 +27,3 @@ {# bootstrap admin field backbone application #} - - diff --git a/templates/web/admin/fields/templates.html.twig b/templates/web/admin/fields/templates.html.twig index a021a296c2..c8802adc9a 100644 --- a/templates/web/admin/fields/templates.html.twig +++ b/templates/web/admin/fields/templates.html.twig @@ -263,7 +263,7 @@ -
> +
From a20962e9bac2c852e7de316f1222d2834f04b5f9 Mon Sep 17 00:00:00 2001 From: aynsix Date: Fri, 7 Feb 2020 12:35:57 +0300 Subject: [PATCH 03/25] hide chkbox --- templates/web/admin/fields/index.html.twig | 14 ++++++++++++++ templates/web/admin/fields/templates.html.twig | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/templates/web/admin/fields/index.html.twig b/templates/web/admin/fields/index.html.twig index e8198a324f..531ef168eb 100644 --- a/templates/web/admin/fields/index.html.twig +++ b/templates/web/admin/fields/index.html.twig @@ -27,3 +27,17 @@ {# bootstrap admin field backbone application #} + + diff --git a/templates/web/admin/fields/templates.html.twig b/templates/web/admin/fields/templates.html.twig index c8802adc9a..63117d56af 100644 --- a/templates/web/admin/fields/templates.html.twig +++ b/templates/web/admin/fields/templates.html.twig @@ -263,12 +263,13 @@ -
+
>
+
From e75a483cd5ba72938b9ce06ad9161c946be8e5bd Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Thu, 6 Feb 2020 19:40:59 +0100 Subject: [PATCH 04/25] PHRAS-2913_create-tokens-faster_4.1 - create token by sql - create feedentry tokens in bulk-sql --- .../Controller/Prod/FeedController.php | 5 + .../Event/Subscriber/FeedEntrySubscriber.php | 69 ++++++----- .../Model/Manipulator/TokenManipulator.php | 107 ++++++++++++++---- .../Model/Repositories/TokenRepository.php | 5 + 4 files changed, 140 insertions(+), 46 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Prod/FeedController.php b/lib/Alchemy/Phrasea/Controller/Prod/FeedController.php index 2bc333f45c..39a0a8f48d 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/FeedController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/FeedController.php @@ -19,8 +19,10 @@ use Alchemy\Phrasea\Core\PhraseaEvents; use Alchemy\Phrasea\Feed\Aggregate; use Alchemy\Phrasea\Feed\Link\AggregateLinkGenerator; use Alchemy\Phrasea\Feed\Link\FeedLinkGenerator; +use Alchemy\Phrasea\Model\Entities\Feed; use Alchemy\Phrasea\Model\Entities\FeedEntry; use Alchemy\Phrasea\Model\Entities\FeedItem; +use Alchemy\Phrasea\Model\Entities\FeedPublisher; use Alchemy\Phrasea\Model\Repositories\FeedEntryRepository; use Alchemy\Phrasea\Model\Repositories\FeedItemRepository; use Alchemy\Phrasea\Model\Repositories\FeedPublisherRepository; @@ -46,6 +48,7 @@ class FeedController extends Controller } public function createFeedEntryAction(Request $request) { + /** @var Feed $feed */ $feed = $this->getFeedRepository()->find($request->request->get('feed_id')); if (null === $feed) { @@ -53,6 +56,8 @@ class FeedController extends Controller } $user = $this->getAuthenticatedUser(); + + /** @var FeedPublisher $publisher */ $publisher = $this->getFeedPublisherRepository()->findOneBy([ 'feed' => $feed, 'user' => $user, diff --git a/lib/Alchemy/Phrasea/Core/Event/Subscriber/FeedEntrySubscriber.php b/lib/Alchemy/Phrasea/Core/Event/Subscriber/FeedEntrySubscriber.php index 03a72a95b4..8bc7cb2b52 100644 --- a/lib/Alchemy/Phrasea/Core/Event/Subscriber/FeedEntrySubscriber.php +++ b/lib/Alchemy/Phrasea/Core/Event/Subscriber/FeedEntrySubscriber.php @@ -13,7 +13,9 @@ namespace Alchemy\Phrasea\Core\Event\Subscriber; use Alchemy\Phrasea\Core\Event\FeedEntryEvent; use Alchemy\Phrasea\Core\PhraseaEvents; +use Alchemy\Phrasea\Model\Entities\User; use Alchemy\Phrasea\Model\Entities\WebhookEvent; +use Alchemy\Phrasea\Model\Manipulator\TokenManipulator; use Alchemy\Phrasea\Notification\Receiver; use Alchemy\Phrasea\Notification\Mail\MailInfoNewPublication; @@ -53,36 +55,43 @@ class FeedEntrySubscriber extends AbstractNotificationSubscriber do { $results = $Query->limit($start, $perLoop)->execute()->get_results(); - foreach ($results as $user_to_notif) { - $mailed = false; + $users_emailed = []; // for all users + $users_to_email = []; // list only users who must be emailed (=create tokens) - if ($params['notify_email'] && $this->shouldSendNotificationFor($user_to_notif, 'eventsmanager_notify_feed')) { - $readyToSend = false; - try { - $token = $this->app['manipulator.token']->createFeedEntryToken($user_to_notif, $entry); - $url = $this->app->url('lightbox', ['LOG' => $token->getValue()]); - - $receiver = Receiver::fromUser($user_to_notif); - $readyToSend = true; - } catch (\Exception $e) { - - } - - if ($readyToSend) { - $mail = MailInfoNewPublication::create($this->app, $receiver); - $mail->setButtonUrl($url); - $mail->setAuthor($entry->getAuthorName()); - $mail->setTitle($entry->getTitle()); - - $this->deliver($mail); - $mailed = true; - } + /** @var User $user */ + foreach ($results as $user) { + $users_emailed[$user->getId()] = false; + if ($params['notify_email'] && $this->shouldSendNotificationFor($user, 'eventsmanager_notify_feed')) { + $users_to_email[$user->getId()] = $user; } - - $this->app['events-manager']->notify($user_to_notif->getId(), 'eventsmanager_notify_feed', $datas, $mailed); } + + // get many tokens in one shot + $tokens = $this->getTokenManipulator()->createFeedEntryTokens($users_to_email, $entry); + foreach($tokens as $token) { + try { + $url = $this->app->url('lightbox', ['LOG' => $token->getValue()]); + $receiver = Receiver::fromUser($token->getUser()); + + $mail = MailInfoNewPublication::create($this->app, $receiver); + $mail->setButtonUrl($url); + $mail->setAuthor($entry->getAuthorName()); + $mail->setTitle($entry->getTitle()); + + $this->deliver($mail); + $users_emailed[$token->getUser()->getId()] = true; + } + catch (\Exception $e) { + // no-op + } + } + foreach($users_emailed as $id => $emailed) { + $this->app['events-manager']->notify($id, 'eventsmanager_notify_feed', $datas, $emailed); + } + $start += $perLoop; - } while (count($results) > 0); + } + while (count($results) > 0); } public static function getSubscribedEvents() @@ -91,4 +100,12 @@ class FeedEntrySubscriber extends AbstractNotificationSubscriber PhraseaEvents::FEED_ENTRY_CREATE => 'onCreate', ]; } + + /** + * @return TokenManipulator + */ + private function getTokenManipulator() + { + return $this->app['manipulator.token']; + } } diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index 2a43363693..c2e6786fe4 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -62,29 +62,45 @@ class TokenManipulator implements ManipulatorInterface * * @return Token */ - public function create(User $user = null, $type, \DateTime $expiration = null, $data = null) + public function create(User $user, $type, \DateTime $expiration = null, $data = null) { - $this->removeExpiredTokens(); + static $stmt = null; - $n = 0; - do { - if ($n++ > 1024) { - throw new \RuntimeException('Unable to create a token.'); + // $this->removeExpiredTokens(); + + if($stmt === null) { + $conn = $this->repository->getEntityManager()->getConnection(); + $sql = "INSERT INTO Tokens (value, user_id, type, data, created, updated, expiration)\n" + . " VALUES(:value, :user_id, :type, :data, :created, :updated, :expiration)"; + $stmt = $conn->prepare($sql); + } + + $token = null; + $now = (new \DateTime())->format('Y-m-d H:i:s'); + $stmtParms = [ + ':value' => null, + ':user_id' => $user->getId(), + ':type' => $type, + ':data' => $data, + ':created' => $now, + ':updated' => $now, + ':expiration' => $expiration + ]; + for($try=0; $try<1024; $try++) { + $stmtParms['value'] = $this->random->generateString(32, self::LETTERS_AND_NUMBERS); + if($stmt->execute($stmtParms) === true) { + $token = new Token(); + $token->setUser($user) + ->setType($type) + ->setValue($stmtParms['value']) + ->setExpiration($expiration) + ->setData($data); + break; } - $value = $this->random->generateString(32, self::LETTERS_AND_NUMBERS); - $found = null !== $this->om->getRepository('Phraseanet:Token')->find($value); - } while ($found); - - $token = new Token(); - - $token->setUser($user) - ->setType($type) - ->setValue($value) - ->setExpiration($expiration) - ->setData($data); - - $this->om->persist($token); - $this->om->flush(); + } + if ($token === null) { + throw new \RuntimeException('Unable to create a token.'); + } return $token; } @@ -126,6 +142,57 @@ class TokenManipulator implements ManipulatorInterface return $this->create($user, self::TYPE_FEED_ENTRY, null, $entry->getId()); } + /** + * Create feedEntryTokens for many users in one shot + * + * @param User[] $users + * @param FeedEntry $entry + * @return Token[] + * @throws \Doctrine\DBAL\DBALException + */ + public function createFeedEntryTokens($users, FeedEntry $entry) + { + // $this->removeExpiredTokens(); + + $conn = $this->repository->getEntityManager()->getConnection(); + + // use an optimized tmp table we can fill fast (only 2 values changes by row, others are default) + $now = $conn->quote((new \DateTime())->format('Y-m-d H:i:s')); + $conn->executeQuery("CREATE TEMPORARY TABLE `tmpTokens` (\n" + . " `value` char(128),\n" + . " `user_id` int(11),\n" + . " `type` char(32) DEFAULT " . $conn->quote(self::TYPE_FEED_ENTRY) . ",\n" + . " `data` int(11) DEFAULT " . $conn->quote($entry->getId()) . ",\n" + . " `created` datetime DEFAULT " . $now . ",\n" + . " `updated` datetime DEFAULT " . $now . ",\n" + . " `expiration` datetime DEFAULT NULL\n" + . ") ENGINE=MEMORY;" + ); + + $tokens = []; + $sql = ""; + foreach ($users as $user) { + $value = $this->random->generateString(32, self::LETTERS_AND_NUMBERS) . $user->getId(); + // todo: don't build a too long sql, we should flush/run into temp table if l>limit. + // But for now we trust that 100 (see FeedEntrySsuscriber) tokens is ok + $sql .= ($sql?',':'') . ('(' . $conn->quote($value) . ',' . $conn->quote($user->getId()) . ')'); + + $token = new Token(); + $token->setUser($user) + ->setType(self::TYPE_FEED_ENTRY) + ->setValue($value) + ->setExpiration(null) + ->setData($entry->getId()); + $tokens[] = $token; + } + + $conn->executeQuery("INSERT INTO tmpTokens (`value`, `user_id`) VALUES " . $sql); + $conn->executeQuery("INSERT INTO Tokens SELECT * FROM tmpTokens"); + $conn->executeQuery("DROP TABLE tmpTokens"); + + return $tokens; + } + /** * @param User $user * @param $data diff --git a/lib/Alchemy/Phrasea/Model/Repositories/TokenRepository.php b/lib/Alchemy/Phrasea/Model/Repositories/TokenRepository.php index ad45425f7c..8e40f9ebd8 100644 --- a/lib/Alchemy/Phrasea/Model/Repositories/TokenRepository.php +++ b/lib/Alchemy/Phrasea/Model/Repositories/TokenRepository.php @@ -72,4 +72,9 @@ class TokenRepository extends EntityRepository return $query->getResult(); } + + public function getEntityManager() + { + return parent::getEntityManager(); + } } From 746c9076d4e4d9d57ba9ec0a63e5576b09e1ad10 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Fri, 7 Feb 2020 19:34:30 +0100 Subject: [PATCH 05/25] fix malformed sql --- lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index c2e6786fe4..e727f8dada 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -84,7 +84,7 @@ class TokenManipulator implements ManipulatorInterface ':data' => $data, ':created' => $now, ':updated' => $now, - ':expiration' => $expiration + ':expiration' => ($expiration === null ? null : $expiration->format('Y-m-d H:i:s')) ]; for($try=0; $try<1024; $try++) { $stmtParms['value'] = $this->random->generateString(32, self::LETTERS_AND_NUMBERS); From 18993043f24d331ae20858b12c2492f42fc64d88 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Fri, 7 Feb 2020 22:14:16 +0100 Subject: [PATCH 06/25] fake change to re-run tests --- lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index e727f8dada..b3cec7dfc9 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -22,7 +22,6 @@ use RandomLib\Generator; class TokenManipulator implements ManipulatorInterface { const LETTERS_AND_NUMBERS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; - const TYPE_FEED_ENTRY = 'FEED_ENTRY'; const TYPE_PASSWORD = 'password'; const TYPE_ACCOUNT_UNLOCK = 'account-unlock'; From 419440f5c2e4d8a4a0ae64184ff602fbe99e8528 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Fri, 7 Feb 2020 22:33:18 +0100 Subject: [PATCH 07/25] fix for failing test --- lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index b3cec7dfc9..9e1475cd4c 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -61,7 +61,7 @@ class TokenManipulator implements ManipulatorInterface * * @return Token */ - public function create(User $user, $type, \DateTime $expiration = null, $data = null) + public function create($user, $type, \DateTime $expiration = null, $data = null) { static $stmt = null; From 83484e3bd06ca3df59c6b5e59020ee4ed8eb6c98 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Sat, 8 Feb 2020 09:58:57 +0100 Subject: [PATCH 08/25] fix for unit-testing on null user (to dig : why ?...) --- lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index 9e1475cd4c..6314e9e614 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -78,12 +78,12 @@ class TokenManipulator implements ManipulatorInterface $now = (new \DateTime())->format('Y-m-d H:i:s'); $stmtParms = [ ':value' => null, - ':user_id' => $user->getId(), + ':user_id' => $user ? $user->getId() : null, ':type' => $type, ':data' => $data, ':created' => $now, ':updated' => $now, - ':expiration' => ($expiration === null ? null : $expiration->format('Y-m-d H:i:s')) + ':expiration' => $expiration ? $expiration->format('Y-m-d H:i:s') : null ]; for($try=0; $try<1024; $try++) { $stmtParms['value'] = $this->random->generateString(32, self::LETTERS_AND_NUMBERS); From e4d3da93365ff452799fbffb9a03969392161486 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Sat, 8 Feb 2020 22:29:52 +0100 Subject: [PATCH 09/25] rollback : can't create token in sql since the om will cry during update. Can't mix om and sql. --- .../Model/Manipulator/TokenManipulator.php | 56 +++++++------------ 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index 6314e9e614..607cbc6cc6 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -61,45 +61,29 @@ class TokenManipulator implements ManipulatorInterface * * @return Token */ - public function create($user, $type, \DateTime $expiration = null, $data = null) + public function create(User $user = null, $type, \DateTime $expiration = null, $data = null) { - static $stmt = null; + $this->removeExpiredTokens(); - // $this->removeExpiredTokens(); - - if($stmt === null) { - $conn = $this->repository->getEntityManager()->getConnection(); - $sql = "INSERT INTO Tokens (value, user_id, type, data, created, updated, expiration)\n" - . " VALUES(:value, :user_id, :type, :data, :created, :updated, :expiration)"; - $stmt = $conn->prepare($sql); - } - - $token = null; - $now = (new \DateTime())->format('Y-m-d H:i:s'); - $stmtParms = [ - ':value' => null, - ':user_id' => $user ? $user->getId() : null, - ':type' => $type, - ':data' => $data, - ':created' => $now, - ':updated' => $now, - ':expiration' => $expiration ? $expiration->format('Y-m-d H:i:s') : null - ]; - for($try=0; $try<1024; $try++) { - $stmtParms['value'] = $this->random->generateString(32, self::LETTERS_AND_NUMBERS); - if($stmt->execute($stmtParms) === true) { - $token = new Token(); - $token->setUser($user) - ->setType($type) - ->setValue($stmtParms['value']) - ->setExpiration($expiration) - ->setData($data); - break; + $n = 0; + do { + if ($n++ > 1024) { + throw new \RuntimeException('Unable to create a token.'); } - } - if ($token === null) { - throw new \RuntimeException('Unable to create a token.'); - } + $value = $this->random->generateString(32, self::LETTERS_AND_NUMBERS); + $found = null !== $this->om->getRepository('Phraseanet:Token')->find($value); + } while ($found); + + $token = new Token(); + + $token->setUser($user) + ->setType($type) + ->setValue($value) + ->setExpiration($expiration) + ->setData($data); + + $this->om->persist($token); + $this->om->flush(); return $token; } From 5f7ca7c9a6c7914586b621b0e42f5aa3a5887256 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Mon, 10 Feb 2020 11:16:41 +0100 Subject: [PATCH 10/25] add indexes to table "Tokens" --- lib/Alchemy/Phrasea/Core/Version.php | 2 +- lib/Alchemy/Phrasea/Model/Entities/Token.php | 9 ++- lib/classes/patch/410alpha21a.php | 75 ++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 lib/classes/patch/410alpha21a.php diff --git a/lib/Alchemy/Phrasea/Core/Version.php b/lib/Alchemy/Phrasea/Core/Version.php index 5f488cdd5a..3a3ff63afc 100644 --- a/lib/Alchemy/Phrasea/Core/Version.php +++ b/lib/Alchemy/Phrasea/Core/Version.php @@ -16,7 +16,7 @@ class Version /** * @var string */ - private $number = '4.1.0-alpha.20a'; + private $number = '4.1.0-alpha.21a'; /** * @var string diff --git a/lib/Alchemy/Phrasea/Model/Entities/Token.php b/lib/Alchemy/Phrasea/Model/Entities/Token.php index 576cf05d36..81c632f97d 100644 --- a/lib/Alchemy/Phrasea/Model/Entities/Token.php +++ b/lib/Alchemy/Phrasea/Model/Entities/Token.php @@ -15,7 +15,14 @@ use Doctrine\ORM\Mapping as ORM; use Gedmo\Mapping\Annotation as Gedmo; /** - * @ORM\Table(name="Tokens") + * @ORM\Table(name="Tokens", + * indexes={ + * @ORM\index(name="type", columns={"type"}), + * @ORM\index(name="created", columns={"created"}), + * @ORM\index(name="updated", columns={"updated"}), + * @ORM\index(name="expiration", columns={"expiration"}) + * } + * ) * @ORM\Entity(repositoryClass="Alchemy\Phrasea\Model\Repositories\TokenRepository") */ class Token diff --git a/lib/classes/patch/410alpha21a.php b/lib/classes/patch/410alpha21a.php new file mode 100644 index 0000000000..f691520b1a --- /dev/null +++ b/lib/classes/patch/410alpha21a.php @@ -0,0 +1,75 @@ +release; + } + + /** + * {@inheritdoc} + */ + public function concern() + { + return $this->concern; + } + + /** + * {@inheritdoc} + */ + public function require_all_upgrades() + { + return false; + } + + /** + * {@inheritdoc} + */ + public function getDoctrineMigrations() + { + return []; + } + + /** + * {@inheritdoc} + */ + public function apply(base $appbox, Application $app) + { + foreach(['type', 'created', 'updated', 'expiration'] as $t) { + $sql = "ALTER TABLE `Tokens` ADD INDEX `".$t."` (`".$t."`);"; + try { + $stmt = $appbox->get_connection()->prepare($sql); + $stmt->execute(); + $stmt->closeCursor(); + } + catch (\Exception $e) { + // the inex already exists ? + } + } + + return true; + } +} From 211ba02f595f0ab52ad6d9861aa63db008766c53 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Mon, 10 Feb 2020 11:39:42 +0100 Subject: [PATCH 11/25] bump patch to 22a --- lib/Alchemy/Phrasea/Core/Version.php | 2 +- lib/classes/patch/{410alpha21a.php => 410alpha22a.php} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename lib/classes/patch/{410alpha21a.php => 410alpha22a.php} (93%) diff --git a/lib/Alchemy/Phrasea/Core/Version.php b/lib/Alchemy/Phrasea/Core/Version.php index 3a3ff63afc..d17062de4c 100644 --- a/lib/Alchemy/Phrasea/Core/Version.php +++ b/lib/Alchemy/Phrasea/Core/Version.php @@ -16,7 +16,7 @@ class Version /** * @var string */ - private $number = '4.1.0-alpha.21a'; + private $number = '4.1.0-alpha.22a'; /** * @var string diff --git a/lib/classes/patch/410alpha21a.php b/lib/classes/patch/410alpha22a.php similarity index 93% rename from lib/classes/patch/410alpha21a.php rename to lib/classes/patch/410alpha22a.php index f691520b1a..6a80bd058e 100644 --- a/lib/classes/patch/410alpha21a.php +++ b/lib/classes/patch/410alpha22a.php @@ -11,10 +11,10 @@ use Alchemy\Phrasea\Application; -class patch_410alpha21a implements patchInterface +class patch_410alpha22a implements patchInterface { /** @var string */ - private $release = '4.1.0-alpha.21a'; + private $release = '4.1.0-alpha.22a'; /** @var array */ private $concern = [base::APPLICATION_BOX]; From 5a794c493a61b9434acfee62a3942afb909603f4 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Tue, 11 Feb 2020 15:17:13 +0100 Subject: [PATCH 12/25] create feedEntriesTokens in bulk / doctrine. No more sql --- .../Model/Manipulator/TokenManipulator.php | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index 607cbc6cc6..56c4e9fc36 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -137,28 +137,9 @@ class TokenManipulator implements ManipulatorInterface { // $this->removeExpiredTokens(); - $conn = $this->repository->getEntityManager()->getConnection(); - - // use an optimized tmp table we can fill fast (only 2 values changes by row, others are default) - $now = $conn->quote((new \DateTime())->format('Y-m-d H:i:s')); - $conn->executeQuery("CREATE TEMPORARY TABLE `tmpTokens` (\n" - . " `value` char(128),\n" - . " `user_id` int(11),\n" - . " `type` char(32) DEFAULT " . $conn->quote(self::TYPE_FEED_ENTRY) . ",\n" - . " `data` int(11) DEFAULT " . $conn->quote($entry->getId()) . ",\n" - . " `created` datetime DEFAULT " . $now . ",\n" - . " `updated` datetime DEFAULT " . $now . ",\n" - . " `expiration` datetime DEFAULT NULL\n" - . ") ENGINE=MEMORY;" - ); - $tokens = []; - $sql = ""; foreach ($users as $user) { $value = $this->random->generateString(32, self::LETTERS_AND_NUMBERS) . $user->getId(); - // todo: don't build a too long sql, we should flush/run into temp table if l>limit. - // But for now we trust that 100 (see FeedEntrySsuscriber) tokens is ok - $sql .= ($sql?',':'') . ('(' . $conn->quote($value) . ',' . $conn->quote($user->getId()) . ')'); $token = new Token(); $token->setUser($user) @@ -167,11 +148,11 @@ class TokenManipulator implements ManipulatorInterface ->setExpiration(null) ->setData($entry->getId()); $tokens[] = $token; - } - $conn->executeQuery("INSERT INTO tmpTokens (`value`, `user_id`) VALUES " . $sql); - $conn->executeQuery("INSERT INTO Tokens SELECT * FROM tmpTokens"); - $conn->executeQuery("DROP TABLE tmpTokens"); + $this->om->persist($token); + } + $this->om->flush(); + $this->om->clear(); return $tokens; } From 2ccdcfed27969db0a5e19ad5fd7cddf6e46284b0 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 22 Jan 2020 12:26:34 +0100 Subject: [PATCH 13/25] PHRAS-2879_facet-order_4.1 save all facet settings in conf, order is preserved in admin/search-engine-settings. todo : ux to re-order facets todo : return facets into query results in this setting order. todo : migration from dbox settings (field struct) to conf ; remove field setting from admin --- .../Admin/SearchEngineController.php | 6 +- .../Elastic/ElasticsearchOptions.php | 73 ++++----- .../Elastic/ElasticsearchSettingsFormType.php | 141 ++++++++++++------ 3 files changed, 135 insertions(+), 85 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php index 8cf04e7839..8073027302 100644 --- a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php +++ b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php @@ -13,6 +13,7 @@ namespace Alchemy\Phrasea\Controller\Admin; use Alchemy\Phrasea\Controller\Controller; use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchSettingsFormType; use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions; +use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Symfony\Component\Form\FormInterface; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; @@ -85,7 +86,10 @@ class SearchEngineController extends Controller */ private function getConfigurationForm(ElasticsearchOptions $options) { - return $this->app->form(new ElasticsearchSettingsFormType(), $options, [ + /** @var GlobalStructure $g */ + $g = $this->app['search_engine.structure']; + + return $this->app->form(new ElasticsearchSettingsFormType($g, $options), $options, [ 'action' => $this->app->url('admin_searchengine_form'), ]); } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index 0532f85c30..b9b3d5a054 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -9,6 +9,9 @@ */ namespace Alchemy\Phrasea\SearchEngine\Elastic; +use igorw; + + class ElasticsearchOptions { const POPULATE_ORDER_RID = "RECORD_ID"; @@ -58,13 +61,8 @@ class ElasticsearchOptions 'populate_direction' => self::POPULATE_DIRECTION_DESC, 'activeTab' => null, ]; - - foreach(self::getAggregableTechnicalFields() as $k => $f) { - $defaultOptions[$k.'_limit'] = 0; - } $options = array_replace($defaultOptions, $options); - $self = new self(); $self->setHost($options['host']); $self->setPort($options['port']); @@ -76,11 +74,10 @@ class ElasticsearchOptions $self->setPopulateOrder($options['populate_order']); $self->setPopulateDirection($options['populate_direction']); $self->setActiveTab($options['activeTab']); - foreach(self::getAggregableTechnicalFields() as $k => $f) { - $self->setAggregableFieldLimit($k, $options[$k.'_limit']); + foreach($options['aggregates'] as $fieldname=>$attributes) { + $self->setAggregableField($fieldname, $attributes); } - return $self; } @@ -99,10 +96,11 @@ class ElasticsearchOptions 'highlight' => $this->highlight, 'populate_order' => $this->populateOrder, 'populate_direction' => $this->populateDirection, - 'activeTab' => $this->activeTab + 'activeTab' => $this->activeTab, + 'aggregates' => [] ]; - foreach(self::getAggregableTechnicalFields() as $k => $f) { - $ret[$k.'_limit'] = $this->getAggregableFieldLimit($k); + foreach($this->_customValues['aggregates'] as $fieldname=>$attributes) { + $ret['aggregates'][$fieldname] = $attributes; } return $ret; @@ -220,14 +218,19 @@ class ElasticsearchOptions $this->highlight = $highlight; } - public function setAggregableFieldLimit($key, $value) + public function setAggregableField($key, $attributes) { - $this->_customValues[$key.'_limit'] = $value; + $this->_customValues['aggregates'][$key] = $attributes; } - public function getAggregableFieldLimit($key) + public function getAggregableField($key) { - return $this->_customValues[$key.'_limit']; + return $this->_customValues['aggregates'][$key]; + } + + public function getAggregableFields() + { + return $this->_customValues['aggregates']; } public function getActiveTab() @@ -241,56 +244,56 @@ class ElasticsearchOptions public function __get($key) { - if(!array_key_exists($key, $this->_customValues)) { - $this->_customValues[$key] = 0; - } - return $this->_customValues[$key]; + $keys = explode(':', $key); + + return igorw\get_in($this->_customValues, $keys); } public function __set($key, $value) { - $this->_customValues[$key] = $value; + $keys = explode(':', $key); + $this->_customValues = igorw\assoc_in($this->_customValues, $keys, $value); } public static function getAggregableTechnicalFields() { return [ - 'base_aggregate' => [ + 'base' => [ 'type' => 'string', 'label' => 'prod::facet:base_label', 'field' => "database", 'esfield' => 'databox_name', 'query' => 'database:%s', ], - 'collection_aggregate' => [ + 'collection' => [ 'type' => 'string', 'label' => 'prod::facet:collection_label', 'field' => "collection", 'esfield' => 'collection_name', 'query' => 'collection:%s', ], - 'doctype_aggregate' => [ + 'doctype' => [ 'type' => 'string', 'label' => 'prod::facet:doctype_label', 'field' => "type", 'esfield' => 'type', 'query' => 'type:%s', ], - 'camera_model_aggregate' => [ + 'camera_model' => [ 'type' => 'string', 'label' => 'Camera Model', 'field' => "meta.CameraModel", 'esfield' => 'metadata_tags.CameraModel', 'query' => 'meta.CameraModel:%s', ], - 'iso_aggregate' => [ + 'iso' => [ 'type' => 'number', 'label' => 'ISO', 'field' => "meta.ISO", 'esfield' => 'metadata_tags.ISO', 'query' => 'meta.ISO=%s', ], - 'aperture_aggregate' => [ + 'aperture' => [ 'type' => 'number', 'label' => 'Aperture', 'field' => "meta.Aperture", @@ -300,7 +303,7 @@ class ElasticsearchOptions return round($value, 1); }, ], - 'shutterspeed_aggregate' => [ + 'shutterspeed' => [ 'type' => 'number', 'label' => 'Shutter speed', 'field' => "meta.ShutterSpeed", @@ -313,7 +316,7 @@ class ElasticsearchOptions return $value . ' s.'; }, ], - 'flashfired_aggregate' => [ + 'flashfired' => [ 'type' => 'boolean', 'label' => 'FlashFired', 'field' => "meta.FlashFired", @@ -327,49 +330,49 @@ class ElasticsearchOptions return array_key_exists($value, $map) ? $map[$value] : $value; }, ], - 'framerate_aggregate' => [ + 'framerate' => [ 'type' => 'number', 'label' => 'FrameRate', 'field' => "meta.FrameRate", 'esfield' => 'metadata_tags.FrameRate', 'query' => 'meta.FrameRate=%s', ], - 'audiosamplerate_aggregate' => [ + 'audiosamplerate' => [ 'type' => 'number', 'label' => 'Audio Samplerate', 'field' => "meta.AudioSamplerate", 'esfield' => 'metadata_tags.AudioSamplerate', 'query' => 'meta.AudioSamplerate=%s', ], - 'videocodec_aggregate' => [ + 'videocodec' => [ 'type' => 'string', 'label' => 'Video codec', 'field' => "meta.VideoCodec", 'esfield' => 'metadata_tags.VideoCodec', 'query' => 'meta.VideoCodec:%s', ], - 'audiocodec_aggregate' => [ + 'audiocodec' => [ 'type' => 'string', 'label' => 'Audio codec', 'field' => "meta.AudioCodec", 'esfield' => 'metadata_tags.AudioCodec', 'query' => 'meta.AudioCodec:%s', ], - 'orientation_aggregate' => [ + 'orientation' => [ 'type' => 'string', 'label' => 'Orientation', 'field' => "meta.Orientation", 'esfield' => 'metadata_tags.Orientation', 'query' => 'meta.Orientation=%s', ], - 'colorspace_aggregate' => [ + 'colorspace' => [ 'type' => 'string', 'label' => 'Colorspace', 'field' => "meta.ColorSpace", 'esfield' => 'metadata_tags.ColorSpace', 'query' => 'meta.ColorSpace:%s', ], - 'mimetype_aggregate' => [ + 'mimetype' => [ 'type' => 'string', 'label' => 'MimeType', 'field' => "meta.MimeType", diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php index e1aaa3e378..985bda7f0e 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php @@ -9,6 +9,7 @@ */ namespace Alchemy\Phrasea\SearchEngine\Elastic; +use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Symfony\Component\Form\AbstractType; use Symfony\Component\Form\Extension\Core\Type\ChoiceType; use Symfony\Component\Form\FormBuilderInterface; @@ -17,6 +18,18 @@ use Symfony\Component\Validator\Constraints\Range; class ElasticsearchSettingsFormType extends AbstractType { + /** @var GlobalStructure */ + private $globalStructure; + + /** @var ElasticsearchOptions */ + private $esSettings; + + public function __construct(GlobalStructure $g, ElasticsearchOptions $settings) + { + $this->globalStructure = $g; + $this->esSettings = $settings; + } + public function buildForm(FormBuilderInterface $builder, array $options) { $builder @@ -56,59 +69,89 @@ class ElasticsearchSettingsFormType extends AbstractType ->add('minScore', 'integer', [ 'label' => 'Thesaurus Min score', 'constraints' => new Range(['min' => 0]), - ]); + ]) + ->add('highlight', 'checkbox', [ + 'label' => 'Activate highlight', + 'required' => false + ]) + // ->add('save', 'submit', [ + // 'attr' => ['class' => 'btn btn-primary'] + // ]) + ->add('esSettingFromIndex', 'button', [ + 'label' => 'Get setting form index', + 'attr' => [ + 'onClick' => 'esSettingFromIndex()', + 'class' => 'btn' + ] + ]) + ->add('dumpField', 'textarea', [ + 'label' => false, + 'required' => false, + 'mapped' => false, + 'attr' => ['class' => 'dumpfield hide'] + ]) + ->add('activeTab', 'hidden'); - foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { - if(array_key_exists('choices', $f)) { - // choices[] : choice_key => choice_value - $choices = $f['choices']; - } - else { - $choices = [ - "10 values" => 10, - "20 values" => 20, - "50 values" => 50, - "100 values" => 100, - "all values" => -1 - ]; - } - // array_unshift($choices, "not aggregated"); // always as first choice - $choices = array_merge(["not aggregated" => 0], $choices); - $builder - ->add($k.'_limit', ChoiceType::class, [ - // 'label' => $f['label'],// . ' ' . 'aggregate limit', - 'choices_as_values' => true, - 'choices' => $choices, - 'attr' => [ - 'class' => 'aggregate' - ] - ]); + // keep aggregates in configuration order with this intermediate array + $aggs = []; + + // helper fct to add aggregate to a tmp list + $addAgg = function($k, $label, $help, $disabled=false, $choices=null) use (&$aggs) { + if(!$choices) { + $choices = [ + "10 values" => 10, + "50 values" => 50, + "100 values" => 100, + "all values" => -1 + ]; } + $choices = array_merge(["not aggregated" => 0], $choices); // add this option always as first choice + $aggs[$k] = [ // default value will be replaced by hardcoded tech fields & all databoxes fields + 'label' => $label, + 'choices_as_values' => true, + 'choices' => $choices, + 'attr' => [ + 'class' => 'aggregate' + ], + 'disabled' => $disabled, + 'help_message' => $help // todo : not displayed ? + ]; + }; - $builder - ->add('highlight', 'checkbox', [ - 'label' => 'Activate highlight', - 'required' => false - ]) -// ->add('save', 'submit', [ -// 'attr' => ['class' => 'btn btn-primary'] -// ]) - ->add('esSettingFromIndex', 'button', [ - 'label' => 'Get setting form index', - 'attr' => [ - 'onClick' => 'esSettingFromIndex()', - 'class' => 'btn' - ] - ]) - ->add('dumpField', 'textarea', [ - 'label' => false, - 'required' => false, - 'mapped' => false, - 'attr' => ['class' => 'dumpfield hide'] - ]) - ->add('activeTab', 'hidden'); + // all fields fron conf + foreach($this->esSettings->getAggregableFields() as $k=>$f) { + // default value will be replaced by hardcoded tech fields & all databoxes fields + $addAgg($k, "/?\\ " . $k, "This field does not exists in current databoxes.", true); + } + + // add or replace hardcoded tech fields + foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { + $choices = array_key_exists('choices', $f) ? $f['choices'] : null; // a tech-field can publish it's own choices + $help = null; + $label = '#' . $k; + if(!array_key_exists('_'.$k, $aggs)) { + $label = "/!\\ " . $label; + $help = "New field, please confirm setting."; + } + $addAgg('_'.$k, $label, $help, false, $choices); + } + + // add or replace all databoxes fields (nb: new db field - unknown in conf - will be a the end) + foreach($this->globalStructure->getAllFields() as $field) { + $k = $label = $field->getName(); + $help = null; + if(!array_key_exists($field->getName(), $aggs)) { + $label = "/!\\ " . $label; + $help = "New field, please confirm setting."; + } + $addAgg($k, $label, $help); // default choices + } + + // populate aggs to form + foreach($aggs as $k=>$agg) { + $builder->add('aggregates:' . $k . ':limit', ChoiceType::class, $agg); + } - ; } public function getName() From 6838198a6928c1677f483d2786356d2a70b7dedf Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 22 Jan 2020 19:09:19 +0100 Subject: [PATCH 14/25] PHRAS-2879_facet-order_4.1 fix setup:install. --- config/configuration.sample.yml | 3 ++- .../Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/configuration.sample.yml b/config/configuration.sample.yml index fbaa368ed9..b7fef2618a 100644 --- a/config/configuration.sample.yml +++ b/config/configuration.sample.yml @@ -28,7 +28,8 @@ main: port: 11211 search-engine: type: phrasea - options: [] + options: + facets: [] task-manager: status: started enabled: true diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index b9b3d5a054..72c2c6674d 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -60,6 +60,7 @@ class ElasticsearchOptions 'populate_order' => self::POPULATE_ORDER_RID, 'populate_direction' => self::POPULATE_DIRECTION_DESC, 'activeTab' => null, + 'facets' => [] ]; $options = array_replace($defaultOptions, $options); From 4305441a0318c923646c6bcc1cb8198db4692bff Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 22 Jan 2020 19:24:31 +0100 Subject: [PATCH 15/25] PHRAS-2879_facet-order_4.1 fix setup:install (cont') --- .../Elastic/ElasticsearchOptions.php | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index 72c2c6674d..22918b8b3a 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -100,8 +100,8 @@ class ElasticsearchOptions 'activeTab' => $this->activeTab, 'aggregates' => [] ]; - foreach($this->_customValues['aggregates'] as $fieldname=>$attributes) { - $ret['aggregates'][$fieldname] = $attributes; + foreach($this->getAggregableFields() as $fieldname=>$attributes) { + $ret['facets'][$fieldname] = $attributes; } return $ret; @@ -221,17 +221,22 @@ class ElasticsearchOptions public function setAggregableField($key, $attributes) { - $this->_customValues['aggregates'][$key] = $attributes; + $facets = $this->getAggregableFields(); + $facets[$key] = $attributes; } public function getAggregableField($key) { - return $this->_customValues['aggregates'][$key]; + $facets = $this->getAggregableFields(); + return array_key_exists($key, $facets) ? $facets[$key] : null; } - public function getAggregableFields() + public function &getAggregableFields() { - return $this->_customValues['aggregates']; + if(!array_key_exists('facets', $this->_customValues) || !is_array($this->_customValues['facets'])) { + $this->_customValues['facets'] = []; + } + return $this->_customValues['facets']; } public function getActiveTab() From 8171b7c1f12af24da4f0f2e5410cd0e959dd0d62 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 22 Jan 2020 19:35:31 +0100 Subject: [PATCH 16/25] PHRAS-2879_facet-order_4.1 fix setup:install (cont' 2) --- .../Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index 22918b8b3a..d65221dc8f 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -38,7 +38,7 @@ class ElasticsearchOptions private $populateDirection; /** @var int[] */ - private $_customValues; + private $_customValues = []; private $activeTab; /** From 53f17e18979abffe7709d26cb7a60553b0dfc308 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Tue, 28 Jan 2020 17:56:30 +0100 Subject: [PATCH 17/25] save modifications in conf, and also write back to databoxes --- .../Admin/SearchEngineController.php | 11 +++++++++ .../Provider/SearchEngineServiceProvider.php | 3 ++- .../Elastic/ElasticSearchEngine.php | 2 +- .../Elastic/ElasticsearchOptions.php | 24 ++++++++++++++++--- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php index 8073027302..565c2d759b 100644 --- a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php +++ b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php @@ -17,6 +17,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure; use Symfony\Component\Form\FormInterface; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; +use databox_descriptionStructure; class SearchEngineController extends Controller { @@ -77,6 +78,16 @@ class SearchEngineController extends Controller */ private function saveElasticSearchOptions(ElasticsearchOptions $configuration) { + // save to databoxes fields for backward compatibility (useless ?) + foreach($configuration->getAggregableFields() as $fname=>$aggregableField) { + foreach ($this->app->getDataboxes() as $databox) { + if(!is_null($f = $databox->get_meta_structure()->get_element_by_name($fname, databox_descriptionStructure::STRICT_COMPARE))) { + $f->set_aggregable($aggregableField['limit'])->save(); + } + } + } + + // save to conf $this->getConf()->set(['main', 'search-engine', 'options'], $configuration->toArray()); } diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index 58f77475a1..faddfae370 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -228,7 +228,8 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['elasticsearch.options'] = $app->share(function ($app) { - $options = ElasticsearchOptions::fromArray($app['conf']->get(['main', 'search-engine', 'options'], [])); + $conf = $app['conf']->get(['main', 'search-engine', 'options'], []); + $options = ElasticsearchOptions::fromArray($conf); if (empty($options->getIndexName())) { $options->setIndexName(strtolower(sprintf('phraseanet_%s', str_replace( diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 63d9ec108e..15482f31d4 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -652,7 +652,7 @@ class ElasticSearchEngine implements SearchEngineInterface $aggs = []; // technical aggregates (enable + optional limit) foreach (ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { - $size = $this->options->getAggregableFieldLimit($k); + $size = $this->options->getAggregableFieldLimit('_'.$k); if ($size !== databox_field::FACET_DISABLED) { if ($size === databox_field::FACET_NO_LIMIT) { $size = ESField::FACET_NO_LIMIT; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index d65221dc8f..14ed77d008 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -9,6 +9,7 @@ */ namespace Alchemy\Phrasea\SearchEngine\Elastic; +use databox_field; use igorw; @@ -219,10 +220,24 @@ class ElasticsearchOptions $this->highlight = $highlight; } + public function setAggregableFieldLimit($key, $value) + { + if(is_null($this->getAggregableField($key))) { + $this->_customValues['facets'][$key] = []; + } + $this->_customValues['facets'][$key]['limit'] = $value; + } + public function setAggregableField($key, $attributes) { - $facets = $this->getAggregableFields(); - $facets[$key] = $attributes; + $this->getAggregableFields(); // ensure facets exists + $this->_customValues['facets'][$key] = $attributes; + } + + public function getAggregableFieldLimit($key) + { + $facet = $this->getAggregableField($key); + return (is_array($facet) && array_key_exists('limit', $facet)) ? $facet['limit'] : databox_field::FACET_DISABLED; } public function getAggregableField($key) @@ -231,7 +246,10 @@ class ElasticsearchOptions return array_key_exists($key, $facets) ? $facets[$key] : null; } - public function &getAggregableFields() + /** + * @return array + */ + public function getAggregableFields() { if(!array_key_exists('facets', $this->_customValues) || !is_array($this->_customValues['facets'])) { $this->_customValues['facets'] = []; From 20839f1dd7bee23296092fefdf1669ab375f472a Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 29 Jan 2020 18:39:15 +0100 Subject: [PATCH 18/25] return facets following the order defined in conf. --- .../Controller/Prod/QueryController.php | 10 +------ .../Provider/SearchEngineServiceProvider.php | 2 +- .../Elastic/ElasticSearchEngine.php | 27 ++++++++++--------- .../Elastic/Search/FacetsResponse.php | 11 ++++++-- .../Elastic/Structure/GlobalStructure.php | 15 ++--------- .../Elastic/Structure/LimitedStructure.php | 8 ------ .../Elastic/Structure/Structure.php | 5 ---- 7 files changed, 28 insertions(+), 50 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php index d327e860c5..82fab9e671 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php @@ -360,6 +360,7 @@ class QueryController extends Controller // add technical fields $fieldsInfosByName = []; foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { + $k = '_'.$k; $fieldsInfosByName[$k] = $f; $fieldsInfosByName[$k]['trans_label'] = $this->app->trans($f['label']); $fieldsInfosByName[$k]['labels'] = []; @@ -433,24 +434,15 @@ class QueryController extends Controller // populates facets (aggregates) $facets = []; - // $facetClauses = []; foreach ($result->getFacets() as $facet) { $facetName = $facet['name']; if(array_key_exists($facetName, $fieldsInfosByName)) { - $f = $fieldsInfosByName[$facetName]; - $facet['label'] = $f['trans_label']; $facet['labels'] = $f['labels']; $facet['type'] = strtoupper($f['type']) . "-AGGREGATE"; $facets[] = $facet; - - // $facetClauses[] = [ - // 'type' => strtoupper($f['type']) . "-AGGREGATE", - // 'field' => $f['field'], - // 'facet' => $facet - // ]; } } diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index faddfae370..e342bf8380 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -97,7 +97,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface }); $app['elasticsearch.facets_response.factory'] = $app->protect(function (array $response) use ($app) { - return new FacetsResponse(new Escaper(), $response, $app['search_engine.structure']); + return new FacetsResponse($app['elasticsearch.options'], new Escaper(), $response, $app['search_engine.structure']); }); return $app; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 15482f31d4..df89adf167 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -663,22 +663,25 @@ class ElasticSearchEngine implements SearchEngineInterface 'size' => $size ] ]; - $aggs[$k] = $agg; + $aggs['_'.$k] = $agg; } } // fields aggregates $structure = $this->context_factory->getLimitedStructure($options); - foreach ($structure->getFacetFields() as $name => $field) { - // 2015-05-26 (mdarse) Removed databox filtering. - // It was already done by the ACL filter in the query scope, so no - // document that shouldn't be displayed can go this far. - $agg = [ - 'terms' => [ - 'field' => $field->getIndexField(true), - 'size' => $field->getFacetValuesLimit() - ] - ]; - $aggs[$name] = AggregationHelper::wrapPrivateFieldAggregation($field, $agg); + foreach($structure->getAllFields() as $name => $field) { + $size = $this->options->getAggregableFieldLimit($name); + if ($size !== databox_field::FACET_DISABLED) { + if ($size === databox_field::FACET_NO_LIMIT) { + $size = ESField::FACET_NO_LIMIT; + } + $agg = [ + 'terms' => [ + 'field' => $field->getIndexField(true), + 'size' => $size + ] + ]; + $aggs[$name] = AggregationHelper::wrapPrivateFieldAggregation($field, $agg); + } } return $aggs; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/FacetsResponse.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/FacetsResponse.php index 2f2d966c51..efcc6d1c76 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/FacetsResponse.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/FacetsResponse.php @@ -15,7 +15,7 @@ class FacetsResponse private $escaper; private $facets = array(); - public function __construct(Escaper $escaper, array $response, GlobalStructure $structure) + public function __construct(ElasticsearchOptions $options, Escaper $escaper, array $response, GlobalStructure $structure) { $this->escaper = $escaper; @@ -25,7 +25,13 @@ class FacetsResponse $atf = ElasticsearchOptions::getAggregableTechnicalFields(); - foreach ($response['aggregations'] as $name => $aggregation) { + // sort facets respecting the order defined in options + foreach($options->getAggregableFields() as $name=>$foptions) { + if(!array_key_exists($name, $response['aggregations'])) { + continue; + } + $aggregation = $response['aggregations'][$name]; + $tf = null; $valueFormatter = function($v){ return $v; }; // default equality formatter @@ -78,6 +84,7 @@ class FacetsResponse ]; } } + } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php index ea4022dffb..0793c91c7b 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php @@ -35,11 +35,6 @@ final class GlobalStructure implements Structure */ private $private = array(); - /** - * @var Field[] - */ - private $facets = array(); - /** * @var Flag[] */ @@ -145,9 +140,11 @@ final class GlobalStructure implements Structure $this->private[$name] = $field; } + /* if ($field->isFacet() && $field->isSearchable()) { $this->facets[$name] = $field; } + */ if ($field->hasConceptInference()) { $this->thesaurus_fields[$name] = $field; @@ -183,14 +180,6 @@ final class GlobalStructure implements Structure return $this->private; } - /** - * @return Field[] - */ - public function getFacetFields() - { - return $this->facets; - } - /** * @return Field[] */ diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/LimitedStructure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/LimitedStructure.php index 671bf87c93..053ca6b0e0 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/LimitedStructure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/LimitedStructure.php @@ -47,14 +47,6 @@ final class LimitedStructure implements Structure return $this->limit($this->structure->getPrivateFields()); } - /** - * @return Field[] - */ - public function getFacetFields() - { - return $this->limit($this->structure->getFacetFields()); - } - public function getThesaurusEnabledFields() { return $this->limit($this->structure->getThesaurusEnabledFields()); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php index 3c2be701e1..44d58e9d5f 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Structure.php @@ -33,11 +33,6 @@ interface Structure */ public function getPrivateFields(); - /** - * @return Field[] - */ - public function getFacetFields(); - /** * @return Field[] */ From 8312be6fc8adc015b0088a87310dac7b9decbfe0 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Wed, 29 Jan 2020 19:00:00 +0100 Subject: [PATCH 19/25] removed useless tests --- .../SearchEngine/Structure/StructureTest.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/Alchemy/Tests/Phrasea/SearchEngine/Structure/StructureTest.php b/tests/Alchemy/Tests/Phrasea/SearchEngine/Structure/StructureTest.php index 27f01811c1..b3bde05cd9 100644 --- a/tests/Alchemy/Tests/Phrasea/SearchEngine/Structure/StructureTest.php +++ b/tests/Alchemy/Tests/Phrasea/SearchEngine/Structure/StructureTest.php @@ -20,7 +20,6 @@ class StructureTest extends \PHPUnit_Framework_TestCase $this->assertEmpty($structure->getAllFields()); $this->assertEmpty($structure->getUnrestrictedFields()); $this->assertEmpty($structure->getPrivateFields()); - $this->assertEmpty($structure->getFacetFields()); $this->assertEmpty($structure->getThesaurusEnabledFields()); $this->assertEmpty($structure->getDateFields()); } @@ -95,19 +94,6 @@ class StructureTest extends \PHPUnit_Framework_TestCase $this->assertNotContains($unrestricted_field, $private_fields); } - public function testGetFacetFields() - { - $facet = new Field('foo', FieldMapping::TYPE_STRING, ['facet' => Field::FACET_NO_LIMIT]); - $not_facet = new Field('bar', FieldMapping::TYPE_STRING, ['facet' => Field::FACET_DISABLED]); - $structure = new Structure(); - $structure->add($facet); - $this->assertContains($facet, $structure->getFacetFields()); - $structure->add($not_facet); - $facet_fields = $structure->getFacetFields(); - $this->assertContains($facet, $facet_fields); - $this->assertNotContains($not_facet, $facet_fields); - } - public function testGetDateFields() { $string = new Field('foo', FieldMapping::TYPE_STRING); From b1b35cc92b0a5d657f42000209258b30659ca291 Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Thu, 30 Jan 2020 15:25:40 +0100 Subject: [PATCH 20/25] fix malformed query on tech fields ('_' in query) --- .../Controller/Prod/QueryController.php | 1 - .../Elastic/ElasticSearchEngine.php | 4 +-- .../Elastic/ElasticsearchOptions.php | 30 +++++++++---------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php index 82fab9e671..8afd22d2b3 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/QueryController.php @@ -360,7 +360,6 @@ class QueryController extends Controller // add technical fields $fieldsInfosByName = []; foreach(ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { - $k = '_'.$k; $fieldsInfosByName[$k] = $f; $fieldsInfosByName[$k]['trans_label'] = $this->app->trans($f['label']); $fieldsInfosByName[$k]['labels'] = []; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index df89adf167..2424330031 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -652,7 +652,7 @@ class ElasticSearchEngine implements SearchEngineInterface $aggs = []; // technical aggregates (enable + optional limit) foreach (ElasticsearchOptions::getAggregableTechnicalFields() as $k => $f) { - $size = $this->options->getAggregableFieldLimit('_'.$k); + $size = $this->options->getAggregableFieldLimit($k); if ($size !== databox_field::FACET_DISABLED) { if ($size === databox_field::FACET_NO_LIMIT) { $size = ESField::FACET_NO_LIMIT; @@ -663,7 +663,7 @@ class ElasticSearchEngine implements SearchEngineInterface 'size' => $size ] ]; - $aggs['_'.$k] = $agg; + $aggs[$k] = $agg; } } // fields aggregates diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index 14ed77d008..b469af4d92 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -282,42 +282,42 @@ class ElasticsearchOptions public static function getAggregableTechnicalFields() { return [ - 'base' => [ + '_base' => [ 'type' => 'string', 'label' => 'prod::facet:base_label', 'field' => "database", 'esfield' => 'databox_name', 'query' => 'database:%s', ], - 'collection' => [ + '_collection' => [ 'type' => 'string', 'label' => 'prod::facet:collection_label', 'field' => "collection", 'esfield' => 'collection_name', 'query' => 'collection:%s', ], - 'doctype' => [ + '_doctype' => [ 'type' => 'string', 'label' => 'prod::facet:doctype_label', 'field' => "type", 'esfield' => 'type', 'query' => 'type:%s', ], - 'camera_model' => [ + '_camera_model' => [ 'type' => 'string', 'label' => 'Camera Model', 'field' => "meta.CameraModel", 'esfield' => 'metadata_tags.CameraModel', 'query' => 'meta.CameraModel:%s', ], - 'iso' => [ + '_iso' => [ 'type' => 'number', 'label' => 'ISO', 'field' => "meta.ISO", 'esfield' => 'metadata_tags.ISO', 'query' => 'meta.ISO=%s', ], - 'aperture' => [ + '_aperture' => [ 'type' => 'number', 'label' => 'Aperture', 'field' => "meta.Aperture", @@ -327,7 +327,7 @@ class ElasticsearchOptions return round($value, 1); }, ], - 'shutterspeed' => [ + '_shutterspeed' => [ 'type' => 'number', 'label' => 'Shutter speed', 'field' => "meta.ShutterSpeed", @@ -340,7 +340,7 @@ class ElasticsearchOptions return $value . ' s.'; }, ], - 'flashfired' => [ + '_flashfired' => [ 'type' => 'boolean', 'label' => 'FlashFired', 'field' => "meta.FlashFired", @@ -354,49 +354,49 @@ class ElasticsearchOptions return array_key_exists($value, $map) ? $map[$value] : $value; }, ], - 'framerate' => [ + '_framerate' => [ 'type' => 'number', 'label' => 'FrameRate', 'field' => "meta.FrameRate", 'esfield' => 'metadata_tags.FrameRate', 'query' => 'meta.FrameRate=%s', ], - 'audiosamplerate' => [ + '_audiosamplerate' => [ 'type' => 'number', 'label' => 'Audio Samplerate', 'field' => "meta.AudioSamplerate", 'esfield' => 'metadata_tags.AudioSamplerate', 'query' => 'meta.AudioSamplerate=%s', ], - 'videocodec' => [ + '_videocodec' => [ 'type' => 'string', 'label' => 'Video codec', 'field' => "meta.VideoCodec", 'esfield' => 'metadata_tags.VideoCodec', 'query' => 'meta.VideoCodec:%s', ], - 'audiocodec' => [ + '_audiocodec' => [ 'type' => 'string', 'label' => 'Audio codec', 'field' => "meta.AudioCodec", 'esfield' => 'metadata_tags.AudioCodec', 'query' => 'meta.AudioCodec:%s', ], - 'orientation' => [ + '_orientation' => [ 'type' => 'string', 'label' => 'Orientation', 'field' => "meta.Orientation", 'esfield' => 'metadata_tags.Orientation', 'query' => 'meta.Orientation=%s', ], - 'colorspace' => [ + '_colorspace' => [ 'type' => 'string', 'label' => 'Colorspace', 'field' => "meta.ColorSpace", 'esfield' => 'metadata_tags.ColorSpace', 'query' => 'meta.ColorSpace:%s', ], - 'mimetype' => [ + '_mimetype' => [ 'type' => 'string', 'label' => 'MimeType', 'field' => "meta.MimeType", From 3b8d75d39d34acafa60ef282c32e8f3d85ee2b7a Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Mon, 3 Feb 2020 20:46:53 +0100 Subject: [PATCH 21/25] fix : tech facets did appear twice. --- .../SearchEngine/Elastic/ElasticsearchSettingsFormType.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php index 985bda7f0e..4dbf6899bf 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php @@ -129,11 +129,11 @@ class ElasticsearchSettingsFormType extends AbstractType $choices = array_key_exists('choices', $f) ? $f['choices'] : null; // a tech-field can publish it's own choices $help = null; $label = '#' . $k; - if(!array_key_exists('_'.$k, $aggs)) { + if(!array_key_exists($k, $aggs)) { $label = "/!\\ " . $label; $help = "New field, please confirm setting."; } - $addAgg('_'.$k, $label, $help, false, $choices); + $addAgg($k, $label, $help, false, $choices); } // add or replace all databoxes fields (nb: new db field - unknown in conf - will be a the end) From c7fc2b64222fc9472bf9a2f6118c2cf27bdce1ae Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Thu, 6 Feb 2020 16:25:40 +0100 Subject: [PATCH 22/25] change patch number --- lib/classes/patch/{410alpha20a.php => 410alpha21a.php} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename lib/classes/patch/{410alpha20a.php => 410alpha21a.php} (96%) diff --git a/lib/classes/patch/410alpha20a.php b/lib/classes/patch/410alpha21a.php similarity index 96% rename from lib/classes/patch/410alpha20a.php rename to lib/classes/patch/410alpha21a.php index ff02a35c96..58091e62f2 100644 --- a/lib/classes/patch/410alpha20a.php +++ b/lib/classes/patch/410alpha21a.php @@ -11,10 +11,10 @@ use Alchemy\Phrasea\Application; -class patch_410alpha20a implements patchInterface +class patch_410alpha21a implements patchInterface { /** @var string */ - private $release = '4.1.0-alpha.20a'; + private $release = '4.1.0-alpha.21a'; /** @var array */ private $concern = [base::DATA_BOX]; From eeec1ab716c7d6f838f80a2946fdd1f764f8067b Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Mon, 10 Feb 2020 20:56:46 +0100 Subject: [PATCH 23/25] save facets following the order of the admin form. --- .../Controller/Admin/SearchEngineController.php | 14 +++++++++++++- .../SearchEngine/Elastic/ElasticsearchOptions.php | 12 ++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php index 565c2d759b..04762b3aef 100644 --- a/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php +++ b/lib/Alchemy/Phrasea/Controller/Admin/SearchEngineController.php @@ -33,7 +33,19 @@ class SearchEngineController extends Controller $form->handleRequest($request); if ($form->isValid()) { - $this->saveElasticSearchOptions($form->getData()); + /** @var ElasticsearchOptions $data */ + $data = $form->getData(); + // $q = $request->request->get('elasticsearch_settings'); + $facetNames = []; // rebuild the data "_customValues/facets" list following the form order + foreach($request->request->get('elasticsearch_settings') as $name=>$value) { + $matches = null; + if(preg_match('/^facets:(.+):limit$/', $name, $matches) === 1) { + $facetNames[] = $matches[1]; + } + } + $data->reorderAggregableFields($facetNames); + + $this->saveElasticSearchOptions($data); return $this->app->redirectPath('admin_searchengine_form'); } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index b469af4d92..c471980752 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -257,6 +257,18 @@ class ElasticsearchOptions return $this->_customValues['facets']; } + // set to change the facets order during admin/form save + public function reorderAggregableFields($facetNames) + { + $newFacets = []; + foreach ($facetNames as $name) { + if(($facet = $this->getAggregableField($name)) !== null) { + $newFacets[$name] = $facet; + } + } + $this->_customValues['facets'] = $newFacets; + } + public function getActiveTab() { return $this->activeTab; From 4f7be4c0a97bd7bbf32562147b9d68e44759019c Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Mon, 10 Feb 2020 21:15:02 +0100 Subject: [PATCH 24/25] fix bad merge happened cause last rebase (?) --- .../Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php index c471980752..07e54449eb 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php @@ -76,7 +76,7 @@ class ElasticsearchOptions $self->setPopulateOrder($options['populate_order']); $self->setPopulateDirection($options['populate_direction']); $self->setActiveTab($options['activeTab']); - foreach($options['aggregates'] as $fieldname=>$attributes) { + foreach($options['facets'] as $fieldname=>$attributes) { $self->setAggregableField($fieldname, $attributes); } @@ -99,7 +99,7 @@ class ElasticsearchOptions 'populate_order' => $this->populateOrder, 'populate_direction' => $this->populateDirection, 'activeTab' => $this->activeTab, - 'aggregates' => [] + 'facets' => [] ]; foreach($this->getAggregableFields() as $fieldname=>$attributes) { $ret['facets'][$fieldname] = $attributes; From 722b1e37505e39d4e3b67a4b3056e92a839f7a9d Mon Sep 17 00:00:00 2001 From: Jean-Yves Gaulier Date: Mon, 17 Feb 2020 20:16:19 +0100 Subject: [PATCH 25/25] include Harrys's new templates fix form restore fix Version --- lib/Alchemy/Phrasea/Core/Version.php | 2 +- .../Elastic/ElasticsearchSettingsFormType.php | 2 +- .../web/admin/fields/templates.html.twig | 14 ++-- .../general-aggregation.html.twig | 66 +++++++++++++++++-- 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/lib/Alchemy/Phrasea/Core/Version.php b/lib/Alchemy/Phrasea/Core/Version.php index 5f488cdd5a..3a3ff63afc 100644 --- a/lib/Alchemy/Phrasea/Core/Version.php +++ b/lib/Alchemy/Phrasea/Core/Version.php @@ -16,7 +16,7 @@ class Version /** * @var string */ - private $number = '4.1.0-alpha.20a'; + private $number = '4.1.0-alpha.21a'; /** * @var string diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php index 4dbf6899bf..1f2fa3d1e2 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchSettingsFormType.php @@ -149,7 +149,7 @@ class ElasticsearchSettingsFormType extends AbstractType // populate aggs to form foreach($aggs as $k=>$agg) { - $builder->add('aggregates:' . $k . ':limit', ChoiceType::class, $agg); + $builder->add('facets:' . $k . ':limit', ChoiceType::class, $agg); } } diff --git a/templates/web/admin/fields/templates.html.twig b/templates/web/admin/fields/templates.html.twig index 0dc661fc5c..016534fd52 100644 --- a/templates/web/admin/fields/templates.html.twig +++ b/templates/web/admin/fields/templates.html.twig @@ -238,14 +238,12 @@ - + <%= field['aggregable'] == "0" ? '{% trans %}Not aggregated{% endtrans %}' : '' %> + <%= field['aggregable'] == "10" ? '10 values' : '' %> + <%= field['aggregable'] == "20" ? '20 values' : '' %> + <%= field['aggregable'] == "50" ? '50 values' : '' %> + <%= field['aggregable'] == "100" ? '100 values' : '' %> + <%= field['aggregable'] == "-1" ? 'All values' : '' %> diff --git a/templates/web/admin/search-engine/general-aggregation.html.twig b/templates/web/admin/search-engine/general-aggregation.html.twig index d4b83ba31b..0c377d2ecd 100644 --- a/templates/web/admin/search-engine/general-aggregation.html.twig +++ b/templates/web/admin/search-engine/general-aggregation.html.twig @@ -1,12 +1,37 @@
+ +
+ {#{{ 'See' | trans }} : #} + + + + + + + +
    {% for formdata in form %} {% set attr = formdata.vars['attr']|join(',') %} + {% set label = formdata.vars['label']|join(',') %} {% if attr == 'aggregate' %} -
  • +
  • + +
    + + {{ form_label(formdata, null, { 'label_attr': {'class': 'label-aggregation'} @@ -22,6 +47,39 @@ {% endif %} {% endfor %} - - \ No newline at end of file + + + + \ No newline at end of file