PHRAS-2913_create-tokens-faster_4.1

- create token by sql
- create feedentry tokens in bulk-sql

lib/Alchemy/Phrasea/Controller/Prod/FeedController.php

@@ -19,8 +19,10 @@ use Alchemy\Phrasea\Core\PhraseaEvents;
 use Alchemy\Phrasea\Feed\Aggregate;
 use Alchemy\Phrasea\Feed\Link\AggregateLinkGenerator;
 use Alchemy\Phrasea\Feed\Link\FeedLinkGenerator;
+use Alchemy\Phrasea\Model\Entities\Feed;
 use Alchemy\Phrasea\Model\Entities\FeedEntry;
 use Alchemy\Phrasea\Model\Entities\FeedItem;
+use Alchemy\Phrasea\Model\Entities\FeedPublisher;
 use Alchemy\Phrasea\Model\Repositories\FeedEntryRepository;
 use Alchemy\Phrasea\Model\Repositories\FeedItemRepository;
 use Alchemy\Phrasea\Model\Repositories\FeedPublisherRepository;
@@ -46,6 +48,7 @@ class FeedController extends Controller
     }
 
     public function createFeedEntryAction(Request $request) {
+        /** @var Feed $feed */
         $feed = $this->getFeedRepository()->find($request->request->get('feed_id'));
 
         if (null === $feed) {
@@ -53,6 +56,8 @@ class FeedController extends Controller
         }
 
         $user = $this->getAuthenticatedUser();
+
+        /** @var FeedPublisher $publisher */
         $publisher = $this->getFeedPublisherRepository()->findOneBy([
             'feed' => $feed,
             'user' => $user,

lib/Alchemy/Phrasea/Core/Event/Subscriber/FeedEntrySubscriber.php

@@ -13,7 +13,9 @@ namespace Alchemy\Phrasea\Core\Event\Subscriber;
 
 use Alchemy\Phrasea\Core\Event\FeedEntryEvent;
 use Alchemy\Phrasea\Core\PhraseaEvents;
+use Alchemy\Phrasea\Model\Entities\User;
 use Alchemy\Phrasea\Model\Entities\WebhookEvent;
+use Alchemy\Phrasea\Model\Manipulator\TokenManipulator;
 use Alchemy\Phrasea\Notification\Receiver;
 use Alchemy\Phrasea\Notification\Mail\MailInfoNewPublication;
 
@@ -53,36 +55,43 @@ class FeedEntrySubscriber extends AbstractNotificationSubscriber
         do {
             $results = $Query->limit($start, $perLoop)->execute()->get_results();
 
-            foreach ($results as $user_to_notif) {
+            $users_emailed = [];     // for all users
-                $mailed = false;
+            $users_to_email = [];    // list only users who must be emailed (=create tokens)
-
-                if ($params['notify_email'] && $this->shouldSendNotificationFor($user_to_notif, 'eventsmanager_notify_feed')) {
-                    $readyToSend = false;
-                    try {
-                        $token = $this->app['manipulator.token']->createFeedEntryToken($user_to_notif, $entry);
-                        $url = $this->app->url('lightbox', ['LOG' => $token->getValue()]);
-
-                        $receiver = Receiver::fromUser($user_to_notif);
-                        $readyToSend = true;
-                    } catch (\Exception $e) {
 
+            /** @var User $user */
+            foreach ($results as $user) {
+                $users_emailed[$user->getId()] = false;
+                if ($params['notify_email'] && $this->shouldSendNotificationFor($user, 'eventsmanager_notify_feed')) {
+                    $users_to_email[$user->getId()] = $user;
+                }
             }
 
-                    if ($readyToSend) {
+            // get many tokens in one shot
+            $tokens = $this->getTokenManipulator()->createFeedEntryTokens($users_to_email, $entry);
+            foreach($tokens as $token) {
+                try {
+                    $url = $this->app->url('lightbox', ['LOG' => $token->getValue()]);
+                    $receiver = Receiver::fromUser($token->getUser());
+
                     $mail = MailInfoNewPublication::create($this->app, $receiver);
                     $mail->setButtonUrl($url);
                     $mail->setAuthor($entry->getAuthorName());
                     $mail->setTitle($entry->getTitle());
 
                     $this->deliver($mail);
-                        $mailed = true;
+                    $users_emailed[$token->getUser()->getId()] = true;
                 }
+                catch (\Exception $e) {
+                    // no-op
+                }
+            }
+            foreach($users_emailed as $id => $emailed) {
+                $this->app['events-manager']->notify($id, 'eventsmanager_notify_feed', $datas, $emailed);
             }
 
-                $this->app['events-manager']->notify($user_to_notif->getId(), 'eventsmanager_notify_feed', $datas, $mailed);
-            }
             $start += $perLoop;
-        } while (count($results) > 0);
+        }
+        while (count($results) > 0);
     }
 
     public static function getSubscribedEvents()
@@ -91,4 +100,12 @@ class FeedEntrySubscriber extends AbstractNotificationSubscriber
             PhraseaEvents::FEED_ENTRY_CREATE => 'onCreate',
         ];
     }
+
+    /**
+     * @return TokenManipulator
+     */
+    private function getTokenManipulator()
+    {
+        return $this->app['manipulator.token'];
+    }
 }

lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php

@@ -62,29 +62,45 @@ class TokenManipulator implements ManipulatorInterface
      *
      * @return Token
      */
-    public function create(User $user = null, $type, \DateTime $expiration = null, $data = null)
+    public function create(User $user, $type, \DateTime $expiration = null, $data = null)
     {
-        $this->removeExpiredTokens();
+        static $stmt = null;
 
-        $n = 0;
+        // $this->removeExpiredTokens();
-        do {
+
-            if ($n++ > 1024) {
+        if($stmt === null) {
-                throw new \RuntimeException('Unable to create a token.');
+            $conn = $this->repository->getEntityManager()->getConnection();
+            $sql = "INSERT INTO Tokens (value, user_id, type, data, created, updated, expiration)\n"
+                . " VALUES(:value, :user_id, :type, :data, :created, :updated, :expiration)";
+            $stmt = $conn->prepare($sql);
         }
-            $value = $this->random->generateString(32, self::LETTERS_AND_NUMBERS);
-            $found = null !== $this->om->getRepository('Phraseanet:Token')->find($value);
-        } while ($found);
 
+        $token = null;
+        $now = (new \DateTime())->format('Y-m-d H:i:s');
+        $stmtParms = [
+            ':value' => null,
+            ':user_id' => $user->getId(),
+            ':type' => $type,
+            ':data' => $data,
+            ':created' => $now,
+            ':updated' => $now,
+            ':expiration' => $expiration
+        ];
+        for($try=0; $try<1024; $try++) {
+            $stmtParms['value'] = $this->random->generateString(32, self::LETTERS_AND_NUMBERS);
+            if($stmt->execute($stmtParms) === true) {
                 $token = new Token();
-
                 $token->setUser($user)
                     ->setType($type)
-            ->setValue($value)
+                    ->setValue($stmtParms['value'])
                     ->setExpiration($expiration)
                     ->setData($data);
-
+                break;
-        $this->om->persist($token);
+            }
-        $this->om->flush();
+        }
+        if ($token === null) {
+            throw new \RuntimeException('Unable to create a token.');
+        }
 
         return $token;
     }
@@ -126,6 +142,57 @@ class TokenManipulator implements ManipulatorInterface
         return $this->create($user, self::TYPE_FEED_ENTRY, null, $entry->getId());
     }
 
+    /**
+     * Create feedEntryTokens for many users in one shot
+     *
+     * @param User[] $users
+     * @param FeedEntry $entry
+     * @return Token[]
+     * @throws \Doctrine\DBAL\DBALException
+     */
+    public function createFeedEntryTokens($users, FeedEntry $entry)
+    {
+        // $this->removeExpiredTokens();
+
+        $conn = $this->repository->getEntityManager()->getConnection();
+
+        // use an optimized tmp table we can fill fast (only 2 values changes by row, others are default)
+        $now = $conn->quote((new \DateTime())->format('Y-m-d H:i:s'));
+        $conn->executeQuery("CREATE TEMPORARY TABLE `tmpTokens` (\n"
+            . " `value` char(128),\n"
+            . " `user_id` int(11),\n"
+            . " `type` char(32) DEFAULT " . $conn->quote(self::TYPE_FEED_ENTRY) . ",\n"
+            . " `data` int(11) DEFAULT " . $conn->quote($entry->getId()) . ",\n"
+            . " `created` datetime DEFAULT " . $now . ",\n"
+            . " `updated` datetime DEFAULT " . $now . ",\n"
+            . " `expiration` datetime DEFAULT NULL\n"
+            . ") ENGINE=MEMORY;"
+        );
+
+        $tokens = [];
+        $sql = "";
+        foreach ($users as $user) {
+            $value = $this->random->generateString(32, self::LETTERS_AND_NUMBERS) . $user->getId();
+            // todo: don't build a too long sql, we should flush/run into temp table if l>limit.
+            // But for now we trust that 100 (see FeedEntrySsuscriber) tokens is ok
+            $sql .= ($sql?',':'') . ('(' . $conn->quote($value) . ',' . $conn->quote($user->getId()) . ')');
+
+            $token = new Token();
+            $token->setUser($user)
+                ->setType(self::TYPE_FEED_ENTRY)
+                ->setValue($value)
+                ->setExpiration(null)
+                ->setData($entry->getId());
+            $tokens[] = $token;
+        }
+
+        $conn->executeQuery("INSERT INTO tmpTokens (`value`, `user_id`) VALUES " . $sql);
+        $conn->executeQuery("INSERT INTO Tokens SELECT * FROM tmpTokens");
+        $conn->executeQuery("DROP TABLE tmpTokens");
+
+        return $tokens;
+    }
+
     /**
      * @param User $user
      * @param $data

lib/Alchemy/Phrasea/Model/Repositories/TokenRepository.php

@@ -72,4 +72,9 @@ class TokenRepository extends EntityRepository
 
         return $query->getResult();
     }
+
+    public function getEntityManager()
+    {
+        return parent::getEntityManager();
+    }
 }