From 37445b3151a7a5b110f6b808f9b1b3407d928a8d Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 22 Oct 2014 11:15:05 +0200 Subject: [PATCH 01/21] Add hoa compiler --- composer.json | 7 +- composer.lock | 766 ++++++++++++++++++++++++++++++++++++++++++++---- grammar/json.pp | 36 +++ 3 files changed, 750 insertions(+), 59 deletions(-) create mode 100644 grammar/json.pp diff --git a/composer.json b/composer.json index 524858dfd2..6aefcc37bc 100644 --- a/composer.json +++ b/composer.json @@ -73,7 +73,8 @@ "vierbergenlars/php-semver" : "~2.1", "zend/gdata" : "~1.12.1", "doctrine/migrations" : "1.0.x-dev@dev", - "willdurand/negotiation" : "~1.3" + "willdurand/negotiation" : "~1.3", + "hoa/compiler": "2.14.09.23" }, "require-dev": { "phpunit/phpunit" : "~3.7", @@ -83,7 +84,9 @@ "behat/mink-extension" : "~1.0", "behat/mink-goutte-driver" : "~1.0", "behat/mink-selenium2-driver" : "~1.0", - "fabpot/goutte" : "~1.0" + "fabpot/goutte" : "~1.0", + "hoa/dispatcher": "0.14.09.23", + "hoa/console": "2.14.09.23" }, "autoload": { "psr-0": { diff --git a/composer.lock b/composer.lock index 1bdc24a8a6..74f08b1144 100644 --- a/composer.lock +++ b/composer.lock @@ -4,22 +4,16 @@ "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "69bf40de460d2f7a3407f1b905dde913", + "hash": "73cfb65537610dfd0edc9cf6485c4ebb", "packages": [ { "name": "alchemy-fr/tcpdf-clone", "version": "6.0.039", "source": { "type": "git", - "url": "https://github.com/alchemy-fr/tcpdf-clone.git", + "url": "https://github.com/alchemy-fr/tcpdf-clone", "reference": "2ba0248a7187f1626df6c128750650416267f0e7" }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/alchemy-fr/tcpdf-clone/zipball/2ba0248a7187f1626df6c128750650416267f0e7", - "reference": "2ba0248a7187f1626df6c128750650416267f0e7", - "shasum": "" - }, "require": { "php": ">=5.3.0" }, @@ -66,10 +60,6 @@ "qrcode", "tcpdf" ], - "support": { - "source": "https://github.com/alchemy-fr/tcpdf-clone/tree/6.0.039", - "issues": "https://github.com/alchemy-fr/tcpdf-clone/issues" - }, "time": "2013-10-13 16:11:17" }, { @@ -113,7 +103,7 @@ "homepage": "http://www.lickmychip.com/" }, { - "name": "nlegoff", + "name": "Nicolas Le Goff", "email": "legoff.n@gmail.com" }, { @@ -301,7 +291,7 @@ } }, "notification-url": "https://packagist.org/downloads/", - "time": "2014-07-07 16:25:07" + "time": "2014-07-07 14:01:51" }, { "name": "alchemy/task-manager", @@ -372,12 +362,12 @@ "source": { "type": "git", "url": "https://github.com/alchemy-fr/Zippy.git", - "reference": "08008f82957b7dc2b54574b506687b33ecfe0589" + "reference": "d2f5e88f2436b9c1294e8819d951822abe39e9a7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/alchemy-fr/Zippy/zipball/08008f82957b7dc2b54574b506687b33ecfe0589", - "reference": "08008f82957b7dc2b54574b506687b33ecfe0589", + "url": "https://api.github.com/repos/alchemy-fr/Zippy/zipball/d2f5e88f2436b9c1294e8819d951822abe39e9a7", + "reference": "d2f5e88f2436b9c1294e8819d951822abe39e9a7", "shasum": "" }, "require": { @@ -426,7 +416,7 @@ "tar", "zip" ], - "time": "2014-09-03 08:10:41" + "time": "2014-05-05 13:39:00" }, { "name": "cboden/ratchet", @@ -747,7 +737,7 @@ { "name": "Johannes Schmitt", "email": "schmittjoh@gmail.com", - "homepage": "https://github.com/schmittjoh", + "homepage": "http://jmsyst.com", "role": "Developer of wrapped JMSSerializerBundle" } ], @@ -935,8 +925,7 @@ { "name": "Jonathan Wage", "email": "jonwage@gmail.com", - "homepage": "http://www.jwage.com/", - "role": "Creator" + "homepage": "http://www.jwage.com/" }, { "name": "Guilherme Blanco", @@ -954,7 +943,7 @@ { "name": "Johannes Schmitt", "email": "schmittjoh@gmail.com", - "homepage": "https://github.com/schmittjoh", + "homepage": "http://jmsyst.com", "role": "Developer of wrapped JMSSerializerBundle" } ], @@ -1008,7 +997,7 @@ { "name": "Johannes Schmitt", "email": "schmittjoh@gmail.com", - "homepage": "https://github.com/schmittjoh", + "homepage": "http://jmsyst.com", "role": "Developer of wrapped JMSSerializerBundle" } ], @@ -1026,12 +1015,12 @@ "source": { "type": "git", "url": "https://github.com/doctrine/migrations.git", - "reference": "1a9dffa64e33fdc10f4b4c3f5d7230b74d4a1021" + "reference": "4256449c5e2603a6b6ee5a78c7c4521d4d4430b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/migrations/zipball/1a9dffa64e33fdc10f4b4c3f5d7230b74d4a1021", - "reference": "1a9dffa64e33fdc10f4b4c3f5d7230b74d4a1021", + "url": "https://api.github.com/repos/doctrine/migrations/zipball/4256449c5e2603a6b6ee5a78c7c4521d4d4430b8", + "reference": "4256449c5e2603a6b6ee5a78c7c4521d4d4430b8", "shasum": "" }, "require": { @@ -1062,12 +1051,14 @@ ], "authors": [ { - "name": "Benjamin Eberlei", - "email": "kontakt@beberlei.de" + "name": "Jonathan Wage", + "email": "jonwage@gmail.com", + "homepage": "http://www.jwage.com/", + "role": "Creator" }, { - "name": "Jonathan Wage", - "email": "jonwage@gmail.com" + "name": "Benjamin Eberlei", + "email": "kontakt@beberlei.de" } ], "description": "Database Schema migrations using Doctrine DBAL", @@ -1076,7 +1067,7 @@ "database", "migrations" ], - "time": "2014-08-18 18:03:07" + "time": "2014-07-09 07:58:02" }, { "name": "doctrine/orm", @@ -1213,13 +1204,13 @@ "version": "v1.0.0", "source": { "type": "git", - "url": "https://github.com/igorw/evenement.git", - "reference": "fa966683e7df3e5dd5929d984a44abfbd6bafe8d" + "url": "https://github.com/igorw/evenement", + "reference": "v1.0.0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/igorw/evenement/zipball/fa966683e7df3e5dd5929d984a44abfbd6bafe8d", - "reference": "fa966683e7df3e5dd5929d984a44abfbd6bafe8d", + "url": "https://github.com/igorw/evenement/zipball/v1.0.0", + "reference": "v1.0.0", "shasum": "" }, "require": { @@ -1246,7 +1237,7 @@ "keywords": [ "event-dispatcher" ], - "time": "2012-05-30 15:01:08" + "time": "2012-05-30 08:01:08" }, { "name": "facebook/php-sdk", @@ -1299,12 +1290,12 @@ "version": "v2.3.9", "source": { "type": "git", - "url": "https://github.com/Atlantic18/DoctrineExtensions.git", + "url": "https://github.com/l3pp4rd/DoctrineExtensions.git", "reference": "35adcaae1a3f50d0d5b73aa50ed8fd28ee35ce54" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Atlantic18/DoctrineExtensions/zipball/35adcaae1a3f50d0d5b73aa50ed8fd28ee35ce54", + "url": "https://api.github.com/repos/l3pp4rd/DoctrineExtensions/zipball/35adcaae1a3f50d0d5b73aa50ed8fd28ee35ce54", "reference": "35adcaae1a3f50d0d5b73aa50ed8fd28ee35ce54", "shasum": "" }, @@ -1372,7 +1363,7 @@ "tree", "uploadable" ], - "time": "2014-01-12 16:34:06" + "time": "2013-08-18 07:18:44" }, { "name": "goodby/csv", @@ -1429,7 +1420,7 @@ "export", "import" ], - "time": "2013-11-22 19:10:34" + "time": "2014-01-12 16:34:06" }, { "name": "guzzle/guzzle", @@ -1523,6 +1514,433 @@ ], "time": "2014-08-11 04:32:36" }, + { + "name": "hoa/compiler", + "version": "2.14.09.23", + "target-dir": "Hoa/Compiler", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Compiler.git", + "reference": "51cdc8b21d13f2fcaa3f3a0d114247534849f8cb" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Compiler/zipball/51cdc8b21d13f2fcaa3f3a0d114247534849f8cb", + "reference": "51cdc8b21d13f2fcaa3f3a0d114247534849f8cb", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0", + "hoa/file": "~0.0", + "hoa/iterator": "~0.0", + "hoa/math": "~0.0", + "hoa/visitor": "~0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Compiler": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Compiler library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "algebraic", + "ast", + "compiler", + "context-free", + "coverage", + "exhaustive", + "grammar", + "isotropic", + "language", + "lexer", + "library", + "ll1", + "llk", + "parser", + "pp", + "random", + "regular", + "rule", + "sampler", + "syntax", + "token", + "trace", + "uniform" + ], + "time": "2014-09-23 09:50:46" + }, + { + "name": "hoa/core", + "version": "2.14.09.23", + "target-dir": "Hoa/Core", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Core.git", + "reference": "e50354e69e451478223d1d0c1ce4f5d741ea7576" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Core/zipball/e50354e69e451478223d1d0c1ce4f5d741ea7576", + "reference": "e50354e69e451478223d1d0c1ce4f5d741ea7576", + "shasum": "" + }, + "require": { + "ext-spl": "*", + "php": ">=5.4.0" + }, + "suggest": { + "ext-mbstring": "ext/mbstring must be present (or a third implementation).", + "hoa/console": "To use the `hoa` script.", + "hoa/dispatcher": "To use the `hoa` script.", + "hoa/router": "To use the `hoa` script." + }, + "bin": [ + "Bin/hoa" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Core": "." + }, + "files": [ + "Core.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Core library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "consistency", + "core", + "data", + "event", + "library", + "listener", + "parameter", + "protocol" + ], + "time": "2014-09-23 09:45:22" + }, + { + "name": "hoa/file", + "version": "0.14.09.23", + "target-dir": "Hoa/File", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/File.git", + "reference": "a39f62a28256180606115416cf27774966cf73e9" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/File/zipball/a39f62a28256180606115416cf27774966cf73e9", + "reference": "a39f62a28256180606115416cf27774966cf73e9", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0", + "hoa/iterator": "~0.0", + "hoa/stream": "~0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "0.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\File": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\File library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "Socket", + "directory", + "file", + "finder", + "library", + "link", + "temporary" + ], + "time": "2014-09-23 09:50:42" + }, + { + "name": "hoa/iterator", + "version": "0.14.09.23", + "target-dir": "Hoa/Iterator", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Iterator.git", + "reference": "1ca570cab25ca359a1a9f4b4c449d49771fc6a5e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Iterator/zipball/1ca570cab25ca359a1a9f4b4c449d49771fc6a5e", + "reference": "1ca570cab25ca359a1a9f4b4c449d49771fc6a5e", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "0.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Iterator": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Iterator library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "iterator", + "library" + ], + "time": "2014-09-23 09:50:40" + }, + { + "name": "hoa/math", + "version": "0.14.09.23", + "target-dir": "Hoa/Math", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Math.git", + "reference": "b52764f602095b4595658f581a504f039cef8d56" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Math/zipball/b52764f602095b4595658f581a504f039cef8d56", + "reference": "b52764f602095b4595658f581a504f039cef8d56", + "shasum": "" + }, + "require": { + "hoa/compiler": "~2.0", + "hoa/core": "~2.0", + "hoa/iterator": "~0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "0.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Math": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Math library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "arrangement", + "combination", + "combinatorics", + "counting", + "library", + "math", + "permutation", + "sampler", + "set" + ], + "time": "2014-09-23 14:02:37" + }, + { + "name": "hoa/stream", + "version": "0.14.09.23", + "target-dir": "Hoa/Stream", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Stream.git", + "reference": "eaf9bfeb633b8a6bf0fba55e9c035db431024869" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Stream/zipball/eaf9bfeb633b8a6bf0fba55e9c035db431024869", + "reference": "eaf9bfeb633b8a6bf0fba55e9c035db431024869", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "0.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Stream": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Stream library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "Context", + "bucket", + "composite", + "filter", + "in", + "library", + "out", + "protocol", + "stream", + "wrapper" + ], + "time": "2014-09-23 09:50:38" + }, + { + "name": "hoa/visitor", + "version": "0.14.09.23", + "target-dir": "Hoa/Visitor", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Visitor.git", + "reference": "071523b6677466979e57a9f9ef61a46b76221935" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Visitor/zipball/071523b6677466979e57a9f9ef61a46b76221935", + "reference": "071523b6677466979e57a9f9ef61a46b76221935", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "0.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Visitor": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Visitor library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "library", + "structure", + "visit", + "visitor" + ], + "time": "2014-09-23 09:50:51" + }, { "name": "igorw/get-in", "version": "v1.0.2", @@ -1596,7 +2014,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-develop": "0.7-dev" + "dev-develop": "0.6-dev" } }, "autoload": { @@ -1623,7 +2041,7 @@ "image processing" ], "support": { - "source": "https://github.com/nlegoff/Imagine/tree/flatten-layer" + "source": "https://github.com/alchemy-fr/Imagine/tree/0.6.1-flatten-layer" }, "time": "2014-10-08 16:23:33" }, @@ -1932,9 +2350,9 @@ ], "authors": [ { - "name": "Johannes Schmitt", + "name": "Johannes M. Schmitt", "email": "schmittjoh@gmail.com", - "homepage": "https://github.com/schmittjoh", + "homepage": "http://jmsyst.com", "role": "Developer of wrapped JMSSerializerBundle" } ], @@ -2249,7 +2667,7 @@ ], "authors": [ { - "name": "Steve Clay", + "name": "Stephen Clay", "email": "steve@mrclay.org", "homepage": "http://www.mrclay.org/", "role": "Developer" @@ -2435,21 +2853,21 @@ "source": { "type": "git", "url": "https://github.com/romainneutron/Imagine-Silex-Service-Provider.git", - "reference": "a8a7862ae90419f2b23746cd8436c2310e4eb084" + "reference": "0.1.2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/romainneutron/Imagine-Silex-Service-Provider/zipball/a8a7862ae90419f2b23746cd8436c2310e4eb084", - "reference": "a8a7862ae90419f2b23746cd8436c2310e4eb084", + "url": "https://api.github.com/repos/romainneutron/Imagine-Silex-Service-Provider/zipball/0.1.2", + "reference": "0.1.2", "shasum": "" }, "require": { "imagine/imagine": "*", "php": ">=5.3.3", - "silex/silex": "~1.0" + "silex/silex": ">=1.0,<2.0" }, "require-dev": { - "symfony/browser-kit": "~2.0" + "symfony/browser-kit": ">=2.0,<3.0" }, "type": "library", "autoload": { @@ -2963,7 +3381,7 @@ "metadata" ], "support": { - "source": "https://github.com/alchemy-fr/PHPExiftool/tree/dev" + "source": "https://github.com/alchemy-fr/PHPExiftool/tree/0.4.1-mwg-metadata-copy" }, "time": "2014-10-08 16:09:02" }, @@ -3001,9 +3419,9 @@ ], "authors": [ { - "name": "Johannes Schmitt", + "name": "Johannes M. Schmitt", "email": "schmittjoh@gmail.com", - "homepage": "https://github.com/schmittjoh", + "homepage": "http://jmsyst.com", "role": "Developer of wrapped JMSSerializerBundle" } ], @@ -3560,7 +3978,7 @@ }, { "name": "Phraseanet Team", - "email": "info@alchemy.fr", + "email": "support@alchemy.fr", "homepage": "http://www.phraseanet.com/" } ], @@ -3607,9 +4025,7 @@ "authors": [ { "name": "Fabien Potencier", - "email": "fabien@symfony.com", - "homepage": "http://fabien.potencier.org", - "role": "Lead Developer" + "email": "fabien@symfony.com" }, { "name": "Chris Corbyn" @@ -4522,6 +4938,242 @@ ], "time": "2014-10-09 15:52:51" }, + { + "name": "hoa/console", + "version": "2.14.09.23", + "target-dir": "Hoa/Console", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Console.git", + "reference": "8466f74ddb5bd323357fe5464629c948a1fd8d25" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Console/zipball/8466f74ddb5bd323357fe5464629c948a1fd8d25", + "reference": "8466f74ddb5bd323357fe5464629c948a1fd8d25", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0", + "hoa/stream": "~0.0", + "hoa/string": "~2.0" + }, + "suggest": { + "ext-pcntl": "To enable hoa://Event/Console/Window:resize.", + "hoa/dispatcher": "To use the console kit.", + "hoa/router": "To use the console kit." + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Console": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Console library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "autocompletion", + "chrome", + "cli", + "console", + "cursor", + "getoption", + "library", + "option", + "parser", + "processus", + "readline", + "terminfo", + "tput", + "window" + ], + "time": "2014-09-23 14:17:08" + }, + { + "name": "hoa/dispatcher", + "version": "0.14.09.23", + "target-dir": "Hoa/Dispatcher", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Dispatcher.git", + "reference": "82924823fa9cedad9775f1ab4d51075e980ac2c6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Dispatcher/zipball/82924823fa9cedad9775f1ab4d51075e980ac2c6", + "reference": "82924823fa9cedad9775f1ab4d51075e980ac2c6", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0" + }, + "suggest": { + "hoa/router": "Provide routers.", + "hoa/view": "Provide view interface." + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "0.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Dispatcher": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Dispatcher library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "dispatcher", + "kit", + "library" + ], + "time": "2014-09-23 14:12:43" + }, + { + "name": "hoa/router", + "version": "2.14.09.23", + "target-dir": "Hoa/Router", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Router.git", + "reference": "8937785aecf7ca3b6dbd5f668eb97e15c672758d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Router/zipball/8937785aecf7ca3b6dbd5f668eb97e15c672758d", + "reference": "8937785aecf7ca3b6dbd5f668eb97e15c672758d", + "shasum": "" + }, + "require": { + "hoa/core": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\Router": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\Router library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "cli", + "http", + "library", + "router" + ], + "time": "2014-09-23 09:50:57" + }, + { + "name": "hoa/string", + "version": "2.14.09.23", + "target-dir": "Hoa/String", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/String.git", + "reference": "c385ffb1382d919c63ab97acd6f62058179c5f2a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/String/zipball/c385ffb1382d919c63ab97acd6f62058179c5f2a", + "reference": "c385ffb1382d919c63ab97acd6f62058179c5f2a", + "shasum": "" + }, + "require": { + "ext-iconv": "*", + "hoa/core": "~2.0" + }, + "suggest": { + "ext-intl": "To get a better Hoa\\String::toAscii() and Hoa\\String::compareTo()." + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-0": { + "Hoa\\String": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "http://hoa-project.net/" + } + ], + "description": "The Hoa\\String library.", + "homepage": "http://hoa-project.net/", + "keywords": [ + "library", + "search", + "string", + "unicode" + ], + "time": "2014-09-23 09:55:55" + }, { "name": "instaclick/php-webdriver", "version": "1.4.0", diff --git a/grammar/json.pp b/grammar/json.pp new file mode 100644 index 0000000000..5a21ef5288 --- /dev/null +++ b/grammar/json.pp @@ -0,0 +1,36 @@ +%skip space \s + +%token true true +%token false false +%token null null +%token quote_ " -> string +%token string:escaped \\(["\\/bfnrt]|u[0-9a-fA-F]{4}) +%token string:string [^"\\]+ +%token string:_quote " -> default +%token brace_ { +%token _brace } +%token bracket_ \[ +%token _bracket \] +%token colon : +%token comma , +%token number \-?(0|[1-9]\d*)(\.\d+)?([eE][\+\-]?\d+)? + +value: + | | | string() | object() | array() | number() + +string: + ::quote_:: + + ::_quote:: + +number: + + +#object: + ::brace_:: pair() ( ::comma:: pair() )* ::_brace:: + +#pair: + string() ::colon:: value() + +#array: + ::bracket_:: value() ( ::comma:: value() )* ::_bracket:: \ No newline at end of file From 3266d8d9ac8f03b1d12fe4d57a46e02a7e9b61da Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 22 Oct 2014 11:15:56 +0200 Subject: [PATCH 02/21] WIP on PP grammar --- grammar/query.pp | 21 +++++++++++++++++++ .../Provider/SearchEngineServiceProvider.php | 16 ++++++++++++++ .../Elastic/ElasticSearchEngine.php | 9 ++++++++ 3 files changed, 46 insertions(+) create mode 100644 grammar/query.pp diff --git a/grammar/query.pp b/grammar/query.pp new file mode 100644 index 0000000000..fa6b32a9ed --- /dev/null +++ b/grammar/query.pp @@ -0,0 +1,21 @@ +%skip space \s + +%token and AND +%token or OR +%token in IN +%token word \S+ + +query: + expression() * + +expression: + in_expression() | text() + +in_expression: + text() ::in:: field() + +field: + + +text: + + diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index e848d29a01..f845309446 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -22,6 +22,8 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus; use Alchemy\Phrasea\SearchEngine\Phrasea\PhraseaEngine; use Alchemy\Phrasea\SearchEngine\Phrasea\PhraseaEngineSubscriber; use Elasticsearch\Client; +use Hoa\Compiler\Llk\Llk; +use Hoa\File\Read; use Monolog\Handler\RotatingFileHandler; use Monolog\Logger; use Silex\Application; @@ -126,6 +128,20 @@ class SearchEngineServiceProvider implements ServiceProviderInterface $app['elasticsearch.options']['index'] ); }); + + $app['query_parser.grammar_path'] = function ($app) { + $configPath = ['registry', 'searchengine', 'query-grammar-path']; + $grammarPath = $app['conf']->get($configPath, 'grammar/query.pp'); + $projectRoot = '../../../../..'; + + return realpath(implode('/', [__DIR__, $projectRoot, $grammarPath])); + }; + + $app['query_parser'] = $app->share(function ($app) { + $grammarPath = $app['query_parser.grammar_path']; + + return Llk::load(new Read($grammarPath)); + }); } public function boot(Application $app) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 4e348de097..0bb0eac251 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -271,6 +271,15 @@ class ElasticSearchEngine implements SearchEngineInterface { $options = $options ?: new SearchEngineOptions(); + $ast = $this->app['query_parser']->parse($string); + + $searchQuery = SearchQuery::fromAST($ast); + + $dump = new \Hoa\Compiler\Visitor\Dump(); + echo $dump->visit($ast); + + die(); + $parser = new QueryParser(); $ast = $parser->parse($string); From 626b57ba633bbbdd399935a5cae2d2032127506f Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 22 Oct 2014 11:16:44 +0200 Subject: [PATCH 03/21] Add config watcher requires `entr` binary --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..2d06cdc528 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +config: + @php bin/console compile:configuration + +watch: + @echo 'config/configuration.yml' | entr make config + +.PHONY: config watch From 1b120e6569abe36177a98cec046c0df1a2c4b9aa Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Tue, 28 Oct 2014 11:08:21 +0100 Subject: [PATCH 04/21] WIP query grammar --- grammar/query.pp | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/grammar/query.pp b/grammar/query.pp index fa6b32a9ed..bb715abaa5 100644 --- a/grammar/query.pp +++ b/grammar/query.pp @@ -1,21 +1,25 @@ %skip space \s -%token and AND -%token or OR +%token quote_ " -> string +%token string:string [^"]+ +%token string:_quote " -> default %token in IN %token word \S+ -query: - expression() * +#query: + expression()+ expression: - in_expression() | text() + in_expression() | text() | unrestricted_text() -in_expression: - text() ::in:: field() +#in_expression: + text() ::in:: ( | string() ) -field: - +#text: + ( | string() )+ -text: - + +#unrestricted_text: + ( | string() | )+ + +string: + ::quote_:: ::_quote:: From bdf6dbd12224f8839da8d12119d189bbf3400a00 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Tue, 28 Oct 2014 11:08:45 +0100 Subject: [PATCH 05/21] Sample PP grammar --- grammar/arithmetic.pp | 90 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 grammar/arithmetic.pp diff --git a/grammar/arithmetic.pp b/grammar/arithmetic.pp new file mode 100644 index 0000000000..b1e110874c --- /dev/null +++ b/grammar/arithmetic.pp @@ -0,0 +1,90 @@ +// +// Hoa +// +// +// @license +// +// New BSD License +// +// Copyright © 2007-2014, Ivan Enderlin. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Hoa nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Grammar \Hoa\Math\Arithmetic. +// +// Provide a grammar for arithmetic expressions. +// +// @author Stéphane Py +// @author Sébastien Houze +// @author Ivan Enderlin +// @copyright Copyright © 2007-2014 Stéphane Py, Sébastien Houze, Ivan Enderlin. +// @license New BSD License +// + + +%skip space \s +%token bracket_ \( +%token _bracket \) +%token comma , +%token number (0|[1-9]\d*)(\.\d+)?([eE][\+\-]?\d+)? +%token plus \+ +%token minus \-|− +%token times \*|× +%token div /|÷ +%token constant [A-Z_]+[A-Z0-9_]+ +%token id \w+ + +expression: + primary() ( ::plus:: #addition expression() )? + +primary: + secondary() ( ::minus:: #substraction expression() )? + +secondary: + ternary() ( ::times:: #multiplication expression() )? + +ternary: + term() ( ::div:: #division expression() )? + +term: + ( ::bracket_:: expression() ::_bracket:: #group ) + | number() + | constant() + | variable() + | ( ::minus:: #negative | ::plus:: ) term() + | function() + +number: + + +constant: + + +#variable: + + +#function: + ::bracket_:: + ( expression() ( ::comma:: expression() )* )? + ::_bracket:: From 3ec520ed119fbcf4eca426b0afd7d5d7562cb3df Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 6 Nov 2014 18:37:53 +0100 Subject: [PATCH 06/21] New query parser between Hoa compiler and search engine --- .../Provider/SearchEngineServiceProvider.php | 4 ++- .../Elastic/ElasticSearchEngine.php | 3 -- .../Elastic/Search/QueryParser.php | 28 +++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index f845309446..860337ddf6 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -18,6 +18,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\ElasticSearchEngine; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\RecordIndexer; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\TermIndexer; +use Alchemy\Phrasea\SearchEngine\Elastic\Search\QueryParser; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus; use Alchemy\Phrasea\SearchEngine\Phrasea\PhraseaEngine; use Alchemy\Phrasea\SearchEngine\Phrasea\PhraseaEngineSubscriber; @@ -139,8 +140,9 @@ class SearchEngineServiceProvider implements ServiceProviderInterface $app['query_parser'] = $app->share(function ($app) { $grammarPath = $app['query_parser.grammar_path']; + $parser = Llk::load(new Read($grammarPath)); - return Llk::load(new Read($grammarPath)); + return new QueryParser($parser); }); } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 0bb0eac251..74d7c2f6d4 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -273,10 +273,7 @@ class ElasticSearchEngine implements SearchEngineInterface $ast = $this->app['query_parser']->parse($string); - $searchQuery = SearchQuery::fromAST($ast); - $dump = new \Hoa\Compiler\Visitor\Dump(); - echo $dump->visit($ast); die(); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php new file mode 100644 index 0000000000..c9a68e92a1 --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php @@ -0,0 +1,28 @@ +parser = $parser; + } + + public function parse($string) + { + $ast = $this->parser->parse($string); + + $dump = new \Hoa\Compiler\Visitor\Dump(); + echo $dump->visit($ast); + } +} + From 0bbd35dc024ecf97a13027016ab7639631b4c7ec Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 6 Nov 2014 18:48:32 +0100 Subject: [PATCH 07/21] AST revamp --- .../Elastic/AST/AndExpression.php | 39 +++---------------- .../Elastic/AST/BinaryOperator.php | 32 +++++++++++++++ .../Elastic/AST/ExceptExpression.php | 21 ++++++++++ .../AST/{KeywordNode.php => FieldNode.php} | 2 +- .../SearchEngine/Elastic/AST/InExpression.php | 4 +- .../SearchEngine/Elastic/AST/OrExpression.php | 39 +++---------------- .../Elastic/AST/QuotedTextNode.php | 3 +- 7 files changed, 68 insertions(+), 72 deletions(-) create mode 100644 lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php create mode 100644 lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/ExceptExpression.php rename lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/{KeywordNode.php => FieldNode.php} (94%) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/AndExpression.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/AndExpression.php index 576851d38a..99e5579c64 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/AndExpression.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/AndExpression.php @@ -2,48 +2,19 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST; -class AndExpression extends Node +class AndExpression extends BinaryOperator { - protected $members = array(); - - public function __construct(Node $left, Node $right) - { - $this->members[] = $left; - $this->members[] = $right; - } - - public function getMembers() - { - return $this->members; - } + protected $operator = 'AND'; public function getQuery($fields = ['_all']) { - $rules = array(); - foreach ($this->members as $member) { - $rules[] = $member->getQuery($fields); - } + $left = $this->left->getQuery($fields); + $right = $this->right->getQuery($fields); return array( 'bool' => array( - 'must' => count($rules) > 1 ? $rules : $rules[0] + 'must' => array($left, $right) ) ); } - - public function __toString() - { - return sprintf('(%s)', implode(' AND ', $this->members)); - } - - public function isFullTextOnly() - { - foreach ($this->members as $member) { - if (!$member->isFullTextOnly()) { - return false; - } - } - - return true; - } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php new file mode 100644 index 0000000000..9e4a0f9edb --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php @@ -0,0 +1,32 @@ +left = $left; + $this->right = $right; + } + + public function __toString() + { + return sprintf('(%s %s %s)', $this->left, $this->operator, $this->right); + } + + public function isFullTextOnly() + { + foreach ($this->members as $member) { + if (!$member->isFullTextOnly()) { + return false; + } + } + + return true; + } +} diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/ExceptExpression.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/ExceptExpression.php new file mode 100644 index 0000000000..a1c812b444 --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/ExceptExpression.php @@ -0,0 +1,21 @@ +left->getQuery($fields); + $right = $this->right->getQuery($fields); + + return array( + 'bool' => array( + 'must' => $left, + 'must_not' => $right + ) + ); + } +} diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeywordNode.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/FieldNode.php similarity index 94% rename from lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeywordNode.php rename to lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/FieldNode.php index 0bdaca7d65..f15de1b18e 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/KeywordNode.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/FieldNode.php @@ -2,7 +2,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST; -class KeywordNode extends Node +class FieldNode extends Node { protected $keyword; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/InExpression.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/InExpression.php index 5c9c7283a4..fb8f8fad54 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/InExpression.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/InExpression.php @@ -7,9 +7,9 @@ class InExpression extends Node protected $field; protected $expression; - public function __construct(KeywordNode $keyword, $expression) + public function __construct(FieldNode $field, $expression) { - $this->field = $keyword; + $this->field = $field; $this->expression = $expression; } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/OrExpression.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/OrExpression.php index bbf9ab65ef..6a345688e5 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/OrExpression.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/OrExpression.php @@ -2,48 +2,19 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\AST; -class OrExpression extends Node +class OrExpression extends BinaryOperator { - protected $members = array(); - - public function __construct(Node $left, Node $right) - { - $this->members[] = $left; - $this->members[] = $right; - } - - public function getMembers() - { - return $this->members; - } + protected $operator = 'OR'; public function getQuery($fields = ['_all']) { - $rules = array(); - foreach ($this->members as $member) { - $rules[] = $member->getQuery($fields); - } + $left = $this->left->getQuery($fields); + $right = $this->right->getQuery($fields); return array( 'bool' => array( - 'should' => count($rules) > 1 ? $rules : $rules[0] + 'should' => array($left, $right) ) ); } - - public function __toString() - { - return sprintf('(%s)', implode(' OR ', $this->members)); - } - - public function isFullTextOnly() - { - foreach ($this->members as $member) { - if (!$member->isFullTextOnly()) { - return false; - } - } - - return true; - } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php index ba06da1212..35df144d6c 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/QuotedTextNode.php @@ -8,9 +8,10 @@ class QuotedTextNode extends TextNode { return array( 'multi_match' => array( + 'type' => 'phrase', 'fields' => $fields, 'query' => $this->text, - 'operator' => 'and' + // 'operator' => 'and' ) ); } From 3cf483256f2d0ded4e4a938248e6d25eb021d204 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 6 Nov 2014 19:09:40 +0100 Subject: [PATCH 08/21] Updated grammar (IN/AND/EXCEPT) --- grammar/query.pp | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/grammar/query.pp b/grammar/query.pp index bb715abaa5..e87670a762 100644 --- a/grammar/query.pp +++ b/grammar/query.pp @@ -4,22 +4,36 @@ %token string:string [^"]+ %token string:_quote " -> default %token in IN +%token and AND +%token or OR +%token except EXCEPT %token word \S+ +// relative order of precedence is NOT > XOR > AND > OR + #query: - expression()+ + primary() -expression: - in_expression() | text() | unrestricted_text() +primary: + secondary() ( ::except:: #except primary() )? -#in_expression: - text() ::in:: ( | string() ) +secondary: + ternary() ( ::or:: #or primary() )? + +ternary: + quaternary() ( ::and:: #and primary() )? + +quaternary: + text() ( ::in:: #in word() )? #text: - ( | string() )+ + ( word() | keyword() )+ -#unrestricted_text: - ( | string() | )+ +word: + | string() string: ::quote_:: ::_quote:: + +keyword: + | | | From 532f240ffd47686c1f828f8e3f8b357175b80a82 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 6 Nov 2014 19:11:06 +0100 Subject: [PATCH 09/21] Parse tree visitor to build ES query --- .../SearchEngine/Elastic/AST/TextNode.php | 5 + .../Elastic/ElasticSearchEngine.php | 5 +- .../Elastic/Search/QueryParser.php | 10 ++ .../Elastic/Search/QueryVisitor.php | 101 ++++++++++++++++++ .../Elastic/Search/SearchQuery.php | 27 +++++ 5 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php create mode 100644 lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php index cc31ee038d..c96aad7d4f 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php @@ -30,4 +30,9 @@ class TextNode extends Node { return true; } + + public function getText() + { + return $this->text; + } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 74d7c2f6d4..438ef7c0bb 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -13,6 +13,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\RecordIndexer; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\TermIndexer; +use Alchemy\Phrasea\SearchEngine\Elastic\Search\SearchQuery; use Alchemy\Phrasea\SearchEngine\SearchEngineInterface; use Alchemy\Phrasea\SearchEngine\SearchEngineOptions; use Alchemy\Phrasea\SearchEngine\SearchEngineResult; @@ -271,9 +272,11 @@ class ElasticSearchEngine implements SearchEngineInterface { $options = $options ?: new SearchEngineOptions(); - $ast = $this->app['query_parser']->parse($string); + $searchQuery = $this->app['query_parser']->parse($string); + $esQuery = $searchQuery->getElasticsearchQuery(); + echo json_encode($esQuery, JSON_PRETTY_PRINT); die(); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php index c9a68e92a1..e725b4362e 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php @@ -23,6 +23,16 @@ class QueryParser $dump = new \Hoa\Compiler\Visitor\Dump(); echo $dump->visit($ast); + + // Create query from syntax tree + $visitor = new QueryVisitor(); + $query = $visitor->visit($ast); + + echo "--------------------\n"; + echo (string) $query . "\n"; + echo "--------------------\n"; + + return $query; } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php new file mode 100644 index 0000000000..fd2cac1052 --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php @@ -0,0 +1,101 @@ +getValue()) { + switch ($value['token']) { + case self::NODE_TOKEN_WORD: + return new AST\TextNode($value['value']); + + case self::NODE_TOKEN_STRING: + return new AST\QuotedTextNode($value['value']); + + default: + // Generic handling off other tokens for unresctricted text + return new AST\TextNode($value['value']); + } + } + + switch ($element->getId()) { + case self::NODE_TYPE_QUERY: + $root = null; + foreach ($element->getChildren() as $child) { + $node = $child->accept($this, $handle, $eldnah); + if ($root) { + $root = new AST\AndExpression($root, $node); + } else { + $root = $node; + } + } + return new SearchQuery($root); + + case self::NODE_TYPE_IN_EXPR: + if ($element->getChildrenNumber() !== 2) { + throw new \Exception('IN expression can only have 2 childs.'); + } + $expression = $element->getChild(0)->accept($this); + $field = new AST\FieldNode($element->getChild(1)->getValue()['value']); + return new AST\InExpression($field, $expression); + + case self::NODE_TYPE_AND_EXPR: + if ($element->getChildrenNumber() !== 2) { + throw new \Exception('AND expression can only have 2 childs.'); + } + $left = $element->getChild(0)->accept($this); + $right = $element->getChild(1)->accept($this); + return new AST\AndExpression($left, $right); + + case self::NODE_TYPE_OR_EXPR: + if ($element->getChildrenNumber() !== 2) { + throw new \Exception('OR expression can only have 2 childs.'); + } + $left = $element->getChild(0)->accept($this); + $right = $element->getChild(1)->accept($this); + return new AST\OrExpression($left, $right); + + case self::NODE_TYPE_TEXT: + case self::NODE_TYPE_UNRESTRICTED_TEXT: + $root = null; + foreach ($element->getChildren() as $child) { + $node = $child->accept($this, $handle, $eldnah); + if ($root) { + // $root = new AST\AndExpression($root, $node); + // Merge text nodes together, but not with quoted ones + if ($root instanceof AST\TextNode && + !$root instanceof AST\QuotedTextNode && + !$node instanceof AST\QuotedTextNode) { + $root = new AST\TextNode(sprintf('%s %s', $root->getText(), $node->getText())); + } else { + $root = new AST\AndExpression($root, $node); + } + } else { + $root = $node; + } + } + return $root; + + default: + throw new \Exception(sprintf('Unknown node type "%s".', $element->getId())); + } + } +} diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php new file mode 100644 index 0000000000..25731c7649 --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php @@ -0,0 +1,27 @@ +root = $root; + } + + public function getElasticsearchQuery() + { + return $this->root->getQuery(); + } + + public function __toString() + { + return (string) $this->root; + } +} From 7d9b633043d0752e35e047aa96466a55daea98c8 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Mon, 10 Nov 2014 18:08:39 +0100 Subject: [PATCH 10/21] New query debug command --- bin/console | 2 + .../SearchEngine/Debug/QueryParseCommand.php | 61 +++++++++++++++++++ .../Elastic/Search/SearchQuery.php | 5 ++ 3 files changed, 68 insertions(+) mode change 100644 => 100755 bin/console create mode 100644 lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php diff --git a/bin/console b/bin/console old mode 100644 new mode 100755 index 7845c5e2de..3c087c5d48 --- a/bin/console +++ b/bin/console @@ -14,6 +14,7 @@ namespace KonsoleKommander; use Alchemy\Phrasea\Command\Plugin\ListPlugin; use Alchemy\Phrasea\Command\Setup\H264ConfigurationDumper; use Alchemy\Phrasea\Command\Setup\H264MappingGenerator; +use Alchemy\Phrasea\Command\SearchEngine\Debug\QueryParseCommand; use Alchemy\Phrasea\Command\SearchEngine\IndexCreateCommand; use Alchemy\Phrasea\Command\SearchEngine\IndexDropCommand; use Alchemy\Phrasea\Command\SearchEngine\IndexFull; @@ -127,6 +128,7 @@ if ($cli['search_engine.type'] === SearchEngineInterface::TYPE_ELASTICSEARCH) { $cli->command(new IndexCreateCommand()); $cli->command(new IndexDropCommand()); $cli->command(new IndexPopulateCommand()); + $cli->command(new QueryParseCommand()); } $cli->command(new WebsocketServer('ws-server:run')); diff --git a/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php b/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php new file mode 100644 index 0000000000..a7e7249deb --- /dev/null +++ b/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php @@ -0,0 +1,61 @@ +setName('searchengine:query:parse') + ->setDescription('Debug a search query') + ->addArgument( + 'query', + InputArgument::REQUIRED, + 'The query to debug' + ) + ->addOption( + 'raw', + false, + InputOption::VALUE_NONE, + 'Only output query dump' + ) + ; + } + + protected function doExecute(InputInterface $input, OutputInterface $output) + { + $string = $input->getArgument('query'); + $raw = $input->getOption('raw'); + + if (!$raw) { + $output->writeln(sprintf('Parsing search query: %s', $string)); + $output->writeln(str_repeat('-', 20)); + } + + $query = $this->container['query_parser']->parse($string); + $dump = $query->dump(); + + if (!$raw) { + $output->writeln($dump); + } else { + $output->write($dump); + } + } +} diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php index 25731c7649..2748f1e97c 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php @@ -24,4 +24,9 @@ class SearchQuery { return (string) $this->root; } + + public function dump() + { + return (string) $this; + } } From 0908b38a406b9740d1c31ddc99b116907537e10b Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 20 Nov 2014 18:13:58 +0100 Subject: [PATCH 11/21] Cleanup QueryParser & SearchQuery --- .../Elastic/Search/QueryParser.php | 29 ++++++++++--------- .../Elastic/Search/SearchQuery.php | 7 +---- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php index e725b4362e..a0982e7874 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php @@ -4,10 +4,9 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Search; use Alchemy\Phrasea\SearchEngine\Elastic\AST; use Hoa\Compiler\Llk\Parser; -use Hoa\Visitor\Element; +use Hoa\Compiler\Visitor\Dump as DumpVisitor; use Hoa\Visitor\Visit; - class QueryParser { private $parser; @@ -17,22 +16,24 @@ class QueryParser $this->parser = $parser; } + /** + * Creates a Query object from a string + */ public function parse($string) + { + return $this->visitString($string, new QueryVisitor()); + } + + public function dump($string) + { + return $this->visitString($string, new DumpVisitor()); + } + + private function visitString($string, Visit $visitor) { $ast = $this->parser->parse($string); - $dump = new \Hoa\Compiler\Visitor\Dump(); - echo $dump->visit($ast); - - // Create query from syntax tree - $visitor = new QueryVisitor(); - $query = $visitor->visit($ast); - - echo "--------------------\n"; - echo (string) $query . "\n"; - echo "--------------------\n"; - - return $query; + return $visitor->visit($ast); } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php index 2748f1e97c..6fe33de9d1 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php @@ -20,13 +20,8 @@ class SearchQuery return $this->root->getQuery(); } - public function __toString() - { - return (string) $this->root; - } - public function dump() { - return (string) $this; + return $this->root->__toString(); } } From c3867d412e1835b8e1f3e49d1debfd7322382544 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 20 Nov 2014 18:22:16 +0100 Subject: [PATCH 12/21] Search\SearchQuery -> Search\Query --- .../SearchEngine/Elastic/Search/{SearchQuery.php => Query.php} | 2 +- .../Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/{SearchQuery.php => Query.php} (96%) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Query.php similarity index 96% rename from lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php rename to lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Query.php index 6fe33de9d1..b2facf5bd9 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/SearchQuery.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Query.php @@ -6,7 +6,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\AST\Node; use Alchemy\Phrasea\SearchEngine\Elastic\AST\AndOperator; use Hoa\Compiler\Llk\TreeNode; -class SearchQuery +class Query { private $root; diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php index fd2cac1052..b75c88e998 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php @@ -47,7 +47,7 @@ class QueryVisitor implements Visit $root = $node; } } - return new SearchQuery($root); + return new Query($root); case self::NODE_TYPE_IN_EXPR: if ($element->getChildrenNumber() !== 2) { From 810a49652170997653c7946eee1f008b17411632 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 26 Nov 2014 11:42:39 +0100 Subject: [PATCH 13/21] Refactor query visitor --- .../Elastic/Search/QueryVisitor.php | 154 +++++++++++------- 1 file changed, 97 insertions(+), 57 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php index b75c88e998..cfdc9dfc79 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php @@ -23,79 +23,119 @@ class QueryVisitor implements Visit public function visit(Element $element, &$handle = null, $eldnah = null) { if (null !== $value = $element->getValue()) { - switch ($value['token']) { - case self::NODE_TOKEN_WORD: - return new AST\TextNode($value['value']); - - case self::NODE_TOKEN_STRING: - return new AST\QuotedTextNode($value['value']); - - default: - // Generic handling off other tokens for unresctricted text - return new AST\TextNode($value['value']); - } + return $this->visitToken($value['token'], $value['value']); + } else { + return $this->visitNode($element); } + } + private function visitToken($token, $value) + { + switch ($token) { + case self::NODE_TOKEN_WORD: + return new AST\TextNode($value); + + case self::NODE_TOKEN_STRING: + return new AST\QuotedTextNode($value); + + default: + // Generic handling off other tokens for unresctricted text + return new AST\TextNode($value); + } + } + + private function visitNode(Element $element) + { switch ($element->getId()) { case self::NODE_TYPE_QUERY: - $root = null; - foreach ($element->getChildren() as $child) { - $node = $child->accept($this, $handle, $eldnah); - if ($root) { - $root = new AST\AndExpression($root, $node); - } else { - $root = $node; - } - } - return new Query($root); + return $this->visitQuery($element); case self::NODE_TYPE_IN_EXPR: - if ($element->getChildrenNumber() !== 2) { - throw new \Exception('IN expression can only have 2 childs.'); - } - $expression = $element->getChild(0)->accept($this); - $field = new AST\FieldNode($element->getChild(1)->getValue()['value']); - return new AST\InExpression($field, $expression); + return $this->visitInNode($element); case self::NODE_TYPE_AND_EXPR: - if ($element->getChildrenNumber() !== 2) { - throw new \Exception('AND expression can only have 2 childs.'); - } - $left = $element->getChild(0)->accept($this); - $right = $element->getChild(1)->accept($this); - return new AST\AndExpression($left, $right); + return $this->visitAndNode($element); case self::NODE_TYPE_OR_EXPR: - if ($element->getChildrenNumber() !== 2) { - throw new \Exception('OR expression can only have 2 childs.'); - } - $left = $element->getChild(0)->accept($this); - $right = $element->getChild(1)->accept($this); - return new AST\OrExpression($left, $right); + return $this->visitOrNode($element); case self::NODE_TYPE_TEXT: case self::NODE_TYPE_UNRESTRICTED_TEXT: - $root = null; - foreach ($element->getChildren() as $child) { - $node = $child->accept($this, $handle, $eldnah); - if ($root) { - // $root = new AST\AndExpression($root, $node); - // Merge text nodes together, but not with quoted ones - if ($root instanceof AST\TextNode && - !$root instanceof AST\QuotedTextNode && - !$node instanceof AST\QuotedTextNode) { - $root = new AST\TextNode(sprintf('%s %s', $root->getText(), $node->getText())); - } else { - $root = new AST\AndExpression($root, $node); - } - } else { - $root = $node; - } - } - return $root; + return $this->visitText($element); default: throw new \Exception(sprintf('Unknown node type "%s".', $element->getId())); } } + + private function visitQuery(Element $element) + { + $root = null; + foreach ($element->getChildren() as $child) { + $node = $child->accept($this); + if ($root) { + $root = new AST\AndExpression($root, $node); + } else { + $root = $node; + } + } + return new Query($root); + } + + private function visitInNode(Element $element) + { + if ($element->getChildrenNumber() !== 2) { + throw new \Exception('IN expression can only have 2 childs.'); + } + $expression = $element->getChild(0)->accept($this); + $field = new AST\FieldNode($element->getChild(1)->getValue()['value']); + return new AST\InExpression($field, $expression); + } + + private function visitAndNode(Element $element) + { + return $this->handleBinaryNode($element, function($left, $right) { + return new AST\AndExpression($left, $right); + }); + } + + private function visitOrNode(Element $element) + { + return $this->handleBinaryNode($element, function($left, $right) { + return new AST\OrExpression($left, $right); + }); + } + + private function handleBinaryNode(Element $element, \Closure $factory) + { + if ($element->getChildrenNumber() !== 2) { + throw new \Exception('Binary expression can only have 2 childs.'); + } + $left = $element->getChild(0)->accept($this); + $right = $element->getChild(1)->accept($this); + + return $factory($left, $right); + } + + private function visitText(Element $element) + { + $root = null; + foreach ($element->getChildren() as $child) { + $node = $child->accept($this); + if ($root) { + // Merge text nodes together, but not with quoted ones + if ($root instanceof AST\TextNode && + !$root instanceof AST\QuotedTextNode && + !$node instanceof AST\QuotedTextNode) { + $root = new AST\TextNode(sprintf('%s %s', $root->getText(), $node->getText())); + } else { + $root = new AST\AndExpression($root, $node); + } + } else { + $root = $node; + } + } + + return $root; + } } From c535664c0f3a94e784da3bd57e76d017c1dae00b Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 26 Nov 2014 13:11:58 +0100 Subject: [PATCH 14/21] Add compiler AST dump to debug command --- .../SearchEngine/Debug/QueryParseCommand.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php b/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php index a7e7249deb..3c8d08817b 100644 --- a/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php +++ b/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php @@ -30,6 +30,12 @@ class QueryParseCommand extends Command InputArgument::REQUIRED, 'The query to debug' ) + ->addOption( + 'compiler-dump', + false, + InputOption::VALUE_NONE, + 'Output underlying compiler AST before visitor processing' + ) ->addOption( 'raw', false, @@ -49,8 +55,13 @@ class QueryParseCommand extends Command $output->writeln(str_repeat('-', 20)); } - $query = $this->container['query_parser']->parse($string); - $dump = $query->dump(); + $parser = $this->container['query_parser']; + if ($input->getOption('compiler-dump')) { + $dump = $parser->dump($string); + } else { + $query = $parser->parse($string); + $dump = $query->dump(); + } if (!$raw) { $output->writeln($dump); From 03926ead81a34135f4d0a2446cbf93d029650b08 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 26 Nov 2014 14:10:55 +0100 Subject: [PATCH 15/21] Brackets support in queries --- grammar/query.pp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/grammar/query.pp b/grammar/query.pp index e87670a762..39a54971c1 100644 --- a/grammar/query.pp +++ b/grammar/query.pp @@ -1,5 +1,7 @@ %skip space \s +%token bracket_ \( +%token _bracket \) %token quote_ " -> string %token string:string [^"]+ %token string:_quote " -> default @@ -7,7 +9,7 @@ %token and AND %token or OR %token except EXCEPT -%token word \S+ +%token word [^\s\(\)]+ // relative order of precedence is NOT > XOR > AND > OR @@ -24,10 +26,13 @@ ternary: quaternary() ( ::and:: #and primary() )? quaternary: - text() ( ::in:: #in word() )? + term() ( ::in:: #in word() )? + +term: + ( ::bracket_:: primary() ::_bracket:: ) | text() #text: - ( word() | keyword() )+ + ( word() | keyword() | symbol() )+ word: | string() @@ -37,3 +42,6 @@ string: keyword: | | | + +symbol: + ::bracket_:: | ::_bracket:: From e9c9fe2376d6a7a40ea2692eb9de781866789408 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 27 Nov 2014 18:27:30 +0100 Subject: [PATCH 16/21] Add Except query support --- grammar/query.pp | 7 ++- .../Elastic/Search/QueryVisitor.php | 44 +++++++++++++------ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/grammar/query.pp b/grammar/query.pp index 39a54971c1..fdd383c21f 100644 --- a/grammar/query.pp +++ b/grammar/query.pp @@ -17,7 +17,12 @@ primary() primary: - secondary() ( ::except:: #except primary() )? + secondary() primary_prime() + +// primary_prime rule is used to remove left-recursion at the expense +// of losing associativity +primary_prime: + ( primary() )? secondary: ternary() ( ::or:: #or primary() )? diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php index cfdc9dfc79..ea8583d7be 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php @@ -6,7 +6,6 @@ use Alchemy\Phrasea\SearchEngine\Elastic\AST; use Hoa\Visitor\Element; use Hoa\Visitor\Visit; - class QueryVisitor implements Visit { const NODE_TYPE_QUERY = '#query'; @@ -19,14 +18,25 @@ class QueryVisitor implements Visit const NODE_TYPE_TOKEN = 'token'; const NODE_TOKEN_WORD = 'word'; const NODE_TOKEN_STRING = 'string'; + const NODE_TOKEN_EXCEPT = 'except'; + + private $leftNode; + private $leftOp; public function visit(Element $element, &$handle = null, $eldnah = null) { if (null !== $value = $element->getValue()) { return $this->visitToken($value['token'], $value['value']); - } else { - return $this->visitNode($element); } + + $node = $this->visitNode($element); + if ($this->leftOp) { + $node = $this->leftOp->__invoke($node); + $this->leftOp = null; + } + $this->leftNode = $node; + + return $node; } private function visitToken($token, $value) @@ -38,6 +48,20 @@ class QueryVisitor implements Visit case self::NODE_TOKEN_STRING: return new AST\QuotedTextNode($value); + case self::NODE_TOKEN_EXCEPT: + // Schedule the operation at the next node visit using also + // previous node to build the "except" expression. + // (we don't have the next node yet). + // + // Tokens taking part in an "except" expression are emited by + // the compiler as a flat list, not a tree, because we can't + // maintain left-associativity required by EXCEPT operator. + $left = $this->leftNode; + $this->leftOp = function ($right) use ($left) { + return new AST\ExceptExpression($left, $right); + }; + break; + default: // Generic handling off other tokens for unresctricted text return new AST\TextNode($value); @@ -70,14 +94,8 @@ class QueryVisitor implements Visit private function visitQuery(Element $element) { - $root = null; foreach ($element->getChildren() as $child) { - $node = $child->accept($this); - if ($root) { - $root = new AST\AndExpression($root, $node); - } else { - $root = $node; - } + $root = $child->accept($this); } return new Query($root); } @@ -94,19 +112,19 @@ class QueryVisitor implements Visit private function visitAndNode(Element $element) { - return $this->handleBinaryNode($element, function($left, $right) { + return $this->handleBinaryOperator($element, function($left, $right) { return new AST\AndExpression($left, $right); }); } private function visitOrNode(Element $element) { - return $this->handleBinaryNode($element, function($left, $right) { + return $this->handleBinaryOperator($element, function($left, $right) { return new AST\OrExpression($left, $right); }); } - private function handleBinaryNode(Element $element, \Closure $factory) + private function handleBinaryOperator(Element $element, \Closure $factory) { if ($element->getChildrenNumber() !== 2) { throw new \Exception('Binary expression can only have 2 childs.'); From a08508ae857fd22701f4569dfbe3289515a1c5f4 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Thu, 27 Nov 2014 18:33:16 +0100 Subject: [PATCH 17/21] Better query error handling --- .../Elastic/Exception/QueryException.php | 16 ++++++++++++++++ .../SearchEngine/Elastic/Search/QueryParser.php | 8 +++++++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 lib/Alchemy/Phrasea/SearchEngine/Elastic/Exception/QueryException.php diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Exception/QueryException.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Exception/QueryException.php new file mode 100644 index 0000000000..b38ea30a72 --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Exception/QueryException.php @@ -0,0 +1,16 @@ +parser->parse($string); + try { + $ast = $this->parser->parse($string); + } catch (CompilerException $e) { + throw new QueryException('Provided query is not valid', 0, $e); + } return $visitor->visit($ast); } From 2aa0b0f1a894c9852fdb3caffac1b73db9ea8eed Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 3 Dec 2014 17:16:22 +0100 Subject: [PATCH 18/21] Proper left-associativity in query operators Remove previous lambda/curring based method Grammar was simplified for EXCEPT case Cleanup unused constants --- grammar/query.pp | 7 +-- .../SearchEngine/Elastic/Search/NodeTypes.php | 18 ++++++ .../Elastic/Search/QueryParser.php | 62 ++++++++++++++++++ .../Elastic/Search/QueryVisitor.php | 63 ++++++------------- 4 files changed, 99 insertions(+), 51 deletions(-) create mode 100644 lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/NodeTypes.php diff --git a/grammar/query.pp b/grammar/query.pp index fdd383c21f..39a54971c1 100644 --- a/grammar/query.pp +++ b/grammar/query.pp @@ -17,12 +17,7 @@ primary() primary: - secondary() primary_prime() - -// primary_prime rule is used to remove left-recursion at the expense -// of losing associativity -primary_prime: - ( primary() )? + secondary() ( ::except:: #except primary() )? secondary: ternary() ( ::or:: #or primary() )? diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/NodeTypes.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/NodeTypes.php new file mode 100644 index 0000000000..63c7ea0e04 --- /dev/null +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/NodeTypes.php @@ -0,0 +1,18 @@ +parser = $parser; @@ -39,7 +46,62 @@ class QueryParser throw new QueryException('Provided query is not valid', 0, $e); } + + $this->fixOperatorAssociativity($ast); + return $visitor->visit($ast); } + + /** + * Walks the tree to restore left-associativity of some operators + * + * @param TreeNode $root AST root node + */ + private function fixOperatorAssociativity(TreeNode &$root) + { + switch ($root->getChildrenNumber()) { + case 0: + // Leaf nodes can't be rotated, and have no childs + return; + + case 2: + // We only want to rotate tree contained in the left associative + // subset of operators + $rootType = $root->getId(); + if (!in_array($rootType, self::$leftAssociativeOperators)) { + break; + } + // Do not break operator precedence + $pivot = $root->getChild(1); + if ($pivot->getId() !== $rootType) { + break; + } + $this->leftRotateTree($root); + break; + } + + // Recursively fix tree branches + $children = $root->getChildren(); + foreach ($children as $index => $_) { + $this->fixOperatorAssociativity($children[$index]); + } + $root->setChildren($children); + } + + private function leftRotateTree(TreeNode &$root) + { + // Pivot = Root.Left + $pivot = $root->getChild(1); + // Root.Right = Pivot.Left + $children = $root->getChildren(); + $children[1] = $pivot->getChild(0); // Pivot, rotation side + $root->setChildren($children); + // Pivot.Left = Root + $children = $pivot->getChildren(); + $children[0] = $root; + $pivot->setChildren($children); + // Root = Pivot + $root = $pivot; + } } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php index ea8583d7be..a60dba1b85 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryVisitor.php @@ -8,60 +8,24 @@ use Hoa\Visitor\Visit; class QueryVisitor implements Visit { - const NODE_TYPE_QUERY = '#query'; - const NODE_TYPE_IN_EXPR = '#in'; - const NODE_TYPE_AND_EXPR = '#and'; - const NODE_TYPE_OR_EXPR = '#or'; - const NODE_TYPE_FIELD = '#field'; - const NODE_TYPE_TEXT = '#text'; - const NODE_TYPE_UNRESTRICTED_TEXT = '#unrestricted_text'; - const NODE_TYPE_TOKEN = 'token'; - const NODE_TOKEN_WORD = 'word'; - const NODE_TOKEN_STRING = 'string'; - const NODE_TOKEN_EXCEPT = 'except'; - - private $leftNode; - private $leftOp; - public function visit(Element $element, &$handle = null, $eldnah = null) { if (null !== $value = $element->getValue()) { return $this->visitToken($value['token'], $value['value']); } - $node = $this->visitNode($element); - if ($this->leftOp) { - $node = $this->leftOp->__invoke($node); - $this->leftOp = null; - } - $this->leftNode = $node; - - return $node; + return $this->visitNode($element); } private function visitToken($token, $value) { switch ($token) { - case self::NODE_TOKEN_WORD: + case NodeTypes::TOKEN_WORD: return new AST\TextNode($value); - case self::NODE_TOKEN_STRING: + case NodeTypes::TOKEN_STRING: return new AST\QuotedTextNode($value); - case self::NODE_TOKEN_EXCEPT: - // Schedule the operation at the next node visit using also - // previous node to build the "except" expression. - // (we don't have the next node yet). - // - // Tokens taking part in an "except" expression are emited by - // the compiler as a flat list, not a tree, because we can't - // maintain left-associativity required by EXCEPT operator. - $left = $this->leftNode; - $this->leftOp = function ($right) use ($left) { - return new AST\ExceptExpression($left, $right); - }; - break; - default: // Generic handling off other tokens for unresctricted text return new AST\TextNode($value); @@ -71,20 +35,22 @@ class QueryVisitor implements Visit private function visitNode(Element $element) { switch ($element->getId()) { - case self::NODE_TYPE_QUERY: + case NodeTypes::QUERY: return $this->visitQuery($element); - case self::NODE_TYPE_IN_EXPR: + case NodeTypes::IN_EXPR: return $this->visitInNode($element); - case self::NODE_TYPE_AND_EXPR: + case NodeTypes::AND_EXPR: return $this->visitAndNode($element); - case self::NODE_TYPE_OR_EXPR: + case NodeTypes::OR_EXPR: return $this->visitOrNode($element); - case self::NODE_TYPE_TEXT: - case self::NODE_TYPE_UNRESTRICTED_TEXT: + case NodeTypes::EXCEPT_EXPR: + return $this->visitExceptNode($element); + + case NodeTypes::TEXT: return $this->visitText($element); default: @@ -124,6 +90,13 @@ class QueryVisitor implements Visit }); } + private function visitExceptNode(Element $element) + { + return $this->handleBinaryOperator($element, function($left, $right) { + return new AST\ExceptExpression($left, $right); + }); + } + private function handleBinaryOperator(Element $element, \Closure $factory) { if ($element->getChildrenNumber() !== 2) { From b057efb290f8992757c61e6396be0fc962d91f3b Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 3 Dec 2014 17:23:00 +0100 Subject: [PATCH 19/21] Query command option to disable AST postprocessing --- .../SearchEngine/Debug/QueryParseCommand.php | 16 ++++++++++++---- .../SearchEngine/Elastic/Search/QueryParser.php | 15 ++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php b/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php index 3c8d08817b..4ab3bad78b 100644 --- a/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php +++ b/lib/Alchemy/Phrasea/Command/SearchEngine/Debug/QueryParseCommand.php @@ -32,13 +32,19 @@ class QueryParseCommand extends Command ) ->addOption( 'compiler-dump', - false, + 'd', InputOption::VALUE_NONE, 'Output underlying compiler AST before visitor processing' ) + ->addOption( + 'no-compiler-postprocessing', + 'p', + InputOption::VALUE_NONE, + 'Prevent AST optimization before visitor processing' + ) ->addOption( 'raw', - false, + null, InputOption::VALUE_NONE, 'Only output query dump' ) @@ -55,11 +61,13 @@ class QueryParseCommand extends Command $output->writeln(str_repeat('-', 20)); } + $postprocessing = !$input->getOption('no-compiler-postprocessing'); + $parser = $this->container['query_parser']; if ($input->getOption('compiler-dump')) { - $dump = $parser->dump($string); + $dump = $parser->dump($string, $postprocessing); } else { - $query = $parser->parse($string); + $query = $parser->parse($string, $postprocessing); $dump = $query->dump(); } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php index 0cc79f3521..f4be7af627 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryParser.php @@ -28,17 +28,17 @@ class QueryParser /** * Creates a Query object from a string */ - public function parse($string) + public function parse($string, $postprocessing = true) { - return $this->visitString($string, new QueryVisitor()); + return $this->visitString($string, new QueryVisitor(), $postprocessing); } - public function dump($string) + public function dump($string, $postprocessing = true) { - return $this->visitString($string, new DumpVisitor()); + return $this->visitString($string, new DumpVisitor(), $postprocessing); } - private function visitString($string, Visit $visitor) + private function visitString($string, Visit $visitor, $postprocessing = true) { try { $ast = $this->parser->parse($string); @@ -46,8 +46,9 @@ class QueryParser throw new QueryException('Provided query is not valid', 0, $e); } - - $this->fixOperatorAssociativity($ast); + if ($postprocessing) { + $this->fixOperatorAssociativity($ast); + } return $visitor->visit($ast); } From e53e92da633ab7868319fb1170c8312424ec85f7 Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 3 Dec 2014 19:35:10 +0100 Subject: [PATCH 20/21] Plug new parser on search engine --- .../Elastic/ElasticSearchEngine.php | 17 ++++------------- .../SearchEngine/Elastic/Search/Query.php | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php index 438ef7c0bb..4adb26495d 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticSearchEngine.php @@ -274,24 +274,15 @@ class ElasticSearchEngine implements SearchEngineInterface $searchQuery = $this->app['query_parser']->parse($string); - $esQuery = $searchQuery->getElasticsearchQuery(); - - echo json_encode($esQuery, JSON_PRETTY_PRINT); - - die(); - - $parser = new QueryParser(); - $ast = $parser->parse($string); - // Contains the full thesaurus paths to search on $pathsToFilter = []; // Contains the thesaurus values by fields (synonyms, translations, etc) $collectFields = []; // Only search in thesaurus for full text search - if ($ast->isFullTextOnly()) { + if ($searchQuery->isFullTextOnly()) { $termFiels = $this->expendToAnalyzedFieldsNames('value'); - $termsQuery = $ast->getQuery($termFiels); + $termsQuery = $searchQuery->getElasticsearchQuery($termFiels); $params = $this->createTermQueryParams($termsQuery, $options); $terms = $this->doExecute('search', $params); @@ -318,7 +309,7 @@ class ElasticSearchEngine implements SearchEngineInterface $recordQuery = [ 'bool' => [ 'should' => [ - $ast->getQuery($recordFields) + $searchQuery->getElasticsearchQuery($recordFields) ] ] ]; @@ -371,7 +362,7 @@ class ElasticSearchEngine implements SearchEngineInterface $results[] = new \record_adapter($this->app, $databoxId, $recordId, $n++); } - $query['_ast'] = (string) $ast; + $query['_ast'] = $searchQuery->dump(); $query['_paths'] = $pathsToFilter; $query['_richFields'] = $collectFields; $query['query'] = json_encode($params); diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Query.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Query.php index b2facf5bd9..8e91458957 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Query.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Query.php @@ -15,9 +15,19 @@ class Query $this->root = $root; } - public function getElasticsearchQuery() + /* + * This method seems weird to me, the implementation returns true when the + * query doesn't contain IN statements, but that doesn't define a full text + * search. + */ + public function isFullTextOnly() { - return $this->root->getQuery(); + return $this->root->isFullTextOnly(); + } + + public function getElasticsearchQuery($fields = array()) + { + return $this->root->getQuery($fields); } public function dump() From 7cce87b1c0ba694bc0bfcdfe1ea8eef284d6366b Mon Sep 17 00:00:00 2001 From: Mathieu Darse Date: Wed, 3 Dec 2014 19:35:46 +0100 Subject: [PATCH 21/21] Fix isFullTextOnly() --- .../Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php index 9e4a0f9edb..54c41a4514 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/BinaryOperator.php @@ -21,12 +21,7 @@ abstract class BinaryOperator extends Node public function isFullTextOnly() { - foreach ($this->members as $member) { - if (!$member->isFullTextOnly()) { - return false; - } - } - - return true; + return $this->left->isFullTextOnly() + && $this->right->isFullTextOnly(); } }