diff --git a/README.md b/README.md index f394ace118..377d5ceee6 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Phraseanet is licensed under GPL-v3 license. https://docs.phraseanet.com/ +For development with Phraseanet API see https://docs.phraseanet.com/4.0/en/Devel/index.html + # Installation : You **must** not download the source from GitHub, but download a packaged version here : @@ -26,13 +28,7 @@ https://www.phraseanet.com/download/ And follow the install steps described at https://docs.phraseanet.com/4.0/en/Admin/Install.html -# Try Phraseanet : - -You can also download a testing pre installed Virtual Machine in OVA format here : - -https://www.phraseanet.com/download/ - -# With Docker +# Phraseanet with Docker: ## Prerequisites @@ -92,9 +88,27 @@ Where `` can be: - `bin/console worker:execute -m 2` - ... - The default parameters allow you to reach the app with : `http://localhost:8082` +### Use Phraseanet images from docker hub + +Retrieve on Docker hub prebuilt images for Phraseanet. + +https://hub.docker.com/r/alchemyfr/phraseanet-fpm + +https://hub.docker.com/r/alchemyfr/phraseanet-worker + +https://hub.docker.com/r/alchemyfr/phraseanet-nginx + +To use them and not build the images locally, we advise to override the properties in file: env.local + +```bash +# Registry from where you pull Docker images +PHRASEANET_DOCKER_REGISTRY=alchemyfr +# Tag of the Docker images +PHRASEANET_DOCKER_TAG= +``` + ## Development mode The development mode uses the `docker-compose-override.yml` file. @@ -175,6 +189,12 @@ export PHRASEANET_SSH_PRIVATE_KEY=$(cat ~/.ssh/id_rsa) export PHRASEANET_SSH_PRIVATE_KEY=$(openssl rsa -in ~/.ssh/id_rsa -out /tmp/id_rsa_raw && cat /tmp/id_rsa_raw && rm /tmp/id_rsa_raw) ``` +# Try Phraseanet with Pre installed VM (deprecated) + +You can also download a testing pre installed Virtual Machine in OVA format here : + +https://www.phraseanet.com/download/ + # With Vagrant (deprecated) ## Development : @@ -194,4 +214,5 @@ Ex: - vagrant up --provision //// 5.6 ///// 1 >> Build the alchemy/phraseanet-php-5.6 box -For development with Phraseanet API see https://docs.phraseanet.com/4.0/en/Devel/index.html + + diff --git a/composer.json b/composer.json index 46ff98203e..0fd22effb0 100644 --- a/composer.json +++ b/composer.json @@ -96,7 +96,7 @@ "league/flysystem": "^1.0", "league/flysystem-aws-s3-v2": "^1.0", "league/fractal": "dev-webgalleries#af1acc0275438571bc8c1d08a05a4b5af92c9f97 as 0.13.0", - "media-alchemyst/media-alchemyst": "^0.5.5", + "media-alchemyst/media-alchemyst": "^0.5.6", "monolog/monolog": "~1.3", "mrclay/minify": "~2.1.6", "neutron/process-manager": "2.0.x-dev@dev", @@ -105,7 +105,7 @@ "neutron/silex-imagine-provider": "~0.1.0", "neutron/temporary-filesystem": "~2.1", "pagerfanta/pagerfanta": "^1.0", - "php-ffmpeg/php-ffmpeg": "~0.5.0", + "php-ffmpeg/php-ffmpeg": "^v0.15", "php-xpdf/php-xpdf": "~0.2.1", "exiftool/exiftool": "^11", "ramsey/uuid": "^3.0", diff --git a/composer.lock b/composer.lock index 41a8b05beb..112a4ecd0f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5a4a0be62b13071a6b06893b7ce08372", + "content-hash": "008ff0b5d3d13b4f0ce5d34348ded83a", "packages": [ { "name": "alchemy-fr/tcpdf-clone", @@ -4331,16 +4331,16 @@ }, { "name": "media-alchemyst/media-alchemyst", - "version": "0.5.5", + "version": "0.5.6", "source": { "type": "git", "url": "https://github.com/alchemy-fr/Media-Alchemyst.git", - "reference": "3bd3204b69882f495adfb617383a077face92ed0" + "reference": "2b9f7697997f7863bbc3d08344c559a1cba519c2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/alchemy-fr/Media-Alchemyst/zipball/3bd3204b69882f495adfb617383a077face92ed0", - "reference": "3bd3204b69882f495adfb617383a077face92ed0", + "url": "https://api.github.com/repos/alchemy-fr/Media-Alchemyst/zipball/2b9f7697997f7863bbc3d08344c559a1cba519c2", + "reference": "2b9f7697997f7863bbc3d08344c559a1cba519c2", "shasum": "" }, "require": { @@ -4350,7 +4350,7 @@ "monolog/monolog": "~1.0", "neutron/temporary-filesystem": "^2.1.1", "php": ">=5.3.3", - "php-ffmpeg/php-ffmpeg": ">=0.4.2,<0.6", + "php-ffmpeg/php-ffmpeg": "^v0.15", "php-mp4box/php-mp4box": "~0.3.0", "php-unoconv/php-unoconv": "~0.3.1", "pimple/pimple": "~1.0", @@ -4401,7 +4401,7 @@ "video", "video processing" ], - "time": "2019-12-11T07:20:45+00:00" + "time": "2020-04-01T08:51:55+00:00" }, { "name": "monolog/monolog", @@ -5147,29 +5147,29 @@ }, { "name": "php-ffmpeg/php-ffmpeg", - "version": "0.5.1", + "version": "v0.15", "source": { "type": "git", "url": "https://github.com/PHP-FFMpeg/PHP-FFMpeg.git", - "reference": "c8949fe3df89edd7692368cc110a51a27971f28a" + "reference": "984dbd046b6d8c285f9e7419fc7645f197513bfa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHP-FFMpeg/PHP-FFMpeg/zipball/c8949fe3df89edd7692368cc110a51a27971f28a", - "reference": "c8949fe3df89edd7692368cc110a51a27971f28a", + "url": "https://api.github.com/repos/PHP-FFMpeg/PHP-FFMpeg/zipball/984dbd046b6d8c285f9e7419fc7645f197513bfa", + "reference": "984dbd046b6d8c285f9e7419fc7645f197513bfa", "shasum": "" }, "require": { - "alchemy/binary-driver": "~1.5", - "doctrine/cache": "~1.0", - "evenement/evenement": "~1.0", - "neutron/temporary-filesystem": "~2.1, >=2.1.1", - "php": ">=5.3.3" + "alchemy/binary-driver": "^1.5 || ~2.0.0 || ^5.0", + "doctrine/cache": "^1.0", + "evenement/evenement": "^3.0 || ^2.0 || ^1.0", + "neutron/temporary-filesystem": "^2.1.1", + "php": "^5.3.9 || ^7.0" }, "require-dev": { - "phpunit/phpunit": "~3.7", "sami/sami": "~1.0", - "silex/silex": "~1.0" + "silex/silex": "~1.0", + "symfony/phpunit-bridge": "^5.0.4" }, "suggest": { "php-ffmpeg/extras": "A compilation of common audio & video drivers for PHP-FFMpeg" @@ -5177,7 +5177,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "0.5-dev" + "dev-master": "0.7-dev" } }, "autoload": { @@ -5199,6 +5199,21 @@ "name": "Phraseanet Team", "email": "info@alchemy.fr", "homepage": "http://www.phraseanet.com/" + }, + { + "name": "Patrik Karisch", + "email": "patrik@karisch.guru", + "homepage": "http://www.karisch.guru" + }, + { + "name": "Romain Biard", + "email": "romain.biard@gmail.com", + "homepage": "https://www.strime.io/" + }, + { + "name": "Jens Hausdorf", + "email": "hello@jens-hausdorf.de", + "homepage": "https://jens-hausdorf.de" } ], "description": "FFMpeg PHP, an Object Oriented library to communicate with AVconv / ffmpeg", @@ -5212,7 +5227,7 @@ "video", "video processing" ], - "time": "2014-08-26T08:46:56+00:00" + "time": "2020-03-23T09:32:09+00:00" }, { "name": "php-mp4box/php-mp4box", @@ -7772,7 +7787,7 @@ ], "packages-dev": [ { - "name": "mikey179/vfsStream", + "name": "mikey179/vfsstream", "version": "v1.6.4", "source": { "type": "git", diff --git a/docker/phraseanet/entrypoint.sh b/docker/phraseanet/entrypoint.sh index 037fe0aa02..2e0f07519b 100755 --- a/docker/phraseanet/entrypoint.sh +++ b/docker/phraseanet/entrypoint.sh @@ -15,6 +15,7 @@ chown -R app:app \ FILE=config/configuration.yml if [ -f "$FILE" ]; then + bin/setup system:config set registry.general.title $PHRASEANET_PROJECT_NAME echo "$FILE exists, skip setup." else echo "$FILE doesn't exist, entering setup..." diff --git a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php index e342bf8380..8968ef5f0a 100644 --- a/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php +++ b/lib/Alchemy/Phrasea/Core/Provider/SearchEngineServiceProvider.php @@ -43,6 +43,7 @@ use Silex\ServiceProviderInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\HttpKernel\KernelEvents; + class SearchEngineServiceProvider implements ServiceProviderInterface { public function register(Application $app) @@ -145,6 +146,7 @@ class SearchEngineServiceProvider implements ServiceProviderInterface $app['elasticsearch.indexer.databox_fetcher_factory'] = $app->share(function ($app) { return new DataboxFetcherFactory( + $app['conf'], $app['elasticsearch.record_helper'], $app['elasticsearch.options'], $app, diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/DataboxFetcherFactory.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/DataboxFetcherFactory.php index b4eb070896..469ef65fc9 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/DataboxFetcherFactory.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/DataboxFetcherFactory.php @@ -2,6 +2,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic; +use Alchemy\Phrasea\Core\Configuration\PropertyAccess; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Delegate\FetcherDelegateInterface; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Fetcher; use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator\CoreHydrator; @@ -13,8 +14,14 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator\TitleHydrator; use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure; use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\CandidateTerms; + class DataboxFetcherFactory { + /** + * @var PropertyAccess phraseanet configuration + */ + private $conf; + /** * @var \ArrayAccess */ @@ -39,14 +46,16 @@ class DataboxFetcherFactory private $options; /** + * @param PropertyAccess $conf * @param RecordHelper $recordHelper * @param ElasticsearchOptions $options * @param \ArrayAccess $container * @param string $structureKey * @param string $thesaurusKey */ - public function __construct(RecordHelper $recordHelper, ElasticsearchOptions $options, \ArrayAccess $container, $structureKey, $thesaurusKey) + public function __construct(PropertyAccess $conf, RecordHelper $recordHelper, ElasticsearchOptions $options, \ArrayAccess $container, $structureKey, $thesaurusKey) { + $this->conf = $conf; $this->recordHelper = $recordHelper; $this->options = $options; $this->container = $container; @@ -70,7 +79,7 @@ class DataboxFetcherFactory [ new CoreHydrator($databox->get_sbas_id(), $databox->get_viewname(), $this->recordHelper), new TitleHydrator($connection, $this->recordHelper), - new MetadataHydrator($connection, $this->getStructure(), $this->recordHelper), + new MetadataHydrator($this->conf, $connection, $this->getStructure(), $this->recordHelper), new FlagHydrator($this->getStructure(), $databox), new ThesaurusHydrator($this->getStructure(), $this->getThesaurus(), $candidateTerms), new SubDefinitionHydrator($connection) diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/GpsPosition.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/GpsPosition.php index ba0dcf573f..9d4f1aabf7 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/GpsPosition.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/GpsPosition.php @@ -15,6 +15,7 @@ use Assert\Assertion; class GpsPosition { + const FULL_GEO_NOTATION = 'FullNotation'; const LONGITUDE_TAG_NAME = 'Longitude'; const LONGITUDE_REF_TAG_NAME = 'LongitudeRef'; const LONGITUDE_REF_WEST = 'W'; @@ -29,6 +30,16 @@ class GpsPosition private $latitude; private $latitude_ref; + public function __construct() + { + $this->clear(); + } + + public function clear() + { + $this->longitude = $this->longitude_ref = $this->latitude = $this->latitude_ref = null; + } + public function set($tag_name, $value) { switch ($tag_name) { @@ -64,6 +75,53 @@ class GpsPosition $this->latitude_ref = $normalized; break; + case self::FULL_GEO_NOTATION: + $re = '/(-?\d+(?:\.\d+)?°?)\s*(\d+(?:\.\d+)?\')?\s*(\d+(?:\.\d+)?")?\s*(N|S|E|W)?/um'; + $normalized = trim(strtoupper($value)); + $matches = null; + preg_match_all($re, $normalized, $matches, PREG_SET_ORDER, 0); + if(count($matches) === 2) { // we need lat and lon + $lat = $lon = null; + foreach ($matches as $imatch => $match) { + if(count($match) != 5) { + continue; + } + $v = 0.0; + for($part=1, $div=1.0; $part<=3; $part++, $div*=60.0) { + $v += floatval($match[$part]) / $div; + } + switch($match[4]) { // N S E W + case 'N': + $lat = $v; + break; + case 'S': + $lat = -$v; + break; + case 'E': + $lon = $v; + break; + case 'W': + $lon = -$v; + break; + case '': // no ref -> lat lon (first=lat, second=lon) + if($imatch === 0) { + $lat = $v; + } + else { + $lon = $v; + } + break; + default: + throw new \InvalidArgumentException(sprintf('Unsupported reference "%s", should be N|S|E|W.', $match[4])); + } + } + if($lat !== null && $lon != null) { + $this->set(self::LATITUDE_TAG_NAME, $lat); + $this->set(self::LONGITUDE_TAG_NAME, $lon); + } + } + break; + default: throw new \InvalidArgumentException(sprintf('Unsupported tag name "%s".', $tag_name)); } @@ -95,19 +153,11 @@ class GpsPosition public function getCompositeLongitude() { - if ($this->longitude === null) { - return null; - } - return $this->longitude ; } public function getCompositeLatitude() { - if ($this->latitude === null) { - return null; - } - return $this->latitude; } diff --git a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php index 5dc2471cb9..f930fce740 100644 --- a/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php +++ b/lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/MetadataHydrator.php @@ -11,6 +11,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator; +use Alchemy\Phrasea\Core\Configuration\PropertyAccess; use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception; use Alchemy\Phrasea\SearchEngine\Elastic\FieldMapping; use Alchemy\Phrasea\SearchEngine\Elastic\Mapping; @@ -24,28 +25,48 @@ use InvalidArgumentException; class MetadataHydrator implements HydratorInterface { + private $conf; private $connection; private $structure; private $helper; - private $gps_position_buffer = []; + private $position_fields_mapping; // get from conf - public function __construct(DriverConnection $connection, Structure $structure, RecordHelper $helper) + private $caption_gps_position; + private $exif_gps_position; + + public function __construct(PropertyAccess $conf, DriverConnection $connection, Structure $structure, RecordHelper $helper) { + $this->conf = $conf; $this->connection = $connection; $this->structure = $structure; $this->helper = $helper; + + // get the fieldnames of source of lat / lon geo fields (defined in instance conf) + $this->position_fields_mapping = []; + foreach($conf->get(['geocoding-providers'], []) as $provider) { + if($provider['enabled'] && array_key_exists('position-fields', $provider)) { + foreach ($provider['position-fields'] as $position_field) { + $this->position_fields_mapping[$position_field['name']] = $position_field['type']; + } + } + } + + $this->caption_gps_position = new GpsPosition(); + $this->exif_gps_position = new GpsPosition(); } public function hydrateRecords(array &$records) { - $sql = "(SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private\n" + $sql = "SELECT * FROM (" + . "(SELECT record_id, ms.name AS `key`, m.value AS value, 'caption' AS type, ms.business AS private\n" . " FROM metadatas AS m INNER JOIN metadatas_structure AS ms ON (ms.id = m.meta_struct_id)\n" . " WHERE record_id IN (?))\n" . "UNION\n" . "(SELECT record_id, t.name AS `key`, t.value AS value, 'exif' AS type, 0 AS private\n" . " FROM technical_datas AS t\n" - . " WHERE record_id IN (?))\n"; + . " WHERE record_id IN (?))\n" + . ") AS t ORDER BY record_id"; $ids = array_keys($records); $statement = $this->connection->executeQuery( @@ -54,7 +75,17 @@ class MetadataHydrator implements HydratorInterface array(Connection::PARAM_INT_ARRAY, Connection::PARAM_INT_ARRAY) ); + $record_id = -1; while ($metadata = $statement->fetch()) { + + if($metadata['record_id'] !== $record_id) { + // record has changed, don't mix with previous one + $this->caption_gps_position->clear(); + $this->exif_gps_position->clear(); + + $record_id = $metadata['record_id']; + } + // Store metadata value $key = $metadata['key']; $value = trim($metadata['value']); @@ -64,10 +95,10 @@ class MetadataHydrator implements HydratorInterface continue; } - $id = $metadata['record_id']; - if (isset($records[$id])) { - $record =& $records[$id]; - } else { + if (isset($records[$record_id])) { + $record =& $records[$record_id]; + } + else { throw new Exception('Received metadata from unexpected record'); } @@ -89,11 +120,31 @@ class MetadataHydrator implements HydratorInterface $record[$field] = array(); } $record[$field][] = $value; + + if(array_key_exists($key, $this->position_fields_mapping)) { + // this field is mapped as a position part (lat, lon, latlon), push it + switch($this->position_fields_mapping[$key]) { + case 'lat': + $this->handleGpsPosition($this->caption_gps_position, $record, GpsPosition::LATITUDE_TAG_NAME, $value); + break; + case 'lng': + case 'lon': + $this->handleGpsPosition($this->caption_gps_position, $record, GpsPosition::LONGITUDE_TAG_NAME, $value); + break; + case 'latlng': + case 'latlon': + $this->handleGpsPosition($this->caption_gps_position, $record, GpsPosition::FULL_GEO_NOTATION, $value); + break; + } + } + break; case 'exif': - if (GpsPosition::isSupportedTagName($key)) { - $this->handleGpsPosition($records, $id, $key, $value); + // exif gps is a first-chance if caption is not yet set + // anyway if caption is set later, it will override the exif values + if (GpsPosition::isSupportedTagName($key) && !$this->caption_gps_position->isCompleteComposite()) { + $this->handleGpsPosition($this->exif_gps_position, $record, $key, $value); break; } $tag = $this->structure->getMetadataTagByName($key); @@ -109,38 +160,25 @@ class MetadataHydrator implements HydratorInterface break; } } - - $this->clearGpsPositionBuffer(); } - private function handleGpsPosition(&$records, $id, $tag_name, $value) + private function handleGpsPosition(GpsPosition &$position, &$record, $tag_name, $value) { - // Get position object - if (!isset($this->gps_position_buffer[$id])) { - $this->gps_position_buffer[$id] = new GpsPosition(); - } - $position = $this->gps_position_buffer[$id]; // Push this tag into object $position->set($tag_name, $value); + // Try to output complete position if ($position->isCompleteComposite()) { $lon = $position->getCompositeLongitude(); $lat = $position->getCompositeLatitude(); - $records[$id]['metadata_tags']['Longitude'] = $lon; - $records[$id]['metadata_tags']['Latitude'] = $lat; + $record['metadata_tags']['Longitude'] = $lon; + $record['metadata_tags']['Latitude'] = $lat; - $records[$id]["location"] = [ + $record["location"] = [ "lat" => $lat, "lon" => $lon ]; - - unset($this->gps_position_buffer[$id]); } } - - private function clearGpsPositionBuffer() - { - $this->gps_position_buffer = []; - } } diff --git a/lib/conf.d/data_templates/DublinCore.xml b/lib/conf.d/data_templates/DublinCore.xml index eae43d5461..db3039df7f 100644 --- a/lib/conf.d/data_templates/DublinCore.xml +++ b/lib/conf.d/data_templates/DublinCore.xml @@ -79,7 +79,7 @@ 748 video yes - libmp3lame + libfdk_aac libx264 screen 1000 diff --git a/lib/conf.d/data_templates/en-simple.xml b/lib/conf.d/data_templates/en-simple.xml index 44b0afa69f..81ce397169 100644 --- a/lib/conf.d/data_templates/en-simple.xml +++ b/lib/conf.d/data_templates/en-simple.xml @@ -79,7 +79,7 @@ 748 video yes - libmp3lame + libfdk_aac libx264 screen 1000 diff --git a/lib/conf.d/data_templates/fr-simple.xml b/lib/conf.d/data_templates/fr-simple.xml index cc0a2b582c..48d628cedd 100644 --- a/lib/conf.d/data_templates/fr-simple.xml +++ b/lib/conf.d/data_templates/fr-simple.xml @@ -79,7 +79,7 @@ 748 video yes - libmp3lame + libfdk_aac libx264 screen 1000