Merge branch 'master' into PHRAS-2739-incorporate-subdefwebhook

Dockerfile

@@ -86,6 +86,18 @@ COPY grammar /var/alchemy/grammar
 COPY templates-profiler /var/alchemy/templates-profiler
 COPY templates /var/alchemy/templates
 COPY tests /var/alchemy/tests
+RUN mkdir -p /var/alchemy/Phraseanet/logs \
+    && chmod -R 777 /var/alchemy/Phraseanet/logs \
+    && mkdir -p /var/alchemy/Phraseanet/cache \
+    && chmod -R 777 /var/alchemy/Phraseanet/cache \
+    && mkdir -p /var/alchemy/Phraseanet/datas \
+    && chmod -R 777 /var/alchemy/Phraseanet/datas \
+    && mkdir -p /var/alchemy/Phraseanet/tmp \
+    && chmod -R 777 /var/alchemy/Phraseanet/tmp \
+    && mkdir -p /var/alchemy/Phraseanet/www/custom \
+    && chmod -R 777 /var/alchemy/Phraseanet/www/custom \
+    && mkdir -p /var/alchemy/Phraseanet/config \
+    && chmod -R 777 /var/alchemy/Phraseanet/config
 
 # Phraseanet
 FROM php:7.0-fpm-stretch as phraseanet-fpm
@@ -129,6 +141,23 @@ RUN apt-get update \
     && docker-php-source delete \
     && rm -rf /var/lib/apt/lists/*
 
+RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" \
+    && php -r "if (hash_file('sha384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" \
+    && php composer-setup.php --install-dir=/usr/local/bin --filename=composer \
+    && php -r "unlink('composer-setup.php');"
+
+# Node Installation (node + yarn)
+# Reference :
+# https://linuxize.com/post/how-to-install-node-js-on-ubuntu-18.04/
+# https://yarnpkg.com/lang/en/docs/install/#debian-stable
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+    && apt install -y nodejs \
+    && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+    && apt-get update && apt-get install -y --no-install-recommends yarn \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/
+
 RUN mkdir /entrypoint /var/alchemy \
     && useradd -u 1000 app \
     && mkdir -p /home/app/.composer \
@@ -136,18 +165,6 @@ RUN mkdir /entrypoint /var/alchemy \
 
 COPY --from=builder --chown=app /var/alchemy /var/alchemy/Phraseanet
 ADD ./docker/phraseanet/ /
-RUN mkdir -p /var/alchemy/Phraseanet/logs \
-    && chmod -R 777 /var/alchemy/Phraseanet/logs \
-    && mkdir -p /var/alchemy/Phraseanet/cache \
-    && chmod -R 777 /var/alchemy/Phraseanet/cache \
-    && mkdir -p /var/alchemy/Phraseanet/datas \
-    && chmod -R 777 /var/alchemy/Phraseanet/datas \
-    && mkdir -p /var/alchemy/Phraseanet/tmp \
-    && chmod -R 777 /var/alchemy/Phraseanet/tmp \
-    && mkdir -p /var/alchemy/Phraseanet/www/custom \
-    && chmod -R 777 /var/alchemy/Phraseanet/www/custom \
-    && mkdir -p /var/alchemy/Phraseanet/config \
-    && chmod -R 777 /var/alchemy/Phraseanet/config
 WORKDIR /var/alchemy/Phraseanet
 ENTRYPOINT ["/phraseanet-entrypoint.sh"]
 CMD ["/boot.sh"]

README.md

@@ -63,14 +63,12 @@ The docker distribution come with 3 differents containers :
 
 ## How to build
 
-The three images can be built respectively with these commands :
+You can build all the images with the following command at the root directory, choosing an arbirary TAG name :
 
-    # nginx server
+    ./build.sh <TAG>
-    docker build --target phraseanet-nginx -t local/phraseanet-nginx .
 
-    # php-fpm application
+It will build and tag the following images :
-    docker build --target phraseanet-fpm -t local/phraseanet-fpm .
-
-    # worker
-    docker build --target phraseanet-worker -t local/phraseanet-worker .
 
+    local/phraseanet-worker:<TAG>
+    local/phraseanet-fpm:<TAG>
+    local/phraseanet-nginx:<TAG>

build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# nginx server
+docker build --target phraseanet-nginx -t local/phraseanet-nginx:$1 .
+
+# php-fpm application
+docker build --target phraseanet-fpm -t local/phraseanet-fpm:$1 .
+
+# worker
+docker build --target phraseanet-worker -t local/phraseanet-worker:$1 .
+

docker/nginx/boot.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 
-cat nginx.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" > /etc/nginx/nginx.conf
+set -xe
+
+cat nginx.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" > /etc/nginx/conf.d/default.conf
 nginx -g "daemon off;"

docker/nginx/etc/nginx/nginx.conf

@@ -0,0 +1,31 @@
+
+user  app;
+worker_processes  1;
+
+error_log  /var/log/ngnix_error.log info;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+
+}

docker/nginx/nginx.conf.sample

@@ -1,85 +1,39 @@
-user  app;
+upstream backend {
-worker_processes  auto;
+    server phraseanet:9000;
-
-#error_log  /var/log/ngnix_error.log info;
-error_log /dev/stdout info;
-
-pid        /var/run/nginx.pid;
-#daemon off;
-
-events {
-    worker_connections  1024;
-    multi_accept on;
 }
 
-http {
+server {
-    include       /etc/nginx/mime.types;
+    listen       80;
-    default_type  application/octet-stream;
+    root         /var/alchemy/Phraseanet/www;
-    server_tokens off;
 
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+    index        index.php;
-                      '$status $body_bytes_sent "$http_referer" '
+    client_max_body_size $MAX_BODY_SIZE;
-                      '"$http_user_agent" "$http_x_forwarded_for"';
 
-    access_log  /dev/stdout main;
+    location /api {
-
+        rewrite ^(.*)$ /api.php/$1 last;
-    sendfile        on;
-    #tcp_nopush     on;
-
-    keepalive_timeout  65;
-
-    #gzip  on;
-
-    reset_timedout_connection on;
-
-    proxy_connect_timeout  300s;
-    proxy_send_timeout  300s;
-    proxy_read_timeout  300s;
-    fastcgi_send_timeout 300s;
-    fastcgi_read_timeout 300;
-
-    resolver 127.0.0.11;
-
-    upstream backend {
-        server phraseanet:9000;
     }
 
-    server {
+    location / {
-        listen       80;
+        # First attempt to serve request as file, then
-        server_name  localhost;
+        # as directory, then fall back to index.html
-        error_log    on;
+        try_files $uri $uri/ @rewriteapp;
-        access_log   on;
+    }
-        root         /var/alchemy/Phraseanet/www;
 
-        index        index.php;
+    location @rewriteapp {
-        client_max_body_size $MAX_BODY_SIZE;
+        rewrite ^(.*)$ /index.php/$1 last;
+    }
 
-        location /api {
+    # PHP scripts -> PHP-FPM server listening on 127.0.0.1:9000
-            rewrite ^(.*)$ /api.php/$1 last;
+    location ~ ^/(index|index_dev|api)\.php(/|$) {
-        }
+        fastcgi_pass   backend;
+        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
+        include        fastcgi_params;
+    }
 
-        location / {
+    location ~ ^/(status|ping)$ {
-            # First attempt to serve request as file, then
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-            # as directory, then fall back to index.html
+        fastcgi_index index.php;
-            try_files $uri $uri/ @rewriteapp;
+        include fastcgi_params;
-        }
+        fastcgi_pass backend;
-
-        location @rewriteapp {
-            rewrite ^(.*)$ /index.php/$1 last;
-        }
-
-        # PHP scripts -> PHP-FPM server listening on 127.0.0.1:9000
-        location ~ ^/(index|index_dev|api)\.php(/|$) {
-            fastcgi_pass   backend;
-            fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
-            include        fastcgi_params;
-        }
-
-        location ~ ^/(status|ping)$ {
-            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-            fastcgi_index index.php;
-            include fastcgi_params;
-            fastcgi_pass backend;
-        }
     }
 }

docker/phraseanet/auto-install.sh

@@ -2,6 +2,16 @@
 
 set -xe
 
+if [ $INSTALL_ACCOUNT_EMAIL = ""]; then
+    echo "INSTALL_ACCOUNT_EMAIL var is not set."
+    exit 1
+fi
+
+if [ $INSTALL_ACCOUNT_PASSWORD = ""]; then
+    echo "INSTALL_ACCOUNT_PASSWORD var is not set."
+    exit 1
+fi
+
 /var/alchemy/Phraseanet/bin/setup system:install \
     --email=$INSTALL_ACCOUNT_EMAIL \
     --password=$INSTALL_ACCOUNT_PASSWORD \

docker/phraseanet/boot.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+set -xe
+
+chown -R app:app /var/alchemy/Phraseanet/config
+chown -R app:app /var/alchemy/Phraseanet/datas
+chown -R app:app /var/alchemy/Phraseanet/tmp
+chown -R app:app /var/alchemy/Phraseanet/www/thumbnails
 FILE=/var/alchemy/Phraseanet/config/configuration.yml
 if [ -f "$FILE" ]; then
     echo "$FILE exist, skip setup."

docker/phraseanet/phraseanet-entrypoint.sh

@@ -5,4 +5,4 @@ set -e
 envsubst < /php.ini.sample > /usr/local/etc/php/php.ini
 envsubst < /php-fpm.conf.sample > /usr/local/etc/php-fpm.conf
 
-docker-php-entrypoint $@
+bash -e docker-php-entrypoint $@

docker/phraseanet/worker-boot.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
+mkdir /var/alchemy/Phraseanet/tmp/locks && chown -R app:app /var/alchemy/Phraseanet/tmp
 runuser app -c 'php /var/alchemy/Phraseanet/bin/console task-manager:scheduler:run'

lib/Alchemy/Phrasea/Cache/RedisCache.php

@@ -73,7 +73,7 @@ class RedisCache extends CacheProvider implements Cache
      */
     protected function doDelete($id)
     {
-        return $this->_redis->delete($id);
+        return $this->_redis->del($id);
     }
 
     /**

lib/Alchemy/Phrasea/Controller/Admin/FieldsController.php

@@ -316,6 +316,7 @@ class FieldsController extends Controller
             ->set_tbranch($data['tbranch'])
             ->set_generate_cterms($data['generate_cterms'])
             ->set_gui_editable($data['gui_editable'])
+            ->set_gui_visible($data['gui_visible'])
             ->set_report($data['report'])
             ->setVocabularyControl(null)
             ->setVocabularyRestricted(false);
@@ -351,7 +352,7 @@ class FieldsController extends Controller
     {
         return [
             'name', 'multi', 'thumbtitle', 'tag', 'business', 'indexable', 'aggregable',
-            'required', 'separator', 'readonly', 'gui_editable', 'type', 'tbranch', 'generate_cterms', 'report',
+            'required', 'separator', 'readonly', 'gui_editable', 'gui_visible' , 'type', 'tbranch', 'generate_cterms', 'report',
             'vocabulary-type', 'vocabulary-restricted', 'dces-element', 'labels'
         ];
     }

lib/Alchemy/Phrasea/Controller/Api/V1Controller.php

@@ -596,6 +596,7 @@ class V1Controller extends Controller
                 'thesaurus_branch' => $databox_field->get_tbranch(),
                 'generate_cterms'  => $databox_field->get_generate_cterms(),
                 'gui_editable'     => $databox_field->get_gui_editable(),
+                'gui_visible'      => $databox_field->get_gui_visible(),
                 'type'             => $databox_field->get_type(),
                 'indexable'        => $databox_field->is_indexable(),
                 'multivalue'       => $databox_field->is_multi(),

lib/Alchemy/Phrasea/Controller/Prod/EditController.php

@@ -77,6 +77,7 @@ class EditController extends Controller
                     'tbranch'              => $meta->get_tbranch(),
                     'generate_cterms'       => $meta->get_generate_cterms(),
                     'gui_editable'         => $meta->get_gui_editable(),
+                    'gui_visible'         => $meta->get_gui_visible(),
                     'maxLength'            => $meta->get_tag()
                         ->getMaxLength(),
                     'minLength'            => $meta->get_tag()

lib/Alchemy/Phrasea/Controller/Prod/QueryController.php

@@ -382,7 +382,6 @@ class QueryController extends Controller
                             'labels'      => $field->get_labels(),
                             'type'        => $field->get_type(),
                             'field'       => $field->get_name(),
-                            'query'       => "field." . $field->get_name() . ":%s",
                             'trans_label' => $field->get_label($this->app['locale']),
                         ];
                         $field->get_label($this->app['locale']);

lib/Alchemy/Phrasea/Core/Version.php

@@ -16,7 +16,7 @@ class Version
     /**
      * @var string
      */
-    private $number = '4.1.0-alpha.17a';
+    private $number = '4.1.0-alpha.18a';
 
     /**
      * @var string

lib/Alchemy/Phrasea/Databox/Field/DbalDataboxFieldRepository.php

@@ -38,6 +38,7 @@ final class DbalDataboxFieldRepository implements DataboxFieldRepository
         'label_nl',
         'generate_cterms',
         'gui_editable',
+        'gui_visible',
     ];
 
     /** @var DataboxFieldFactory */

lib/Alchemy/Phrasea/SearchEngine/Elastic/AST/TextNode.php

@@ -45,6 +45,9 @@ class TextNode extends AbstractTermNode implements ContextAbleInterface
                 foreach ($context->localizeField($field) as $f) {
                     $index_fields[] = $f;
                 }
+                foreach ($context->truncationField($field) as $f) {
+                    $index_fields[] = $f;
+                }
             }
             if (!$index_fields) {
                 return null;

lib/Alchemy/Phrasea/SearchEngine/Elastic/ElasticsearchOptions.php

@@ -323,7 +323,7 @@ class ElasticsearchOptions
                     "aggregated (2 values: fired = 0 or 1)" => -1,
                 ],
                 'output_formatter' => function($value) {
-                    static $map = ['0'=>"No flash", '1'=>"Flash"];
+                    static $map = ["false"=>"No flash", "true"=>"Flash", '0'=>"No flash", '1'=>"Flash"];
                     return array_key_exists($value, $map) ? $map[$value] : $value;
                 },
             ],

lib/Alchemy/Phrasea/SearchEngine/Elastic/Index.php

@@ -90,6 +90,16 @@ class Index
                     // TODO Maybe replace nfkc_normalizer + asciifolding with icu_folding
                     'filter' => ['nfkc_normalizer', 'asciifolding']
                 ],
+                'truncation_analyzer' => [
+                    'type'      => 'custom',
+                    'tokenizer' => 'truncation_tokenizer',
+                    'filter'    => ['lowercase', 'stop', 'kstem']
+                ],
+                'truncation_analyzer#search' => [
+                    'type'      => 'custom',
+                    'tokenizer' => 'truncation_tokenizer',
+                    'filter'    => ['lowercase', 'stop', 'kstem']
+                ],
                 // Lang specific
                 'fr_full' => [
                     'type' => 'custom',
@@ -145,6 +155,12 @@ class Index
                 ]
             ],
             'tokenizer' => [
+                'truncation_tokenizer' => [
+                    "type" => "edgeNGram",
+                    "min_gram" => "2",
+                    "max_gram" => "15",
+                    "token_chars" => [ "letter", "digit", "punctuation", "symbol" ]
+                ],
                 'thesaurus_path' => [
                     'type' => 'path_hierarchy'
                 ]

lib/Alchemy/Phrasea/SearchEngine/Elastic/Indexer/Record/Hydrator/ThesaurusHydrator.php

@@ -13,6 +13,7 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Indexer\Record\Hydrator;
 
 use Alchemy\Phrasea\SearchEngine\Elastic\Exception\Exception;
 use Alchemy\Phrasea\SearchEngine\Elastic\RecordHelper;
+use Alchemy\Phrasea\SearchEngine\Elastic\Structure\GlobalStructure;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\CandidateTerms;
 use Alchemy\Phrasea\SearchEngine\Elastic\Thesaurus\Concept;
@@ -27,7 +28,7 @@ class ThesaurusHydrator implements HydratorInterface
     private $thesaurus;
     private $candidate_terms;
 
-    public function __construct(Structure $structure, Thesaurus $thesaurus, CandidateTerms $candidate_terms)
+    public function __construct(GlobalStructure $structure, Thesaurus $thesaurus, CandidateTerms $candidate_terms)
     {
         $this->structure = $structure;
         $this->thesaurus = $thesaurus;
@@ -67,7 +68,13 @@ class ThesaurusHydrator implements HydratorInterface
         $terms = array();
         $filters = array();
         $field_names = array();
+        /** @var Field[] $dbFields */
+        $dbFields = $this->structure->getAllFieldsByDatabox($record['databox_id']);
         foreach ($fields as $name => $field) {
+            if(!array_key_exists($name, $dbFields) || !$dbFields[$name]->get_generate_cterms()) {
+                continue;
+            }
+
             $root_concepts = $field->getThesaurusRoots();
             // Loop through all values to prepare bulk query
             $field_values = \igorw\get_in($record, explode('.', $index_fields[$name]));
@@ -98,10 +105,7 @@ class ThesaurusHydrator implements HydratorInterface
                 }
             }
             else {
-                $field = $fields[$name];
+                $this->candidate_terms->insert($field_names[$offset], $values[$offset]);
-                if($field->get_generate_cterms()) {
-                    $this->candidate_terms->insert($field_names[$offset], $values[$offset]);
-                }
             }
         }
     }

lib/Alchemy/Phrasea/SearchEngine/Elastic/Mapping/StringFieldMapping.php

@@ -55,8 +55,13 @@ class StringFieldMapping extends ComplexFieldMapping
     {
         $child = new StringFieldMapping('light');
         $child->setAnalyzer('general_light');
-
         $this->addChild($child);
+
+        $child = new StringFieldMapping('truncated');
+        $child->setAnalyzer('truncation_analyzer', 'indexing');
+        $child->setAnalyzer('truncation_analyzer#search', 'searching');
+        $this->addChild($child);
+
         $this->addLocalizedChildren($locales);
 
         return $this;

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/Escaper.php

@@ -4,6 +4,11 @@ namespace Alchemy\Phrasea\SearchEngine\Elastic\Search;
 
 class Escaper
 {
+    public function quoteWord($value)
+    {
+        return '"' . $this->escapeRaw($value) . '"';
+    }
+
     public function escapeWord($value)
     {
         // Strip double quotes from values to prevent broken queries

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/FacetsResponse.php

@@ -62,7 +62,7 @@ class FacetsResponse
                         'value'     => $key,
                         'raw_value' => $key,
                         'count'     => $bucket['doc_count'],
-                        'query'     => sprintf('field.%s:%s', $this->escaper->escapeWord($name), $this->escaper->escapeWord($key))
+                        'query'     => sprintf('field.%s=%s', $this->escaper->escapeWord($name), $this->escaper->quoteWord($key))
                     ];
                 }
 

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContext.php

@@ -9,6 +9,7 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Field;
 use Alchemy\Phrasea\SearchEngine\Elastic\AST\Field as ASTField;
 use Alchemy\Phrasea\SearchEngine\Elastic\AST\Flag;
 use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Structure;
+use Alchemy\Phrasea\SearchEngine\SearchEngineOptions;
 
 /**
  * @todo Check for private fields and only search on them if allowed
@@ -23,13 +24,23 @@ class QueryContext
     private $queryLocale;
     /** @var array */
     private $fields;
+    /** @var  SearchEngineOptions */
+    private $options;
 
-    public function __construct(Structure $structure, array $locales, $queryLocale, array $fields = null)
+    /**
+     * @param SearchEngineOptions|null $options
+     * @param Structure $structure
+     * @param array $locales
+     * @param $queryLocale
+     * @param array $fields
+     */
+    public function __construct($options, Structure $structure, array $locales, $queryLocale, array $fields = null)
     {
         $this->structure = $structure;
         $this->locales = $locales;
         $this->queryLocale = $queryLocale;
         $this->fields = $fields;
+        $this->options = $options;
     }
 
     public function narrowToFields(array $fields)
@@ -43,7 +54,7 @@ class QueryContext
             }
         }
 
-        return new static($this->structure, $this->locales, $this->queryLocale, $fields);
+        return new static($this->options, $this->structure, $this->locales, $this->queryLocale, $fields);
     }
 
     /**
@@ -131,6 +142,16 @@ class QueryContext
         return $ret;
     }
 
+    public function truncationField(Field $field)
+    {
+        if($this->options && $this->options->useTruncation()) {
+            return [sprintf('%s.truncated', $field->getIndexField())];
+        }
+        else {
+            return [];
+        }
+    }
+
     private function localizeFieldName($field)
     {
         $fields = array();

lib/Alchemy/Phrasea/SearchEngine/Elastic/Search/QueryContextFactory.php

@@ -23,7 +23,7 @@ class QueryContextFactory
             ? $this->getLimitedStructure($options)
             : $this->structure;
 
-        $context = new QueryContext($structure, $this->locales, $this->current_locale);
+        $context = new QueryContext($options, $structure, $this->locales, $this->current_locale);
 
         if ($options) {
             $fields = $this->getSearchedFields($options);

lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/Field.php

@@ -24,6 +24,11 @@ class Field implements Typed
      */
     private $name;
 
+    /**
+     * @var int
+     */
+    private $databox_id;
+
     /**
      * @var string
      */
@@ -73,6 +78,7 @@ class Field implements Typed
         }
 
         return new self($field->get_name(), $type, [
+            'databox_id' => $databox->get_sbas_id(),
             'searchable' => $field->is_indexable(),
             'private' => $field->isBusiness(),
             'facet' => $facet,
@@ -102,6 +108,7 @@ class Field implements Typed
     {
         $this->name = (string) $name;
         $this->type = $type;
+        $this->databox_id      = \igorw\get_in($options, ['databox_id'], 0);
         $this->is_searchable   = \igorw\get_in($options, ['searchable'], true);
         $this->is_private      = \igorw\get_in($options, ['private'], false);
         $this->facet           = \igorw\get_in($options, ['facet']);
@@ -126,6 +133,7 @@ class Field implements Typed
     public function withOptions(array $options)
     {
         return new self($this->name, $this->type, $options + [
+            'databox_id' => $this->databox_id,
             'searchable' => $this->is_searchable,
             'private' => $this->is_private,
             'facet' => $this->facet,
@@ -155,6 +163,11 @@ class Field implements Typed
         return sprintf('concept_path.%s', $this->name);
     }
 
+    public function get_databox_id()
+    {
+        return $this->databox_id;
+    }
+
     public function getType()
     {
         return $this->type;

lib/Alchemy/Phrasea/SearchEngine/Elastic/Structure/GlobalStructure.php

@@ -14,6 +14,12 @@ final class GlobalStructure implements Structure
      */
     private $fields = array();
 
+
+    /**
+     * @var Field[][]
+     */
+    private $fieldsByDatabox = [];
+
     /**
      * @var Field[]
      * */
@@ -119,6 +125,10 @@ final class GlobalStructure implements Structure
 
     public function add(Field $field)
     {
+        // store info for each field, not still merged by databox
+        $this->fieldsByDatabox[$field->get_databox_id()][$field->getName()] = $field;
+
+        // store merged infos (same field name)
         $name = $field->getName();
 
         if (isset($this->fields[$name])) {
@@ -152,6 +162,11 @@ final class GlobalStructure implements Structure
         return $this->fields;
     }
 
+    public function getAllFieldsByDatabox($databox_id)
+    {
+        return $this->fieldsByDatabox[$databox_id];
+    }
+
     /**
      * @return Field[]
      */

lib/Alchemy/Phrasea/SearchEngine/SearchEngineOptions.php

@@ -71,6 +71,8 @@ class SearchEngineOptions
     protected $i18n;
     /** @var bool */
     protected $stemming = true;
+    /** @var bool */
+    protected $use_truncation = false;
     /** @var string */
     protected $sort_by;
 
@@ -105,7 +107,8 @@ class SearchEngineOptions
         'sort_ord',
         'business_fields',
         'max_results',
-        'first_result'
+        'first_result',
+        'use_truncation',
     ];
 
     /**
@@ -217,6 +220,29 @@ class SearchEngineOptions
         return $this;
     }
 
+    /**
+     * Tells whether to use truncation or not
+     *
+     * @param  boolean             $boolean
+     * @return $this
+     */
+    public function setUseTruncation($boolean)
+    {
+        $this->use_truncation = !!$boolean;
+
+        return $this;
+    }
+
+    /**
+     * Return wheter the use of truncation is enabled or not
+     *
+     * @return boolean
+     */
+    public function useTruncation()
+    {
+        return $this->use_truncation;
+    }
+
     /**
      * Return wheter the use of stemming is enabled or not
      *
@@ -542,6 +568,8 @@ class SearchEngineOptions
         $options->setFields($databoxFields);
         $options->setDateFields($databoxDateFields);
 
+        $options->setUseTruncation((Boolean) $request->get('truncation'));
+
         return $options;
     }
 
@@ -628,6 +656,7 @@ class SearchEngineOptions
                 }
             },
             'stemming' => $optionSetter('setStemming'),
+            'use_truncation' => $optionSetter('setUseTruncation'),
             'date_fields' => function ($value, SearchEngineOptions $options) use ($fieldNormalizer) {
                 $options->setDateFields($fieldNormalizer($value));
             },

lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php

@@ -48,6 +48,7 @@ class PhraseanetExtension extends \Twig_Extension
             new \Twig_SimpleFunction('border_checker_from_fqcn', array($this, 'getCheckerFromFQCN')),
             new \Twig_SimpleFunction('caption_field', array($this, 'getCaptionField')),
             new \Twig_SimpleFunction('caption_field_label', array($this, 'getCaptionFieldLabel')),
+            new \Twig_SimpleFunction('caption_field_gui_visible', array($this, 'getCaptionFieldGuiVisible')),
             new \Twig_SimpleFunction('caption_field_order', array($this, 'getCaptionFieldOrder')),
 
             new \Twig_SimpleFunction('flag_slugify', array(Flag::class, 'normalizeName')),
@@ -77,6 +78,29 @@ class PhraseanetExtension extends \Twig_Extension
         return '';
     }
 
+    /**
+     * get localized field's gui_visible
+     * @param RecordInterface $record
+     * @param $fieldName
+     * @return string - the name gui_visible
+     */
+    public function getCaptionFieldGuiVisible(RecordInterface $record, $fieldName)
+    {
+        if ($record) {
+            /** @var \appbox $appbox */
+            $appbox = $this->app['phraseanet.appbox'];
+            $databox = $appbox->get_databox($record->getDataboxId());
+            foreach ($databox->get_meta_structure() as $meta) {
+                /** @var \databox_field $meta */
+                if ($meta->get_name() === $fieldName) {
+                    return $meta->get_gui_visible($this->app['locale']);
+                }
+            }
+        }
+
+        return '';
+    }
+
     public function getCaptionField(RecordInterface $record, $field, $value)
     {
         if ($record instanceof ElasticsearchRecord) {

lib/Alchemy/Phrasea/Utilities/RedisSessionHandler.php

@@ -80,7 +80,7 @@ class RedisSessionHandler implements \SessionHandlerInterface
      */
     public function destroy($sessionId)
     {
-        return 1 === $this->redis->delete($this->prefix.$sessionId);
+        return 1 === $this->redis->del($this->prefix.$sessionId);
     }
 
     /**

lib/classes/databox.php

@@ -464,6 +464,7 @@ class databox extends base implements ThumbnailedElement
                 ->set_tbranch(isset($field['tbranch']) ? (string) $field['tbranch'] : '')
                 ->set_generate_cterms((isset($field['generate_cterms']) && (string) $field['generate_cterms'] == 1))
                 ->set_gui_editable((isset($field['gui_editable']) && (string) $field['gui_editable'] == 1))
+                ->set_gui_visible((isset($field['gui_editable']) && (string) $field['gui_visible'] == 1))
                 ->set_thumbtitle(isset($field['thumbtitle']) ? (string) $field['thumbtitle'] : (isset($field['thumbTitle']) ? $field['thumbTitle'] : '0'))
                 ->set_report(isset($field['report']) ? (string) $field['report'] : '1')
                 ->save();

lib/classes/databox/field.php

@@ -45,6 +45,7 @@ class databox_field implements cache_cacheableInterface
     protected $tbranch;
     protected $generate_cterms;
     protected $gui_editable;
+    protected $gui_visible;
     protected $separator;
     protected $thumbtitle;
 
@@ -170,6 +171,7 @@ class databox_field implements cache_cacheableInterface
         $this->tbranch = $row['tbranch'];
         $this->generate_cterms = (bool)$row['generate_cterms'];
         $this->gui_editable = (bool)$row['gui_editable'];
+        $this->gui_visible = (bool)$row['gui_visible'];
         $this->VocabularyType = $row['VocabularyControlType'];
         $this->VocabularyRestriction = (bool)$row['RestrictToVocabularyControl'];
 
@@ -312,6 +314,7 @@ class databox_field implements cache_cacheableInterface
           `tbranch` = :tbranch,
           `generate_cterms` = :generate_cterms,
           `gui_editable` = :gui_editable,
+          `gui_visible` = :gui_visible,
           `sorter` = :position,
           `thumbtitle` = :thumbtitle,
           `VocabularyControlType` = :VocabularyControlType,
@@ -337,6 +340,7 @@ class databox_field implements cache_cacheableInterface
             ':tbranch'               => $this->tbranch,
             ':generate_cterms'        => $this->generate_cterms ? '1' : '0',
             ':gui_editable'          => $this->gui_editable ? '1' : '0',
+            ':gui_visible'          => $this->gui_visible ? '1' : '0',
             ':position'              => $this->position,
             ':thumbtitle'            => $this->thumbtitle,
             ':VocabularyControlType' => $this->getVocabularyControl() ? $this->getVocabularyControl()->getType() : null,
@@ -390,6 +394,7 @@ class databox_field implements cache_cacheableInterface
         $meta->setAttribute('tbranch', $this->tbranch);
         $meta->setAttribute('generate_cterms', $this->generate_cterms ? '1' : '0');
         $meta->setAttribute('gui_editable', $this->gui_editable ? '1' : '0');
+        $meta->setAttribute('gui_visible', $this->gui_visible ? '1' : '0');
         if ($this->multi) {
             $meta->setAttribute('separator', $this->separator);
         }
@@ -743,6 +748,17 @@ class databox_field implements cache_cacheableInterface
         return $this;
     }
 
+    /**
+     * @param  boolean       $gui_visible
+     * @return databox_field
+     */
+    public function set_gui_visible($gui_visible)
+    {
+        $this->gui_visible = $gui_visible;
+
+        return $this;
+    }
+
     /**
      *
      * @param  string        $separator
@@ -845,6 +861,15 @@ class databox_field implements cache_cacheableInterface
         return $this->gui_editable;
     }
 
+    /**
+     *
+     * @return boolean
+     */
+    public function get_gui_visible()
+    {
+        return $this->gui_visible;
+    }
+
     /**
      * @param Boolean $all If set to false, returns a one-char separator to use for serialiation
      *
@@ -957,6 +982,7 @@ class databox_field implements cache_cacheableInterface
             'tbranch'               => $this->tbranch,
             'generate_cterms'        => $this->generate_cterms,
             'gui_editable'          => $this->gui_editable,
+            'gui_visible'          => $this->gui_visible,
             'separator'             => $this->separator,
             'required'              => $this->required,
             'report'                => $this->report,
@@ -995,10 +1021,10 @@ class databox_field implements cache_cacheableInterface
         }
 
         $sql = "INSERT INTO metadatas_structure
-        (`id`, `name`, `src`, `readonly`, `gui_editable`, `required`, `indexable`, `type`, `tbranch`, `generate_cterms`,
+        (`id`, `name`, `src`, `readonly`, `gui_editable`,`gui_visible`, `required`, `indexable`, `type`, `tbranch`, `generate_cterms`,
           `thumbtitle`, `multi`, `business`, `aggregable`,
           `report`, `sorter`, `separator`)
-        VALUES (null, :name, '', 0, 1, 0, 1, 'string', '', 1,
+        VALUES (null, :name, '', 0, 1, 1, 0, 1, 'string', '', 1,
           null, 0, 0, 0,
            1, :sorter, '')";
 

lib/conf.d/bases_structure.xml

@@ -2049,6 +2049,14 @@
             <default>1</default>
             <comment></comment>
           </field>
+          <field>
+            <name>gui_visible</name>
+            <type>int(1) unsigned</type>
+            <null></null>
+            <extra></extra>
+            <default>1</default>
+            <comment></comment>
+          </field>
         </fields>
         <indexes>
           <index>

resources/www/common/styles/main.scss

@@ -50,7 +50,9 @@ $mainMenuLinkBackgroundHoverColor: transparent;
   }
 }
 
-
+[class*=" icon-"].fa, [class^=icon-].fa, .fa {
+  font-family: Fontawesome!important;
+}
 [class^="icon-"], [class*=" icon-"].icomoon {
   display: inline-block;
   width: inherit;

templates/web/admin/fields/templates.html.twig

@@ -197,6 +197,14 @@
                         </label>
                     </td>
                 </tr>
+                <tr>
+                    <td>
+                        <label for="gui_visible" class="checkbox">
+                            <input id="gui_visible" type="checkbox" <%= field.gui_visible ? "checked='checked'" : "" %> />
+                            {% trans %}gui_visible{% endtrans %}
+                        </label>
+                    </td>
+                </tr>
                 <tr>
                     <td colspan="2">
                         <label for="business" class="checkbox">

templates/web/common/macro_caption.html.twig

@@ -1,8 +1,10 @@
 {% macro caption(record, can_see_business, display_exif, limitedWidth = false) %}
     <dl class="{% if limitedWidth %}{% else %}dl-horizontal{% endif %}">
         {% for name, value in record.getCaption(caption_field_order(record, can_see_business)) %}
-            <dt>{{ caption_field_label(record, name) }}</dt>
+           {% if caption_field_gui_visible(record, name) == 1 %}
-            <dd>{{ caption_field(record, name, value)|e|highlight|linkify|parseColor }}</dd>
+                <dt>{{ caption_field_label(record, name) }}</dt>
+                <dd>{{ caption_field(record, name, value)|e|highlight|linkify|parseColor }}</dd>
+           {% endif %}
         {% endfor %}
     </dl>
     {% if display_exif|default(true) and app.getAuthenticator().user is not none and user_setting('technical_display') == 'group' %}

templates/web/common/menubar.html.twig

@@ -302,8 +302,18 @@
     $(document).ready(function () {
         $('body').on('click', '#help-trigger', function (event) {
             $('#mainMenu .helpcontextmenu').toggleClass('shown');
-            console.log('mety');
+
         });
+
+        $(document).on('click', function (e) {
+            if($('#mainMenu .helpcontextmenu').hasClass('shown')) {
+                var $this = $(e.target);
+                if ($this.closest('#mainMenu .helpcontextmenu.shown, .help-trigger').length == 0 && $this[0].id != "toggle") {
+                    $('#mainMenu .helpcontextmenu').removeClass('shown');
+                }
+            }
+        });
+
         // var key = 'help';
         var configurationSettingLinks = {{ configuration|json_encode|raw }};
         //seperate array based on location of link

tests/Alchemy/Tests/Phrasea/Controller/Admin/FieldsTest.php

@@ -230,6 +230,7 @@ class FieldsTest extends \PhraseanetAuthenticatedWebTestCase
             'vocabulary-type' => 'User',
             'vocabulary-restricted' => true,
             'gui_editable' => true,
+            'gui_visible' => true,
             'generate_cterms' => true,
         ]);
 

tests/Alchemy/Tests/Phrasea/SearchEngine/AST/TextNodeTest.php

@@ -48,6 +48,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context->getUnrestrictedFields()->willReturn([$field]);
         $query_context->getPrivateFields()->willReturn([]);
         $query_context->localizeField($field)->willReturn(['foo.fr', 'foo.en']);
+        $query_context->truncationField($field)->willReturn([]);
 
         $node = new TextNode('bar', new Context('baz'));
         $query = $node->buildQuery($query_context->reveal());
@@ -80,12 +81,18 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context
             ->localizeField($public_field)
             ->willReturn(['foo.fr', 'foo.en']);
+        $query_context
+            ->truncationField($public_field)
+            ->willReturn([]);
         $query_context
             ->getPrivateFields()
             ->willReturn([$private_field]);
         $query_context
             ->localizeField($private_field)
             ->willReturn(['private_caption.bar.fr', 'private_caption.bar.en']);
+        $query_context
+            ->truncationField($private_field)
+            ->willReturn([]);
 
         $node = new TextNode('baz');
         $query = $node->buildQuery($query_context->reveal());
@@ -136,6 +143,7 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context->getUnrestrictedFields()->willReturn([$field]);
         $query_context->getPrivateFields()->willReturn([]);
         $query_context->localizeField($field)->willReturn(['foo.fr', 'foo.en']);
+        $query_context->truncationField($field)->willReturn([]);
 
         $node = new TextNode('bar');
         $node->setConcepts([
@@ -180,12 +188,18 @@ class TextNodeTest extends \PHPUnit_Framework_TestCase
         $query_context
             ->localizeField($public_field)
             ->willReturn(['foo.fr', 'foo.en']);
+        $query_context
+            ->truncationField($public_field)
+            ->willReturn([]);
         $query_context
             ->getPrivateFields()
             ->willReturn([$private_field]);
         $query_context
             ->localizeField($private_field)
             ->willReturn(['private_caption.bar.fr', 'private_caption.bar.en']);
+        $query_context
+            ->truncationField($private_field)
+            ->willReturn([]);
 
         $node = new TextNode('baz');
         $node->setConcepts([

tests/Alchemy/Tests/Phrasea/SearchEngine/Search/QueryContextTest.php

@@ -18,7 +18,7 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase
     {
         $structure = $this->prophesize(Structure::class)->reveal();
         $available_locales = ['ab', 'cd', 'ef'];
-        $context = new QueryContext($structure, $available_locales, 'fr');
+        $context = new QueryContext(null, $structure, $available_locales, 'fr');
         $narrowed = $context->narrowToFields(['some_field']);
         $this->assertEquals(['some_field'], $narrowed->getFields());
     }
@@ -33,10 +33,10 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase
             'bar' => $bar_field
         ]);
 
-        $context = new QueryContext($structure->reveal(), [], 'fr');
+        $context = new QueryContext(null, $structure->reveal(), [], 'fr');
         $this->assertEquals([$foo_field, $bar_field], $context->getUnrestrictedFields());
 
-        $narrowed_context = new QueryContext($structure->reveal(), [], 'fr', ['foo']);
+        $narrowed_context = new QueryContext(null, $structure->reveal(), [], 'fr', ['foo']);
         $this->assertEquals([$foo_field], $narrowed_context->getUnrestrictedFields());
     }
 
@@ -50,10 +50,10 @@ class QueryContextTest extends \PHPUnit_Framework_TestCase
             'bar' => $bar_field
         ]);
 
-        $context = new QueryContext($structure->reveal(), [], 'fr');
+        $context = new QueryContext(null, $structure->reveal(), [], 'fr');
         $this->assertEquals([$foo_field, $bar_field], $context->getPrivateFields());
 
-        $narrowed_context = new QueryContext($structure->reveal(), [], 'fr', ['foo']);
+        $narrowed_context = new QueryContext(null, $structure->reveal(), [], 'fr', ['foo']);
         $this->assertEquals([$foo_field], $narrowed_context->getPrivateFields());
     }
 }

tests/Alchemy/Tests/Phrasea/SearchEngine/Structure/StructureTest.php

@@ -36,6 +36,7 @@ class StructureTest extends \PHPUnit_Framework_TestCase
         $field->isFacet()->willReturn(false);
         $field->hasConceptInference()->willReturn(false);
         $field->getDependantCollections()->willReturn(['1']);
+        $field->get_databox_id()->willReturn('1');
 
         $structure->add($field->reveal());
         $this->assertCount(1, $structure->getAllFields());
@@ -60,6 +61,7 @@ class StructureTest extends \PHPUnit_Framework_TestCase
         $field->isPrivate()->willReturn(false);
         $field->isFacet()->willReturn(false);
         $field->hasConceptInference()->willReturn(false);
+        $field->get_databox_id()->willReturn('1');
 
         $other = new Field('foo', FieldMapping::TYPE_STRING);
 

tests/Alchemy/Tests/Phrasea/SearchEngine/Structure/ValueCheckerTest.php

@@ -14,14 +14,14 @@ use Alchemy\Phrasea\SearchEngine\Elastic\Structure\Typed;
 class ValueCheckerTest extends \PHPUnit_Framework_TestCase
 {
     /**
-     * @dataProvider escapeRawProvider
+     * @dataProvider valueCheckerProvider
      */
     public function testValueCompatibility($subject, $value, $compatible)
     {
         $this->assertEquals($compatible, ValueChecker::isValueCompatible($subject, $value));
     }
 
-    public function escapeRawProvider()
+    public function valueCheckerProvider()
     {
         $values = [
             [FieldMapping::TYPE_FLOAT  ,  42         , true ],

www/scripts/models/field.js

@@ -28,6 +28,7 @@ define([
             "tbranch": "",
             "generate_cterms": false,
             "gui_editable": true,
+            "gui_visible": true,
             "separator": "",
             "required": false,
             "report": true,

www/scripts/tests/specs/models.js

@@ -52,6 +52,10 @@ define([
                     this.field.get('gui_editable').should.equal("1");
                 });
 
+                it("should default gui_visible property to '1'", function () {
+                    this.field.get('gui_visible').should.equal("1");
+                });
+
                 it("should default separator property to 'empty'", function () {
                     this.field.get('separator').should.equal("");
                 });

yarn.lock

@@ -5283,10 +5283,14 @@ jquery-simplecolorpicker@^0.3.1:
   resolved "https://registry.yarnpkg.com/jquery-simplecolorpicker/-/jquery-simplecolorpicker-0.3.1.tgz#4f6befd380ab05470f585d5482e5180556e460eb"
   integrity sha1-T2vv04CrBUcPWF1UguUYBVbkYOs=
 
-"jquery-treeview@git+https://github.com/alchemy-fr/jquery-treeview.git", "jquery-treeview@git+https://github.com/alchemy-fr/jquery-treeview.git#1e9e5a49d2875b878801e904cd08c2d25e85af1e":
+"jquery-treeview@git+https://github.com/alchemy-fr/jquery-treeview.git#1e9e5a49d2875b878801e904cd08c2d25e85af1e":
+  version "1.4.2"
+  resolved "git+https://github.com/alchemy-fr/jquery-treeview.git#1e9e5a49d2875b878801e904cd08c2d25e85af1e"
+
+"jquery-treeview@https://github.com/alchemy-fr/jquery-treeview.git":
   version "1.4.2"
   uid "1e9e5a49d2875b878801e904cd08c2d25e85af1e"
-  resolved "git+https://github.com/alchemy-fr/jquery-treeview.git#1e9e5a49d2875b878801e904cd08c2d25e85af1e"
+  resolved "https://github.com/alchemy-fr/jquery-treeview.git#1e9e5a49d2875b878801e904cd08c2d25e85af1e"
 
 jquery-ui-datepicker-with-i18n@^1.10.4:
   version "1.10.4"
@@ -5974,9 +5978,9 @@ lodash.keys@~2.4.1:
     lodash.isobject "~2.4.1"
 
 lodash.merge@^4.6.0:
-  version "4.6.1"
+  version "4.6.2"
-  resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.1.tgz#adc25d9cb99b9391c59624f379fbba60d7111d54"
+  resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
-  integrity sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==
+  integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==
 
 lodash.restparam@^3.0.0:
   version "3.6.1"