Merge pull request #1090 from tdonohue/update_docker

Update Docker Compose scripts to resync with DSpace/DSpace

docker/cli.assetstore.yml

@@ -22,7 +22,8 @@ services:
     networks:
       dspacenet: {}
     environment:
-      - LOADASSETS=https://www.dropbox.com/s/v3ahfcuatklbmi0/assetstore-2019-11-28.tar.gz?dl=1
+      # This assetstore zip is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data
+      - LOADASSETS=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/assetstore.tar.gz
     entrypoint:
       - /bin/bash
       - '-c'

docker/db.entities.yml

@@ -19,4 +19,33 @@ services:
     image: dspace/dspace-postgres-pgcrypto:loadsql
     environment:
       # This LOADSQL should be kept in sync with the URL in DSpace/DSpace
-      - LOADSQL=https://www.dropbox.com/s/4ap1y6deseoc8ws/dspace7-entities-2019-11-28.sql?dl=1
+      # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data
+      - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-2021-04-14.sql
+  dspace:
+    ### OVERRIDE default 'entrypoint' in 'docker-compose-rest.yml' ####
+    # Ensure that the database is ready BEFORE starting tomcat
+    # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep
+    # 2. Then, run database migration to init database tables
+    # 3. (Custom for Entities) enable Entity-specific collection submission mappings in item-submission.xml
+    #    This 'sed' command inserts the sample configurations specific to the Entities data set, see:
+    #    https://github.com/DSpace/DSpace/blob/main/dspace/config/item-submission.xml#L36-L49
+    # 4. Finally, start Tomcat
+    entrypoint:
+      - /bin/bash
+      - '-c'
+      - |
+        while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
+        /dspace/bin/dspace database migrate
+        sed -i '/name-map collection-handle="default".*/a \\n <name-map collection-handle="123456789/3" submission-name="Publication"/> \
+          <name-map collection-handle="123456789/4" submission-name="Publication"/> \
+          <name-map collection-handle="123456789/281" submission-name="Publication"/> \
+          <name-map collection-handle="123456789/5" submission-name="Publication"/> \
+          <name-map collection-handle="123456789/8" submission-name="OrgUnit"/> \
+          <name-map collection-handle="123456789/6" submission-name="Person"/> \
+          <name-map collection-handle="123456789/279" submission-name="Person"/> \
+          <name-map collection-handle="123456789/7" submission-name="Project"/> \
+          <name-map collection-handle="123456789/280" submission-name="Project"/> \
+          <name-map collection-handle="123456789/28" submission-name="Journal"/> \
+          <name-map collection-handle="123456789/29" submission-name="JournalVolume"/> \
+          <name-map collection-handle="123456789/30" submission-name="JournalIssue"/>' /dspace/config/item-submission.xml
+        catalina.sh run

docker/docker-compose-ci.yml

@@ -8,9 +8,13 @@
 
 # Docker Compose for running the DSpace backend for e2e testing in a CI environment
 # This is used by our GitHub CI at .github/workflows/build.yml
+# It is based heavily on the Backend's Docker Compose:
+# https://github.com/DSpace/DSpace/blob/main/docker-compose.yml
+version: '3.7'
 networks:
   dspacenet:
 services:
+  # DSpace (backend) webapp container
   dspace:
     container_name: dspace
     depends_on:
@@ -26,12 +30,28 @@ services:
     volumes:
     - assetstore:/dspace/assetstore
     - "./local.cfg:/dspace/config/local.cfg"
+    # Mount DSpace's solr configs to a volume, so that we can share to 'dspacesolr' container (see below)
+    - solr_configs:/dspace/solr
+    # Ensure that the database is ready BEFORE starting tomcat
+    # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep
+    # 2. Then, run database migration to init database tables
+    # 3. Finally, start Tomcat
+    entrypoint:
+    - /bin/bash
+    - '-c'
+    - |
+      while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
+      /dspace/bin/dspace database migrate
+      catalina.sh run
+  # DSpace database container
+  # NOTE: This is customized to use our loadsql image, so that we are using a database with existing test data
   dspacedb:
     container_name: dspacedb
     environment:
       # This LOADSQL should be kept in sync with the LOADSQL in
       # https://github.com/DSpace/DSpace/blob/main/dspace/src/main/docker-compose/db.entities.yml
-      LOADSQL: https://www.dropbox.com/s/4ap1y6deseoc8ws/dspace7-entities-2019-11-28.sql?dl=1
+      # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data
+      LOADSQL: https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-2021-04-14.sql
       PGDATA: /pgdata
     image: dspace/dspace-postgres-pgcrypto:loadsql
     networks:
@@ -40,9 +60,14 @@ services:
     tty: true
     volumes:
     - pgdata:/pgdata
+  # DSpace Solr container
   dspacesolr:
     container_name: dspacesolr
-    image: dspace/dspace-solr
+    # Uses official Solr image at https://hub.docker.com/_/solr/
+    image: solr:8.8
+    # Needs main 'dspace' container to start first to guarantee access to solr_configs
+    depends_on:
+    - dspace
     networks:
       dspacenet:
     ports:
@@ -50,16 +75,27 @@ services:
       target: 8983
     stdin_open: true
     tty: true
+    working_dir: /var/solr/data
     volumes:
-    - solr_authority:/opt/solr/server/solr/authority/data
+    # Mount our "solr_configs" volume available under the Solr's configsets folder (in a 'dspace' subfolder)
-    - solr_oai:/opt/solr/server/solr/oai/data
+    # This copies the Solr configs from main 'dspace' container into 'dspacesolr' via that volume
-    - solr_search:/opt/solr/server/solr/search/data
+    - solr_configs:/opt/solr/server/solr/configsets/dspace
-    - solr_statistics:/opt/solr/server/solr/statistics/data
+    # Keep Solr data directory between reboots
-version: '3.7'
+    - solr_data:/var/solr/data
+    # Initialize all DSpace Solr cores using the mounted configsets (see above), then start Solr
+    entrypoint:
+    - /bin/bash
+    - '-c'
+    - |
+      init-var-solr
+      precreate-core authority /opt/solr/server/solr/configsets/dspace/authority
+      precreate-core oai /opt/solr/server/solr/configsets/dspace/oai
+      precreate-core search /opt/solr/server/solr/configsets/dspace/search
+      precreate-core statistics /opt/solr/server/solr/configsets/dspace/statistics
+      exec solr -f
 volumes:
   assetstore:
   pgdata:
-  solr_authority:
+  solr_data:
-  solr_oai:
+  # Special volume used to share Solr configs from 'dspace' to 'dspacesolr' container (see above)
-  solr_search:
+  solr_configs:
-  solr_statistics:

docker/docker-compose-rest.yml

@@ -14,6 +14,7 @@ version: '3.7'
 networks:
   dspacenet:
 services:
+  # DSpace (backend) webapp container
   dspace:
     container_name: dspace
     image: dspace/dspace:dspace-7_x-test
@@ -29,6 +30,8 @@ services:
     volumes:
     - assetstore:/dspace/assetstore
     - "./local.cfg:/dspace/config/local.cfg"
+    # Mount DSpace's solr configs to a volume, so that we can share to 'dspacesolr' container (see below)
+    - solr_configs:/dspace/solr
     # Ensure that the database is ready BEFORE starting tomcat
     # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep
     # 2. Then, run database migration to init database tables
@@ -40,6 +43,7 @@ services:
       while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
       /dspace/bin/dspace database migrate
       catalina.sh run
+  # DSpace database container    
   dspacedb:
     container_name: dspacedb
     environment:
@@ -54,9 +58,14 @@ services:
     tty: true
     volumes:
     - pgdata:/pgdata
+  # DSpace Solr container  
   dspacesolr:
     container_name: dspacesolr
-    image: dspace/dspace-solr
+    # Uses official Solr image at https://hub.docker.com/_/solr/
+    image: solr:8.8
+    # Needs main 'dspace' container to start first to guarantee access to solr_configs
+    depends_on:
+    - dspace
     networks:
       dspacenet:
     ports:
@@ -64,15 +73,27 @@ services:
       target: 8983
     stdin_open: true
     tty: true
+    working_dir: /var/solr/data
     volumes:
-    - solr_authority:/opt/solr/server/solr/authority/data
+    # Mount our "solr_configs" volume available under the Solr's configsets folder (in a 'dspace' subfolder)
-    - solr_oai:/opt/solr/server/solr/oai/data
+    # This copies the Solr configs from main 'dspace' container into 'dspacesolr' via that volume
-    - solr_search:/opt/solr/server/solr/search/data
+    - solr_configs:/opt/solr/server/solr/configsets/dspace
-    - solr_statistics:/opt/solr/server/solr/statistics/data
+    # Keep Solr data directory between reboots
+    - solr_data:/var/solr/data
+    # Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr
+    entrypoint:
+    - /bin/bash
+    - '-c'
+    - |
+      init-var-solr
+      precreate-core authority /opt/solr/server/solr/configsets/dspace/authority
+      precreate-core oai /opt/solr/server/solr/configsets/dspace/oai
+      precreate-core search /opt/solr/server/solr/configsets/dspace/search
+      precreate-core statistics /opt/solr/server/solr/configsets/dspace/statistics
+      exec solr -f
 volumes:
   assetstore:
   pgdata:
-  solr_authority:
+  solr_data:
-  solr_oai:
+  # Special volume used to share Solr configs from 'dspace' to 'dspacesolr' container (see above)
-  solr_search:
+  solr_configs:
-  solr_statistics: