From 1741678807aeb9b6a56a521dc6fdeaaf69ae3657 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 7 Apr 2021 13:57:25 -0500 Subject: [PATCH 1/3] Update Docker Compose scripts to resync with DSpace/DSpace --- docker/cli.assetstore.yml | 3 +- docker/db.entities.yml | 3 +- docker/docker-compose-ci.yml | 53 +++++++++++++++++++++++++++------- docker/docker-compose-rest.yml | 34 ++++++++++++++++------ 4 files changed, 71 insertions(+), 22 deletions(-) diff --git a/docker/cli.assetstore.yml b/docker/cli.assetstore.yml index 592194a752..c2846286d7 100644 --- a/docker/cli.assetstore.yml +++ b/docker/cli.assetstore.yml @@ -22,7 +22,8 @@ services: networks: dspacenet: {} environment: - - LOADASSETS=https://www.dropbox.com/s/v3ahfcuatklbmi0/assetstore-2019-11-28.tar.gz?dl=1 + # This assetstore zip is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data + - LOADASSETS=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/assetstore.tar.gz entrypoint: - /bin/bash - '-c' diff --git a/docker/db.entities.yml b/docker/db.entities.yml index d39eedc5c6..3d4831e151 100644 --- a/docker/db.entities.yml +++ b/docker/db.entities.yml @@ -19,4 +19,5 @@ services: image: dspace/dspace-postgres-pgcrypto:loadsql environment: # This LOADSQL should be kept in sync with the URL in DSpace/DSpace - - LOADSQL=https://www.dropbox.com/s/4ap1y6deseoc8ws/dspace7-entities-2019-11-28.sql?dl=1 + # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data + - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-2021-04-14.sql \ No newline at end of file diff --git a/docker/docker-compose-ci.yml b/docker/docker-compose-ci.yml index ec2d0912cf..5397e15a8b 100644 --- a/docker/docker-compose-ci.yml +++ b/docker/docker-compose-ci.yml @@ -8,9 +8,13 @@ # Docker Compose for running the DSpace backend for e2e testing in a CI environment # This is used by our GitHub CI at .github/workflows/build.yml +# It is based heavily on the Backend's Docker Compose: +# https://github.com/DSpace/DSpace/blob/main/docker-compose.yml +version: '3.7' networks: dspacenet: services: + # DSpace (backend) webapp container dspace: container_name: dspace depends_on: @@ -26,12 +30,26 @@ services: volumes: - assetstore:/dspace/assetstore - "./local.cfg:/dspace/config/local.cfg" + # Ensure that the database is ready BEFORE starting tomcat + # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep + # 2. Then, run database migration to init database tables + # 3. Finally, start Tomcat + entrypoint: + - /bin/bash + - '-c' + - | + while (! /dev/null 2>&1; do sleep 1; done; + /dspace/bin/dspace database migrate + catalina.sh run + # DSpace database container + # NOTE: This is customized to use our loadsql image, so that we are using a database with existing test data dspacedb: container_name: dspacedb environment: # This LOADSQL should be kept in sync with the LOADSQL in # https://github.com/DSpace/DSpace/blob/main/dspace/src/main/docker-compose/db.entities.yml - LOADSQL: https://www.dropbox.com/s/4ap1y6deseoc8ws/dspace7-entities-2019-11-28.sql?dl=1 + # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data + LOADSQL: https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-2021-04-14.sql PGDATA: /pgdata image: dspace/dspace-postgres-pgcrypto:loadsql networks: @@ -40,9 +58,11 @@ services: tty: true volumes: - pgdata:/pgdata + # DSpace Solr container dspacesolr: container_name: dspacesolr - image: dspace/dspace-solr + # Uses official Solr image at https://hub.docker.com/_/solr/ + image: solr:8.8 networks: dspacenet: ports: @@ -50,16 +70,27 @@ services: target: 8983 stdin_open: true tty: true + working_dir: /var/solr/data volumes: - - solr_authority:/opt/solr/server/solr/authority/data - - solr_oai:/opt/solr/server/solr/oai/data - - solr_search:/opt/solr/server/solr/search/data - - solr_statistics:/opt/solr/server/solr/statistics/data -version: '3.7' + # Mount our local Solr core configs so that they are available as Solr configsets on container + - ./dspace/solr/authority:/opt/solr/server/solr/configsets/authority + - ./dspace/solr/oai:/opt/solr/server/solr/configsets/oai + - ./dspace/solr/search:/opt/solr/server/solr/configsets/search + - ./dspace/solr/statistics:/opt/solr/server/solr/configsets/statistics + # Keep Solr data directory between reboots + - solr_data:/var/solr/data + # Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr + entrypoint: + - /bin/bash + - '-c' + - | + init-var-solr + precreate-core authority /opt/solr/server/solr/configsets/authority + precreate-core oai /opt/solr/server/solr/configsets/oai + precreate-core search /opt/solr/server/solr/configsets/search + precreate-core statistics /opt/solr/server/solr/configsets/statistics + exec solr -f volumes: assetstore: pgdata: - solr_authority: - solr_oai: - solr_search: - solr_statistics: + solr_data: \ No newline at end of file diff --git a/docker/docker-compose-rest.yml b/docker/docker-compose-rest.yml index e6668f3f58..9797bd26e9 100644 --- a/docker/docker-compose-rest.yml +++ b/docker/docker-compose-rest.yml @@ -14,6 +14,7 @@ version: '3.7' networks: dspacenet: services: + # DSpace (backend) webapp container dspace: container_name: dspace image: dspace/dspace:dspace-7_x-test @@ -40,6 +41,7 @@ services: while (! /dev/null 2>&1; do sleep 1; done; /dspace/bin/dspace database migrate catalina.sh run + # DSpace database container dspacedb: container_name: dspacedb environment: @@ -54,9 +56,11 @@ services: tty: true volumes: - pgdata:/pgdata + # DSpace Solr container dspacesolr: container_name: dspacesolr - image: dspace/dspace-solr + # Uses official Solr image at https://hub.docker.com/_/solr/ + image: solr:8.8 networks: dspacenet: ports: @@ -64,15 +68,27 @@ services: target: 8983 stdin_open: true tty: true + working_dir: /var/solr/data volumes: - - solr_authority:/opt/solr/server/solr/authority/data - - solr_oai:/opt/solr/server/solr/oai/data - - solr_search:/opt/solr/server/solr/search/data - - solr_statistics:/opt/solr/server/solr/statistics/data + # Mount our local Solr core configs so that they are available as Solr configsets on container + - ./dspace/solr/authority:/opt/solr/server/solr/configsets/authority + - ./dspace/solr/oai:/opt/solr/server/solr/configsets/oai + - ./dspace/solr/search:/opt/solr/server/solr/configsets/search + - ./dspace/solr/statistics:/opt/solr/server/solr/configsets/statistics + # Keep Solr data directory between reboots + - solr_data:/var/solr/data + # Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr + entrypoint: + - /bin/bash + - '-c' + - | + init-var-solr + precreate-core authority /opt/solr/server/solr/configsets/authority + precreate-core oai /opt/solr/server/solr/configsets/oai + precreate-core search /opt/solr/server/solr/configsets/search + precreate-core statistics /opt/solr/server/solr/configsets/statistics + exec solr -f volumes: assetstore: pgdata: - solr_authority: - solr_oai: - solr_search: - solr_statistics: + solr_data: From b998f82b8dad07137fa68101d77646d1ab703f9f Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 14 Apr 2021 16:30:26 -0500 Subject: [PATCH 2/3] Bug fix: 'dspacesolr' container must have a copy of the Solr configs from 'dspace' container as they aren't in the dspace-angular codebase --- docker/docker-compose-ci.yml | 24 +++++++++++++----------- docker/docker-compose-rest.yml | 20 +++++++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/docker/docker-compose-ci.yml b/docker/docker-compose-ci.yml index 5397e15a8b..760b558322 100644 --- a/docker/docker-compose-ci.yml +++ b/docker/docker-compose-ci.yml @@ -30,6 +30,8 @@ services: volumes: - assetstore:/dspace/assetstore - "./local.cfg:/dspace/config/local.cfg" + # Mount DSpace's solr configs to a volume, so that we can share to 'dspacesolr' container (see below) + - solr_configs:/dspace/solr # Ensure that the database is ready BEFORE starting tomcat # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep # 2. Then, run database migration to init database tables @@ -72,25 +74,25 @@ services: tty: true working_dir: /var/solr/data volumes: - # Mount our local Solr core configs so that they are available as Solr configsets on container - - ./dspace/solr/authority:/opt/solr/server/solr/configsets/authority - - ./dspace/solr/oai:/opt/solr/server/solr/configsets/oai - - ./dspace/solr/search:/opt/solr/server/solr/configsets/search - - ./dspace/solr/statistics:/opt/solr/server/solr/configsets/statistics + # Mount our "solr_configs" volume available under the Solr's configsets folder (in a 'dspace' subfolder) + # This copies the Solr configs from main 'dspace' container into 'dspacesolr' via that volume + - solr_configs:/opt/solr/server/solr/configsets/dspace # Keep Solr data directory between reboots - solr_data:/var/solr/data - # Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr + # Initialize all DSpace Solr cores using the mounted configsets (see above), then start Solr entrypoint: - /bin/bash - '-c' - | init-var-solr - precreate-core authority /opt/solr/server/solr/configsets/authority - precreate-core oai /opt/solr/server/solr/configsets/oai - precreate-core search /opt/solr/server/solr/configsets/search - precreate-core statistics /opt/solr/server/solr/configsets/statistics + precreate-core authority /opt/solr/server/solr/configsets/dspace/authority + precreate-core oai /opt/solr/server/solr/configsets/dspace/oai + precreate-core search /opt/solr/server/solr/configsets/dspace/search + precreate-core statistics /opt/solr/server/solr/configsets/dspace/statistics exec solr -f volumes: assetstore: pgdata: - solr_data: \ No newline at end of file + solr_data: + # Special volume used to share Solr configs from 'dspace' to 'dspacesolr' container (see above) + solr_configs: \ No newline at end of file diff --git a/docker/docker-compose-rest.yml b/docker/docker-compose-rest.yml index 9797bd26e9..7188768e14 100644 --- a/docker/docker-compose-rest.yml +++ b/docker/docker-compose-rest.yml @@ -30,6 +30,8 @@ services: volumes: - assetstore:/dspace/assetstore - "./local.cfg:/dspace/config/local.cfg" + # Mount DSpace's solr configs to a volume, so that we can share to 'dspacesolr' container (see below) + - solr_configs:/dspace/solr # Ensure that the database is ready BEFORE starting tomcat # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep # 2. Then, run database migration to init database tables @@ -70,11 +72,9 @@ services: tty: true working_dir: /var/solr/data volumes: - # Mount our local Solr core configs so that they are available as Solr configsets on container - - ./dspace/solr/authority:/opt/solr/server/solr/configsets/authority - - ./dspace/solr/oai:/opt/solr/server/solr/configsets/oai - - ./dspace/solr/search:/opt/solr/server/solr/configsets/search - - ./dspace/solr/statistics:/opt/solr/server/solr/configsets/statistics + # Mount our "solr_configs" volume available under the Solr's configsets folder (in a 'dspace' subfolder) + # This copies the Solr configs from main 'dspace' container into 'dspacesolr' via that volume + - solr_configs:/opt/solr/server/solr/configsets/dspace # Keep Solr data directory between reboots - solr_data:/var/solr/data # Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr @@ -83,12 +83,14 @@ services: - '-c' - | init-var-solr - precreate-core authority /opt/solr/server/solr/configsets/authority - precreate-core oai /opt/solr/server/solr/configsets/oai - precreate-core search /opt/solr/server/solr/configsets/search - precreate-core statistics /opt/solr/server/solr/configsets/statistics + precreate-core authority /opt/solr/server/solr/configsets/dspace/authority + precreate-core oai /opt/solr/server/solr/configsets/dspace/oai + precreate-core search /opt/solr/server/solr/configsets/dspace/search + precreate-core statistics /opt/solr/server/solr/configsets/dspace/statistics exec solr -f volumes: assetstore: pgdata: solr_data: + # Special volume used to share Solr configs from 'dspace' to 'dspacesolr' container (see above) + solr_configs: From 8dd8b06409461dc0f66517335d025bbbfb6d0b13 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Thu, 15 Apr 2021 12:37:09 -0500 Subject: [PATCH 3/3] Ensure Entities submission config is setup & ensure Solr image starts after main one --- docker/db.entities.yml | 30 +++++++++++++++++++++++++++++- docker/docker-compose-ci.yml | 3 +++ docker/docker-compose-rest.yml | 3 +++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docker/db.entities.yml b/docker/db.entities.yml index 3d4831e151..818d14877c 100644 --- a/docker/db.entities.yml +++ b/docker/db.entities.yml @@ -20,4 +20,32 @@ services: environment: # This LOADSQL should be kept in sync with the URL in DSpace/DSpace # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data - - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-2021-04-14.sql \ No newline at end of file + - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-2021-04-14.sql + dspace: + ### OVERRIDE default 'entrypoint' in 'docker-compose-rest.yml' #### + # Ensure that the database is ready BEFORE starting tomcat + # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep + # 2. Then, run database migration to init database tables + # 3. (Custom for Entities) enable Entity-specific collection submission mappings in item-submission.xml + # This 'sed' command inserts the sample configurations specific to the Entities data set, see: + # https://github.com/DSpace/DSpace/blob/main/dspace/config/item-submission.xml#L36-L49 + # 4. Finally, start Tomcat + entrypoint: + - /bin/bash + - '-c' + - | + while (! /dev/null 2>&1; do sleep 1; done; + /dspace/bin/dspace database migrate + sed -i '/name-map collection-handle="default".*/a \\n \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + ' /dspace/config/item-submission.xml + catalina.sh run \ No newline at end of file diff --git a/docker/docker-compose-ci.yml b/docker/docker-compose-ci.yml index 760b558322..d2d02f0a55 100644 --- a/docker/docker-compose-ci.yml +++ b/docker/docker-compose-ci.yml @@ -65,6 +65,9 @@ services: container_name: dspacesolr # Uses official Solr image at https://hub.docker.com/_/solr/ image: solr:8.8 + # Needs main 'dspace' container to start first to guarantee access to solr_configs + depends_on: + - dspace networks: dspacenet: ports: diff --git a/docker/docker-compose-rest.yml b/docker/docker-compose-rest.yml index 7188768e14..c99e469941 100644 --- a/docker/docker-compose-rest.yml +++ b/docker/docker-compose-rest.yml @@ -63,6 +63,9 @@ services: container_name: dspacesolr # Uses official Solr image at https://hub.docker.com/_/solr/ image: solr:8.8 + # Needs main 'dspace' container to start first to guarantee access to solr_configs + depends_on: + - dspace networks: dspacenet: ports: