Merge pull request #11396 from TexasDigitalLibrary/port_11329_to_7x

[Port dspace-7_x] Fix Hibernate syntax bugs in the CollectionDAO and BitstreamDAO

.github/dependabot.yml

@@ -0,0 +1,118 @@
+#-------------------
+# DSpace's dependabot rules. Enables maven updates for all dependencies on a weekly basis
+# for main and any maintenance branches. Security updates only apply to main.
+#-------------------
+version: 2
+updates:
+  - package-ecosystem: "maven"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    # Allow up to 10 open PRs for dependencies
+    open-pull-requests-limit: 10
+    # Group together some upgrades in a single PR
+    groups:
+      # Group together all Build Tools in a single PR
+      build-tools:
+        applies-to: version-updates
+        patterns:
+        - "org.apache.maven.plugins:*"
+        - "*:*-maven-plugin"
+        - "*:maven-*-plugin"
+        - "com.github.spotbugs:spotbugs"
+        - "com.google.code.findbugs:*"
+        - "com.google.errorprone:*"
+        - "com.puppycrawl.tools:checkstyle"
+        - "org.sonatype.plugins:*"
+        exclude-patterns:
+        # Exclude anything from Spring, as that is in a separate group
+        - "org.springframework.*:*"
+        update-types:
+        - "minor"
+        - "patch"
+      test-tools:
+        applies-to: version-updates
+        patterns:
+        - "junit:*"
+        - "com.github.stefanbirker:system-rules"
+        - "com.h2database:*"
+        - "io.findify:s3mock*"
+        - "io.netty:*"
+        - "org.hamcrest:*"
+        - "org.mock-server:*"
+        - "org.mockito:*"
+        update-types:
+        - "minor"
+        - "patch"
+      # Group together all Apache Commons deps in a single PR
+      apache-commons:
+        applies-to: version-updates
+        patterns:
+        - "org.apache.commons:*"
+        - "commons-*:commons-*"
+        update-types:
+        - "minor"
+        - "patch"
+      # Group together all fasterxml deps in a single PR
+      fasterxml:
+        applies-to: version-updates
+        patterns:
+        - "com.fasterxml:*"
+        - "com.fasterxml.*:*"
+        update-types:
+        - "minor"
+        - "patch"
+       # Group together all Hibernate deps in a single PR
+      hibernate:
+        applies-to: version-updates
+        patterns:
+        - "org.hibernate.*:*"
+        update-types:
+        - "minor"
+        - "patch"
+      # Group together all Jakarta deps in a single PR
+      jakarta:
+        applies-to: version-updates
+        patterns:
+        - "jakarta.*:*"
+        - "org.eclipse.angus:jakarta.mail"
+        - "org.glassfish.jaxb:jaxb-runtime"
+        update-types:
+        - "minor"
+        - "patch"
+      # Group together all Google deps in a single PR
+      google-apis:
+        applies-to: version-updates
+        patterns:
+        - "com.google.apis:*"
+        - "com.google.api-client:*"
+        - "com.google.http-client:*"
+        - "com.google.oauth-client:*"
+        update-types:
+        - "minor"
+        - "patch"
+      # Group together all Spring deps in a single PR
+      spring:
+        applies-to: version-updates
+        patterns:
+        - "org.springframework:*"
+        - "org.springframework.*:*"
+        update-types:
+        - "minor"
+        - "patch"
+      # Group together all WebJARs deps in a single PR
+      webjars:
+        applies-to: version-updates
+        patterns:
+        - "org.webjars:*"
+        - "org.webjars.*:*"
+        update-types:
+        - "minor"
+        - "patch"
+    ignore:
+      # Don't try to auto-update any DSpace dependencies
+      - dependency-name: "org.dspace:*"
+      - dependency-name: "org.dspace.*:*"
+      # Ignore all major version updates for all dependencies. We'll only automate minor/patch updates.
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/build.yml

@@ -49,7 +49,7 @@ jobs:
 
       # https://github.com/actions/setup-java
       - name: Install JDK ${{ matrix.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: 'temurin'
@@ -65,14 +65,14 @@ jobs:
       # (This artifact is downloadable at the bottom of any job's summary page)
       - name: Upload Results of ${{ matrix.type }} to Artifact
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.type }} results
           path: ${{ matrix.resultsdir }}
 
       # Upload code coverage report to artifact, so that it can be shared with the 'codecov' job (see below)
       - name: Upload code coverage report to Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.type }} coverage report
           path: 'dspace/target/site/jacoco-aggregate/jacoco.xml'
@@ -91,7 +91,7 @@ jobs:
 
       # Download artifacts from previous 'tests' job
       - name: Download coverage artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
 
       # Now attempt upload to Codecov using its action.
       # NOTE: We use a retry action to retry the Codecov upload if it fails the first time.
@@ -101,10 +101,11 @@ jobs:
       - name: Upload coverage to Codecov.io
         uses: Wandalen/wretry.action@v1.3.0
         with:
-          action: codecov/codecov-action@v3
+          action: codecov/codecov-action@v4
           # Ensure codecov-action throws an error when it fails to upload
           with: |
             fail_ci_if_error: true
+            token: ${{ secrets.CODECOV_TOKEN }}
           # Try re-running action 5 times max
           attempt_limit: 5
           # Run again in 30 seconds

.github/workflows/codescan.yml

@@ -39,7 +39,7 @@ jobs:
 
       # https://github.com/actions/setup-java
       - name: Install JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
           distribution: 'temurin'
@@ -47,7 +47,7 @@ jobs:
       # Initializes the CodeQL tools for scanning.
       # https://github.com/github/codeql-action
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           # Codescan Javascript as well since a few JS files exist in REST API's interface
           languages: java, javascript
@@ -56,8 +56,8 @@ jobs:
       # NOTE: Based on testing, this autobuild process works well for DSpace. A custom
       # DSpace build w/caching (like in build.yml) was about the same speed as autobuild.
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # Perform GitHub Code Scanning.
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/docker.yml

@@ -15,6 +15,7 @@ on:
 
 permissions:
   contents: read  #  to fetch code (actions/checkout)
+  packages: write #  to write images to GitHub Container Registry (GHCR)
 
 jobs:
   ####################################################
@@ -43,7 +44,7 @@ jobs:
     needs: dspace-dependencies
     uses: ./.github/workflows/reusable-docker-build.yml
     with:
-      build_id: dspace
+      build_id: dspace-prod
       image_name: dspace/dspace
       dockerfile_path: ./Dockerfile
     secrets:
@@ -102,6 +103,8 @@ jobs:
       build_id: dspace-solr
       image_name: dspace/dspace-solr
       dockerfile_path: ./dspace/src/main/docker/dspace-solr/Dockerfile
+      # Must pass solrconfigs to the Dockerfile so that it can find the required Solr config files
+      dockerfile_additional_contexts: 'solrconfigs=./dspace/solr/'
     secrets:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
       DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
@@ -118,7 +121,7 @@ jobs:
     if: github.repository == 'dspace/dspace'
     uses: ./.github/workflows/reusable-docker-build.yml
     with:
-      build_id: dspace-postgres-pgcrypto
+      build_id: dspace-postgres-pgcrypto-prod
       image_name: dspace/dspace-postgres-pgcrypto
       # Must build out of subdirectory to have access to install script for pgcrypto.
       # NOTE: this context will build the image based on the Dockerfile in the specified directory
@@ -145,4 +148,114 @@ jobs:
       tags_flavor: suffix=-loadsql
     secrets:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+      DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+  #################################################################################
+  # Test Deployment via Docker to ensure newly built images are working properly
+  #################################################################################
+  docker-deploy:
+    # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace'
+    if: github.repository == 'dspace/dspace'
+    runs-on: ubuntu-latest
+    # Must run after all major images are built
+    needs: [dspace, dspace-test, dspace-cli, dspace-postgres-pgcrypto, dspace-solr]
+    env:
+      # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080
+      dspace__P__server__P__url: http://127.0.0.1:8080/server
+      # Enable all optional modules / controllers for this test deployment.
+      # This helps check for errors in deploying these modules via Spring Boot
+      iiif__P__enabled: true
+      oai__P__enabled: true
+      rdf__P__enabled: true
+      signposting__P__enabled: true
+      sword__D__server__P__enabled: true
+      swordv2__D__server__P__enabled: true
+      # If this is a PR against main (default branch), use "latest".
+      # Else if this is a PR against a different branch, used the base branch name.
+      # Else if this is a commit on main (default branch), use the "latest" tag.
+      # Else, just use the branch name.
+      # NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image.
+      DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }}
+      # Docker Registry to use for Docker compose scripts below.
+      # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub.
+      DOCKER_REGISTRY: ghcr.io
+    steps:
+      # Checkout our codebase (to get access to Docker Compose scripts)
+      - name: Checkout codebase
+        uses: actions/checkout@v4
+      # Download Docker image artifacts (which were just built by reusable-docker-build.yml)
+      - name: Download Docker image artifacts
+        uses: actions/download-artifact@v4
+        with:
+          # Download all amd64 Docker images (TAR files) into the /tmp/docker directory
+          pattern: docker-image-*-linux-amd64
+          path: /tmp/docker
+          merge-multiple: true
+      # Load each of the images into Docker by calling "docker image load" for each.
+      # This ensures we are using the images just built & not any prior versions on DockerHub
+      - name: Load all downloaded Docker images
+        run: |
+          find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \;
+          docker image ls -a
+      # Start backend using our compose script in the codebase.
+      - name: Start backend in Docker
+        run: |
+          docker compose -f docker-compose.yml up -d
+          sleep 10
+          docker container ls
+      # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml
+      - name: Load test data into Backend
+        run: |
+          docker compose -f docker-compose-cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en
+          docker compose -f docker-compose-cli.yml -f dspace/src/main/docker-compose/cli.ingest.yml run --rm dspace-cli
+      # Verify backend started successfully.
+      # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml)
+      # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs)
+      - name: Verify backend is responding properly
+        run: |
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api)
+          echo "$result"
+          echo "$result" |  grep -oE "\"DSpace Started with Docker Compose\","
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections)
+          echo "$result"
+          echo "$result" |  grep -oE "\"Dog in Yard\","
+      # Verify basic backend logging is working.
+      # 1. Access the top communities list. Verify that the "Before request" INFO statement is logged
+      # 2. Access an invalid endpoint (and ignore 404 response). Verify that a "status:404" WARN statement is logged
+      - name: Verify backend is logging properly
+        run: |
+          wget -O/dev/null -q http://127.0.0.1:8080/server/api/core/communities/search/top
+          logs=$(docker compose -f docker-compose.yml logs -n 5 dspace)
+          echo "$logs"
+          echo "$logs" | grep -o "Before request \[GET /server/api/core/communities/search/top\]"
+          wget -O/dev/null -q http://127.0.0.1:8080/server/api/does/not/exist || true
+          logs=$(docker compose -f docker-compose.yml logs -n 5 dspace)
+          echo "$logs"
+          echo "$logs" | grep -o "status:404 exception: The repository type does.not was not found"
+      # Verify Handle Server can be stared and is working properly
+      # 1. First generate the "[dspace]/handle-server" folder with the sitebndl.zip
+      # 2. Start the Handle Server (and wait 20 seconds to let it start up)
+      # 3. Verify logs do NOT include "Exception" in the text (as that means an error occurred)
+      # 4. Check that Handle Proxy HTML page is responding on default port (8000)
+      - name: Verify Handle Server is working properly
+        run: |
+          docker exec -i dspace /dspace/bin/make-handle-config
+          echo "Starting Handle Server..."
+          docker exec -i dspace /dspace/bin/start-handle-server
+          sleep 20
+          echo "Checking for errors in error.log"
+          result=$(docker exec -i dspace sh -c "cat /dspace/handle-server/logs/error.log* || echo ''")
+          echo "$result"
+          echo "$result" | grep -vqz "Exception"
+          echo "Checking for errors in handle-server.log..."
+          result=$(docker exec -i dspace cat /dspace/log/handle-server.log)
+          echo "$result"
+          echo "$result" | grep -vqz "Exception"
+          echo "Checking to see if Handle Proxy webpage is available..."
+          result=$(wget -O- -q http://127.0.0.1:8000/)
+          echo "$result"
+          echo "$result" | grep -oE "Handle Proxy"
+      # Shutdown our containers
+      - name: Shutdown Docker containers
+        run: |
+          docker compose -f docker-compose.yml down

.github/workflows/issue_opened.yml

@@ -16,7 +16,7 @@ jobs:
       # Only add to project board if issue is flagged as "needs triage" or has no labels
       # NOTE: By default we flag new issues as "needs triage" in our issue template
       if: (contains(github.event.issue.labels.*.name, 'needs triage') || join(github.event.issue.labels.*.name) == '')
-      uses: actions/add-to-project@v0.5.0
+      uses: actions/add-to-project@v1.0.0
       # Note, the authentication token below is an ORG level Secret. 
       # It must be created/recreated manually via a personal access token with admin:org, project, public_repo permissions
       # See: https://docs.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token#permissions-for-the-github_token

.github/workflows/pull_request_opened.yml

@@ -21,4 +21,4 @@ jobs:
     # Assign the PR to whomever created it. This is useful for visualizing assignments on project boards
     # See https://github.com/toshimaru/auto-author-assign
     - name: Assign PR to creator
-      uses: toshimaru/auto-author-assign@v2.0.1
+      uses: toshimaru/auto-author-assign@v2.1.0

.github/workflows/reusable-docker-build.yml

@@ -24,6 +24,12 @@ on:
       dockerfile_context:
         required: false
         type: string
+        default: '.'
+      # Optionally a list of "additional_contexts" to pass to Dockerfile. Defaults to empty
+      dockerfile_additional_contexts:
+        required: false
+        type: string
+        default: ''
       # If Docker image should have additional tag flavor details (e.g. a suffix), it may be passed in.
       tags_flavor:
         required: false
@@ -48,10 +54,13 @@ env:
   # For a new commit on default branch (main), use the literal tag 'latest' on Docker image.
   # For a new commit on other branches, use the branch name as the tag for Docker image.
   # For a new tag, copy that tag name as the tag for Docker image.
+  # For a pull request, use the name of the base branch that the PR was created against or "latest" (for main).
+  #   e.g. PR against 'main' will use "latest". a PR against 'dspace-7_x' will use 'dspace-7_x'.
   IMAGE_TAGS: |
     type=raw,value=latest,enable=${{ github.ref_name == github.event.repository.default_branch }}
     type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }}
     type=ref,event=tag
+    type=raw,value=${{ (github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || github.event.pull_request.base.ref }},enable=${{ github.event_name == 'pull_request' }}
   # Define default tag "flavor" for docker/metadata-action per
   # https://github.com/docker/metadata-action#flavor-input
   # We manage the 'latest' tag ourselves to the 'main' branch (see settings above)
@@ -64,8 +73,12 @@ env:
   REDEPLOY_DEMO_URL: ${{ secrets.REDEPLOY_DEMO_URL }}
   # Current DSpace maintenance branch (and architecture) which is deployed to demo.dspace.org / sandbox.dspace.org
   # (NOTE: No deployment branch specified for sandbox.dspace.org as it uses the default_branch)
-  DEPLOY_DEMO_BRANCH: 'dspace-7_x'
+  DEPLOY_DEMO_BRANCH: 'dspace-9_x'
+  DEPLOY_SANDBOX_BRANCH: 'main'
   DEPLOY_ARCH: 'linux/amd64'
+  # Registry used during building of Docker images. (All images are later copied to docker.io registry)
+  # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub.
+  DOCKER_BUILD_REGISTRY: ghcr.io
 
 jobs:
   docker-build:
@@ -74,66 +87,87 @@ jobs:
       matrix:
         # Architectures / Platforms for which we will build Docker images
         arch: [ 'linux/amd64', 'linux/arm64' ]
-        os: [ ubuntu-latest ]
         isPr:
           - ${{ github.event_name == 'pull_request' }}
         # If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work.
         # The below exclude therefore ensures we do NOT build ARM64 for PRs.
         exclude:
           - isPr: true
-            os: ubuntu-latest
             arch: linux/arm64
 
-    runs-on: ${{ matrix.os }}
+    # If ARM64, then use the Ubuntu ARM64 runner. Otherwise, use the Ubuntu AMD64 runner
+    runs-on: ${{ matrix.arch == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
 
     steps:
+      # This step converts the slashes in the "arch" matrix values above into dashes & saves to env.ARCH_NAME
+      # E.g. "linux/amd64" becomes "linux-amd64"
+      # This is necessary because all upload artifacts CANNOT have special chars (like slashes)
+      # NOTE: The regex-like syntax below is Bash Parameter Substitution
+      - name: Prepare
+        run: |
+          platform=${{ matrix.arch }}
+          echo "ARCH_NAME=${platform//\//-}" >> $GITHUB_ENV
+
       # https://github.com/actions/checkout
       - name: Checkout codebase
         uses: actions/checkout@v4
 
+      # https://github.com/docker/login-action
+      # NOTE: This login occurs for BOTH non-PRs or PRs. PRs *must* also login to access private images from GHCR
+      # during the build process
+      - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.DOCKER_BUILD_REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       # https://github.com/docker/setup-buildx-action
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      # https://github.com/docker/setup-qemu-action
-      - name: Set up QEMU emulation to build for multiple architectures
-        uses: docker/setup-qemu-action@v3
-
-      # https://github.com/docker/login-action
-      - name: Login to DockerHub
-        # Only login if not a PR, as PRs only trigger a Docker build and not a push
-        if: ${{ ! matrix.isPr }}
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
       # https://github.com/docker/metadata-action
-      # Get Metadata for docker_build_deps step below
-      - name: Sync metadata (tags, labels) from GitHub to Docker for image
+      # Extract metadata used for Docker images in all build steps below
+      - name: Extract metadata (tags, labels) from GitHub for Docker image
         id: meta_build
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.IMAGE_NAME }}
+          images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: ${{ env.IMAGE_TAGS }}
           flavor: ${{ env.TAGS_FLAVOR }}
 
+      #--------------------------------------------------------------------
+      # First, for all branch commits (non-PRs) we build the image & upload
+      # to GitHub Container Registry (GHCR). After uploading the image
+      # to GHCR, we store the image digest in an artifact, so we can
+      # create a merged manifest later (see 'docker-build_manifest' job).
+      #
+      # NOTE: We use GHCR in order to avoid aggressive rate limits at DockerHub.
+      #--------------------------------------------------------------------
       # https://github.com/docker/build-push-action
-      - name: Build and push image
+      - name: Build and push image to ${{ env.DOCKER_BUILD_REGISTRY }}
+        if: ${{ ! matrix.isPr }}
         id: docker_build
         uses: docker/build-push-action@v5
         with:
-          context: ${{ inputs.dockerfile_context || '.' }}
+          build-contexts: |
+            ${{ inputs.dockerfile_additional_contexts }}
+          context: ${{ inputs.dockerfile_context }}
           file: ${{ inputs.dockerfile_path }}
+          # Tell DSpace's Docker files to use the build registry instead of DockerHub
+          build-args:
+            DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }}
           platforms: ${{ matrix.arch }}
-          # For pull requests, we run the Docker build (to ensure no PR changes break the build),
-          # but we ONLY do an image push to DockerHub if it's NOT a PR
-          push: ${{ ! matrix.isPr }}
+          push: true
           # Use tags / labels provided by 'docker/metadata-action' above
           tags: ${{ steps.meta_build.outputs.tags }}
           labels: ${{ steps.meta_build.outputs.labels }}
+          # Use GitHub cache to load cached Docker images and cache the results of this build
+          # This decreases the number of images we need to fetch from DockerHub
+          cache-from: type=gha,scope=${{ inputs.build_id }}
+          cache-to: type=gha,scope=${{ inputs.build_id }},mode=min
 
-      # Export the digest of Docker build locally (for non PRs only)
+      # Export the digest of Docker build locally
       - name: Export Docker build digest
         if: ${{ ! matrix.isPr }}
         run: |
@@ -141,53 +175,90 @@ jobs:
             digest="${{ steps.docker_build.outputs.digest }}"
             touch "/tmp/digests/${digest#sha256:}"
 
-      # Upload digest to an artifact, so that it can be used in manifest below
+      # Upload digest to an artifact, so that it can be used in combined manifest below
+      # (The purpose of the combined manifest is to list both amd64 and arm64 builds under same tag)
       - name: Upload Docker build digest to artifact
         if: ${{ ! matrix.isPr }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: digests-${{ inputs.build_id }}
+          name: digests-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
 
-      # If this build is NOT a PR and passed in a REDEPLOY_SANDBOX_URL secret,
-      # Then redeploy https://sandbox.dspace.org if this build is for our deployment architecture and 'main' branch.
-      - name: Redeploy sandbox.dspace.org (based on main branch)
-        if: |
-          !matrix.isPR &&
-          env.REDEPLOY_SANDBOX_URL != '' &&
-          matrix.arch == env.DEPLOY_ARCH &&
-          github.ref_name == github.event.repository.default_branch
-        run: |
-          curl -X POST $REDEPLOY_SANDBOX_URL
+      #------------------------------------------------------------------------------
+      # Second, we build the image again in order to store it in a local TAR file.
+      # This TAR of the image is cached/saved as an artifact, so that it can be used
+      # by later jobs to install the brand-new images for automated testing.
+      # This TAR build is performed BOTH for PRs and for branch commits (non-PRs).
+      #
+      # (This approach has the advantage of avoiding having to download the newly built
+      # image from DockerHub or GHCR during automated testing.)
+      #
+      # See the 'docker-deploy' job in docker.yml as an example of where this TAR is used.
+      #-------------------------------------------------------------------------------
+      # Build local image (again) and store in a TAR file in /tmp directory
+      # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
+      # NOTE: This step cannot be combined with the build above as it's a different type of output.
+      - name: Build and push image to local TAR file
+        if: ${{ matrix.arch == 'linux/amd64'}}
+        uses: docker/build-push-action@v5
+        with:
+          build-contexts: |
+            ${{ inputs.dockerfile_additional_contexts }}
+          context: ${{ inputs.dockerfile_context }}
+          file: ${{ inputs.dockerfile_path }}
+          # Tell DSpace's Docker files to use the build registry instead of DockerHub
+          build-args:
+            DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }}
+          platforms: ${{ matrix.arch }}
+          tags: ${{ steps.meta_build.outputs.tags }}
+          labels: ${{ steps.meta_build.outputs.labels }}
+          # Use GitHub cache to load cached Docker images and cache the results of this build
+          # This decreases the number of images we need to fetch from DockerHub
+          cache-from: type=gha,scope=${{ inputs.build_id }}
+          cache-to: type=gha,scope=${{ inputs.build_id }},mode=min
+          # Export image to a local TAR file
+          outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar
 
-      # If this build is NOT a PR and passed in a REDEPLOY_DEMO_URL secret,
-      # Then redeploy https://demo.dspace.org if this build is for our deployment architecture and demo branch.
-      - name: Redeploy demo.dspace.org (based on maintenace branch)
-        if: |
-          !matrix.isPR &&
-          env.REDEPLOY_DEMO_URL != '' &&
-          matrix.arch == env.DEPLOY_ARCH &&
-          github.ref_name == env.DEPLOY_DEMO_BRANCH
-        run: |
-          curl -X POST $REDEPLOY_DEMO_URL
+      # Upload the local docker image (in TAR file) to a build Artifact
+      # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
+      - name: Upload local image TAR to artifact
+        if: ${{ matrix.arch == 'linux/amd64'}}
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
+          path: /tmp/${{ inputs.build_id }}.tar
+          if-no-files-found: error
+          retention-days: 1
 
-  # Merge Docker digests (from various architectures) into a manifest.
-  # This runs after all Docker builds complete above, and it tells hub.docker.com
-  # that these builds should be all included in the manifest for this tag.
-  # (e.g. AMD64 and ARM64 should be listed as options under the same tagged Docker image)
+  ##########################################################################################
+  # Merge Docker digests (from various architectures) into a single manifest.
+  # This runs after all Docker builds complete above. The purpose is to include all builds
+  # under a single manifest for this tag.
+  # (e.g. both linux/amd64 and linux/arm64 should be listed under the same tagged Docker image)
+  ##########################################################################################
   docker-build_manifest:
+    # Only run if this is NOT a PR
     if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
     needs:
       - docker-build
     steps:
       - name: Download Docker build digests
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: digests-${{ inputs.build_id }}
           path: /tmp/digests
+          # Download digests for both AMD64 and ARM64 into same directory
+          pattern: digests-${{ inputs.build_id }}-*
+          merge-multiple: true
+
+      - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.DOCKER_BUILD_REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -195,23 +266,89 @@ jobs:
       - name: Add Docker metadata for image
         id: meta
         uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: ${{ env.IMAGE_TAGS }}
+          flavor: ${{ env.TAGS_FLAVOR }}
+
+      - name: Create manifest list from digests and push to ${{ env.DOCKER_BUILD_REGISTRY }}
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          $(printf '${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect manifest in ${{ env.DOCKER_BUILD_REGISTRY }}
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+
+  ##########################################################################################
+  # Copy images / manifest to DockerHub.
+  # This MUST run after *both* images (AMD64 and ARM64) are built and uploaded to GitHub
+  # Container Registry (GHCR). Attempting to run this in parallel to GHCR builds can result
+  # in a race condition...i.e. the copy to DockerHub may fail if GHCR image has been updated
+  # at the moment when the copy occurs.
+  ##########################################################################################
+  docker-copy_to_dockerhub:
+    # Only run if this is NOT a PR
+    if: ${{ github.event_name != 'pull_request' }}
+    runs-on: ubuntu-latest
+    needs:
+      - docker-build_manifest
+
+    steps:
+      # 'regctl' is used to more easily copy the image to DockerHub and obtain the digest from DockerHub
+      # See https://github.com/regclient/regclient/blob/main/docs/regctl.md
+      - name: Install regctl for Docker registry tools
+        uses: regclient/actions/regctl-installer@main
+        with:
+          release: 'v0.8.0'
+
+      # This recreates Docker tags for DockerHub
+      - name: Add Docker metadata for image
+        id: meta_dockerhub
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.IMAGE_NAME }}
           tags: ${{ env.IMAGE_TAGS }}
           flavor: ${{ env.TAGS_FLAVOR }}
 
-      - name: Login to Docker Hub
+      # Login to source registry first, as this is where we are copying *from*
+      - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.DOCKER_BUILD_REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to DockerHub, since this is where we are copying *to*
+      - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
-      - name: Create manifest list from digests and push
-        working-directory: /tmp/digests
+      # Copy the image from source to DockerHub
+      - name: Copy image from ${{ env.DOCKER_BUILD_REGISTRY }} to docker.io
         run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-          $(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *)
+          regctl image copy ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} docker.io/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }}
 
-      - name: Inspect image
+      #--------------------------------------------------------------------
+      # Finally, check whether demo.dspace.org or sandbox.dspace.org need
+      # to be redeployed based on these new DockerHub images.
+      #--------------------------------------------------------------------
+      # If this build is for the branch that Sandbox uses and passed in a REDEPLOY_SANDBOX_URL secret,
+      # Then redeploy https://sandbox.dspace.org
+      - name: Redeploy sandbox.dspace.org (based on main branch)
+        if: |
+          env.REDEPLOY_SANDBOX_URL != '' &&
+          github.ref_name == env.DEPLOY_SANDBOX_BRANCH
         run: |
-          docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+          curl -X POST $REDEPLOY_SANDBOX_URL
+      # If this build is for the branch that Demo uses and passed in a REDEPLOY_DEMO_URL secret,
+      # Then redeploy https://demo.dspace.org
+      - name: Redeploy demo.dspace.org (based on maintenance branch)
+        if: |
+          env.REDEPLOY_DEMO_URL != '' &&
+          github.ref_name == env.DEPLOY_DEMO_BRANCH
+        run: |
+          curl -X POST $REDEPLOY_DEMO_URL

.gitignore

@@ -10,6 +10,7 @@ tags
 .project
 .classpath
 .checkstyle
+.factorypath
 
 ## Ignore project files created by IntelliJ IDEA
 *.iml

Dockerfile

@@ -6,9 +6,14 @@
 # This Dockerfile uses JDK11 by default, but has also been tested with JDK17.
 # To build with JDK17, use "--build-arg JDK_VERSION=17"
 ARG JDK_VERSION=11
+# The Docker version tag to build from
+ARG DSPACE_VERSION=dspace-7_x
+# The Docker registry to use for DSpace images. Defaults to "docker.io"
+# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument
+ARG DOCKER_REGISTRY=docker.io
 
 # Step 1 - Run Maven Build
-FROM dspace/dspace-dependencies:dspace-7_x as build
+FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build
 ARG TARGET_DIR=dspace-installer
 WORKDIR /app
 # The dspace-installer directory will be written to /install
@@ -28,35 +33,38 @@ RUN mvn --no-transfer-progress package ${MAVEN_FLAGS} && \
   mvn clean
 
 # Step 2 - Run Ant Deploy
-FROM openjdk:${JDK_VERSION}-slim as ant_build
+FROM docker.io/eclipse-temurin:${JDK_VERSION} AS ant_build
 ARG TARGET_DIR=dspace-installer
 # COPY the /install directory from 'build' container to /dspace-src in this container
 COPY --from=build /install /dspace-src
 WORKDIR /dspace-src
 # Create the initial install deployment using ANT
-ENV ANT_VERSION 1.10.13
-ENV ANT_HOME /tmp/ant-$ANT_VERSION
-ENV PATH $ANT_HOME/bin:$PATH
-# Need wget to install ant
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends wget \
-    && apt-get purge -y --auto-remove \
-    && rm -rf /var/lib/apt/lists/*
+ENV ANT_VERSION=1.10.13
+ENV ANT_HOME=/tmp/ant-$ANT_VERSION
+ENV PATH=$ANT_HOME/bin:$PATH
 # Download and install 'ant'
 RUN mkdir $ANT_HOME && \
-    wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
+    curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \
+      https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \
+    tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \
+    rm /tmp/apache-ant.tar.gz
 # Run necessary 'ant' deploy scripts
 RUN ant init_installation update_configs update_code update_webapps
 
 # Step 3 - Run tomcat
 # Create a new tomcat image that does not retain the the build directory contents
-FROM tomcat:9-jdk${JDK_VERSION}
+FROM docker.io/tomcat:9-jdk${JDK_VERSION}
 # NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration.
 ENV DSPACE_INSTALL=/dspace
 # Copy the /dspace directory from 'ant_build' container to /dspace in this container
 COPY --from=ant_build /dspace $DSPACE_INSTALL
-# Expose Tomcat port and AJP port
-EXPOSE 8080 8009
+# Need host command for "[dspace]/bin/make-handle-config"
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends host \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/*
+# Expose Tomcat port (8080) and AJP port (8009) and Handle Server HTTP port (8000)
+EXPOSE 8080 8009 8000
 # Give java extra memory (2GB)
 ENV JAVA_OPTS=-Xmx2000m
 

Dockerfile.cli

@@ -6,9 +6,14 @@
 # This Dockerfile uses JDK11 by default, but has also been tested with JDK17.
 # To build with JDK17, use "--build-arg JDK_VERSION=17"
 ARG JDK_VERSION=11
+# The Docker version tag to build from
+ARG DSPACE_VERSION=dspace-7_x
+# The Docker registry to use for DSpace images. Defaults to "docker.io"
+# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument
+ARG DOCKER_REGISTRY=docker.io
 
 # Step 1 - Run Maven Build
-FROM dspace/dspace-dependencies:dspace-7_x as build
+FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build
 ARG TARGET_DIR=dspace-installer
 WORKDIR /app
 # The dspace-installer directory will be written to /install
@@ -24,31 +29,34 @@ RUN mvn --no-transfer-progress package && \
   mvn clean
 
 # Step 2 - Run Ant Deploy
-FROM openjdk:${JDK_VERSION}-slim as ant_build
+FROM docker.io/eclipse-temurin:${JDK_VERSION} AS ant_build
 ARG TARGET_DIR=dspace-installer
 # COPY the /install directory from 'build' container to /dspace-src in this container
 COPY --from=build /install /dspace-src
 WORKDIR /dspace-src
 # Create the initial install deployment using ANT
-ENV ANT_VERSION 1.10.13
-ENV ANT_HOME /tmp/ant-$ANT_VERSION
-ENV PATH $ANT_HOME/bin:$PATH
-# Need wget to install ant, and unzip for managing AIPs
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends wget unzip \
-    && apt-get purge -y --auto-remove \
-    && rm -rf /var/lib/apt/lists/*
+ENV ANT_VERSION=1.10.13
+ENV ANT_HOME=/tmp/ant-$ANT_VERSION
+ENV PATH=$ANT_HOME/bin:$PATH
 # Download and install 'ant'
 RUN mkdir $ANT_HOME && \
-    wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
+    curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \
+      https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \
+    tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \
+    rm /tmp/apache-ant.tar.gz
 # Run necessary 'ant' deploy scripts
 RUN ant init_installation update_configs update_code
 
 # Step 3 - Run jdk
-FROM openjdk:${JDK_VERSION}
+FROM docker.io/eclipse-temurin:${JDK_VERSION}
 # NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration.
 ENV DSPACE_INSTALL=/dspace
 # Copy the /dspace directory from 'ant_build' container to /dspace in this container
 COPY --from=ant_build /dspace $DSPACE_INSTALL
 # Give java extra memory (1GB)
 ENV JAVA_OPTS=-Xmx1000m
+# Install unzip for AIPs
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends unzip \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/*

Dockerfile.dependencies

@@ -6,8 +6,8 @@
 # To build with JDK17, use "--build-arg JDK_VERSION=17"
 ARG JDK_VERSION=11
 
-# Step 1 - Run Maven Build
-FROM maven:3-openjdk-${JDK_VERSION}-slim as build
+# Step 1 - Download all Dependencies
+FROM docker.io/maven:3-eclipse-temurin-${JDK_VERSION} AS build
 ARG TARGET_DIR=dspace-installer
 WORKDIR /app
 # Create the 'dspace' user account & home directory
@@ -19,16 +19,60 @@ RUN chown -Rv dspace: /app
 # Switch to dspace user & run below commands as that user
 USER dspace
 
-# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents)
-ADD --chown=dspace . /app/
+# This next part may look odd, but it speeds up the build of this image *significantly*.
+# Copy ONLY the POMs to this image (from local machine). This will allow us to download all dependencies *without*
+# performing any code compilation steps.
+
+# Parent POM
+ADD --chown=dspace pom.xml /app/
+RUN mkdir -p /app/dspace
+
+# 'dspace' module POM. Includes 'additions' ONLY, as it's the only submodule that is required to exist.
+ADD --chown=dspace dspace/pom.xml /app/dspace/
+RUN mkdir -p /app/dspace/modules/
+ADD --chown=dspace dspace/modules/pom.xml /app/dspace/modules/
+RUN mkdir -p /app/dspace/modules/additions
+ADD --chown=dspace dspace/modules/additions/pom.xml /app/dspace/modules/additions/
+
+# 'dspace-api' module POM
+RUN mkdir -p /app/dspace-api
+ADD --chown=dspace dspace-api/pom.xml /app/dspace-api/
+
+# 'dspace-iiif' module POM
+RUN mkdir -p /app/dspace-iiif
+ADD --chown=dspace dspace-iiif/pom.xml /app/dspace-iiif/
+
+# 'dspace-oai' module POM
+RUN mkdir -p /app/dspace-oai
+ADD --chown=dspace dspace-oai/pom.xml /app/dspace-oai/
+
+# 'dspace-rdf' module POM
+RUN mkdir -p /app/dspace-rdf
+ADD --chown=dspace dspace-rdf/pom.xml /app/dspace-rdf/
+
+# 'dspace-server-webapp' module POM
+RUN mkdir -p /app/dspace-server-webapp
+ADD --chown=dspace dspace-server-webapp/pom.xml /app/dspace-server-webapp/
+
+# 'dspace-services' module POM
+RUN mkdir -p /app/dspace-services
+ADD --chown=dspace dspace-services/pom.xml /app/dspace-services/
+
+# 'dspace-sword' module POM
+RUN mkdir -p /app/dspace-sword
+ADD --chown=dspace dspace-sword/pom.xml /app/dspace-sword/
+
+# 'dspace-swordv2' module POM
+RUN mkdir -p /app/dspace-swordv2
+ADD --chown=dspace dspace-swordv2/pom.xml /app/dspace-swordv2/
 
 # Trigger the installation of all maven dependencies (hide download progress messages)
 # Maven flags here ensure that we skip final assembly, skip building test environment and skip all code verification checks.
-# These flags speed up this installation as much as reasonably possible.
-ENV MAVEN_FLAGS="-P-assembly -P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true"
-RUN mvn --no-transfer-progress install ${MAVEN_FLAGS}
+# These flags speed up this installation and skip tasks we cannot perform as we don't have the full source code.
+ENV MAVEN_FLAGS="-P-assembly -P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxjc.skip=true -Dxml.skip=true"
+RUN mvn --no-transfer-progress verify ${MAVEN_FLAGS}
 
-# Clear the contents of the /app directory (including all maven builds), so no artifacts remain.
+# Clear the contents of the /app directory (including all maven target folders), so no artifacts remain.
 # This ensures when dspace:dspace is built, it will use the Maven local cache (~/.m2) for dependencies
 USER root
 RUN rm -rf /app/*

Dockerfile.test

@@ -8,9 +8,14 @@
 # This Dockerfile uses JDK11 by default, but has also been tested with JDK17.
 # To build with JDK17, use "--build-arg JDK_VERSION=17"
 ARG JDK_VERSION=11
+# The Docker version tag to build from
+ARG DSPACE_VERSION=dspace-7_x
+# The Docker registry to use for DSpace images. Defaults to "docker.io"
+# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument
+ARG DOCKER_REGISTRY=docker.io
 
 # Step 1 - Run Maven Build
-FROM dspace/dspace-dependencies:dspace-7_x as build
+FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build
 ARG TARGET_DIR=dspace-installer
 WORKDIR /app
 # The dspace-installer directory will be written to /install
@@ -27,33 +32,36 @@ RUN mvn --no-transfer-progress package -Pdspace-rest && \
   mvn clean
 
 # Step 2 - Run Ant Deploy
-FROM openjdk:${JDK_VERSION}-slim as ant_build
+FROM docker.io/eclipse-temurin:${JDK_VERSION} AS ant_build
 ARG TARGET_DIR=dspace-installer
 # COPY the /install directory from 'build' container to /dspace-src in this container
 COPY --from=build /install /dspace-src
 WORKDIR /dspace-src
 # Create the initial install deployment using ANT
-ENV ANT_VERSION 1.10.12
-ENV ANT_HOME /tmp/ant-$ANT_VERSION
-ENV PATH $ANT_HOME/bin:$PATH
-# Need wget to install ant
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends wget \
-    && apt-get purge -y --auto-remove \
-    && rm -rf /var/lib/apt/lists/*
+ENV ANT_VERSION=1.10.12
+ENV ANT_HOME=/tmp/ant-$ANT_VERSION
+ENV PATH=$ANT_HOME/bin:$PATH
 # Download and install 'ant'
 RUN mkdir $ANT_HOME && \
-    wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
+    curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \
+      https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \
+    tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \
+    rm /tmp/apache-ant.tar.gz
 # Run necessary 'ant' deploy scripts
 RUN ant init_installation update_configs update_code update_webapps
 
 # Step 3 - Run tomcat
 # Create a new tomcat image that does not retain the the build directory contents
-FROM tomcat:9-jdk${JDK_VERSION}
+FROM docker.io/tomcat:9-jdk${JDK_VERSION}
 ENV DSPACE_INSTALL=/dspace
 ENV TOMCAT_INSTALL=/usr/local/tomcat
 # Copy the /dspace directory from 'ant_build' containger to /dspace in this container
 COPY --from=ant_build /dspace $DSPACE_INSTALL
+# Need host command for "[dspace]/bin/make-handle-config"
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends host \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/*
 # Enable the AJP connector in Tomcat's server.xml
 # NOTE: secretRequired="false" should only be used when AJP is NOT accessible from an external network. But, secretRequired="true" isn't supported by mod_proxy_ajp until Apache 2.5
 RUN sed -i '/Service name="Catalina".*/a \\n    <Connector protocol="AJP/1.3" port="8009" address="0.0.0.0" redirectPort="8443" URIEncoding="UTF-8" secretRequired="false" />' $TOMCAT_INSTALL/conf/server.xml

checkstyle-suppressions.xml

@@ -7,4 +7,5 @@
     <!-- TODO: We should have these turned on. But, currently there's a known bug with indentation checks
          on JMockIt Expectations blocks and similar. See https://github.com/checkstyle/checkstyle/issues/3739 -->
     <suppress checks="Indentation" files="src[/\\]test[/\\]java"/>
+    <suppress checks="Regexp" files="DSpaceHttpClientFactory\.java"/>
 </suppressions>

checkstyle.xml

@@ -92,7 +92,7 @@ For more information on CheckStyle configurations below, see: http://checkstyle.
         <!-- Requirements for Javadocs for methods -->
         <module name="JavadocMethod">
             <!-- All public methods MUST HAVE Javadocs -->
-            <property name="scope" value="public"/>
+            <property name="accessModifiers" value="public"/>
             <!-- Allow params, throws and return tags to be optional -->
             <property name="allowMissingParamTags" value="true"/>
             <property name="allowMissingReturnTag" value="true"/>
@@ -136,5 +136,22 @@ For more information on CheckStyle configurations below, see: http://checkstyle.
         <module name="OneStatementPerLine"/>
         <!-- Require that "catch" statements are not empty (must at least contain a comment) -->
         <module name="EmptyCatchBlock"/>
+
+        <!-- Require to use DSpaceHttpClientFactory.getClient() statement instead of creating directly the client -->
+        <module name="Regexp">
+            <property name="format" value="HttpClientBuilder\.create\s*\(\s*\)" />
+            <property name="message" value="Use DSpaceHttpClientFactory.getClient() instead of HttpClientBuilder.create()" />
+            <property name="illegalPattern" value="true"/>
+            <property name="ignoreComments" value="true"/>
+        </module>
+        <!-- Require to use DSpaceHttpClientFactory.getClient() statement instead of creating directly the client -->
+        <module name="Regexp">
+            <property name="format" value="HttpClients\.createDefault\s*\(\s*\)" />
+            <property name="message" value="Use DSpaceHttpClientFactory.getClient() instead of HttpClients.createDefault()" />
+            <property name="illegalPattern" value="true"/>
+            <property name="ignoreComments" value="true"/>
+        </module>
+
+
     </module>
 </module>

docker-compose-cli.yml

@@ -1,8 +1,12 @@
-version: "3.7"
-
+networks:
+  # Default to using network named 'dspacenet' from docker-compose.yml.
+  # Its full name will be prepended with the project name (e.g. "-p d7" means it will be named "d7_dspacenet")
+  default:
+    name: ${COMPOSE_PROJECT_NAME}_dspacenet
+    external: true
 services:
   dspace-cli:
-    image: "${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-7_x}"
+    image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-7_x}"
     container_name: dspace-cli
     build:
       context: .
@@ -26,13 +30,8 @@ services:
     - ./dspace/config:/dspace/config
     entrypoint: /dspace/bin/dspace
     command: help
-    networks:
-      - dspacenet
     tty: true
     stdin_open: true
 
 volumes:
   assetstore:
-
-networks:
-  dspacenet:

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.7'
 networks:
   dspacenet:
     ipam:
@@ -29,14 +28,14 @@ services:
       # from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above.
       proxies__P__trusted__P__ipranges: '172.23.0'
       LOGGING_CONFIG: /dspace/config/log4j2-container.xml
-    image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}"
+    image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}"
     build:
       context: .
       dockerfile: Dockerfile.test
     depends_on:
     - dspacedb
     networks:
-      dspacenet:
+      - dspacenet
     ports:
     - published: 8080
       target: 8080
@@ -67,7 +66,7 @@ services:
   dspacedb:
     container_name: dspacedb
     # Uses a custom Postgres image with pgcrypto installed
-    image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7_x}"
+    image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7_x}"
     build:
       # Must build out of subdirectory to have access to install script for pgcrypto
       context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/
@@ -87,10 +86,12 @@ services:
   # DSpace Solr container
   dspacesolr:
     container_name: dspacesolr
-    image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}"
+    image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}"
     build:
-      context: .
-      dockerfile: ./dspace/src/main/docker/dspace-solr/Dockerfile
+      context: ./dspace/src/main/docker/dspace-solr/
+      # Provide path to Solr configs necessary to build Docker image
+      additional_contexts:
+        solrconfigs: ./dspace/solr/
       args:
         SOLR_VERSION: "${SOLR_VER:-8.11}"
     networks:

dspace-api/pom.xml

@@ -12,7 +12,7 @@
     <parent>
         <groupId>org.dspace</groupId>
         <artifactId>dspace-parent</artifactId>
-        <version>7.6.2-SNAPSHOT</version>
+        <version>7.6.6-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
@@ -99,24 +99,10 @@
                 </executions>
             </plugin>
 
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>build-helper-maven-plugin</artifactId>
-                <version>3.4.0</version>
-                <executions>
-                    <execution>
-                        <phase>validate</phase>
-                        <goals>
-                            <goal>maven-version</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>buildnumber-maven-plugin</artifactId>
-                <version>3.2.0</version>
+                <version>3.2.1</version>
                 <configuration>
                     <revisionOnScmFailure>UNKNOWN_REVISION</revisionOnScmFailure>
                 </configuration>
@@ -396,6 +382,13 @@
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-orm</artifactId>
+            <exclusions>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -420,6 +413,16 @@
                     <groupId>org.bouncycastle</groupId>
                     <artifactId>bcprov-jdk15on</artifactId>
                 </exclusion>
+                <!-- Excluded BouncyCastle dependencies because we use a later version of BouncyCastle.
+                     Having two versions of BouncyCastle in the classpath can cause Handle Server to throw errors. -->
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcpkix-jdk15on</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcprov-jdk15on</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <!-- Jetty is needed to run Handle Server -->
@@ -458,10 +461,6 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-dbcp2</artifactId>
         </dependency>
-        <dependency>
-            <groupId>commons-fileupload</groupId>
-            <artifactId>commons-fileupload</artifactId>
-        </dependency>
 
         <dependency>
             <groupId>commons-io</groupId>
@@ -495,12 +494,6 @@
         <dependency>
             <groupId>jaxen</groupId>
             <artifactId>jaxen</artifactId>
-            <exclusions>
-                <exclusion>
-                    <artifactId>xom</artifactId>
-                    <groupId>xom</groupId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.jdom</groupId>
@@ -534,7 +527,7 @@
         </dependency>
         <dependency>
             <groupId>org.hamcrest</groupId>
-            <artifactId>hamcrest-all</artifactId>
+            <artifactId>hamcrest</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -626,7 +619,7 @@
         <dependency>
             <groupId>com.maxmind.geoip2</groupId>
             <artifactId>geoip2</artifactId>
-            <version>2.11.0</version>
+            <version>2.17.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.ant</groupId>
@@ -635,7 +628,7 @@
         <dependency>
             <groupId>dnsjava</groupId>
             <artifactId>dnsjava</artifactId>
-            <version>2.1.7</version>
+            <version>3.6.3</version>
         </dependency>
 
         <dependency>
@@ -671,7 +664,7 @@
         <dependency>
             <groupId>org.flywaydb</groupId>
             <artifactId>flyway-core</artifactId>
-            <version>8.4.4</version>
+            <version>8.5.13</version>
         </dependency>
 
         <!-- Google Analytics -->
@@ -745,7 +738,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-s3</artifactId>
-            <version>1.12.261</version>
+            <version>1.12.791</version>
         </dependency>
 
         <dependency>
@@ -794,7 +787,7 @@
         <dependency>
             <groupId>com.opencsv</groupId>
             <artifactId>opencsv</artifactId>
-            <version>5.6</version>
+            <version>5.12.0</version>
         </dependency>
 
         <!-- Email templating -->
@@ -812,7 +805,7 @@
         <dependency>
             <groupId>org.apache.bcel</groupId>
             <artifactId>bcel</artifactId>
-            <version>6.6.0</version>
+            <version>6.10.0</version>
             <scope>test</scope>
         </dependency>
 
@@ -821,12 +814,19 @@
             <groupId>eu.openaire</groupId>
             <artifactId>funders-model</artifactId>
             <version>2.0.0</version>
+            <exclusions>
+                <!-- Newer version pulled in via Jersey below -->
+                <exclusion>
+                    <groupId>org.javassist</groupId>
+                    <artifactId>javassist</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.mock-server</groupId>
             <artifactId>mockserver-junit-rule</artifactId>
-            <version>5.11.2</version>
+            <version>5.15.0</version>
             <scope>test</scope>
             <exclusions>
                 <!-- Exclude snakeyaml to avoid conflicts with: spring-boot-starter-cache -->
@@ -853,6 +853,12 @@
                 </exclusion>
             </exclusions>
         </dependency>
+
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>mockwebserver</artifactId>
+            <scope>test</scope>
+        </dependency>
     
     </dependencies>
 
@@ -864,42 +870,42 @@
           <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-buffer</artifactId>
-            <version>4.1.94.Final</version>
+            <version>4.2.6.Final</version>
           </dependency>
           <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-transport</artifactId>
-            <version>4.1.94.Final</version>
+            <version>4.2.6.Final</version>
           </dependency>
           <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-transport-native-unix-common</artifactId>
-            <version>4.1.94.Final</version>
+            <version>4.2.6.Final</version>
            </dependency>
           <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-common</artifactId>
-            <version>4.1.94.Final</version>
+            <version>4.2.6.Final</version>
           </dependency>
           <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-handler</artifactId>
-            <version>4.1.94.Final</version>
+            <version>4.2.6.Final</version>
           </dependency>
           <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-codec</artifactId>
-            <version>4.1.94.Final</version>
+            <version>4.2.6.Final</version>
           </dependency>
           <dependency>
             <groupId>org.apache.velocity</groupId>
             <artifactId>velocity-engine-core</artifactId>
-            <version>2.3</version>
+            <version>2.4.1</version>
           </dependency>
           <dependency>
               <groupId>org.xmlunit</groupId>
               <artifactId>xmlunit-core</artifactId>
-              <version>2.8.0</version>
+              <version>2.10.3</version>
               <scope>test</scope>
           </dependency>
           <dependency>
@@ -917,15 +923,10 @@
             <artifactId>validation-api</artifactId>
             <version>2.0.1.Final</version>
           </dependency>
-          <dependency>
-            <groupId>io.swagger</groupId>
-            <artifactId>swagger-core</artifactId>
-            <version>1.6.2</version>
-          </dependency>
           <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
-            <version>2.13.9</version>
+            <version>2.13.16</version>
             <scope>test</scope>
           </dependency>
         </dependencies>

dspace-api/src/main/java/org/dspace/administer/CreateAdministrator.java

@@ -116,6 +116,17 @@ public final class CreateAdministrator {
     protected CreateAdministrator()
             throws Exception {
         context = new Context();
+        try {
+            context.getDBConfig();
+        } catch (NullPointerException npr) {
+            // if database is null, there is no point in continuing. Prior to this exception and catch,
+            // NullPointerException was thrown, that wasn't very helpful.
+            throw new IllegalStateException("Problem connecting to database. This " +
+                    "indicates issue with either network or version (or possibly some other). " +
+                    "If you are running this in docker-compose, please make sure dspace-cli was " +
+                    "built from the same sources as running dspace container AND that they are in " +
+                    "the same project/network.");
+        }
         groupService = EPersonServiceFactory.getInstance().getGroupService();
         ePersonService = EPersonServiceFactory.getInstance().getEPersonService();
     }

dspace-api/src/main/java/org/dspace/administer/RegistryImporter.java

@@ -10,7 +10,6 @@ package org.dspace.administer;
 import java.io.File;
 import java.io.IOException;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
 import javax.xml.xpath.XPath;
@@ -18,6 +17,7 @@ import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 
+import org.dspace.app.util.XMLUtils;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
@@ -49,8 +49,9 @@ public class RegistryImporter {
      */
     public static Document loadXML(String filename)
         throws IOException, ParserConfigurationException, SAXException {
-        DocumentBuilder builder = DocumentBuilderFactory.newInstance()
-                                                        .newDocumentBuilder();
+        // This XML builder will *not* disable external entities as XML
+        // registries are considered trusted content
+        DocumentBuilder builder = XMLUtils.getTrustedDocumentBuilder();
 
         Document document = builder.parse(new File(filename));
 

dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java

@@ -13,7 +13,6 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
 import javax.xml.xpath.XPath;
@@ -21,7 +20,15 @@ import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.BitstreamFormat;
 import org.dspace.content.factory.ContentServiceFactory;
@@ -41,7 +48,7 @@ import org.xml.sax.SAXException;
  * <P>
  * <code>RegistryLoader -bitstream bitstream-formats.xml</code>
  * <P>
- * <code>RegistryLoader -dc dc-types.xml</code>
+ * <code>RegistryLoader -metadata dc-types.xml</code>
  *
  * @author Robert Tansley
  * @version $Revision$
@@ -50,7 +57,7 @@ public class RegistryLoader {
     /**
      * log4j category
      */
-    private static Logger log = org.apache.logging.log4j.LogManager.getLogger(RegistryLoader.class);
+    private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(RegistryLoader.class);
 
     protected static BitstreamFormatService bitstreamFormatService = ContentServiceFactory.getInstance()
                                                                                           .getBitstreamFormatService();
@@ -67,50 +74,99 @@ public class RegistryLoader {
      * @throws Exception if error
      */
     public static void main(String[] argv) throws Exception {
-        String usage = "Usage: " + RegistryLoader.class.getName()
-            + " (-bitstream | -metadata) registry-file.xml";
-
-        Context context = null;
+        // Set up command-line options and parse arguments
+        CommandLineParser parser = new DefaultParser();
+        Options options = createCommandLineOptions();
 
         try {
-            context = new Context();
+            CommandLine line = parser.parse(options, argv);
+
+            // Check if help option was entered or no options provided
+            if (line.hasOption('h') || line.getOptions().length == 0) {
+                printHelp(options);
+                System.exit(0);
+            }
+
+            Context context = new Context();
 
             // Can't update registries anonymously, so we need to turn off
             // authorisation
             context.turnOffAuthorisationSystem();
 
-            // Work out what we're loading
-            if (argv[0].equalsIgnoreCase("-bitstream")) {
-                RegistryLoader.loadBitstreamFormats(context, argv[1]);
-            } else if (argv[0].equalsIgnoreCase("-metadata")) {
-                // Call MetadataImporter, as it handles Metadata schema updates
-                MetadataImporter.loadRegistry(argv[1], true);
-            } else {
-                System.err.println(usage);
+            try {
+                // Work out what we're loading
+                if (line.hasOption('b')) {
+                    String filename = line.getOptionValue('b');
+                    if (StringUtils.isEmpty(filename)) {
+                        System.err.println("No file path provided for bitstream format registry");
+                        printHelp(options);
+                        System.exit(1);
+                    }
+                    RegistryLoader.loadBitstreamFormats(context, filename);
+                } else if (line.hasOption('m')) {
+                    String filename = line.getOptionValue('m');
+                    if (StringUtils.isEmpty(filename)) {
+                        System.err.println("No file path provided for metadata registry");
+                        printHelp(options);
+                        System.exit(1);
+                    }
+                    // Call MetadataImporter, as it handles Metadata schema updates
+                    MetadataImporter.loadRegistry(filename, true);
+                } else {
+                    System.err.println("No registry type specified");
+                    printHelp(options);
+                    System.exit(1);
+                }
+
+                // Commit changes and close Context
+                context.complete();
+                System.exit(0);
+            } catch (Exception e) {
+                log.fatal(LogHelper.getHeader(context, "error_loading_registries", ""), e);
+                System.err.println("Error: \n - " + e.getMessage());
+                System.exit(1);
+            } finally {
+                // Clean up our context, if it still exists & it was never completed
+                if (context != null && context.isValid()) {
+                    context.abort();
+                }
             }
-
-            // Commit changes and close Context
-            context.complete();
-
-            System.exit(0);
-        } catch (ArrayIndexOutOfBoundsException ae) {
-            System.err.println(usage);
-
+        } catch (ParseException e) {
+            System.err.println("Error parsing command-line arguments: " + e.getMessage());
+            printHelp(options);
             System.exit(1);
-        } catch (Exception e) {
-            log.fatal(LogHelper.getHeader(context, "error_loading_registries",
-                                           ""), e);
-
-            System.err.println("Error: \n - " + e.getMessage());
-            System.exit(1);
-        } finally {
-            // Clean up our context, if it still exists & it was never completed
-            if (context != null && context.isValid()) {
-                context.abort();
-            }
         }
     }
 
+    /**
+     * Create the command-line options
+     * @return the command-line options
+     */
+    private static Options createCommandLineOptions() {
+        Options options = new Options();
+
+        options.addOption("b", "bitstream", true, "load bitstream format registry from specified file");
+        options.addOption("m", "metadata", true, "load metadata registry from specified file");
+        options.addOption("h", "help", false, "print this help message");
+
+        return options;
+    }
+
+    /**
+     * Print the help message
+     * @param options the command-line options
+     */
+    private static void printHelp(Options options) {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("RegistryLoader",
+                            "Load bitstream format or metadata registries into the database\n",
+                            options,
+                            "\nExamples:\n" +
+                            " RegistryLoader -b bitstream-formats.xml\n" +
+                            " RegistryLoader -m dc-types.xml",
+                            true);
+    }
+
     /**
      * Load Bitstream Format metadata
      *
@@ -210,8 +266,9 @@ public class RegistryLoader {
      */
     private static Document loadXML(String filename) throws IOException,
         ParserConfigurationException, SAXException {
-        DocumentBuilder builder = DocumentBuilderFactory.newInstance()
-                                                        .newDocumentBuilder();
+        // This XML builder will *not* disable external entities as XML
+        // registries are considered trusted content
+        DocumentBuilder builder = XMLUtils.getTrustedDocumentBuilder();
 
         return builder.parse(new File(filename));
     }
@@ -221,7 +278,7 @@ public class RegistryLoader {
      * contains:
      * <P>
      * <code>
-     * &lt;foo&gt;&lt;mimetype&gt;application/pdf&lt;/mimetype&gt;&lt;/foo&gt;
+     * <foo><mimetype>application/pdf</mimetype></foo>
      * </code>
      * passing this the <code>foo</code> node and <code>mimetype</code> will
      * return <code>application/pdf</code>.
@@ -262,10 +319,10 @@ public class RegistryLoader {
      * document contains:
      * <P>
      * <code>
-     * &lt;foo&gt;
-     * &lt;bar&gt;val1&lt;/bar&gt;
-     * &lt;bar&gt;val2&lt;/bar&gt;
-     * &lt;/foo&gt;
+     * <foo>
+     * <bar>val1</bar>
+     * <bar>val2</bar>
+     * </foo>
      * </code>
      * passing this the <code>foo</code> node and <code>bar</code> will
      * return <code>val1</code> and <code>val2</code>.

dspace-api/src/main/java/org/dspace/administer/StructBuilder.java

@@ -27,7 +27,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
 import javax.xml.xpath.XPath;
@@ -43,6 +42,7 @@ import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.lang3.StringUtils;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.Collection;
 import org.dspace.content.Community;
@@ -613,8 +613,8 @@ public class StructBuilder {
      */
     private static org.w3c.dom.Document loadXML(InputStream input)
         throws IOException, ParserConfigurationException, SAXException {
-        DocumentBuilder builder = DocumentBuilderFactory.newInstance()
-                                                        .newDocumentBuilder();
+        // This builder factory does not disable external DTD, entities, etc.
+        DocumentBuilder builder = XMLUtils.getTrustedDocumentBuilder();
 
         org.w3c.dom.Document document = builder.parse(input);
 
@@ -802,7 +802,7 @@ public class StructBuilder {
 
             // default the short description to the empty string
             collectionService.setMetadataSingleValue(context, collection,
-                    MD_SHORT_DESCRIPTION, Item.ANY, " ");
+                    MD_SHORT_DESCRIPTION, null, " ");
 
             // import the rest of the metadata
             for (Map.Entry<String, MetadataFieldName> entry : collectionMap.entrySet()) {

dspace-api/src/main/java/org/dspace/app/bulkaccesscontrol/BulkAccessControl.java

@@ -18,6 +18,7 @@ import java.io.InputStream;
 import java.sql.SQLException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.time.ZoneOffset;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.Iterator;
@@ -154,7 +155,7 @@ public class BulkAccessControl extends DSpaceRunnable<BulkAccessControlScriptCon
         }
 
         ObjectMapper mapper = new ObjectMapper();
-        mapper.setTimeZone(TimeZone.getTimeZone("UTC"));
+        mapper.setTimeZone(TimeZone.getTimeZone(ZoneOffset.UTC));
         BulkAccessControlInput accessControl;
         context = new Context(Context.Mode.BATCH_EDIT);
         setEPerson(context);
@@ -416,7 +417,7 @@ public class BulkAccessControl extends DSpaceRunnable<BulkAccessControlScriptCon
         discoverQuery.setQuery(query);
         discoverQuery.setStart(start);
         discoverQuery.setMaxResults(limit);
-
+        discoverQuery.setSortField("search.resourceid", DiscoverQuery.SORT_ORDER.asc);
         return discoverQuery;
     }
 

dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java

@@ -188,6 +188,15 @@ public class DSpaceCSV implements Serializable {
                     // Verify that the heading is valid in the metadata registry
                     String[] clean = element.split("\\[");
                     String[] parts = clean[0].split("\\.");
+                    // Check language if present, if it's ANY then throw an exception
+                    if (clean.length > 1 && clean[1].equals(Item.ANY + "]")) {
+                        throw new MetadataImportInvalidHeadingException("Language ANY (*) was found in the heading " +
+                                                                                "of the metadata value to import, " +
+                                                                                "this should never be the case",
+                                                                        MetadataImportInvalidHeadingException.ENTRY,
+                                                                        columnCounter);
+
+                    }
 
                     if (parts.length < 2) {
                         throw new MetadataImportInvalidHeadingException(element,
@@ -223,6 +232,15 @@ public class DSpaceCSV implements Serializable {
                         }
                     }
 
+                    // Verify there isn’t already a header that is the same; if it already exists,
+                    // throw MetadataImportInvalidHeadingException
+                    String header = authorityPrefix + element;
+                    if (headings.contains(header)) {
+                        throw new MetadataImportInvalidHeadingException("Duplicate heading found: " + header,
+                                                                        MetadataImportInvalidHeadingException.ENTRY,
+                                                                        columnCounter);
+                    }
+
                     // Store the heading
                     headings.add(authorityPrefix + element);
                 }

dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearch.java

@@ -14,6 +14,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
 
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.DefaultParser.Builder;
 import org.apache.commons.cli.ParseException;
 import org.dspace.content.Item;
 import org.dspace.content.MetadataDSpaceCsvExportServiceImpl;
@@ -143,7 +145,7 @@ public class MetadataExportSearch extends DSpaceRunnable<MetadataExportSearchScr
 
         Iterator<Item> itemIterator = searchService.iteratorSearch(context, dso, discoverQuery);
         handler.logDebug("creating dspacecsv");
-        DSpaceCSV dSpaceCSV = metadataDSpaceCsvExportService.export(context, itemIterator, true);
+        DSpaceCSV dSpaceCSV = metadataDSpaceCsvExportService.export(context, itemIterator, true, handler);
         handler.logDebug("writing to file " + getFileNameOrExportFile());
         handler.writeFilestream(context, getFileNameOrExportFile(), dSpaceCSV.getInputStream(), EXPORT_CSV);
         context.restoreAuthSystemState();
@@ -167,4 +169,14 @@ public class MetadataExportSearch extends DSpaceRunnable<MetadataExportSearchScr
         }
         return scopeObj;
     }
+
+    @Override
+    protected StepResult parse(String[] args) throws ParseException {
+        commandLine = new DefaultParser().parse(getScriptConfiguration().getOptions(), args);
+        Builder builder = new DefaultParser().builder();
+        builder.setStripLeadingAndTrailingQuotes(false);
+        commandLine = builder.build().parse(getScriptConfiguration().getOptions(), args);
+        setup();
+        return StepResult.Continue;
+    }
 }

dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java

@@ -494,7 +494,7 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
 
                 // Check it has an owning collection
                 List<String> collections = line.get("collection");
-                if (collections == null) {
+                if (collections == null || collections.isEmpty()) {
                     throw new MetadataImportException(
                         "New items must have a 'collection' assigned in the form of a handle");
                 }
@@ -825,8 +825,10 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
                 addRelationships(c, item, element, values);
             } else {
                 itemService.clearMetadata(c, item, schema, element, qualifier, language);
-                itemService.addMetadata(c, item, schema, element, qualifier,
-                                        language, values, authorities, confidences);
+                if (!values.isEmpty()) {
+                    itemService.addMetadata(c, item, schema, element, qualifier,
+                                            language, values, authorities, confidences);
+                }
                 itemService.update(c, item);
             }
         }
@@ -1121,8 +1123,8 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
                     .getAuthoritySeparator() + dcv.getConfidence();
             }
 
-            // Add it
-            if ((value != null) && (!"".equals(value))) {
+            // Add it, if value is not blank
+            if (value != null && StringUtils.isNotBlank(value)) {
                 changes.registerAdd(dcv);
             }
         }

dspace-api/src/main/java/org/dspace/app/client/DSpaceHttpClientFactory.java

@@ -0,0 +1,152 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.client;
+
+import static org.apache.commons.collections4.ListUtils.emptyIfNull;
+
+import java.util.List;
+
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.HttpResponseInterceptor;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.dspace.services.ConfigurationService;
+import org.dspace.utils.DSpace;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Factory of {@link HttpClient} with common configurations.
+ *
+ * @author Luca Giamminonni (luca.giamminonni at 4science.it)
+ *
+ */
+public class DSpaceHttpClientFactory {
+
+    @Autowired
+    private ConfigurationService configurationService;
+
+    @Autowired
+    private DSpaceProxyRoutePlanner proxyRoutePlanner;
+
+    @Autowired(required = false)
+    private List<HttpRequestInterceptor> requestInterceptors;
+
+    @Autowired(required = false)
+    private List<HttpResponseInterceptor> responseInterceptors;
+
+    /**
+     * Get an instance of {@link DSpaceHttpClientFactory} from the Spring context.
+     * @return the bean instance
+     */
+    public static DSpaceHttpClientFactory getInstance() {
+        return new DSpace().getSingletonService(DSpaceHttpClientFactory.class);
+    }
+
+    /**
+     * Build an instance of {@link HttpClient} setting the proxy if configured.
+     *
+     * @return the client
+     */
+    public CloseableHttpClient build() {
+        return build(HttpClientBuilder.create(), true);
+    }
+
+    /**
+     * return a Builder if an instance of {@link HttpClient} pre-setting the proxy if configured.
+     *
+     * @return the client
+     */
+    public HttpClientBuilder builder(boolean setProxy) {
+        HttpClientBuilder clientBuilder = HttpClientBuilder.create();
+        if (setProxy) {
+            clientBuilder.setRoutePlanner(proxyRoutePlanner);
+        }
+        getRequestInterceptors().forEach(clientBuilder::addInterceptorLast);
+        getResponseInterceptors().forEach(clientBuilder::addInterceptorLast);
+        return clientBuilder;
+    }
+
+    /**
+     * Build an instance of {@link HttpClient} without setting the proxy, even if
+     * configured.
+     *
+     * @return the client
+     */
+    public CloseableHttpClient buildWithoutProxy() {
+        return build(HttpClientBuilder.create(), false);
+    }
+
+    /**
+     * Build an instance of {@link HttpClient} setting the proxy if configured,
+     * disabling automatic retries and setting the maximum total connection.
+     *
+     * @param  maxConnTotal the maximum total connection value
+     * @return              the client
+     */
+    public CloseableHttpClient buildWithoutAutomaticRetries(int maxConnTotal) {
+        HttpClientBuilder clientBuilder = HttpClientBuilder.create()
+                .disableAutomaticRetries()
+                .setMaxConnTotal(maxConnTotal);
+        return build(clientBuilder, true);
+    }
+
+    /**
+     * Build an instance of {@link HttpClient} setting the proxy if configured with
+     * the given request configuration.
+     * @param  requestConfig the request configuration
+     * @return               the client
+     */
+    public CloseableHttpClient buildWithRequestConfig(RequestConfig requestConfig) {
+        HttpClientBuilder httpClientBuilder = HttpClientBuilder.create()
+                .setDefaultRequestConfig(requestConfig);
+        return build(httpClientBuilder, true);
+    }
+
+    private CloseableHttpClient build(HttpClientBuilder clientBuilder, boolean setProxy) {
+        if (setProxy) {
+            clientBuilder.setRoutePlanner(proxyRoutePlanner);
+        }
+        getRequestInterceptors().forEach(clientBuilder::addInterceptorLast);
+        getResponseInterceptors().forEach(clientBuilder::addInterceptorLast);
+        return clientBuilder.build();
+    }
+
+    public ConfigurationService getConfigurationService() {
+        return configurationService;
+    }
+
+    public void setConfigurationService(ConfigurationService configurationService) {
+        this.configurationService = configurationService;
+    }
+
+    public List<HttpRequestInterceptor> getRequestInterceptors() {
+        return emptyIfNull(requestInterceptors);
+    }
+
+    public void setRequestInterceptors(List<HttpRequestInterceptor> requestInterceptors) {
+        this.requestInterceptors = requestInterceptors;
+    }
+
+    public List<HttpResponseInterceptor> getResponseInterceptors() {
+        return emptyIfNull(responseInterceptors);
+    }
+
+    public void setResponseInterceptors(List<HttpResponseInterceptor> responseInterceptors) {
+        this.responseInterceptors = responseInterceptors;
+    }
+
+    public DSpaceProxyRoutePlanner getProxyRoutePlanner() {
+        return proxyRoutePlanner;
+    }
+
+    public void setProxyRoutePlanner(DSpaceProxyRoutePlanner proxyRoutePlanner) {
+        this.proxyRoutePlanner = proxyRoutePlanner;
+    }
+}

dspace-api/src/main/java/org/dspace/app/client/DSpaceProxyRoutePlanner.java

@@ -0,0 +1,73 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.client;
+
+import java.util.Arrays;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.impl.conn.DefaultRoutePlanner;
+import org.apache.http.protocol.HttpContext;
+import org.dspace.services.ConfigurationService;
+
+/**
+ * Extension of {@link DefaultRoutePlanner} that determine the proxy based on
+ * the configuration service, ignoring configured hosts.
+ *
+ * @author Luca Giamminonni (luca.giamminonni at 4science.it)
+ *
+ */
+public class DSpaceProxyRoutePlanner extends DefaultRoutePlanner {
+
+    private ConfigurationService configurationService;
+
+    public DSpaceProxyRoutePlanner(ConfigurationService configurationService) {
+        super(null);
+        this.configurationService = configurationService;
+    }
+
+    @Override
+    protected HttpHost determineProxy(HttpHost target, HttpRequest request, HttpContext context) throws HttpException {
+        if (isTargetHostConfiguredToBeIgnored(target)) {
+            return null;
+        }
+        String proxyHost = configurationService.getProperty("http.proxy.host");
+        String proxyPort = configurationService.getProperty("http.proxy.port");
+        if (StringUtils.isAnyBlank(proxyHost, proxyPort)) {
+            return null;
+        }
+        try {
+            return new HttpHost(proxyHost, Integer.parseInt(proxyPort), "http");
+        } catch (NumberFormatException e) {
+            throw new RuntimeException("Invalid proxy port configuration: " + proxyPort);
+        }
+    }
+
+    private boolean isTargetHostConfiguredToBeIgnored(HttpHost target) {
+        String[] hostsToIgnore = configurationService.getArrayProperty("http.proxy.hosts-to-ignore");
+        if (ArrayUtils.isEmpty(hostsToIgnore)) {
+            return false;
+        }
+        return Arrays.stream(hostsToIgnore)
+                .anyMatch(host -> matchesHost(host, target.getHostName()));
+    }
+
+    private boolean matchesHost(String hostPattern, String hostName) {
+        if (hostName.equals(hostPattern)) {
+            return true;
+        } else if (hostPattern.startsWith("*")) {
+            return hostName.endsWith(StringUtils.removeStart(hostPattern, "*"));
+        } else if (hostPattern.endsWith("*")) {
+            return hostName.startsWith(StringUtils.removeEnd(hostPattern, "*"));
+        }
+        return false;
+    }
+}

dspace-api/src/main/java/org/dspace/app/itemexport/ItemExportServiceImpl.java

@@ -352,7 +352,7 @@ public class ItemExportServiceImpl implements ItemExportService {
 
     /**
      * Create the 'collections' file.  List handles of all Collections which
-     * contain this Item.  The "owning" Collection is listed first.
+     * contain this Item. The "owning" Collection is listed first.
      *
      * @param item list collections holding this Item.
      * @param destDir write the file here.
@@ -363,12 +363,14 @@ public class ItemExportServiceImpl implements ItemExportService {
         File outFile = new File(destDir, "collections");
         if (outFile.createNewFile()) {
             try (PrintWriter out = new PrintWriter(new FileWriter(outFile))) {
-                String ownerHandle = item.getOwningCollection().getHandle();
-                out.println(ownerHandle);
+                Collection owningCollection = item.getOwningCollection();
+                // The owning collection is null for workspace and workflow items
+                if (owningCollection != null) {
+                    out.println(owningCollection.getHandle());
+                }
                 for (Collection collection : item.getCollections()) {
-                    String collectionHandle = collection.getHandle();
-                    if (!collectionHandle.equals(ownerHandle)) {
-                        out.println(collectionHandle);
+                    if (!collection.equals(owningCollection)) {
+                        out.println(collection.getHandle());
                     }
                 }
             }
@@ -490,7 +492,7 @@ public class ItemExportServiceImpl implements ItemExportService {
 
         File wkDir = new File(workDir);
         if (!wkDir.exists() && !wkDir.mkdirs()) {
-            logError("Unable to create working direcory");
+            logError("Unable to create working directory");
         }
 
         File dnDir = new File(destDirName);
@@ -498,11 +500,18 @@ public class ItemExportServiceImpl implements ItemExportService {
             logError("Unable to create destination directory");
         }
 
-        // export the items using normal export method
-        exportItem(context, items, workDir, seqStart, migrate, excludeBitstreams);
+        try {
+            // export the items using normal export method (this exports items to our workDir)
+            exportItem(context, items, workDir, seqStart, migrate, excludeBitstreams);
 
-        // now zip up the export directory created above
-        zip(workDir, destDirName + System.getProperty("file.separator") + zipFileName);
+            // now zip up the workDir directory created above
+            zip(workDir, destDirName + System.getProperty("file.separator") + zipFileName);
+        } finally {
+            // Cleanup workDir created above, if it still exists
+            if (wkDir.exists()) {
+                deleteDirectory(wkDir);
+            }
+        }
     }
 
     @Override

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java

@@ -22,6 +22,7 @@ import java.util.UUID;
 
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.tika.Tika;
 import org.dspace.app.itemimport.factory.ItemImportServiceFactory;
@@ -333,33 +334,38 @@ public class ItemImport extends DSpaceRunnable<ItemImportScriptConfiguration> {
     protected void readZip(Context context, ItemImportService itemImportService) throws Exception {
         Optional<InputStream> optionalFileStream = Optional.empty();
         Optional<InputStream> validationFileStream = Optional.empty();
-        if (!remoteUrl) {
-            // manage zip via upload
-            optionalFileStream = handler.getFileStream(context, zipfilename);
-            validationFileStream = handler.getFileStream(context, zipfilename);
-        } else {
-            // manage zip via remote url
-            optionalFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
-            validationFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
-        }
-
-        if (validationFileStream.isPresent()) {
-            // validate zip file
-            if (validationFileStream.isPresent()) {
-                validateZip(validationFileStream.get());
+        try {
+            if (!remoteUrl) {
+                // manage zip via upload
+                optionalFileStream = handler.getFileStream(context, zipfilename);
+                validationFileStream = handler.getFileStream(context, zipfilename);
+            } else {
+                // manage zip via remote url
+                optionalFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
+                validationFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
             }
 
-            workFile = new File(itemImportService.getTempWorkDir() + File.separator
-                    + zipfilename + "-" + context.getCurrentUser().getID());
-            FileUtils.copyInputStreamToFile(optionalFileStream.get(), workFile);
-        } else {
-            throw new IllegalArgumentException(
-                    "Error reading file, the file couldn't be found for filename: " + zipfilename);
-        }
+            if (validationFileStream.isPresent()) {
+                // validate zip file
+                if (validationFileStream.isPresent()) {
+                    validateZip(validationFileStream.get());
+                }
 
-        workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR
-                           + File.separator + context.getCurrentUser().getID());
-        sourcedir = itemImportService.unzip(workFile, workDir.getAbsolutePath());
+                workFile = new File(itemImportService.getTempWorkDir() + File.separator
+                        + zipfilename + "-" + context.getCurrentUser().getID());
+                FileUtils.copyInputStreamToFile(optionalFileStream.get(), workFile);
+            } else {
+                throw new IllegalArgumentException(
+                        "Error reading file, the file couldn't be found for filename: " + zipfilename);
+            }
+
+            workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR
+                    + File.separator + context.getCurrentUser().getID());
+            sourcedir = itemImportService.unzip(workFile, workDir.getAbsolutePath());
+        } finally {
+            optionalFileStream.ifPresent(IOUtils::closeQuietly);
+            validationFileStream.ifPresent(IOUtils::closeQuietly);
+        }
     }
 
     /**

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportCLI.java

@@ -17,6 +17,7 @@ import java.util.Optional;
 import java.util.UUID;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.dspace.app.itemimport.service.ItemImportService;
 import org.dspace.content.Collection;
@@ -111,7 +112,11 @@ public class ItemImportCLI extends ItemImport {
 
                 // validate zip file
                 InputStream validationFileStream = new FileInputStream(myZipFile);
-                validateZip(validationFileStream);
+                try {
+                    validateZip(validationFileStream);
+                } finally {
+                    IOUtils.closeQuietly(validationFileStream);
+                }
 
                 workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR
                         + File.separator + context.getCurrentUser().getID());
@@ -120,22 +125,28 @@ public class ItemImportCLI extends ItemImport {
             } else {
                 // manage zip via remote url
                 Optional<InputStream> optionalFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
-                if (optionalFileStream.isPresent()) {
-                    // validate zip file via url
-                    Optional<InputStream> validationFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
-                    if (validationFileStream.isPresent()) {
-                        validateZip(validationFileStream.get());
-                    }
+                Optional<InputStream> validationFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
+                try {
+                    if (optionalFileStream.isPresent()) {
+                        // validate zip file via url
 
-                    workFile = new File(itemImportService.getTempWorkDir() + File.separator
-                            + zipfilename + "-" + context.getCurrentUser().getID());
-                    FileUtils.copyInputStreamToFile(optionalFileStream.get(), workFile);
-                    workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR
-                                       + File.separator + context.getCurrentUser().getID());
-                    sourcedir = itemImportService.unzip(workFile, workDir.getAbsolutePath());
-                } else {
-                    throw new IllegalArgumentException(
-                            "Error reading file, the file couldn't be found for filename: " + zipfilename);
+                        if (validationFileStream.isPresent()) {
+                            validateZip(validationFileStream.get());
+                        }
+
+                        workFile = new File(itemImportService.getTempWorkDir() + File.separator
+                                + zipfilename + "-" + context.getCurrentUser().getID());
+                        FileUtils.copyInputStreamToFile(optionalFileStream.get(), workFile);
+                        workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR
+                                + File.separator + context.getCurrentUser().getID());
+                        sourcedir = itemImportService.unzip(workFile, workDir.getAbsolutePath());
+                    } else {
+                        throw new IllegalArgumentException(
+                                "Error reading file, the file couldn't be found for filename: " + zipfilename);
+                    }
+                } finally {
+                    optionalFileStream.ifPresent(IOUtils::closeQuietly);
+                    validationFileStream.ifPresent(IOUtils::closeQuietly);
                 }
             }
         }

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportServiceImpl.java

@@ -29,6 +29,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.net.URL;
+import java.nio.file.Path;
 import java.sql.SQLException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -48,7 +49,6 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 import javax.mail.MessagingException;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
 import javax.xml.xpath.XPath;
@@ -67,6 +67,7 @@ import org.apache.logging.log4j.Logger;
 import org.dspace.app.itemimport.service.ItemImportService;
 import org.dspace.app.util.LocalSchemaFilenameFilter;
 import org.dspace.app.util.RelationshipUtils;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.authorize.ResourcePolicy;
 import org.dspace.authorize.service.AuthorizeService;
@@ -179,6 +180,8 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
     @Autowired(required = true)
     protected MetadataValueService metadataValueService;
 
+    protected DocumentBuilder builder;
+
     protected String tempWorkDir;
 
     protected boolean isTest = false;
@@ -742,15 +745,22 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
             myitem = wi.getItem();
         }
 
+        // normalize and validate path to make sure itemname doesn't contain path traversal
+        Path itemPath = new File(path + File.separatorChar + itemname + File.separatorChar)
+            .toPath().normalize();
+        if (!itemPath.startsWith(path)) {
+            throw new IOException("Illegal item metadata path: '" + itemPath);
+        }
+        // Normalization chops off the last separator, and we need to put it back
+        String itemPathDir = itemPath.toString() + File.separatorChar;
+
         // now fill out dublin core for item
-        loadMetadata(c, myitem, path + File.separatorChar + itemname
-            + File.separatorChar);
+        loadMetadata(c, myitem, itemPathDir);
 
         // and the bitstreams from the contents file
         // process contents file, add bistreams and bundles, return any
         // non-standard permissions
-        List<String> options = processContentsFile(c, myitem, path
-            + File.separatorChar + itemname, "contents");
+        List<String> options = processContentsFile(c, myitem, itemPathDir, "contents");
 
         if (useWorkflow) {
             // don't process handle file
@@ -768,8 +778,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
             }
         } else {
             // only process handle file if not using workflow system
-            String myhandle = processHandleFile(c, myitem, path
-                + File.separatorChar + itemname, "handle");
+            String myhandle = processHandleFile(c, myitem, itemPathDir, "handle");
 
             // put item in system
             if (!isTest) {
@@ -1001,6 +1010,34 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
         }
     }
 
+    /**
+     * Ensures a file path does not attempt to access files outside the designated parent directory.
+     *
+     * @param parentDir          The absolute path to the parent directory that should contain the file
+     * @param fileName           The name or path of the file to validate
+     * @throws IOException       If an error occurs while resolving canonical paths, or the file path attempts
+     *                           to access a location outside the parent directory
+     */
+    private void validateFilePath(String parentDir, String fileName) throws IOException {
+        File parent = new File(parentDir);
+        File file = new File(fileName);
+
+        // If the fileName is not an absolute path, we resolve it against the parentDir
+        if (!file.isAbsolute()) {
+            file = new File(parent, fileName);
+        }
+
+        String parentCanonicalPath = parent.getCanonicalPath();
+        String fileCanonicalPath = file.getCanonicalPath();
+
+        if (!fileCanonicalPath.startsWith(parentCanonicalPath)) {
+            log.error("File path outside of canonical root requested: fileCanonicalPath={} does not begin " +
+                "with parentCanonicalPath={}", fileCanonicalPath, parentCanonicalPath);
+            throw new IOException("Illegal file path '" + fileName + "' encountered. This references a path " +
+                "outside of the import package. Please see the system logs for more details.");
+        }
+    }
+
     /**
      * Read the collections file inside the item directory. If there
      * is one and it is not empty return a list of collections in
@@ -1201,6 +1238,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
                             sDescription = sDescription.replaceFirst("description:", "");
                         }
 
+                        validateFilePath(path, sFilePath);
                         registerBitstream(c, i, iAssetstore, sFilePath, sBundle, sDescription);
                         logInfo("\tRegistering Bitstream: " + sFilePath
                             + "\tAssetstore: " + iAssetstore
@@ -1414,6 +1452,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
             return;
         }
 
+        validateFilePath(path, fileName);
         String fullpath = path + File.separatorChar + fileName;
 
         // get an input stream
@@ -1743,7 +1782,8 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
                     } else {
                         logInfo("\tSetting special permissions for "
                             + bitstreamName);
-                        setPermission(c, myGroup, actionID, bs);
+                        String rpType = useWorkflow ? ResourcePolicy.TYPE_SUBMISSION : ResourcePolicy.TYPE_INHERITED;
+                        setPermission(c, myGroup, rpType, actionID, bs);
                     }
                 }
 
@@ -1801,24 +1841,25 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
      *
      * @param c        DSpace Context
      * @param g        Dspace Group
+     * @param rpType   resource policy type string
      * @param actionID action identifier
      * @param bs       Bitstream
      * @throws SQLException       if database error
      * @throws AuthorizeException if authorization error
      * @see org.dspace.core.Constants
      */
-    protected void setPermission(Context c, Group g, int actionID, Bitstream bs)
+    protected void setPermission(Context c, Group g, String rpType, int actionID, Bitstream bs)
         throws SQLException, AuthorizeException {
         if (!isTest) {
             // remove the default policy
             authorizeService.removeAllPolicies(c, bs);
 
             // add the policy
-            ResourcePolicy rp = resourcePolicyService.create(c);
+            ResourcePolicy rp = resourcePolicyService.create(c, null, g);
 
             rp.setdSpaceObject(bs);
             rp.setAction(actionID);
-            rp.setGroup(g);
+            rp.setRpType(rpType);
 
             resourcePolicyService.update(c, rp);
         } else {
@@ -1886,9 +1927,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
      */
     protected Document loadXML(String filename) throws IOException,
         ParserConfigurationException, SAXException {
-        DocumentBuilder builder = DocumentBuilderFactory.newInstance()
-                                                        .newDocumentBuilder();
-
+        DocumentBuilder builder = XMLUtils.getDocumentBuilder();
         return builder.parse(new File(filename));
     }
 
@@ -1957,58 +1996,57 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
         try {
             while (entries.hasMoreElements()) {
                 entry = entries.nextElement();
+                String entryName = entry.getName();
+                File outFile = new File(zipDir + entryName);
+                // Verify that this file/directory will be extracted into our zipDir (and not somewhere else!)
+                if (!outFile.toPath().normalize().startsWith(zipDir)) {
+                    throw new IOException("Bad zip entry: '" + entryName
+                                              + "' in file '" + zipfile.getAbsolutePath() + "'!"
+                                              + " Cannot process this file or directory.");
+                }
                 if (entry.isDirectory()) {
-                    if (!new File(zipDir + entry.getName()).mkdirs()) {
+                    if (!outFile.mkdirs()) {
                         logError("Unable to create contents directory: " + zipDir + entry.getName());
                     }
                 } else {
-                    String entryName = entry.getName();
-                    File outFile = new File(zipDir + entryName);
-                    // Verify that this file will be extracted into our zipDir (and not somewhere else!)
-                    if (!outFile.toPath().normalize().startsWith(zipDir)) {
-                        throw new IOException("Bad zip entry: '" + entryName
-                                                  + "' in file '" + zipfile.getAbsolutePath() + "'!"
-                                                  + " Cannot process this file.");
-                    } else {
-                        logInfo("Extracting file: " + entryName);
+                    logInfo("Extracting file: " + entryName);
 
-                        int index = entryName.lastIndexOf('/');
-                        if (index == -1) {
-                            // Was it created on Windows instead?
-                            index = entryName.lastIndexOf('\\');
-                        }
-                        if (index > 0) {
-                            File dir = new File(zipDir + entryName.substring(0, index));
-                            if (!dir.exists() && !dir.mkdirs()) {
-                                logError("Unable to create directory: " + dir.getAbsolutePath());
-                            }
-
-                            //Entries could have too many directories, and we need to adjust the sourcedir
-                            // file1.zip (SimpleArchiveFormat / item1 / contents|dublin_core|...
-                            //            SimpleArchiveFormat / item2 / contents|dublin_core|...
-                            // or
-                            // file2.zip (item1 / contents|dublin_core|...
-                            //            item2 / contents|dublin_core|...
-
-                            //regex supports either windows or *nix file paths
-                            String[] entryChunks = entryName.split("/|\\\\");
-                            if (entryChunks.length > 2) {
-                                if (StringUtils.equals(sourceDirForZip, sourcedir)) {
-                                    sourceDirForZip = sourcedir + "/" + entryChunks[0];
-                                }
-                            }
-                        }
-                        byte[] buffer = new byte[1024];
-                        int len;
-                        InputStream in = zf.getInputStream(entry);
-                        BufferedOutputStream out = new BufferedOutputStream(
-                            new FileOutputStream(outFile));
-                        while ((len = in.read(buffer)) >= 0) {
-                            out.write(buffer, 0, len);
-                        }
-                        in.close();
-                        out.close();
+                    int index = entryName.lastIndexOf('/');
+                    if (index == -1) {
+                        // Was it created on Windows instead?
+                        index = entryName.lastIndexOf('\\');
                     }
+                    if (index > 0) {
+                        File dir = new File(zipDir + entryName.substring(0, index));
+                        if (!dir.exists() && !dir.mkdirs()) {
+                            logError("Unable to create directory: " + dir.getAbsolutePath());
+                        }
+
+                        //Entries could have too many directories, and we need to adjust the sourcedir
+                        // file1.zip (SimpleArchiveFormat / item1 / contents|dublin_core|...
+                        //            SimpleArchiveFormat / item2 / contents|dublin_core|...
+                        // or
+                        // file2.zip (item1 / contents|dublin_core|...
+                        //            item2 / contents|dublin_core|...
+
+                        //regex supports either windows or *nix file paths
+                        String[] entryChunks = entryName.split("/|\\\\");
+                        if (entryChunks.length > 2) {
+                            if (StringUtils.equals(sourceDirForZip, sourcedir)) {
+                                sourceDirForZip = sourcedir + "/" + entryChunks[0];
+                            }
+                        }
+                    }
+                    byte[] buffer = new byte[1024];
+                    int len;
+                    InputStream in = zf.getInputStream(entry);
+                    BufferedOutputStream out = new BufferedOutputStream(
+                        new FileOutputStream(outFile));
+                    while ((len = in.read(buffer)) >= 0) {
+                        out.write(buffer, 0, len);
+                    }
+                    in.close();
+                    out.close();
                 }
             }
         } finally {
@@ -2233,7 +2271,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
                                     String fileName) throws MessagingException {
         try {
             Locale supportedLocale = I18nUtil.getEPersonLocale(eperson);
-            Email email = Email.getEmail(I18nUtil.getEmailFilename(supportedLocale, "bte_batch_import_success"));
+            Email email = Email.getEmail(I18nUtil.getEmailFilename(supportedLocale, "batch_import_success"));
             email.addRecipient(eperson.getEmail());
             email.addArgument(fileName);
 
@@ -2249,7 +2287,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
         logError("An error occurred during item import, the user will be notified. " + error);
         try {
             Locale supportedLocale = I18nUtil.getEPersonLocale(eperson);
-            Email email = Email.getEmail(I18nUtil.getEmailFilename(supportedLocale, "bte_batch_import_error"));
+            Email email = Email.getEmail(I18nUtil.getEmailFilename(supportedLocale, "batch_import_error"));
             email.addRecipient(eperson.getEmail());
             email.addArgument(error);
             email.addArgument(configurationService.getProperty("dspace.ui.url") + "/feedback");

dspace-api/src/main/java/org/dspace/app/itemupdate/DeleteBitstreamsAction.java

@@ -70,16 +70,19 @@ public class DeleteBitstreamsAction extends UpdateBitstreamsAction {
                                 }
                             }
 
-                            if (alterProvenance) {
+                            if (alterProvenance && !bundles.isEmpty()) {
                                 DtoMetadata dtom = DtoMetadata.create("dc.description.provenance", "en", "");
 
                                 String append = "Bitstream " + bs.getName() + " deleted on " + DCDate
                                     .getCurrent() + "; ";
-                                Item item = bundles.iterator().next().getItems().iterator().next();
-                                ItemUpdate.pr("Append provenance with: " + append);
+                                List<Item> items = bundles.iterator().next().getItems();
+                                if (!items.isEmpty()) {
+                                    Item item = items.iterator().next();
+                                    ItemUpdate.pr("Append provenance with: " + append);
 
-                                if (!isTest) {
-                                    MetadataUtilities.appendMetadata(context, item, dtom, false, append);
+                                    if (!isTest) {
+                                        MetadataUtilities.appendMetadata(context, item, dtom, false, append);
+                                    }
                                 }
                             }
                         }

dspace-api/src/main/java/org/dspace/app/itemupdate/ItemArchive.java

@@ -23,8 +23,6 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.Transformer;
 import javax.xml.transform.TransformerConfigurationException;
@@ -33,6 +31,7 @@ import javax.xml.transform.TransformerFactory;
 
 import org.apache.logging.log4j.Logger;
 import org.dspace.app.util.LocalSchemaFilenameFilter;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.Item;
@@ -52,7 +51,6 @@ public class ItemArchive {
 
     public static final String DUBLIN_CORE_XML = "dublin_core.xml";
 
-    protected static DocumentBuilder builder = null;
     protected Transformer transformer = null;
 
     protected List<DtoMetadata> dtomList = null;
@@ -95,14 +93,14 @@ public class ItemArchive {
         InputStream is = null;
         try {
             is = new FileInputStream(new File(dir, DUBLIN_CORE_XML));
-            itarch.dtomList = MetadataUtilities.loadDublinCore(getDocumentBuilder(), is);
+            itarch.dtomList = MetadataUtilities.loadDublinCore(XMLUtils.getDocumentBuilder(), is);
 
             //The code to search for local schema files was copied from org.dspace.app.itemimport
             // .ItemImportServiceImpl.java
             File file[] = dir.listFiles(new LocalSchemaFilenameFilter());
             for (int i = 0; i < file.length; i++) {
                 is = new FileInputStream(file[i]);
-                itarch.dtomList.addAll(MetadataUtilities.loadDublinCore(getDocumentBuilder(), is));
+                itarch.dtomList.addAll(MetadataUtilities.loadDublinCore(XMLUtils.getDocumentBuilder(), is));
             }
         } finally {
             if (is != null) {
@@ -126,14 +124,6 @@ public class ItemArchive {
         return itarch;
     }
 
-    protected static DocumentBuilder getDocumentBuilder()
-        throws ParserConfigurationException {
-        if (builder == null) {
-            builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
-        }
-        return builder;
-    }
-
     /**
      * Getter for Transformer
      *
@@ -318,7 +308,7 @@ public class ItemArchive {
 
         try {
             out = new FileOutputStream(new File(dir, "dublin_core.xml"));
-            Document doc = MetadataUtilities.writeDublinCore(getDocumentBuilder(), undoDtomList);
+            Document doc = MetadataUtilities.writeDublinCore(XMLUtils.getDocumentBuilder(), undoDtomList);
             MetadataUtilities.writeDocument(doc, getTransformer(), out);
 
             // if undo has delete bitstream

dspace-api/src/main/java/org/dspace/app/launcher/ScriptLauncher.java

@@ -19,6 +19,7 @@ import java.util.TreeMap;
 import org.apache.commons.cli.ParseException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.core.Context;
 import org.dspace.scripts.DSpaceRunnable;
 import org.dspace.scripts.DSpaceRunnable.StepResult;
@@ -314,7 +315,7 @@ public class ScriptLauncher {
         String config = kernelImpl.getConfigurationService().getProperty("dspace.dir") +
             System.getProperty("file.separator") + "config" +
             System.getProperty("file.separator") + "launcher.xml";
-        SAXBuilder saxBuilder = new SAXBuilder();
+        SAXBuilder saxBuilder = XMLUtils.getSAXBuilder();
         Document doc = null;
         try {
             doc = saxBuilder.build(config);

dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java

@@ -33,7 +33,8 @@ public class MediaFilterScriptConfiguration<T extends MediaFilterScript> extends
         options.addOption("v", "verbose", false, "print all extracted text and other details to STDOUT");
         options.addOption("q", "quiet", false, "do not print anything except in the event of errors.");
         options.addOption("f", "force", false, "force all bitstreams to be processed");
-        options.addOption("i", "identifier", true, "ONLY process bitstreams belonging to identifier");
+        options.addOption("i", "identifier", true,
+            "ONLY process bitstreams belonging to the provided handle identifier");
         options.addOption("m", "maximum", true, "process no more than maximum items");
         options.addOption("h", "help", false, "help");
 

dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java

@@ -132,12 +132,18 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
     @Override
     public void applyFiltersCommunity(Context context, Community community)
         throws Exception {   //only apply filters if community not in skip-list
+        // ensure that the community is attached to the current hibernate session
+        // as we are committing after each item (handles, sub-communties and
+        // collections are lazy attributes)
+        community = context.reloadEntity(community);
         if (!inSkipList(community.getHandle())) {
             List<Community> subcommunities = community.getSubcommunities();
             for (Community subcommunity : subcommunities) {
                 applyFiltersCommunity(context, subcommunity);
             }
-
+            // ensure that the community is attached to the current hibernate session
+            // as we are committing after each item
+            community = context.reloadEntity(community);
             List<Collection> collections = community.getCollections();
             for (Collection collection : collections) {
                 applyFiltersCollection(context, collection);
@@ -148,6 +154,9 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
     @Override
     public void applyFiltersCollection(Context context, Collection collection)
         throws Exception {
+        // ensure that the collection is attached to the current hibernate session
+        // as we are committing after each item (handles are lazy attributes)
+        collection = context.reloadEntity(collection);
         //only apply filters if collection not in skip-list
         if (!inSkipList(collection.getHandle())) {
             Iterator<Item> itemIterator = itemService.findAllByCollection(context, collection);
@@ -171,6 +180,8 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
             }
             // clear item objects from context cache and internal cache
             c.uncacheEntity(currentItem);
+            // commit after each item to release DB resources
+            c.commit();
             currentItem = null;
         }
     }

dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java

@@ -18,6 +18,7 @@ import java.nio.charset.StandardCharsets;
 import org.apache.commons.lang.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.poi.util.IOUtils;
 import org.apache.tika.Tika;
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.metadata.Metadata;
@@ -37,6 +38,8 @@ import org.xml.sax.SAXException;
 public class TikaTextExtractionFilter
     extends MediaFilter {
     private final static Logger log = LogManager.getLogger();
+    private static final int DEFAULT_MAX_CHARS = 100_000;
+    private static final int DEFAULT_MAX_ARRAY = 100_000_000;
 
     @Override
     public String getFilteredName(String oldFilename) {
@@ -70,9 +73,12 @@ public class TikaTextExtractionFilter
         }
 
         // Not using temporary file. We'll use Tika's default in-memory parsing.
-        // Get maximum characters to extract. Default is 100,000 chars, which is also Tika's default setting.
         String extractedText;
-        int maxChars = configurationService.getIntProperty("textextractor.max-chars", 100000);
+        // Get maximum characters to extract. Default is 100,000 chars, which is also Tika's default setting.
+        int maxChars = configurationService.getIntProperty("textextractor.max-chars", DEFAULT_MAX_CHARS);
+        // Get maximum size of structure that Tika will try to buffer.
+        int maxArray = configurationService.getIntProperty("textextractor.max-array", DEFAULT_MAX_ARRAY);
+        IOUtils.setByteArrayMaxOverride(maxArray);
         try {
             // Use Tika to extract text from input. Tika will automatically detect the file type.
             Tika tika = new Tika();
@@ -80,13 +86,13 @@ public class TikaTextExtractionFilter
             extractedText = tika.parseToString(source);
         } catch (IOException e) {
             System.err.format("Unable to extract text from bitstream in Item %s%n", currentItem.getID().toString());
-            e.printStackTrace();
+            e.printStackTrace(System.err);
             log.error("Unable to extract text from bitstream in Item {}", currentItem.getID().toString(), e);
             throw e;
         } catch (OutOfMemoryError oe) {
             System.err.format("OutOfMemoryError occurred when extracting text from bitstream in Item %s. " +
                 "You may wish to enable 'textextractor.use-temp-file'.%n", currentItem.getID().toString());
-            oe.printStackTrace();
+            oe.printStackTrace(System.err);
             log.error("OutOfMemoryError occurred when extracting text from bitstream in Item {}. " +
                           "You may wish to enable 'textextractor.use-temp-file'.", currentItem.getID().toString(), oe);
             throw oe;
@@ -138,7 +144,7 @@ public class TikaTextExtractionFilter
                 @Override
                 public void characters(char[] ch, int start, int length) throws SAXException {
                     try {
-                        writer.append(new String(ch), start, length);
+                        writer.append(new String(ch, start, length));
                     } catch (IOException e) {
                         String errorMsg = String.format("Could not append to temporary file at %s " +
                                                             "when performing text extraction",
@@ -156,7 +162,7 @@ public class TikaTextExtractionFilter
                 @Override
                 public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
                     try {
-                        writer.append(new String(ch), start, length);
+                        writer.append(new String(ch, start, length));
                     } catch (IOException e) {
                         String errorMsg = String.format("Could not append to temporary file at %s " +
                                                             "when performing text extraction",
@@ -167,6 +173,10 @@ public class TikaTextExtractionFilter
                 }
             });
 
+            ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();
+            int maxArray = configurationService.getIntProperty("textextractor.max-array", DEFAULT_MAX_ARRAY);
+            IOUtils.setByteArrayMaxOverride(maxArray);
+
             AutoDetectParser parser = new AutoDetectParser();
             Metadata metadata = new Metadata();
             // parse our source InputStream using the above custom handler

dspace-api/src/main/java/org/dspace/app/sfx/SFXFileReaderServiceImpl.java

@@ -18,6 +18,7 @@ import javax.xml.parsers.ParserConfigurationException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.app.sfx.service.SFXFileReaderService;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.content.DCPersonName;
 import org.dspace.content.Item;
 import org.dspace.content.MetadataValue;
@@ -79,9 +80,9 @@ public class SFXFileReaderServiceImpl implements SFXFileReaderService {
         log.info("Parsing XML file... " + fileName);
         DocumentBuilder docBuilder;
         Document doc = null;
-        DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
-        docBuilderFactory.setIgnoringElementContentWhitespace(true);
         try {
+            DocumentBuilderFactory docBuilderFactory = XMLUtils.getDocumentBuilderFactory();
+            docBuilderFactory.setIgnoringElementContentWhitespace(true);
             docBuilder = docBuilderFactory.newDocumentBuilder();
         } catch (ParserConfigurationException e) {
             log.error("Wrong parser configuration: " + e.getMessage());

dspace-api/src/main/java/org/dspace/app/sherpa/SHERPAService.java

@@ -17,15 +17,15 @@ import javax.annotation.PostConstruct;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.client.DSpaceHttpClientFactory;
 import org.dspace.app.sherpa.v2.SHERPAPublisherResponse;
 import org.dspace.app.sherpa.v2.SHERPAResponse;
 import org.dspace.app.sherpa.v2.SHERPAUtils;
@@ -45,8 +45,6 @@ import org.springframework.cache.annotation.Cacheable;
  */
 public class SHERPAService {
 
-    private CloseableHttpClient client = null;
-
     private int maxNumberOfTries;
     private long sleepBetweenTimeouts;
     private int timeout = 5000;
@@ -59,19 +57,6 @@ public class SHERPAService {
     @Autowired
     ConfigurationService configurationService;
 
-    /**
-     * Create a new HTTP builder with sensible defaults in constructor
-     */
-    public SHERPAService() {
-        HttpClientBuilder builder = HttpClientBuilder.create();
-        // httpclient 4.3+ doesn't appear to have any sensible defaults any more. Setting conservative defaults as
-        // not to hammer the SHERPA service too much.
-        client = builder
-            .disableAutomaticRetries()
-            .setMaxConnTotal(5)
-            .build();
-    }
-
     /**
      * Complete initialization of the Bean.
      */
@@ -132,46 +117,47 @@ public class SHERPAService {
                 timeout,
                 sleepBetweenTimeouts));
 
-            try {
+            try (CloseableHttpClient client = DSpaceHttpClientFactory.getInstance().buildWithoutAutomaticRetries(5)) {
                 Thread.sleep(sleepBetweenTimeouts);
 
                 // Construct a default HTTP method (first result)
                 method = constructHttpGet(type, field, predicate, value, start, limit);
 
                 // Execute the method
-                HttpResponse response = client.execute(method);
-                int statusCode = response.getStatusLine().getStatusCode();
+                try (CloseableHttpResponse response = client.execute(method)) {
+                    int statusCode = response.getStatusLine().getStatusCode();
 
-                log.debug(response.getStatusLine().getStatusCode() + ": "
-                    + response.getStatusLine().getReasonPhrase());
+                    log.debug(response.getStatusLine().getStatusCode() + ": "
+                            + response.getStatusLine().getReasonPhrase());
 
-                if (statusCode != HttpStatus.SC_OK) {
-                    sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO return not OK status: "
-                        + statusCode);
-                    String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
-                    log.error("Error from SHERPA HTTP request: " + errorBody);
-                }
-
-                HttpEntity responseBody = response.getEntity();
-
-                // If the response body is valid, pass to SHERPAResponse for parsing as JSON
-                if (null != responseBody) {
-                    log.debug("Non-null SHERPA resonse received for query of " + value);
-                    InputStream content = null;
-                    try {
-                        content = responseBody.getContent();
-                        sherpaResponse =
-                            new SHERPAPublisherResponse(content, SHERPAPublisherResponse.SHERPAFormat.JSON);
-                    } catch (IOException e) {
-                        log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
-                    } finally {
-                        if (content != null) {
-                            content.close();
-                        }
+                    if (statusCode != HttpStatus.SC_OK) {
+                        sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO return not OK status: "
+                                + statusCode);
+                        String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
+                        log.error("Error from SHERPA HTTP request: " + errorBody);
+                    }
+
+                    HttpEntity responseBody = response.getEntity();
+
+                    // If the response body is valid, pass to SHERPAResponse for parsing as JSON
+                    if (null != responseBody) {
+                        log.debug("Non-null SHERPA response received for query of " + value);
+                        InputStream content = null;
+                        try {
+                            content = responseBody.getContent();
+                            sherpaResponse =
+                                    new SHERPAPublisherResponse(content, SHERPAPublisherResponse.SHERPAFormat.JSON);
+                        } catch (IOException e) {
+                            log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
+                        } finally {
+                            if (content != null) {
+                                content.close();
+                            }
+                        }
+                    } else {
+                        log.debug("Empty SHERPA response body for query on " + value);
+                        sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO returned no response");
                     }
-                } else {
-                    log.debug("Empty SHERPA response body for query on " + value);
-                    sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO returned no response");
                 }
             } catch (URISyntaxException e) {
                 String errorMessage = "Error building SHERPA v2 API URI: " + e.getMessage();
@@ -235,45 +221,46 @@ public class SHERPAService {
                 timeout,
                 sleepBetweenTimeouts));
 
-            try {
+            try (CloseableHttpClient client = DSpaceHttpClientFactory.getInstance().buildWithoutAutomaticRetries(5)) {
                 Thread.sleep(sleepBetweenTimeouts);
 
                 // Construct a default HTTP method (first result)
                 method = constructHttpGet(type, field, predicate, value, start, limit);
 
                 // Execute the method
-                HttpResponse response = client.execute(method);
-                int statusCode = response.getStatusLine().getStatusCode();
+                try (CloseableHttpResponse response = client.execute(method)) {
+                    int statusCode = response.getStatusLine().getStatusCode();
 
-                log.debug(response.getStatusLine().getStatusCode() + ": "
-                    + response.getStatusLine().getReasonPhrase());
+                    log.debug(response.getStatusLine().getStatusCode() + ": "
+                            + response.getStatusLine().getReasonPhrase());
 
-                if (statusCode != HttpStatus.SC_OK) {
-                    sherpaResponse = new SHERPAResponse("SHERPA/RoMEO return not OK status: "
-                        + statusCode);
-                    String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
-                    log.error("Error from SHERPA HTTP request: " + errorBody);
-                }
-
-                HttpEntity responseBody = response.getEntity();
-
-                // If the response body is valid, pass to SHERPAResponse for parsing as JSON
-                if (null != responseBody) {
-                    log.debug("Non-null SHERPA resonse received for query of " + value);
-                    InputStream content = null;
-                    try {
-                        content = responseBody.getContent();
-                        sherpaResponse = new SHERPAResponse(content, SHERPAResponse.SHERPAFormat.JSON);
-                    } catch (IOException e) {
-                        log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
-                    } finally {
-                        if (content != null) {
-                            content.close();
-                        }
+                    if (statusCode != HttpStatus.SC_OK) {
+                        sherpaResponse = new SHERPAResponse("SHERPA/RoMEO return not OK status: "
+                                + statusCode);
+                        String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
+                        log.error("Error from SHERPA HTTP request: " + errorBody);
+                    }
+
+                    HttpEntity responseBody = response.getEntity();
+
+                    // If the response body is valid, pass to SHERPAResponse for parsing as JSON
+                    if (null != responseBody) {
+                        log.debug("Non-null SHERPA response received for query of " + value);
+                        InputStream content = null;
+                        try {
+                            content = responseBody.getContent();
+                            sherpaResponse = new SHERPAResponse(content, SHERPAResponse.SHERPAFormat.JSON);
+                        } catch (IOException e) {
+                            log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
+                        } finally {
+                            if (content != null) {
+                                content.close();
+                            }
+                        }
+                    } else {
+                        log.debug("Empty SHERPA response body for query on " + value);
+                        sherpaResponse = new SHERPAResponse("SHERPA/RoMEO returned no response");
                     }
-                } else {
-                    log.debug("Empty SHERPA response body for query on " + value);
-                    sherpaResponse = new SHERPAResponse("SHERPA/RoMEO returned no response");
                 }
             } catch (URISyntaxException e) {
                 String errorMessage = "Error building SHERPA v2 API URI: " + e.getMessage();
@@ -283,7 +270,7 @@ public class SHERPAService {
                 String errorMessage = "Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage();
                 log.error(errorMessage, e);
                 sherpaResponse = new SHERPAResponse(errorMessage);
-            }  catch (InterruptedException e) {
+            } catch (InterruptedException e) {
                 String errorMessage = "Encountered exception while sleeping thread: " + e.getMessage();
                 log.error(errorMessage, e);
                 sherpaResponse = new SHERPAResponse(errorMessage);

dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java

@@ -7,6 +7,8 @@
  */
 package org.dspace.app.sitemap;
 
+import static org.dspace.discovery.SearchUtils.RESOURCE_TYPE_FIELD;
+
 import java.io.File;
 import java.io.IOException;
 import java.sql.SQLException;
@@ -189,7 +191,8 @@ public class GenerateSitemaps {
         try {
             DiscoverQuery discoveryQuery = new DiscoverQuery();
             discoveryQuery.setMaxResults(PAGE_SIZE);
-            discoveryQuery.setQuery("search.resourcetype:Community");
+            discoveryQuery.setQuery("*:*");
+            discoveryQuery.addFilterQueries(RESOURCE_TYPE_FIELD + ":Community");
             do {
                 discoveryQuery.setStart(offset);
                 DiscoverResult discoverResult = searchService.search(c, discoveryQuery);
@@ -213,7 +216,8 @@ public class GenerateSitemaps {
             offset = 0;
             discoveryQuery = new DiscoverQuery();
             discoveryQuery.setMaxResults(PAGE_SIZE);
-            discoveryQuery.setQuery("search.resourcetype:Collection");
+            discoveryQuery.setQuery("*:*");
+            discoveryQuery.addFilterQueries(RESOURCE_TYPE_FIELD + ":Collection");
             do {
                 discoveryQuery.setStart(offset);
                 DiscoverResult discoverResult = searchService.search(c, discoveryQuery);
@@ -237,7 +241,8 @@ public class GenerateSitemaps {
             offset = 0;
             discoveryQuery = new DiscoverQuery();
             discoveryQuery.setMaxResults(PAGE_SIZE);
-            discoveryQuery.setQuery("search.resourcetype:Item");
+            discoveryQuery.setQuery("*:*");
+            discoveryQuery.addFilterQueries(RESOURCE_TYPE_FIELD + ":Item");
             discoveryQuery.addSearchField("search.entitytype");
             do {
 

dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java

@@ -98,42 +98,25 @@ public class SolrDatabaseResyncCli extends DSpaceRunnable<SolrDatabaseResyncCliS
 
     private void performStatusUpdate(Context context) throws SearchServiceException, SolrServerException, IOException {
         SolrQuery solrQuery = new SolrQuery();
-        solrQuery.setQuery(STATUS_FIELD + ":" + STATUS_FIELD_PREDB);
+        solrQuery.setQuery("*:*");
+        solrQuery.addFilterQuery(STATUS_FIELD + ":" + STATUS_FIELD_PREDB);
         solrQuery.addFilterQuery(SearchUtils.RESOURCE_TYPE_FIELD + ":" + IndexableItem.TYPE);
         String dateRangeFilter = SearchUtils.LAST_INDEXED_FIELD + ":[* TO " + maxTime + "]";
         logDebugAndOut("Date range filter used; " + dateRangeFilter);
         solrQuery.addFilterQuery(dateRangeFilter);
         solrQuery.addField(SearchUtils.RESOURCE_ID_FIELD);
         solrQuery.addField(SearchUtils.RESOURCE_UNIQUE_ID);
+        solrQuery.setRows(0);
         QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD);
-
-        if (response != null) {
-            logInfoAndOut(response.getResults().size() + " items found to process");
-
-            for (SolrDocument doc : response.getResults()) {
-                String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD);
-                String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID);
-                logDebugAndOut("Processing item with UUID: " + uuid);
-
-                Optional<IndexableObject> indexableObject = Optional.empty();
-                try {
-                    indexableObject = indexObjectServiceFactory
-                            .getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid);
-                } catch (SQLException e) {
-                    log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid +
-                            "\" from the database, removing related solr document", e);
-                }
-
-                try {
-                    if (indexableObject.isPresent()) {
-                        logDebugAndOut("Item exists in DB, updating solr document");
-                        updateItem(context, indexableObject.get());
-                    } else {
-                        logDebugAndOut("Item doesn't exist in DB, removing solr document");
-                        removeItem(context, uniqueId);
-                    }
-                } catch (SQLException | IOException e) {
-                    log.error(e.getMessage(), e);
+        if (response != null && response.getResults() != null) {
+            long nrOfPreDBResults = response.getResults().getNumFound();
+            if (nrOfPreDBResults > 0) {
+                logInfoAndOut(nrOfPreDBResults + " items found to process");
+                int batchSize = configurationService.getIntProperty("script.solr-database-resync.batch-size", 100);
+                for (int start = 0; start < nrOfPreDBResults; start += batchSize) {
+                    solrQuery.setStart(start);
+                    solrQuery.setRows(batchSize);
+                    performStatusUpdateOnNextBatch(context, solrQuery);
                 }
             }
         }
@@ -141,6 +124,38 @@ public class SolrDatabaseResyncCli extends DSpaceRunnable<SolrDatabaseResyncCliS
         indexingService.commit();
     }
 
+    private void performStatusUpdateOnNextBatch(Context context, SolrQuery solrQuery)
+            throws SolrServerException, IOException {
+        QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD);
+
+        for (SolrDocument doc : response.getResults()) {
+            String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD);
+            String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID);
+            logDebugAndOut("Processing item with UUID: " + uuid);
+
+            Optional<IndexableObject> indexableObject = Optional.empty();
+            try {
+                indexableObject = indexObjectServiceFactory
+                        .getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid);
+            } catch (SQLException e) {
+                log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid +
+                        "\" from the database, removing related solr document", e);
+            }
+
+            try {
+                if (indexableObject.isPresent()) {
+                    logDebugAndOut("Item exists in DB, updating solr document");
+                    updateItem(context, indexableObject.get());
+                } else {
+                    logDebugAndOut("Item doesn't exist in DB, removing solr document");
+                    removeItem(context, uniqueId);
+                }
+            } catch (SQLException | IOException e) {
+                log.error(e.getMessage(), e);
+            }
+        }
+    }
+
     private void updateItem(Context context, IndexableObject indexableObject) throws SolrServerException, IOException {
         Map<String,Object> fieldModifier = new HashMap<>(1);
         fieldModifier.put("remove", STATUS_FIELD_PREDB);

dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java

@@ -281,10 +281,14 @@ public class LogAnalyser {
      */
     private static String fileTemplate = "dspace\\.log.*";
 
+    private static final ConfigurationService configurationService =
+            DSpaceServicesFactory.getInstance().getConfigurationService();
+
     /**
      * the configuration file from which to configure the analyser
      */
-    private static String configFile;
+    private static String configFile = configurationService.getProperty("dspace.dir")
+            + File.separator + "config" + File.separator + "dstat.cfg";
 
     /**
      * the output file to which to write aggregation data
@@ -616,8 +620,6 @@ public class LogAnalyser {
         }
 
         // now do the host name and url lookup
-        ConfigurationService configurationService
-                = DSpaceServicesFactory.getInstance().getConfigurationService();
         hostName = Utils.getHostName(configurationService.getProperty("dspace.ui.url"));
         name = configurationService.getProperty("dspace.name").trim();
         url = configurationService.getProperty("dspace.ui.url").trim();
@@ -658,8 +660,6 @@ public class LogAnalyser {
                                      String myConfigFile, String myOutFile,
                                      Date myStartDate, Date myEndDate,
                                      boolean myLookUp) {
-        ConfigurationService configurationService
-                = DSpaceServicesFactory.getInstance().getConfigurationService();
 
         if (myLogDir != null) {
             logDir = myLogDir;
@@ -673,9 +673,6 @@ public class LogAnalyser {
 
         if (myConfigFile != null) {
             configFile = myConfigFile;
-        } else {
-            configFile = configurationService.getProperty("dspace.dir")
-                    + File.separator + "config" + File.separator + "dstat.cfg";
         }
 
         if (myStartDate != null) {

dspace-api/src/main/java/org/dspace/app/util/DCInput.java

@@ -163,7 +163,7 @@ public class DCInput {
      * The scope of the input sets, this restricts hidden metadata fields from
      * view by the end user during submission.
      */
-    public static final String SUBMISSION_SCOPE = "submit";
+    public static final String SUBMISSION_SCOPE = "submission";
 
     /**
      * Class constructor for creating a DCInput object based on the contents of
@@ -262,7 +262,7 @@ public class DCInput {
 
     /**
      * Is this DCInput for display in the given scope? The scope should be
-     * either "workflow" or "submit", as per the input forms definition. If the
+     * either "workflow" or "submission", as per the input forms definition. If the
      * internal visibility is set to "null" then this will always return true.
      *
      * @param scope String identifying the scope that this input's visibility

dspace-api/src/main/java/org/dspace/app/util/DCInputsReader.java

@@ -14,9 +14,7 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import javax.servlet.ServletException;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.FactoryConfigurationError;
 
 import org.apache.commons.lang3.StringUtils;
@@ -119,15 +117,17 @@ public class DCInputsReader {
         formDefns = new HashMap<String, List<List<Map<String, String>>>>();
         valuePairs = new HashMap<String, List<String>>();
 
-        String uri = "file:" + new File(fileName).getAbsolutePath();
+        File inputFile = new File(fileName);
+        String inputFileDir = inputFile.toPath().normalize().getParent().toString();
+
+        String uri = "file:" + inputFile.getAbsolutePath();
 
         try {
-            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-            factory.setValidating(false);
-            factory.setIgnoringComments(true);
-            factory.setIgnoringElementContentWhitespace(true);
-
-            DocumentBuilder db = factory.newDocumentBuilder();
+            // This document builder will *not* disable external
+            // entities as they can be useful in managing large forms, but
+            // it will restrict them to be within the directory that the
+            // current input form XML file exists (or a sub-directory)
+            DocumentBuilder db = XMLUtils.getTrustedDocumentBuilder(inputFileDir);
             Document doc = db.parse(uri);
             doNodes(doc);
             checkValues();
@@ -150,17 +150,16 @@ public class DCInputsReader {
      * Returns the set of DC inputs used for a particular collection, or the
      * default set if no inputs defined for the collection
      *
-     * @param collectionHandle collection's unique Handle
+     * @param collection collection for which search the set of DC inputs
      * @return DC input set
      * @throws DCInputsReaderException if no default set defined
-     * @throws ServletException
      */
-    public List<DCInputSet> getInputsByCollectionHandle(String collectionHandle)
+    public List<DCInputSet> getInputsByCollection(Collection collection)
         throws DCInputsReaderException {
         SubmissionConfig config;
         try {
             config = SubmissionServiceFactory.getInstance().getSubmissionConfigService()
-                        .getSubmissionConfigByCollection(collectionHandle);
+                        .getSubmissionConfigByCollection(collection);
             String formName = config.getSubmissionName();
             if (formName == null) {
                 throw new DCInputsReaderException("No form designated as default");
@@ -381,7 +380,7 @@ public class DCInputsReader {
         }
         // sanity check number of fields
         if (fields.size() < 1) {
-            throw new DCInputsReaderException("Form " + formName + "row " + rowIdx + " has no fields");
+            throw new DCInputsReaderException("Form " + formName + ", row " + rowIdx + " has no fields");
         }
     }
 
@@ -691,7 +690,7 @@ public class DCInputsReader {
 
     public String getInputFormNameByCollectionAndField(Collection collection, String field)
         throws DCInputsReaderException {
-        List<DCInputSet> inputSets = getInputsByCollectionHandle(collection.getHandle());
+        List<DCInputSet> inputSets = getInputsByCollection(collection);
         for (DCInputSet inputSet : inputSets) {
             String[] tokenized = Utils.tokenize(field);
             String schema = tokenized[0];

dspace-api/src/main/java/org/dspace/app/util/InitializeEntities.java

@@ -11,7 +11,6 @@ import java.io.File;
 import java.io.IOException;
 import java.sql.SQLException;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
 import org.apache.commons.cli.CommandLine;
@@ -64,20 +63,36 @@ public class InitializeEntities {
      */
     public static void main(String[] argv) throws SQLException, AuthorizeException, ParseException {
         InitializeEntities initializeEntities = new InitializeEntities();
+        // Set up command-line options and parse arguments
         CommandLineParser parser = new DefaultParser();
         Options options = createCommandLineOptions();
         CommandLine line = parser.parse(options,argv);
-        String fileLocation = getFileLocationFromCommandLine(line);
+        // First of all, check if the help option was entered or a required argument is missing
         checkHelpEntered(options, line);
+        // Get the file location from the command line
+        String fileLocation = getFileLocationFromCommandLine(line);
+        // Run the script
         initializeEntities.run(fileLocation);
     }
+
+    /**
+     * Check if the help option was entered or a required argument is missing. If so, print help and exit.
+     * @param options the defined command-line options
+     * @param line the parsed command-line arguments
+     */
     private static void checkHelpEntered(Options options, CommandLine line) {
-        if (line.hasOption("h")) {
+        if (line.hasOption("h") || !line.hasOption("f")) {
             HelpFormatter formatter = new HelpFormatter();
             formatter.printHelp("Intialize Entities", options);
             System.exit(0);
         }
     }
+
+    /**
+     * Get the file path from the command-line argument. Exits with exit code 1 if no file argument was entered.
+     * @param line the parsed command-line arguments
+     * @return the file path
+     */
     private static String getFileLocationFromCommandLine(CommandLine line) {
         String query = line.getOptionValue("f");
         if (StringUtils.isEmpty(query)) {
@@ -88,13 +103,25 @@ public class InitializeEntities {
         return query;
     }
 
+    /**
+     * Create the command-line options
+     * @return the command-line options
+     */
     protected static Options createCommandLineOptions() {
         Options options = new Options();
-        options.addOption("f", "file", true, "the location for the file containing the xml data");
+        options.addOption("f", "file", true, "the path to the file containing the " +
+                "relationship definitions (e.g. ${dspace.dir}/config/entities/relationship-types.xml)");
+        options.addOption("h", "help", false, "print this message");
 
         return options;
     }
 
+    /**
+     * Run the script for the given file location
+     * @param fileLocation the file location
+     * @throws SQLException If something goes wrong initializing context or inserting relationship types
+     * @throws AuthorizeException  If the script user fails to authorize while inserting relationship types
+     */
     private void run(String fileLocation) throws SQLException, AuthorizeException {
         Context context = new Context();
         context.turnOffAuthorisationSystem();
@@ -102,11 +129,18 @@ public class InitializeEntities {
         context.complete();
     }
 
+    /**
+     * Parse the XML file at fileLocation to create relationship types in the database
+     * @param context DSpace context
+     * @param fileLocation the full or relative file path to the relationship types XML
+     * @throws AuthorizeException If the script user fails to authorize while inserting relationship types
+     */
     private void parseXMLToRelations(Context context, String fileLocation) throws AuthorizeException {
         try {
             File fXmlFile = new File(fileLocation);
-            DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
-            DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+            // This XML builder will allow external entities, so the relationship types XML should
+            // be considered trusted by administrators
+            DocumentBuilder dBuilder = XMLUtils.getTrustedDocumentBuilder();
             Document doc = dBuilder.parse(fXmlFile);
 
             doc.getDocumentElement().normalize();
@@ -158,15 +192,15 @@ public class InitializeEntities {
 
                     for (int j = 0; j < leftCardinalityList.getLength(); j++) {
                         Node node = leftCardinalityList.item(j);
-                        leftCardinalityMin = getString(leftCardinalityMin,(Element) node, "min");
-                        leftCardinalityMax = getString(leftCardinalityMax,(Element) node, "max");
+                        leftCardinalityMin = getCardinalityMinString(leftCardinalityMin,(Element) node, "min");
+                        leftCardinalityMax = getCardinalityMinString(leftCardinalityMax,(Element) node, "max");
 
                     }
 
                     for (int j = 0; j < rightCardinalityList.getLength(); j++) {
                         Node node = rightCardinalityList.item(j);
-                        rightCardinalityMin = getString(rightCardinalityMin,(Element) node, "min");
-                        rightCardinalityMax = getString(rightCardinalityMax,(Element) node, "max");
+                        rightCardinalityMin = getCardinalityMinString(rightCardinalityMin,(Element) node, "min");
+                        rightCardinalityMax = getCardinalityMinString(rightCardinalityMax,(Element) node, "max");
 
                     }
                     populateRelationshipType(context, leftType, rightType, leftwardType, rightwardType,
@@ -182,13 +216,39 @@ public class InitializeEntities {
         }
     }
 
-    private String getString(String leftCardinalityMin,Element node, String minOrMax) {
+    /**
+     * Extract the min or max value for the left or right cardinality from the node text content
+     * @param leftCardinalityMin current left cardinality min
+     * @param node node to extract the min or max value from
+     * @param minOrMax element tag name to parse
+     * @return final left cardinality min
+     */
+    private String getCardinalityMinString(String leftCardinalityMin, Element node, String minOrMax) {
         if (node.getElementsByTagName(minOrMax).getLength() > 0) {
             leftCardinalityMin = node.getElementsByTagName(minOrMax).item(0).getTextContent();
         }
         return leftCardinalityMin;
     }
 
+    /**
+     * Populate the relationship type based on values parsed from the XML relationship types configuration
+     *
+     * @param context DSpace context
+     * @param leftType left relationship type (e.g. "Publication").
+     * @param rightType right relationship type (e.g. "Journal").
+     * @param leftwardType leftward relationship type (e.g. "isAuthorOfPublication").
+     * @param rightwardType rightward relationship type (e.g. "isPublicationOfAuthor").
+     * @param leftCardinalityMin left cardinality min
+     * @param leftCardinalityMax left cardinality max
+     * @param rightCardinalityMin right cardinality min
+     * @param rightCardinalityMax right cardinality max
+     * @param copyToLeft copy metadata values to left if right side is deleted
+     * @param copyToRight copy metadata values to right if left side is deleted
+     * @param tilted set a tilted relationship side (left or right) if there are many relationships going one way
+     *               to help performance (e.g. authors with 1000s of publications)
+     * @throws SQLException if database error occurs while saving the relationship type
+     * @throws AuthorizeException if authorization error occurs while saving the relationship type
+     */
     private void populateRelationshipType(Context context, String leftType, String rightType, String leftwardType,
                                           String rightwardType, String leftCardinalityMin, String leftCardinalityMax,
                                           String rightCardinalityMin, String rightCardinalityMax,

dspace-api/src/main/java/org/dspace/app/util/OpenSearchServiceImpl.java

@@ -101,6 +101,14 @@ public class OpenSearchServiceImpl implements OpenSearchService {
             configurationService.getProperty("websvc.opensearch.uicontext");
     }
 
+    /**
+     * Get base search UI URL (websvc.opensearch.max_num_of_items_per_request)
+     */
+    public int getMaxNumOfItemsPerRequest() {
+        return configurationService.getIntProperty(
+                "websvc.opensearch.max_num_of_items_per_request", 100);
+    }
+
     @Override
     public String getContentType(String format) {
         return "html".equals(format) ? "text/html" :
@@ -155,7 +163,7 @@ public class OpenSearchServiceImpl implements OpenSearchService {
             format = "atom_1.0";
         }
 
-        SyndicationFeed feed = new SyndicationFeed(labels.get(SyndicationFeed.MSG_UITYPE));
+        SyndicationFeed feed = new SyndicationFeed();
         feed.populate(null, context, scope, results, labels);
         feed.setType(format);
         feed.addModule(openSearchMarkup(query, totalResults, start, pageSize));

dspace-api/src/main/java/org/dspace/app/util/SubmissionConfigReader.java

@@ -11,21 +11,21 @@ import java.io.File;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.FactoryConfigurationError;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.content.Collection;
 import org.dspace.content.DSpaceObject;
+import org.dspace.content.Item;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.content.service.CollectionService;
 import org.dspace.core.Context;
-import org.dspace.discovery.SearchServiceException;
 import org.dspace.handle.factory.HandleServiceFactory;
 import org.dspace.services.factory.DSpaceServicesFactory;
 import org.w3c.dom.Document;
@@ -96,6 +96,13 @@ public class SubmissionConfigReader {
      */
     private Map<String, Map<String, String>> stepDefns = null;
 
+    /**
+     * Hashmap which stores which submission process configuration is used by
+     * which entityType, computed from the item submission config file
+     * (specifically, the 'submission-map' tag)
+     */
+    private Map<String, String> entityTypeToSubmissionConfig = null;
+
     /**
      * Reference to the item submission definitions defined in the
      * "submission-definitions" section
@@ -127,6 +134,7 @@ public class SubmissionConfigReader {
 
     public void reload() throws SubmissionConfigReaderException {
         collectionToSubmissionConfig = null;
+        entityTypeToSubmissionConfig = null;
         stepDefns = null;
         submitDefns = null;
         buildInputs(configDir + SUBMIT_DEF_FILE_PREFIX + SUBMIT_DEF_FILE_SUFFIX);
@@ -145,26 +153,21 @@ public class SubmissionConfigReader {
      */
     private void buildInputs(String fileName) throws SubmissionConfigReaderException {
         collectionToSubmissionConfig = new HashMap<String, String>();
-        submitDefns = new HashMap<String, List<Map<String, String>>>();
+        entityTypeToSubmissionConfig = new HashMap<String, String>();
+        submitDefns = new LinkedHashMap<String, List<Map<String, String>>>();
 
         String uri = "file:" + new File(fileName).getAbsolutePath();
 
         try {
-            DocumentBuilderFactory factory = DocumentBuilderFactory
-                .newInstance();
-            factory.setValidating(false);
-            factory.setIgnoringComments(true);
-            factory.setIgnoringElementContentWhitespace(true);
-
-            DocumentBuilder db = factory.newDocumentBuilder();
+            // This document builder factory will *not* disable external
+            // entities as they can be useful in managing large forms, but
+            // it will restrict them to the config dir containing submission definitions
+            DocumentBuilder db = XMLUtils.getTrustedDocumentBuilder(configDir);
             Document doc = db.parse(uri);
             doNodes(doc);
         } catch (FactoryConfigurationError fe) {
             throw new SubmissionConfigReaderException(
                 "Cannot create Item Submission Configuration parser", fe);
-        } catch (SearchServiceException se) {
-            throw new SubmissionConfigReaderException(
-                "Cannot perform a discovery search for Item Submission Configuration", se);
         } catch (Exception e) {
             throw new SubmissionConfigReaderException(
                 "Error creating Item Submission Configuration: " + e);
@@ -210,18 +213,36 @@ public class SubmissionConfigReader {
      * Returns the Item Submission process config used for a particular
      * collection, or the default if none is defined for the collection
      *
-     * @param collectionHandle collection's unique Handle
+     * @param col collection for which search Submission process config
      * @return the SubmissionConfig representing the item submission config
-     * @throws SubmissionConfigReaderException if no default submission process configuration defined
+     * @throws IllegalStateException if no default submission process configuration defined
      */
-    public SubmissionConfig getSubmissionConfigByCollection(String collectionHandle) {
-        // get the name of the submission process config for this collection
-        String submitName = collectionToSubmissionConfig
-            .get(collectionHandle);
-        if (submitName == null) {
+    public SubmissionConfig getSubmissionConfigByCollection(Collection col) {
+
+        String submitName;
+
+        if (col != null) {
+
+            // get the name of the submission process config for this collection
             submitName = collectionToSubmissionConfig
-                .get(DEFAULT_COLLECTION);
+                .get(col.getHandle());
+            if (submitName != null) {
+                return getSubmissionConfigByName(submitName);
+            }
         }
+
+        // get the name of the submission process based on the entity type of this collections
+        if (!entityTypeToSubmissionConfig.isEmpty()) {
+            String entityType = collectionService.getMetadataFirstValue(col, "dspace", "entity", "type", Item.ANY);
+            submitName = entityTypeToSubmissionConfig
+                .get(entityType);
+            if (submitName != null) {
+                return getSubmissionConfigByName(submitName);
+            }
+        }
+
+        submitName = collectionToSubmissionConfig.get(DEFAULT_COLLECTION);
+
         if (submitName == null) {
             throw new IllegalStateException(
                 "No item submission process configuration designated as 'default' in 'submission-map' section of " +
@@ -300,7 +321,7 @@ public class SubmissionConfigReader {
      * should correspond to the collection-form maps, the form definitions, and
      * the display/storage word pairs.
      */
-    private void doNodes(Node n) throws SAXException, SearchServiceException, SubmissionConfigReaderException {
+    private void doNodes(Node n) throws SAXException, SubmissionConfigReaderException {
         if (n == null) {
             return;
         }
@@ -347,9 +368,7 @@ public class SubmissionConfigReader {
      * the collection handle and item submission name, put name in hashmap keyed
      * by the collection handle.
      */
-    private void processMap(Node e) throws SAXException, SearchServiceException {
-        // create a context
-        Context context = new Context();
+    private void processMap(Node e) throws SAXException {
 
         NodeList nl = e.getChildNodes();
         int len = nl.getLength();
@@ -369,7 +388,7 @@ public class SubmissionConfigReader {
                     throw new SAXException(
                         "name-map element is missing submission-name attribute in 'item-submission.xml'");
                 }
-                if (content != null && content.length() > 0) {
+                if (content != null && !content.isEmpty()) {
                     throw new SAXException(
                         "name-map element has content in 'item-submission.xml', it should be empty.");
                 }
@@ -377,12 +396,7 @@ public class SubmissionConfigReader {
                     collectionToSubmissionConfig.put(id, value);
 
                 } else {
-                    // get all collections for this entity-type
-                    List<Collection> collections = collectionService.findAllCollectionsByEntityType( context,
-                                    entityType);
-                    for (Collection collection : collections) {
-                        collectionToSubmissionConfig.putIfAbsent(collection.getHandle(), value);
-                    }
+                    entityTypeToSubmissionConfig.put(entityType, value);
                 }
             } // ignore any child node that isn't a "name-map"
         }
@@ -663,4 +677,4 @@ public class SubmissionConfigReader {
         }
         return results;
     }
-}
+}

dspace-api/src/main/java/org/dspace/app/util/SyndicationFeed.java

@@ -84,11 +84,6 @@ public class SyndicationFeed {
     public static final String MSG_FEED_TITLE = "feed.title";
     public static final String MSG_FEED_DESCRIPTION = "general-feed.description";
     public static final String MSG_METADATA = "metadata.";
-    public static final String MSG_UITYPE = "ui.type";
-
-    // UI keywords
-    public static final String UITYPE_XMLUI = "xmlui";
-    public static final String UITYPE_JSPUI = "jspui";
 
     // default DC fields for entry
     protected String defaultTitleField = "dc.title";
@@ -145,10 +140,6 @@ public class SyndicationFeed {
     // the feed object we are building
     protected SyndFeed feed = null;
 
-    // memory of UI that called us, "xmlui" or "jspui"
-    // affects Bitstream retrieval URL and I18N keys
-    protected String uiType = null;
-
     protected HttpServletRequest request = null;
 
     protected CollectionService collectionService;
@@ -157,12 +148,9 @@ public class SyndicationFeed {
 
     /**
      * Constructor.
-     *
-     * @param ui either "xmlui" or "jspui"
      */
-    public SyndicationFeed(String ui) {
+    public SyndicationFeed() {
         feed = new SyndFeedImpl();
-        uiType = ui;
         ContentServiceFactory contentServiceFactory = ContentServiceFactory.getInstance();
         itemService = contentServiceFactory.getItemService();
         collectionService = contentServiceFactory.getCollectionService();
@@ -518,8 +506,7 @@ public class SyndicationFeed {
     protected String urlOfBitstream(HttpServletRequest request, Bitstream logo) {
         String name = logo.getName();
         return resolveURL(request, null) +
-            (uiType.equalsIgnoreCase(UITYPE_XMLUI) ? "/bitstream/id/" : "/retrieve/") +
-            logo.getID() + "/" + (name == null ? "" : name);
+            "/bitstreams/" + logo.getID() + "/download";
     }
 
     protected String baseURL = null;  // cache the result for null

dspace-api/src/main/java/org/dspace/app/util/Util.java

@@ -405,21 +405,13 @@ public class Util {
         DCInput myInputs = null;
         boolean myInputsFound = false;
         String formFileName = I18nUtil.getInputFormsFileName(locale);
-        String col_handle = "";
 
         Collection collection = item.getOwningCollection();
 
-        if (collection == null) {
-            // set an empty handle so to get the default input set
-            col_handle = "";
-        } else {
-            col_handle = collection.getHandle();
-        }
-
         // Read the input form file for the specific collection
         DCInputsReader inputsReader = new DCInputsReader(formFileName);
 
-        List<DCInputSet> inputSets = inputsReader.getInputsByCollectionHandle(col_handle);
+        List<DCInputSet> inputSets = inputsReader.getInputsByCollection(collection);
 
         // Replace the values of Metadatum[] with the correct ones in case
         // of
@@ -500,8 +492,8 @@ public class Util {
     public static List<String> differenceInSubmissionFields(Collection fromCollection, Collection toCollection)
         throws DCInputsReaderException {
         DCInputsReader reader = new DCInputsReader();
-        List<DCInputSet> from = reader.getInputsByCollectionHandle(fromCollection.getHandle());
-        List<DCInputSet> to = reader.getInputsByCollectionHandle(toCollection.getHandle());
+        List<DCInputSet> from = reader.getInputsByCollection(fromCollection);
+        List<DCInputSet> to = reader.getInputsByCollection(toCollection);
 
         Set<String> fromFieldName = new HashSet<>();
         Set<String> toFieldName = new HashSet<>();

dspace-api/src/main/java/org/dspace/app/util/WebAppServiceImpl.java

@@ -13,12 +13,12 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
-import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.client.DSpaceHttpClientFactory;
 import org.dspace.app.util.dao.WebAppDAO;
 import org.dspace.app.util.service.WebAppService;
 import org.dspace.core.Context;
@@ -77,8 +77,8 @@ public class WebAppServiceImpl implements WebAppService {
             for (WebApp app : webApps) {
                 method = new HttpHead(app.getUrl());
                 int status;
-                try (CloseableHttpClient client = HttpClientBuilder.create().build()) {
-                    HttpResponse response = client.execute(method);
+                try (CloseableHttpClient client = DSpaceHttpClientFactory.getInstance().build()) {
+                    CloseableHttpResponse response = client.execute(method);
                     status = response.getStatusLine().getStatusCode();
                 }
                 if (status != HttpStatus.SC_OK) {

dspace-api/src/main/java/org/dspace/app/util/XMLUtils.java

@@ -7,12 +7,26 @@
  */
 package org.dspace.app.util;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLInputFactory;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jdom2.input.SAXBuilder;
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
 
 /**
  * Simple class to read information from small XML using DOM manipulation
@@ -161,4 +175,195 @@ public class XMLUtils {
         }
         return result;
     }
+
+    /**
+     * Initialize and return a javax DocumentBuilderFactory with NO security
+     * applied. This is intended only for internal, administrative/configuration
+     * use where external entities and other dangerous features are actually
+     * purposefully included.
+     * The method here is tiny, but may be expanded with other features like
+     * whitespace handling, and calling this method name helps to document
+     * the fact that the caller knows it is trusting the XML source / factory.
+     *
+     * @return document builder factory to generate new builders
+     * @throws ParserConfigurationException
+     */
+    public static DocumentBuilderFactory getTrustedDocumentBuilderFactory()
+            throws ParserConfigurationException {
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        return factory;
+    }
+
+    /**
+     * Initialize and return the javax DocumentBuilderFactory with some basic security
+     * applied to avoid XXE attacks and other unwanted content inclusion
+     * @return document builder factory to generate new builders
+     * @throws ParserConfigurationException
+     */
+    public static DocumentBuilderFactory getDocumentBuilderFactory()
+            throws ParserConfigurationException {
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        // No DOCTYPE / DTDs
+        factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        // No external general entities
+        factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        // No external parameter entities
+        factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        // No external DTDs
+        factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        // Even if entities somehow get defined, they will not be expanded
+        factory.setExpandEntityReferences(false);
+        // Disable "XInclude" markup processing
+        factory.setXIncludeAware(false);
+
+        return factory;
+    }
+
+    /**
+     * Initialize and return a javax DocumentBuilder with less security
+     * applied. This is intended only for internal, administrative/configuration
+     * use where external entities and other dangerous features are actually
+     * purposefully included, but are only allowed from specified paths, e.g.
+     * dspace.dir or some other path specified by the java caller.
+     * The method here is tiny, but may be expanded with other features like
+     * whitespace handling, and calling this method name helps to document
+     * the fact that the caller knows it is trusting the XML source / builder
+     * <p>
+     * If no allowedPaths are passed, then all external entities are rejected
+     *
+     * @return document builder with no security features set
+     * @throws ParserConfigurationException if the builder can not be configured
+     */
+    public static DocumentBuilder getTrustedDocumentBuilder(String... allowedPaths)
+            throws ParserConfigurationException {
+        DocumentBuilderFactory factory = getTrustedDocumentBuilderFactory();
+        factory.setValidating(false);
+        factory.setIgnoringComments(true);
+        factory.setIgnoringElementContentWhitespace(true);
+        DocumentBuilder builder = factory.newDocumentBuilder();
+        builder.setEntityResolver(new PathRestrictedEntityResolver(allowedPaths));
+        return factory.newDocumentBuilder();
+    }
+
+    /**
+     * Initialize and return the javax DocumentBuilder with some basic security applied
+     * to avoid XXE attacks and other unwanted content inclusion
+     * @return document builder for use in XML parsing
+     * @throws ParserConfigurationException if the builder can not be configured
+     */
+    public static DocumentBuilder getDocumentBuilder()
+            throws ParserConfigurationException {
+        return getDocumentBuilderFactory().newDocumentBuilder();
+    }
+
+    /**
+     * Initialize and return the SAX document builder with some basic security applied
+     * to avoid XXE attacks and other unwanted content inclusion
+     * @return SAX document builder for use in XML parsing
+     */
+    public static SAXBuilder getSAXBuilder() {
+        return getSAXBuilder(false);
+    }
+
+    /**
+     * Initialize and return the SAX document builder with some basic security applied
+     * to avoid XXE attacks and other unwanted content inclusion
+     * @param validate whether to use JDOM XSD validation
+     * @return SAX document builder for use in XML parsing
+     */
+    public static SAXBuilder getSAXBuilder(boolean validate) {
+        SAXBuilder saxBuilder = new SAXBuilder();
+        if (validate) {
+            saxBuilder.setValidation(true);
+        }
+        // No DOCTYPE / DTDs
+        saxBuilder.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        // No external general entities
+        saxBuilder.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        // No external parameter entities
+        saxBuilder.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        // No external DTDs
+        saxBuilder.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        // Don't expand entities
+        saxBuilder.setExpandEntities(false);
+
+        return saxBuilder;
+    }
+
+    /**
+     * Initialize and return the Java XML Input Factory with some basic security applied
+     * to avoid XXE attacks and other unwanted content inclusion
+     * @return XML input factory for use in XML parsing
+     */
+    public static XMLInputFactory getXMLInputFactory() {
+        XMLInputFactory xmlInputFactory = XMLInputFactory.newFactory();
+        xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+
+        return xmlInputFactory;
+    }
+
+    /**
+     * This entity resolver accepts one or more path strings in its
+     * constructor and throws a SAXException if the entity systemID
+     * is not within the allowed path (or a subdirectory).
+     * If no parameters are passed, then this effectively disallows
+     * any external entity resolution.
+     */
+    public static class PathRestrictedEntityResolver implements EntityResolver {
+        private final List<String> allowedBasePaths;
+
+        public PathRestrictedEntityResolver(String... allowedBasePaths) {
+            this.allowedBasePaths = Arrays.asList(allowedBasePaths);
+        }
+
+        @Override
+        public InputSource resolveEntity(String publicId, String systemId)
+                throws SAXException, IOException {
+
+            if (systemId == null) {
+                return null;
+            }
+
+            String filePath;
+            if (systemId.startsWith("file://")) {
+                filePath = systemId.substring(7);
+            } else if (systemId.startsWith("file:")) {
+                filePath = systemId.substring(5);
+            } else if (!systemId.contains("://")) {
+                filePath = systemId;
+            } else {
+                throw new SAXException("External resources not allowed: " + systemId +
+                        ". Only local file paths are permitted.");
+            }
+
+            Path resolvedPath;
+            try {
+                resolvedPath = Paths.get(filePath).toAbsolutePath().normalize();
+            } catch (Exception e) {
+                throw new SAXException("Invalid path: " + systemId, e);
+            }
+
+            boolean isAllowed = false;
+            for (String basePath : allowedBasePaths) {
+                Path allowedPath = Paths.get(basePath).toAbsolutePath().normalize();
+                if (resolvedPath.startsWith(allowedPath)) {
+                    isAllowed = true;
+                    break;
+                }
+            }
+
+            if (!isAllowed) {
+                throw new SAXException("Access denied to path: " + resolvedPath);
+            }
+
+            File file = resolvedPath.toFile();
+            if (!file.exists() || !file.canRead()) {
+                throw new SAXException("File not found or not readable: " + resolvedPath);
+            }
+
+            return new InputSource(new FileInputStream(file));
+        }
+    }
+
+
 }

dspace-api/src/main/java/org/dspace/app/util/service/OpenSearchService.java

@@ -117,4 +117,10 @@ public interface OpenSearchService {
 
     public DSpaceObject resolveScope(Context context, String scope) throws SQLException;
 
+    /**
+     * Retrieves the maximum number of items that can be included in a single opensearch request.
+     *
+     * @return the maximum number of items allowed per request
+     */
+    int getMaxNumOfItemsPerRequest();
 }

dspace-api/src/main/java/org/dspace/authenticate/LDAPAuthentication.java

@@ -17,6 +17,7 @@ import java.util.Collections;
 import java.util.Hashtable;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
 import javax.naming.directory.Attribute;
@@ -68,12 +69,8 @@ import org.dspace.services.factory.DSpaceServicesFactory;
  * @author Ivan Masár
  * @author Michael Plate
  */
-public class LDAPAuthentication
-    implements AuthenticationMethod {
+public class LDAPAuthentication implements AuthenticationMethod {
 
-    /**
-     * log4j category
-     */
     private static final Logger log
             = org.apache.logging.log4j.LogManager.getLogger(LDAPAuthentication.class);
 
@@ -130,7 +127,7 @@ public class LDAPAuthentication
         return false;
     }
 
-    /*
+    /**
      * This is an explicit method.
      */
     @Override
@@ -138,7 +135,7 @@ public class LDAPAuthentication
         return false;
     }
 
-    /*
+    /**
      * Add authenticated users to the group defined in dspace.cfg by
      * the login.specialgroup key.
      */
@@ -177,7 +174,7 @@ public class LDAPAuthentication
         return Collections.EMPTY_LIST;
     }
 
-    /*
+    /**
      * Authenticate the given credentials.
      * This is the heart of the authentication method: test the
      * credentials for authenticity, and if accepted, attempt to match
@@ -187,7 +184,7 @@ public class LDAPAuthentication
      * @param context
      *  DSpace context, will be modified (ePerson set) upon success.
      *
-     * @param username
+     * @param netid
      *  Username (or email address) when method is explicit. Use null for
      *  implicit method.
      *
@@ -250,7 +247,7 @@ public class LDAPAuthentication
         }
 
         // Check a DN was found
-        if ((dn == null) || (dn.trim().equals(""))) {
+        if (StringUtils.isBlank(dn)) {
             log.info(LogHelper
                          .getHeader(context, "failed_login", "no DN found for user " + netid));
             return BAD_CREDENTIALS;
@@ -269,6 +266,18 @@ public class LDAPAuthentication
                 context.setCurrentUser(eperson);
                 request.setAttribute(LDAP_AUTHENTICATED, true);
 
+                // update eperson's attributes
+                context.turnOffAuthorisationSystem();
+                setEpersonAttributes(context, eperson, ldap, Optional.empty());
+                try {
+                    ePersonService.update(context, eperson);
+                    context.dispatchEvents();
+                } catch (AuthorizeException e) {
+                    log.warn("update of eperson " + eperson.getID()  + " failed", e);
+                } finally {
+                    context.restoreAuthSystemState();
+                }
+
                 // assign user to groups based on ldap dn
                 assignGroups(dn, ldap.ldapGroup, context);
 
@@ -313,14 +322,13 @@ public class LDAPAuthentication
                             log.info(LogHelper.getHeader(context,
                                                           "type=ldap-login", "type=ldap_but_already_email"));
                             context.turnOffAuthorisationSystem();
-                            eperson.setNetid(netid.toLowerCase());
+                            setEpersonAttributes(context, eperson, ldap, Optional.of(netid));
                             ePersonService.update(context, eperson);
                             context.dispatchEvents();
                             context.restoreAuthSystemState();
                             context.setCurrentUser(eperson);
                             request.setAttribute(LDAP_AUTHENTICATED, true);
 
-
                             // assign user to groups based on ldap dn
                             assignGroups(dn, ldap.ldapGroup, context);
 
@@ -331,20 +339,7 @@ public class LDAPAuthentication
                                 try {
                                     context.turnOffAuthorisationSystem();
                                     eperson = ePersonService.create(context);
-                                    if (StringUtils.isNotEmpty(email)) {
-                                        eperson.setEmail(email);
-                                    }
-                                    if (StringUtils.isNotEmpty(ldap.ldapGivenName)) {
-                                        eperson.setFirstName(context, ldap.ldapGivenName);
-                                    }
-                                    if (StringUtils.isNotEmpty(ldap.ldapSurname)) {
-                                        eperson.setLastName(context, ldap.ldapSurname);
-                                    }
-                                    if (StringUtils.isNotEmpty(ldap.ldapPhone)) {
-                                        ePersonService.setMetadataSingleValue(context, eperson,
-                                                MD_PHONE, ldap.ldapPhone, null);
-                                    }
-                                    eperson.setNetid(netid.toLowerCase());
+                                    setEpersonAttributes(context, eperson, ldap, Optional.of(netid));
                                     eperson.setCanLogIn(true);
                                     authenticationService.initEPerson(context, request, eperson);
                                     ePersonService.update(context, eperson);
@@ -382,6 +377,29 @@ public class LDAPAuthentication
         return BAD_ARGS;
     }
 
+    /**
+     * Update eperson's attributes
+     */
+    private void setEpersonAttributes(Context context, EPerson eperson, SpeakerToLDAP ldap, Optional<String> netid)
+        throws SQLException {
+
+        if (StringUtils.isNotEmpty(ldap.ldapEmail)) {
+            eperson.setEmail(ldap.ldapEmail);
+        }
+        if (StringUtils.isNotEmpty(ldap.ldapGivenName)) {
+            eperson.setFirstName(context, ldap.ldapGivenName);
+        }
+        if (StringUtils.isNotEmpty(ldap.ldapSurname)) {
+            eperson.setLastName(context, ldap.ldapSurname);
+        }
+        if (StringUtils.isNotEmpty(ldap.ldapPhone)) {
+            ePersonService.setMetadataSingleValue(context, eperson, MD_PHONE, ldap.ldapPhone, null);
+        }
+        if (netid.isPresent()) {
+            eperson.setNetid(netid.get().toLowerCase());
+        }
+    }
+
     /**
      * Internal class to manage LDAP query and results, mainly
      * because there are multiple values to return.
@@ -503,6 +521,7 @@ public class LDAPAuthentication
                     } else {
                         searchName = ldap_provider_url + ldap_search_context;
                     }
+                    @SuppressWarnings("BanJNDI")
                     NamingEnumeration<SearchResult> answer = ctx.search(
                         searchName,
                         "(&({0}={1}))", new Object[] {ldap_id_field,
@@ -553,7 +572,7 @@ public class LDAPAuthentication
                             att = atts.get(attlist[4]);
                             if (att != null) {
                                 // loop through all groups returned by LDAP
-                                ldapGroup = new ArrayList<String>();
+                                ldapGroup = new ArrayList<>();
                                 for (NamingEnumeration val = att.getAll(); val.hasMoreElements(); )  {
                                     ldapGroup.add((String) val.next());
                                 }
@@ -633,7 +652,8 @@ public class LDAPAuthentication
                         ctx.addToEnvironment(javax.naming.Context.AUTHORITATIVE, "true");
                         ctx.addToEnvironment(javax.naming.Context.REFERRAL, "follow");
                         // dummy operation to check if authentication has succeeded
-                        ctx.getAttributes("");
+                        @SuppressWarnings("BanJNDI")
+                        Attributes trash = ctx.getAttributes("");
                     } else if (!useTLS) {
                         // Authenticate
                         env.put(javax.naming.Context.SECURITY_AUTHENTICATION, "Simple");
@@ -671,7 +691,7 @@ public class LDAPAuthentication
         }
     }
 
-    /*
+    /**
      * Returns the URL of an external login page which is not applicable for this authn method.
      *
      * Note: Prior to DSpace 7, this method return the page of login servlet.
@@ -699,7 +719,7 @@ public class LDAPAuthentication
         return "ldap";
     }
 
-    /*
+    /**
      * Add authenticated users to the group defined in dspace.cfg by
      * the authentication-ldap.login.groupmap.* key.
      *

dspace-api/src/main/java/org/dspace/authenticate/oidc/impl/OidcClientImpl.java

@@ -22,12 +22,13 @@ import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.NameValuePair;
-import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.methods.RequestBuilder;
-import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.message.BasicNameValuePair;
+import org.dspace.app.client.DSpaceHttpClientFactory;
 import org.dspace.authenticate.oidc.OidcClient;
 import org.dspace.authenticate.oidc.OidcClientException;
 import org.dspace.authenticate.oidc.model.OidcTokenResponseDTO;
@@ -83,21 +84,17 @@ public class OidcClientImpl implements OidcClient {
     }
 
     private <T> T executeAndParseJson(HttpUriRequest httpUriRequest, Class<T> clazz) {
-
-        HttpClient client = HttpClientBuilder.create().build();
-
-        return executeAndReturns(() -> {
-
-            HttpResponse response = client.execute(httpUriRequest);
-
-            if (isNotSuccessfull(response)) {
-                throw new OidcClientException(getStatusCode(response), formatErrorMessage(response));
-            }
-
-            return objectMapper.readValue(getContent(response), clazz);
-
-        });
-
+        try (CloseableHttpClient client = DSpaceHttpClientFactory.getInstance().build()) {
+            return executeAndReturns(() -> {
+                CloseableHttpResponse response = client.execute(httpUriRequest);
+                if (isNotSuccessfull(response)) {
+                    throw new OidcClientException(getStatusCode(response), formatErrorMessage(response));
+                }
+                return objectMapper.readValue(getContent(response), clazz);
+            });
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     private <T> T executeAndReturns(ThrowingSupplier<T, Exception> supplier) {

dspace-api/src/main/java/org/dspace/authority/orcid/Orcidv3SolrAuthorityImpl.java

@@ -7,27 +7,22 @@
  */
 package org.dspace.authority.orcid;
 
-import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.dspace.authority.AuthorityValue;
 import org.dspace.authority.SolrAuthorityInterface;
 import org.dspace.external.OrcidRestConnector;
 import org.dspace.external.provider.orcid.xml.XMLtoBio;
-import org.json.JSONObject;
+import org.dspace.orcid.model.factory.OrcidFactoryUtils;
 import org.orcid.jaxb.model.v3.release.common.OrcidIdentifier;
 import org.orcid.jaxb.model.v3.release.record.Person;
 import org.orcid.jaxb.model.v3.release.search.Result;
@@ -50,6 +45,11 @@ public class Orcidv3SolrAuthorityImpl implements SolrAuthorityInterface {
 
     private String accessToken;
 
+    /**
+     * Maximum retries to allow for the access token retrieval
+     */
+    private int maxClientRetries = 3;
+
     public void setOAUTHUrl(String oAUTHUrl) {
         OAUTHUrl = oAUTHUrl;
     }
@@ -62,46 +62,32 @@ public class Orcidv3SolrAuthorityImpl implements SolrAuthorityInterface {
         this.clientSecret = clientSecret;
     }
 
+    public String getAccessToken() {
+        return accessToken;
+    }
+
+    public void setAccessToken(String accessToken) {
+        this.accessToken = accessToken;
+    }
+
     /**
      *  Initialize the accessToken that is required for all subsequent calls to ORCID
      */
     public void init() {
-        if (StringUtils.isBlank(accessToken)
-                && StringUtils.isNotBlank(clientSecret)
-                && StringUtils.isNotBlank(clientId)
-                && StringUtils.isNotBlank(OAUTHUrl)) {
-            String authenticationParameters = "?client_id=" + clientId +
-                    "&client_secret=" + clientSecret +
-                    "&scope=/read-public&grant_type=client_credentials";
-            try {
-                HttpPost httpPost = new HttpPost(OAUTHUrl + authenticationParameters);
-                httpPost.addHeader("Accept", "application/json");
-                httpPost.addHeader("Content-Type", "application/x-www-form-urlencoded");
+        // Initialize access token at spring instantiation. If it fails, the access token will be null rather
+        // than causing a fatal Spring startup error
+        initializeAccessToken();
+    }
 
-                HttpClient httpClient = HttpClientBuilder.create().build();
-                HttpResponse getResponse = httpClient.execute(httpPost);
-
-                JSONObject responseObject = null;
-                try (InputStream is = getResponse.getEntity().getContent();
-                     BufferedReader streamReader = new BufferedReader(new InputStreamReader(is, "UTF-8"))) {
-                    String inputStr;
-                    while ((inputStr = streamReader.readLine()) != null && responseObject == null) {
-                        if (inputStr.startsWith("{") && inputStr.endsWith("}") && inputStr.contains("access_token")) {
-                            try {
-                                responseObject = new JSONObject(inputStr);
-                            } catch (Exception e) {
-                                //Not as valid as I'd hoped, move along
-                                responseObject = null;
-                            }
-                        }
-                    }
-                }
-                if (responseObject != null && responseObject.has("access_token")) {
-                    accessToken = (String) responseObject.get("access_token");
-                }
-            } catch (Exception e) {
-                throw new RuntimeException("Error during initialization of the Orcid connector", e);
-            }
+    public void initializeAccessToken() {
+        // If we have reaches max retries or the access token is already set, return immediately
+        if (maxClientRetries <= 0 || org.apache.commons.lang3.StringUtils.isNotBlank(accessToken)) {
+            return;
+        }
+        try {
+            accessToken = OrcidFactoryUtils.retrieveAccessToken(clientId, clientSecret, OAUTHUrl).orElse(null);
+        } catch (IOException e) {
+            log.error("Error retrieving ORCID access token, {} retries left", --maxClientRetries);
         }
     }
 
@@ -116,7 +102,7 @@ public class Orcidv3SolrAuthorityImpl implements SolrAuthorityInterface {
      */
     @Override
     public List<AuthorityValue> queryAuthorities(String text, int max) {
-        init();
+        initializeAccessToken();
         List<Person> bios = queryBio(text, max);
         List<AuthorityValue> result = new ArrayList<>();
         for (Person person : bios) {
@@ -135,7 +121,7 @@ public class Orcidv3SolrAuthorityImpl implements SolrAuthorityInterface {
      */
     @Override
     public AuthorityValue queryAuthorityID(String id) {
-        init();
+        initializeAccessToken();
         Person person = getBio(id);
         AuthorityValue valueFromPerson = Orcidv3AuthorityValue.create(person);
         return valueFromPerson;
@@ -151,11 +137,14 @@ public class Orcidv3SolrAuthorityImpl implements SolrAuthorityInterface {
         if (!isValid(id)) {
             return null;
         }
-        init();
+        if (orcidRestConnector == null) {
+            log.error("ORCID REST connector is null, returning null Person");
+            return null;
+        }
+        initializeAccessToken();
         InputStream bioDocument = orcidRestConnector.get(id + ((id.endsWith("/person")) ? "" : "/person"), accessToken);
         XMLtoBio converter = new XMLtoBio();
-        Person person = converter.convertSinglePerson(bioDocument);
-        return person;
+        return converter.convertSinglePerson(bioDocument);
     }
 
 
@@ -167,10 +156,16 @@ public class Orcidv3SolrAuthorityImpl implements SolrAuthorityInterface {
      * @return List<Person>
      */
     public List<Person> queryBio(String text, int start, int rows) {
-        init();
         if (rows > 100) {
             throw new IllegalArgumentException("The maximum number of results to retrieve cannot exceed 100.");
         }
+        // Check REST connector is initialized
+        if (orcidRestConnector == null) {
+            log.error("ORCID REST connector is not initialized, returning empty list");
+            return Collections.emptyList();
+        }
+        // Check / init access token
+        initializeAccessToken();
 
         String searchPath = "search?q=" + URLEncoder.encode(text) + "&start=" + start + "&rows=" + rows;
         log.debug("queryBio searchPath=" + searchPath + " accessToken=" + accessToken);

dspace-api/src/main/java/org/dspace/authorize/AuthorizeServiceImpl.java

@@ -9,6 +9,7 @@ package org.dspace.authorize;
 
 import static org.dspace.app.util.AuthorizeUtil.canCollectionAdminManageAccounts;
 import static org.dspace.app.util.AuthorizeUtil.canCommunityAdminManageAccounts;
+import static org.dspace.discovery.SearchUtils.RESOURCE_TYPE_FIELD;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -550,13 +551,11 @@ public class AuthorizeServiceImpl implements AuthorizeService {
         List<ResourcePolicy> newPolicies = new ArrayList<>(policies.size());
 
         for (ResourcePolicy srp : policies) {
-            ResourcePolicy rp = resourcePolicyService.create(c);
+            ResourcePolicy rp = resourcePolicyService.create(c, srp.getEPerson(), srp.getGroup());
 
             // copy over values
             rp.setdSpaceObject(dest);
             rp.setAction(srp.getAction());
-            rp.setEPerson(srp.getEPerson());
-            rp.setGroup(srp.getGroup());
             rp.setStartDate(srp.getStartDate());
             rp.setEndDate(srp.getEndDate());
             rp.setRpName(srp.getRpName());
@@ -670,11 +669,9 @@ public class AuthorizeServiceImpl implements AuthorizeService {
                 "We need at least an eperson or a group in order to create a resource policy.");
         }
 
-        ResourcePolicy myPolicy = resourcePolicyService.create(context);
+        ResourcePolicy myPolicy = resourcePolicyService.create(context, eperson, group);
         myPolicy.setdSpaceObject(dso);
         myPolicy.setAction(type);
-        myPolicy.setGroup(group);
-        myPolicy.setEPerson(eperson);
         myPolicy.setRpType(rpType);
         myPolicy.setRpName(rpName);
         myPolicy.setRpDescription(rpDescription);
@@ -740,7 +737,7 @@ public class AuthorizeServiceImpl implements AuthorizeService {
      */
     @Override
     public boolean isCommunityAdmin(Context context) throws SQLException {
-        return performCheck(context, "search.resourcetype:" + IndexableCommunity.TYPE);
+        return performCheck(context, RESOURCE_TYPE_FIELD + ":" + IndexableCommunity.TYPE);
     }
 
     /**
@@ -753,7 +750,7 @@ public class AuthorizeServiceImpl implements AuthorizeService {
      */
     @Override
     public boolean isCollectionAdmin(Context context) throws SQLException {
-        return performCheck(context, "search.resourcetype:" + IndexableCollection.TYPE);
+        return performCheck(context, RESOURCE_TYPE_FIELD + ":" + IndexableCollection.TYPE);
     }
 
     /**
@@ -766,7 +763,7 @@ public class AuthorizeServiceImpl implements AuthorizeService {
      */
     @Override
     public boolean isItemAdmin(Context context) throws SQLException {
-        return performCheck(context, "search.resourcetype:" + IndexableItem.TYPE);
+        return performCheck(context, RESOURCE_TYPE_FIELD + ":" + IndexableItem.TYPE);
     }
 
     /**
@@ -780,8 +777,8 @@ public class AuthorizeServiceImpl implements AuthorizeService {
     @Override
     public boolean isComColAdmin(Context context) throws SQLException {
         return performCheck(context,
-            "(search.resourcetype:" + IndexableCommunity.TYPE + " OR search.resourcetype:" +
-                IndexableCollection.TYPE + ")");
+            "(" + RESOURCE_TYPE_FIELD + ":" + IndexableCommunity.TYPE + " OR " +
+            RESOURCE_TYPE_FIELD + ":" + IndexableCollection.TYPE + ")");
     }
 
     /**
@@ -799,7 +796,7 @@ public class AuthorizeServiceImpl implements AuthorizeService {
         throws SearchServiceException, SQLException {
         List<Community> communities = new ArrayList<>();
         query = formatCustomQuery(query);
-        DiscoverResult discoverResult = getDiscoverResult(context, query + "search.resourcetype:" +
+        DiscoverResult discoverResult = getDiscoverResult(context, query + RESOURCE_TYPE_FIELD + ":" +
                                                               IndexableCommunity.TYPE,
             offset, limit, null, null);
         for (IndexableObject solrCollections : discoverResult.getIndexableObjects()) {
@@ -821,9 +818,9 @@ public class AuthorizeServiceImpl implements AuthorizeService {
     public long countAdminAuthorizedCommunity(Context context, String query)
         throws SearchServiceException, SQLException {
         query = formatCustomQuery(query);
-        DiscoverResult discoverResult = getDiscoverResult(context, query + "search.resourcetype:" +
+        DiscoverResult discoverResult = getDiscoverResult(context, query + RESOURCE_TYPE_FIELD + ":" +
                                                               IndexableCommunity.TYPE,
-            null, null, null, null);
+            null, 0, null, null);
         return discoverResult.getTotalSearchResults();
     }
 
@@ -846,7 +843,7 @@ public class AuthorizeServiceImpl implements AuthorizeService {
         }
 
         query = formatCustomQuery(query);
-        DiscoverResult discoverResult = getDiscoverResult(context, query + "search.resourcetype:" +
+        DiscoverResult discoverResult = getDiscoverResult(context, query + RESOURCE_TYPE_FIELD + ":" +
                                                               IndexableCollection.TYPE,
             offset, limit, CollectionService.SOLR_SORT_FIELD, SORT_ORDER.asc);
         for (IndexableObject solrCollections : discoverResult.getIndexableObjects()) {
@@ -868,9 +865,9 @@ public class AuthorizeServiceImpl implements AuthorizeService {
     public long countAdminAuthorizedCollection(Context context, String query)
         throws SearchServiceException, SQLException {
         query = formatCustomQuery(query);
-        DiscoverResult discoverResult = getDiscoverResult(context, query + "search.resourcetype:" +
+        DiscoverResult discoverResult = getDiscoverResult(context, query + RESOURCE_TYPE_FIELD + ":" +
                                                               IndexableCollection.TYPE,
-            null, null, null, null);
+            null, 0, null, null);
         return discoverResult.getTotalSearchResults();
     }
 

dspace-api/src/main/java/org/dspace/authorize/FixDefaultPolicies.java

@@ -126,10 +126,9 @@ public class FixDefaultPolicies {
 
         // now create the default policies for submitted items
         ResourcePolicyService resourcePolicyService = AuthorizeServiceFactory.getInstance().getResourcePolicyService();
-        ResourcePolicy myPolicy = resourcePolicyService.create(c);
+        ResourcePolicy myPolicy = resourcePolicyService.create(c, null, anonymousGroup);
         myPolicy.setdSpaceObject(t);
         myPolicy.setAction(myaction);
-        myPolicy.setGroup(anonymousGroup);
         resourcePolicyService.update(c, myPolicy);
     }
 }

dspace-api/src/main/java/org/dspace/authorize/PolicySet.java

@@ -229,11 +229,10 @@ public class PolicySet {
                         // before create a new policy check if an identical policy is already in place
                         if (!authorizeService.isAnIdenticalPolicyAlreadyInPlace(c, myitem, group, actionID, -1)) {
                             // now add the policy
-                            ResourcePolicy rp = resourcePolicyService.create(c);
+                            ResourcePolicy rp = resourcePolicyService.create(c, null, group);
 
                             rp.setdSpaceObject(myitem);
                             rp.setAction(actionID);
-                            rp.setGroup(group);
 
                             rp.setRpName(name);
                             rp.setRpDescription(description);
@@ -262,11 +261,10 @@ public class PolicySet {
                             // before create a new policy check if an identical policy is already in place
                             if (!authorizeService.isAnIdenticalPolicyAlreadyInPlace(c, bundle, group, actionID, -1)) {
                                 // now add the policy
-                                ResourcePolicy rp = resourcePolicyService.create(c);
+                                ResourcePolicy rp = resourcePolicyService.create(c, null, group);
 
                                 rp.setdSpaceObject(bundle);
                                 rp.setAction(actionID);
-                                rp.setGroup(group);
 
                                 rp.setRpName(name);
                                 rp.setRpDescription(description);
@@ -305,11 +303,10 @@ public class PolicySet {
                                     if (!authorizeService
                                         .isAnIdenticalPolicyAlreadyInPlace(c, bitstream, group, actionID, -1)) {
                                         // now add the policy
-                                        ResourcePolicy rp = resourcePolicyService.create(c);
+                                        ResourcePolicy rp = resourcePolicyService.create(c, null, group);
 
                                         rp.setdSpaceObject(bitstream);
                                         rp.setAction(actionID);
-                                        rp.setGroup(group);
 
                                         rp.setRpName(name);
                                         rp.setRpDescription(description);

dspace-api/src/main/java/org/dspace/authorize/ResourcePolicyServiceImpl.java

@@ -19,6 +19,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.authorize.dao.ResourcePolicyDAO;
+import org.dspace.authorize.service.AuthorizeService;
 import org.dspace.authorize.service.ResourcePolicyService;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.factory.ContentServiceFactory;
@@ -51,6 +52,9 @@ public class ResourcePolicyServiceImpl implements ResourcePolicyService {
     @Autowired
     private GroupService groupService;
 
+    @Autowired
+    private AuthorizeService authorizeService;
+
     protected ResourcePolicyServiceImpl() {
     }
 
@@ -71,14 +75,22 @@ public class ResourcePolicyServiceImpl implements ResourcePolicyService {
      * Create a new ResourcePolicy
      *
      * @param context DSpace context object
+     * @param ePerson
+     * @param group
      * @return ResourcePolicy
      * @throws SQLException if database error
      */
     @Override
-    public ResourcePolicy create(Context context) throws SQLException {
+    public ResourcePolicy create(Context context, EPerson ePerson, Group group) throws SQLException {
         // FIXME: Check authorisation
         // Create a table row
-        ResourcePolicy resourcePolicy = resourcePolicyDAO.create(context, new ResourcePolicy());
+        ResourcePolicy policyToBeCreated = new ResourcePolicy();
+        if (ePerson == null && group == null) {
+            throw new IllegalArgumentException("A resource policy must contain a valid eperson or group");
+        }
+        policyToBeCreated.setEPerson(ePerson);
+        policyToBeCreated.setGroup(group);
+        ResourcePolicy resourcePolicy = resourcePolicyDAO.create(context, policyToBeCreated);
         return resourcePolicy;
     }
 
@@ -205,9 +217,7 @@ public class ResourcePolicyServiceImpl implements ResourcePolicyService {
     @Override
     public ResourcePolicy clone(Context context, ResourcePolicy resourcePolicy)
         throws SQLException, AuthorizeException {
-        ResourcePolicy clone = create(context);
-        clone.setGroup(resourcePolicy.getGroup());
-        clone.setEPerson(resourcePolicy.getEPerson());
+        ResourcePolicy clone = create(context, resourcePolicy.getEPerson(), resourcePolicy.getGroup());
         clone.setStartDate((Date) ObjectUtils.clone(resourcePolicy.getStartDate()));
         clone.setEndDate((Date) ObjectUtils.clone(resourcePolicy.getEndDate()));
         clone.setRpType((String) ObjectUtils.clone(resourcePolicy.getRpType()));
@@ -411,11 +421,11 @@ public class ResourcePolicyServiceImpl implements ResourcePolicyService {
         ResourcePolicy resourcePolicy = resourcePolicyDAO.findOneById(context, id);
         Group group = resourcePolicy.getGroup();
 
-        if (resourcePolicy.getEPerson() != null && resourcePolicy.getEPerson().getID() == eperson.getID()) {
+        if (resourcePolicy.getEPerson() != null && resourcePolicy.getEPerson().getID().equals(eperson.getID())) {
             isMy = true;
         } else if (group != null && groupService.isMember(context, eperson, group)) {
             isMy = true;
         }
-        return isMy;
+        return isMy || authorizeService.isAdmin(context, eperson, resourcePolicy.getdSpaceObject());
     }
 }

dspace-api/src/main/java/org/dspace/authorize/service/ResourcePolicyService.java

@@ -17,7 +17,6 @@ import org.dspace.content.DSpaceObject;
 import org.dspace.core.Context;
 import org.dspace.eperson.EPerson;
 import org.dspace.eperson.Group;
-import org.dspace.service.DSpaceCRUDService;
 
 /**
  * Service interface class for the ResourcePolicy object.
@@ -26,7 +25,34 @@ import org.dspace.service.DSpaceCRUDService;
  *
  * @author kevinvandevelde at atmire.com
  */
-public interface ResourcePolicyService extends DSpaceCRUDService<ResourcePolicy> {
+public interface ResourcePolicyService {
+
+    public ResourcePolicy create(Context context, EPerson eperson, Group group) throws SQLException, AuthorizeException;
+
+    public ResourcePolicy find(Context context, int id) throws SQLException;
+
+    /**
+     * Persist a model object.
+     *
+     * @param context
+     * @param resourcePolicy object to be persisted.
+     * @throws SQLException passed through.
+     * @throws AuthorizeException passed through.
+     */
+    public void update(Context context, ResourcePolicy resourcePolicy) throws SQLException, AuthorizeException;
+
+
+    /**
+     * Persist a collection of model objects.
+     *
+     * @param context
+     * @param resourcePolicies object to be persisted.
+     * @throws SQLException passed through.
+     * @throws AuthorizeException passed through.
+     */
+    public void update(Context context, List<ResourcePolicy> resourcePolicies) throws SQLException, AuthorizeException;
+
+    public void delete(Context context, ResourcePolicy resourcePolicy) throws SQLException, AuthorizeException;
 
 
     public List<ResourcePolicy> find(Context c, DSpaceObject o) throws SQLException;

dspace-api/src/main/java/org/dspace/browse/BrowseEngine.java

@@ -422,9 +422,6 @@ public class BrowseEngine {
                 }
             }
 
-            // this is the total number of results in answer to the query
-            int total = getTotalResults(true);
-
             // set the ordering field (there is only one option)
             dao.setOrderField("sort_value");
 
@@ -444,6 +441,9 @@ public class BrowseEngine {
             dao.setOffset(offset);
             dao.setLimit(scope.getResultsPerPage());
 
+            // this is the total number of results in answer to the query
+            int total = getTotalResults(true);
+
             // Holder for the results
             List<String[]> results = null;
 
@@ -680,33 +680,9 @@ public class BrowseEngine {
         // tell the browse query whether we are distinct
         dao.setDistinct(distinct);
 
-        // ensure that the select is set to "*"
-        String[] select = {"*"};
-        dao.setCountValues(select);
-
-        // FIXME: it would be nice to have a good way of doing this in the DAO
-        // now reset all of the fields that we don't want to have constraining
-        // our count, storing them locally to reinstate later
-        String focusField = dao.getJumpToField();
-        String focusValue = dao.getJumpToValue();
-        int limit = dao.getLimit();
-        int offset = dao.getOffset();
-
-        dao.setJumpToField(null);
-        dao.setJumpToValue(null);
-        dao.setLimit(-1);
-        dao.setOffset(-1);
-
         // perform the query and get the result
         int count = dao.doCountQuery();
 
-        // now put back the values we removed for this method
-        dao.setJumpToField(focusField);
-        dao.setJumpToValue(focusValue);
-        dao.setLimit(limit);
-        dao.setOffset(offset);
-        dao.setCountValues(null);
-
         log.debug(LogHelper.getHeader(context, "get_total_results_return", "return=" + count));
 
         return count;

dspace-api/src/main/java/org/dspace/browse/ItemCountDAO.java

@@ -13,26 +13,17 @@ import org.dspace.core.Context;
 /**
  * Interface for data access of cached community and collection item count
  * information
- *
- * @author Richard Jones
  */
 public interface ItemCountDAO {
-    /**
-     * Set the DSpace Context to use during data access
-     *
-     * @param context DSpace Context
-     * @throws ItemCountException if count error
-     */
-    public void setContext(Context context) throws ItemCountException;
 
     /**
      * Get the number of items in the given DSpaceObject container.  This method will
      * only succeed if the DSpaceObject is an instance of either a Community or a
      * Collection.  Otherwise it will throw an exception.
      *
+     * @param context DSpace context
      * @param dso Dspace Object
      * @return count
-     * @throws ItemCountException if count error
      */
-    public int getCount(DSpaceObject dso) throws ItemCountException;
+    int getCount(Context context, DSpaceObject dso);
 }

dspace-api/src/main/java/org/dspace/browse/ItemCountDAOFactory.java

@@ -1,67 +0,0 @@
-/**
- * The contents of this file are subject to the license and copyright
- * detailed in the LICENSE and NOTICE files at the root of the source
- * tree and available online at
- *
- * http://www.dspace.org/license/
- */
-package org.dspace.browse;
-
-import java.lang.reflect.InvocationTargetException;
-
-import org.dspace.core.Context;
-import org.dspace.services.ConfigurationService;
-import org.dspace.services.factory.DSpaceServicesFactory;
-
-/**
- * Factory class to allow us to load the correct DAO for registering
- * item count information
- *
- * @author Richard Jones
- * @author Ivan Masár
- */
-public class ItemCountDAOFactory {
-
-    /**
-     * Default constructor
-     */
-    private ItemCountDAOFactory() { }
-
-    /**
-     * Get an instance of ItemCountDAO which supports the correct storage backend
-     * for the specific DSpace instance.
-     *
-     * @param context DSpace Context
-     * @return DAO
-     * @throws ItemCountException if count error
-     */
-    public static ItemCountDAO getInstance(Context context)
-        throws ItemCountException {
-
-        /** Log4j logger */
-        ItemCountDAO dao = null;
-
-        ConfigurationService configurationService
-                = DSpaceServicesFactory.getInstance().getConfigurationService();
-        String className = configurationService.getProperty("ItemCountDAO.class");
-
-        // SOLR implementation is the default since DSpace 4.0
-        if (className == null) {
-            dao = new ItemCountDAOSolr();
-        } else {
-            try {
-                dao = (ItemCountDAO) Class.forName(className.trim())
-                        .getDeclaredConstructor()
-                        .newInstance();
-            } catch (ClassNotFoundException | IllegalAccessException
-                    | InstantiationException | NoSuchMethodException
-                    | SecurityException | IllegalArgumentException
-                    | InvocationTargetException e) {
-                throw new ItemCountException("The configuration for ItemCountDAO is invalid: " + className, e);
-            }
-        }
-
-        dao.setContext(context);
-        return dao;
-    }
-}

dspace-api/src/main/java/org/dspace/browse/ItemCountDAOSolr.java

@@ -24,14 +24,12 @@ import org.dspace.discovery.SearchService;
 import org.dspace.discovery.SearchServiceException;
 import org.dspace.discovery.configuration.DiscoveryConfigurationParameters;
 import org.dspace.discovery.indexobject.IndexableItem;
-import org.dspace.services.factory.DSpaceServicesFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 
 /**
  * Discovery (Solr) driver implementing ItemCountDAO interface to look up item
  * count information in communities and collections. Caching operations are
  * intentionally not implemented because Solr already is our cache.
- *
- * @author Ivan Masár, Andrea Bollini
  */
 public class ItemCountDAOSolr implements ItemCountDAO {
     /**
@@ -39,11 +37,6 @@ public class ItemCountDAOSolr implements ItemCountDAO {
      */
     private static Logger log = org.apache.logging.log4j.LogManager.getLogger(ItemCountDAOSolr.class);
 
-    /**
-     * DSpace context
-     */
-    private Context context;
-
     /**
      * Hold the communities item count obtained from SOLR after the first query. This only works
      * well if the ItemCountDAO lifecycle is bound to the request lifecycle as
@@ -61,41 +54,28 @@ public class ItemCountDAOSolr implements ItemCountDAO {
     /**
      * Solr search service
      */
-    SearchService searcher = DSpaceServicesFactory.getInstance().getServiceManager()
-                                                  .getServiceByName(SearchService.class.getName(), SearchService.class);
-
-    /**
-     * Set the dspace context to use
-     *
-     * @param context DSpace Context
-     * @throws ItemCountException if count error
-     */
-    @Override
-    public void setContext(Context context) throws ItemCountException {
-        this.context = context;
-    }
+    @Autowired
+    protected SearchService searchService;
 
     /**
      * Get the count of the items in the given container.
      *
-     * @param dso Dspace Context
+     * @param context DSpace context
+     * @param dso DspaceObject
      * @return count
-     * @throws ItemCountException if count error
      */
     @Override
-    public int getCount(DSpaceObject dso) throws ItemCountException {
-        loadCount();
-        Integer val;
+    public int getCount(Context context, DSpaceObject dso) {
+        loadCount(context);
+        Integer val = null;
         if (dso instanceof Collection) {
-            val = collectionsCount.get(String.valueOf(((Collection) dso).getID()));
+            val = collectionsCount.get(dso.getID().toString());
         } else if (dso instanceof Community) {
-            val = communitiesCount.get(String.valueOf(((Community) dso).getID()));
-        } else {
-            throw new ItemCountException("We can only count items in Communities or Collections");
+            val = communitiesCount.get(dso.getID().toString());
         }
 
         if (val != null) {
-            return val.intValue();
+            return val;
         } else {
             return 0;
         }
@@ -105,15 +85,15 @@ public class ItemCountDAOSolr implements ItemCountDAO {
      * make sure that the counts are actually fetched from Solr (if haven't been
      * cached in a Map yet)
      *
-     * @throws ItemCountException if count error
+     * @param context DSpace Context
      */
-    private void loadCount() throws ItemCountException {
+    private void loadCount(Context context) {
         if (communitiesCount != null || collectionsCount != null) {
             return;
         }
 
-        communitiesCount = new HashMap<String, Integer>();
-        collectionsCount = new HashMap<String, Integer>();
+        communitiesCount = new HashMap<>();
+        collectionsCount = new HashMap<>();
 
         DiscoverQuery query = new DiscoverQuery();
         query.setFacetMinCount(1);
@@ -125,11 +105,13 @@ public class ItemCountDAOSolr implements ItemCountDAO {
                                                    DiscoveryConfigurationParameters.SORT.COUNT));
         query.addFilterQueries("search.resourcetype:" + IndexableItem.TYPE);    // count only items
         query.addFilterQueries("NOT(discoverable:false)");  // only discoverable
+        query.addFilterQueries("withdrawn:false");  // only not withdrawn
+        query.addFilterQueries("archived:true");  // only archived
         query.setMaxResults(0);
 
-        DiscoverResult sResponse = null;
+        DiscoverResult sResponse;
         try {
-            sResponse = searcher.search(context, query);
+            sResponse = searchService.search(context, query);
             List<FacetResult> commCount = sResponse.getFacetResult("location.comm");
             List<FacetResult> collCount = sResponse.getFacetResult("location.coll");
             for (FacetResult c : commCount) {
@@ -139,8 +121,7 @@ public class ItemCountDAOSolr implements ItemCountDAO {
                 collectionsCount.put(c.getAsFilterQuery(), (int) c.getCount());
             }
         } catch (SearchServiceException e) {
-            log.error("caught exception: ", e);
-            throw new ItemCountException(e);
+            log.error("Could not initialize Community/Collection Item Counts from Solr: ", e);
         }
     }
 }

dspace-api/src/main/java/org/dspace/browse/ItemCountException.java

@@ -1,32 +0,0 @@
-/**
- * The contents of this file are subject to the license and copyright
- * detailed in the LICENSE and NOTICE files at the root of the source
- * tree and available online at
- *
- * http://www.dspace.org/license/
- */
-package org.dspace.browse;
-
-/**
- * Exception type to handle item count specific problems
- *
- * @author Richard Jones
- */
-public class ItemCountException extends Exception {
-
-    public ItemCountException() {
-    }
-
-    public ItemCountException(String message) {
-        super(message);
-    }
-
-    public ItemCountException(Throwable cause) {
-        super(cause);
-    }
-
-    public ItemCountException(String message, Throwable cause) {
-        super(message, cause);
-    }
-
-}

dspace-api/src/main/java/org/dspace/browse/ItemCounter.java

@@ -13,26 +13,18 @@ import org.apache.logging.log4j.Logger;
 import org.dspace.content.Collection;
 import org.dspace.content.Community;
 import org.dspace.content.DSpaceObject;
-import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.content.service.ItemService;
 import org.dspace.core.Context;
 import org.dspace.services.ConfigurationService;
 import org.dspace.services.factory.DSpaceServicesFactory;
-import org.dspace.web.ContextUtil;
+import org.springframework.beans.factory.annotation.Autowired;
 
 /**
  * This class provides a standard interface to all item counting
- * operations for communities and collections.  It can be run from the
- * command line to prepare the cached data if desired, simply by
- * running:
+ * operations for communities and collections.
  *
- * java org.dspace.browse.ItemCounter
- *
- * It can also be invoked via its standard API.  In the event that
- * the data cache is not being used, this class will return direct
+ * In the event that the data cache is not being used, this class will return direct
  * real time counts of content.
- *
- * @author Richard Jones
  */
 public class ItemCounter {
     /**
@@ -40,57 +32,15 @@ public class ItemCounter {
      */
     private static Logger log = org.apache.logging.log4j.LogManager.getLogger(ItemCounter.class);
 
-    /**
-     * DAO to use to store and retrieve data
-     */
-    private ItemCountDAO dao;
-
-    /**
-     * DSpace Context
-     */
-    private Context context;
-
-    /**
-     * This field is used to hold singular instance of a class.
-     * Singleton pattern is used but this class should be
-     * refactored to modern DSpace approach (injectible service).
-     */
-
-    private static ItemCounter instance;
-
+    @Autowired
     protected ItemService itemService;
+    @Autowired
     protected ConfigurationService configurationService;
 
-    private boolean showStrengths;
-    private boolean useCache;
-
     /**
-     * Construct a new item counter which will use the given DSpace Context
-     *
-     * @param context current context
-     * @throws ItemCountException if count error
+     * Construct a new item counter
      */
-    public ItemCounter(Context context) throws ItemCountException {
-        this.context = context;
-        this.dao = ItemCountDAOFactory.getInstance(this.context);
-        this.itemService = ContentServiceFactory.getInstance().getItemService();
-        this.configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();
-        this.showStrengths = configurationService.getBooleanProperty("webui.strengths.show", false);
-        this.useCache = configurationService.getBooleanProperty("webui.strengths.cache", true);
-    }
-
-    /**
-     * Get the singular instance of a class.
-     * It creates a new instance at the first usage of this method.
-     *
-     * @return instance af a class
-     * @throws ItemCountException when error occurs
-     */
-    public static ItemCounter getInstance() throws ItemCountException {
-        if (instance == null) {
-            instance = new ItemCounter(ContextUtil.obtainCurrentRequestContext());
-        }
-        return instance;
+    protected ItemCounter() {
     }
 
     /**
@@ -103,17 +53,24 @@ public class ItemCounter {
      * If it is equal to 'false' it will count the number of items
      * in the container in real time.
      *
+     * @param context DSpace Context
      * @param dso DSpaceObject
-     * @return count
-     * @throws ItemCountException when error occurs
+     * @return count (-1 is returned if count could not be determined or is disabled)
      */
-    public int getCount(DSpaceObject dso) throws ItemCountException {
+    public int getCount(Context context, DSpaceObject dso) {
+        boolean showStrengths = configurationService.getBooleanProperty("webui.strengths.show", false);
+        boolean useCache = configurationService.getBooleanProperty("webui.strengths.cache", true);
         if (!showStrengths) {
             return -1;
         }
 
         if (useCache) {
-            return dao.getCount(dso);
+            // NOTE: This bean is NOT Autowired above because it's a "prototype" bean which we want to reload
+            // occasionally. Each time the bean reloads it will update the cached item counts.
+            ItemCountDAO dao =
+                DSpaceServicesFactory.getInstance().getServiceManager().getServiceByName("itemCountDAO",
+                                                                                         ItemCountDAO.class);
+            return dao.getCount(context, dso);
         }
 
         // if we make it this far, we need to manually count
@@ -121,8 +78,8 @@ public class ItemCounter {
             try {
                 return itemService.countItems(context, (Collection) dso);
             } catch (SQLException e) {
-                log.error("caught exception: ", e);
-                throw new ItemCountException(e);
+                log.error("Error counting number of Items in Collection {} :", dso.getID(), e);
+                return -1;
             }
         }
 
@@ -130,8 +87,8 @@ public class ItemCounter {
             try {
                 return itemService.countItems(context, ((Community) dso));
             } catch (SQLException e) {
-                log.error("caught exception: ", e);
-                throw new ItemCountException(e);
+                log.error("Error counting number of Items in Community {} :", dso.getID(), e);
+                return -1;
             }
         }
 

dspace-api/src/main/java/org/dspace/browse/SolrBrowseDAO.java

@@ -7,12 +7,17 @@
  */
 package org.dspace.browse;
 
+import static org.dspace.discovery.SearchUtils.RESOURCE_ID_FIELD;
+import static org.dspace.discovery.SearchUtils.RESOURCE_TYPE_FIELD;
+
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.solr.client.solrj.util.ClientUtils;
@@ -21,7 +26,6 @@ import org.dspace.authorize.service.AuthorizeService;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.Item;
 import org.dspace.core.Context;
-import org.dspace.discovery.DiscoverFacetField;
 import org.dspace.discovery.DiscoverQuery;
 import org.dspace.discovery.DiscoverQuery.SORT_ORDER;
 import org.dspace.discovery.DiscoverResult;
@@ -32,7 +36,6 @@ import org.dspace.discovery.SearchService;
 import org.dspace.discovery.SearchServiceException;
 import org.dspace.discovery.SearchUtils;
 import org.dspace.discovery.configuration.DiscoveryConfiguration;
-import org.dspace.discovery.configuration.DiscoveryConfigurationParameters;
 import org.dspace.discovery.indexobject.IndexableItem;
 import org.dspace.services.factory.DSpaceServicesFactory;
 
@@ -179,19 +182,30 @@ public class SolrBrowseDAO implements BrowseDAO {
             addLocationScopeFilter(query);
             addDefaultFilterQueries(query);
             if (distinct) {
-                DiscoverFacetField dff;
-                if (StringUtils.isNotBlank(startsWith)) {
-                    dff = new DiscoverFacetField(facetField,
-                        DiscoveryConfigurationParameters.TYPE_TEXT, -1,
-                        DiscoveryConfigurationParameters.SORT.VALUE, startsWith);
+                // We use a json.facet query for metadata browsing because it allows us to limit the results
+                // while obtaining the total number of facet values with numBuckets:true and sort in reverse order
+                // Example of json.facet query:
+                // {"<fieldName>": {"type":"terms","field": "<fieldName>_filter", "limit":0, "offset":0,
+                // "sort":"index desc", "numBuckets":true, "prefix":"<startsWith>"}}
+                ObjectNode jsonFacet = JsonNodeFactory.instance.objectNode();
+                ObjectNode entriesFacet = JsonNodeFactory.instance.objectNode();
+                entriesFacet.put("type", "terms");
+                entriesFacet.put("field", facetField + "_filter");
+                entriesFacet.put("limit", limit);
+                entriesFacet.put("offset", offset);
+                entriesFacet.put("numBuckets", true);
+                if (ascending) {
+                    entriesFacet.put("sort", "index");
                 } else {
-                    dff = new DiscoverFacetField(facetField,
-                        DiscoveryConfigurationParameters.TYPE_TEXT, -1,
-                        DiscoveryConfigurationParameters.SORT.VALUE);
+                    entriesFacet.put("sort", "index desc");
                 }
-                query.addFacetField(dff);
-                query.setFacetMinCount(1);
+                if (StringUtils.isNotBlank(startsWith)) {
+                    // Add the prefix to the json facet query
+                    entriesFacet.put("prefix", startsWith);
+                }
+                jsonFacet.set(facetField, entriesFacet);
                 query.setMaxResults(0);
+                query.addProperty("json.facet", jsonFacet.toString());
             } else {
                 query.setMaxResults(limit/* > 0 ? limit : 20*/);
                 if (offset > 0) {
@@ -248,8 +262,7 @@ public class SolrBrowseDAO implements BrowseDAO {
         DiscoverResult resp = getSolrResponse();
         int count = 0;
         if (distinct) {
-            List<FacetResult> facetResults = resp.getFacetResult(facetField);
-            count = facetResults.size();
+            count = (int) resp.getTotalEntries();
         } else {
             // we need to cast to int to respect the BrowseDAO contract...
             count = (int) resp.getTotalSearchResults();
@@ -266,26 +279,15 @@ public class SolrBrowseDAO implements BrowseDAO {
         DiscoverResult resp = getSolrResponse();
         List<FacetResult> facet = resp.getFacetResult(facetField);
         int count = doCountQuery();
-        int start = offset > 0 ? offset : 0;
-        int max = limit > 0 ? limit : count; //if negative, return everything
+        int max = facet.size();
         List<String[]> result = new ArrayList<>();
-        if (ascending) {
-            for (int i = start; i < (start + max) && i < count; i++) {
-                FacetResult c = facet.get(i);
-                String freq = showFrequencies ? String.valueOf(c.getCount())
-                    : "";
-                result.add(new String[] {c.getDisplayedValue(),
-                    c.getAuthorityKey(), freq});
-            }
-        } else {
-            for (int i = count - start - 1; i >= count - (start + max)
-                && i >= 0; i--) {
-                FacetResult c = facet.get(i);
-                String freq = showFrequencies ? String.valueOf(c.getCount())
-                    : "";
-                result.add(new String[] {c.getDisplayedValue(),
-                    c.getAuthorityKey(), freq});
-            }
+
+        for (int i = 0; i < max && i < count; i++) {
+            FacetResult c = facet.get(i);
+            String freq = showFrequencies ? String.valueOf(c.getCount())
+                : "";
+            result.add(new String[] {c.getDisplayedValue(),
+                c.getAuthorityKey(), freq});
         }
 
         return result;
@@ -309,8 +311,10 @@ public class SolrBrowseDAO implements BrowseDAO {
     public String doMaxQuery(String column, String table, int itemID)
         throws BrowseException {
         DiscoverQuery query = new DiscoverQuery();
-        query.setQuery("search.resourceid:" + itemID
-                           + " AND search.resourcetype:" + IndexableItem.TYPE);
+        query.setQuery("*:*");
+        query.addFilterQueries(
+                RESOURCE_ID_FIELD + ":" + itemID,
+                RESOURCE_TYPE_FIELD + ":" + IndexableItem.TYPE);
         query.setMaxResults(1);
         DiscoverResult resp = null;
         try {

dspace-api/src/main/java/org/dspace/checker/CheckerCommand.java

@@ -131,7 +131,7 @@ public final class CheckerCommand {
                 collector.collect(context, info);
             }
 
-            context.uncacheEntity(bitstream);
+            context.commit();
             bitstream = dispatcher.next();
         }
     }

dspace-api/src/main/java/org/dspace/checker/dao/impl/MostRecentChecksumDAOImpl.java

@@ -56,8 +56,8 @@ public class MostRecentChecksumDAOImpl extends AbstractHibernateDAO<MostRecentCh
         criteriaQuery.where(criteriaBuilder.and(
             criteriaBuilder.equal(mostRecentChecksumRoot.get(MostRecentChecksum_.toBeProcessed), false),
             criteriaBuilder
-                .lessThanOrEqualTo(mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), startDate),
-            criteriaBuilder.greaterThan(mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), endDate)
+                .lessThanOrEqualTo(mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), endDate),
+            criteriaBuilder.greaterThan(mostRecentChecksumRoot.get(MostRecentChecksum_.processStartDate), startDate)
                             )
         );
         List<Order> orderList = new LinkedList<>();

dspace-api/src/main/java/org/dspace/content/Bitstream.java

@@ -307,10 +307,18 @@ public class Bitstream extends DSpaceObject implements DSpaceObjectLegacySupport
         return collection;
     }
 
+    public void setCollection(Collection collection) {
+        this.collection = collection;
+    }
+
     public Community getCommunity() {
         return community;
     }
 
+    public void setCommunity(Community community) {
+        this.community = community;
+    }
+
     /**
      * Get the asset store number where this bitstream is stored
      *

dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java

@@ -458,10 +458,15 @@ public class BitstreamServiceImpl extends DSpaceObjectServiceImpl<Bitstream> imp
 
     @Override
     public Bitstream findByIdOrLegacyId(Context context, String id) throws SQLException {
-        if (StringUtils.isNumeric(id)) {
-            return findByLegacyId(context, Integer.parseInt(id));
-        } else {
-            return find(context, UUID.fromString(id));
+        try {
+            if (StringUtils.isNumeric(id)) {
+                return findByLegacyId(context, Integer.parseInt(id));
+            } else {
+                return find(context, UUID.fromString(id));
+            }
+        } catch (IllegalArgumentException e) {
+            // Not a valid legacy ID or valid UUID
+            return null;
         }
     }
 

dspace-api/src/main/java/org/dspace/content/BundleServiceImpl.java

@@ -562,10 +562,15 @@ public class BundleServiceImpl extends DSpaceObjectServiceImpl<Bundle> implement
 
     @Override
     public Bundle findByIdOrLegacyId(Context context, String id) throws SQLException {
-        if (StringUtils.isNumeric(id)) {
-            return findByLegacyId(context, Integer.parseInt(id));
-        } else {
-            return find(context, UUID.fromString(id));
+        try {
+            if (StringUtils.isNumeric(id)) {
+                return findByLegacyId(context, Integer.parseInt(id));
+            } else {
+                return find(context, UUID.fromString(id));
+            }
+        } catch (IllegalArgumentException e) {
+            // Not a valid legacy ID or valid UUID
+            return null;
         }
     }
 

dspace-api/src/main/java/org/dspace/content/Collection.java

@@ -29,7 +29,6 @@ import javax.persistence.Table;
 import javax.persistence.Transient;
 
 import org.dspace.authorize.AuthorizeException;
-import org.dspace.browse.ItemCountException;
 import org.dspace.content.comparator.NameAscendingComparator;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.content.service.CollectionService;
@@ -135,6 +134,9 @@ public class Collection extends DSpaceObject implements DSpaceObjectLegacySuppor
 
     protected void setLogo(Bitstream logo) {
         this.logo = logo;
+        if (logo != null) {
+            logo.setCollection(this);
+        }
         setModified();
     }
 
@@ -231,7 +233,7 @@ public class Collection extends DSpaceObject implements DSpaceObjectLegacySuppor
      * @throws SQLException if database error
      */
     public void setLicense(Context context, String license) throws SQLException {
-        getCollectionService().setMetadataSingleValue(context, this, MD_LICENSE, Item.ANY, license);
+        getCollectionService().setMetadataSingleValue(context, this, MD_LICENSE, null, license);
     }
 
     /**
@@ -336,18 +338,4 @@ public class Collection extends DSpaceObject implements DSpaceObjectLegacySuppor
         }
         return collectionService;
     }
-
-    /**
-     * return count of the collection items
-     *
-     * @return int
-     */
-    public int countArchivedItems() {
-        try {
-            return collectionService.countArchivedItems(this);
-        } catch (ItemCountException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
 }

dspace-api/src/main/java/org/dspace/content/CollectionServiceImpl.java

@@ -31,7 +31,6 @@ import org.dspace.authorize.AuthorizeException;
 import org.dspace.authorize.ResourcePolicy;
 import org.dspace.authorize.service.AuthorizeService;
 import org.dspace.authorize.service.ResourcePolicyService;
-import org.dspace.browse.ItemCountException;
 import org.dspace.browse.ItemCounter;
 import org.dspace.content.dao.CollectionDAO;
 import org.dspace.content.service.BitstreamService;
@@ -122,6 +121,9 @@ public class CollectionServiceImpl extends DSpaceObjectServiceImpl<Collection> i
     @Autowired(required = true)
     protected ConfigurationService configurationService;
 
+    @Autowired
+    protected ItemCounter itemCounter;
+
     protected CollectionServiceImpl() {
         super();
     }
@@ -895,10 +897,15 @@ public class CollectionServiceImpl extends DSpaceObjectServiceImpl<Collection> i
 
     @Override
     public Collection findByIdOrLegacyId(Context context, String id) throws SQLException {
-        if (StringUtils.isNumeric(id)) {
-            return findByLegacyId(context, Integer.parseInt(id));
-        } else {
-            return find(context, UUID.fromString(id));
+        try {
+            if (StringUtils.isNumeric(id)) {
+                return findByLegacyId(context, Integer.parseInt(id));
+            } else {
+                return find(context, UUID.fromString(id));
+            }
+        } catch (IllegalArgumentException e) {
+            // Not a valid legacy ID or valid UUID
+            return null;
         }
     }
 
@@ -1014,7 +1021,8 @@ public class CollectionServiceImpl extends DSpaceObjectServiceImpl<Collection> i
         if (StringUtils.isNotBlank(q)) {
             StringBuilder buildQuery = new StringBuilder();
             String escapedQuery = ClientUtils.escapeQueryChars(q);
-            buildQuery.append("(").append(escapedQuery).append(" OR ").append(escapedQuery).append("*").append(")");
+            buildQuery.append("(").append(escapedQuery).append(" OR dc.title_sort:*")
+                .append(escapedQuery).append("*").append(")");
             discoverQuery.setQuery(buildQuery.toString());
         }
         DiscoverResult resp = searchService.search(context, discoverQuery);
@@ -1049,35 +1057,15 @@ public class CollectionServiceImpl extends DSpaceObjectServiceImpl<Collection> i
         return (int) resp.getTotalSearchResults();
     }
 
-    @Override
-    @SuppressWarnings("rawtypes")
-    public List<Collection> findAllCollectionsByEntityType(Context context, String entityType)
-            throws SearchServiceException {
-        List<Collection> collectionList = new ArrayList<>();
-
-        DiscoverQuery discoverQuery = new DiscoverQuery();
-        discoverQuery.setDSpaceObjectFilter(IndexableCollection.TYPE);
-        discoverQuery.addFilterQueries("dspace.entity.type:" + entityType);
-
-        DiscoverResult discoverResult = searchService.search(context, discoverQuery);
-        List<IndexableObject> solrIndexableObjects = discoverResult.getIndexableObjects();
-
-        for (IndexableObject solrCollection : solrIndexableObjects) {
-            Collection c = ((IndexableCollection) solrCollection).getIndexedObject();
-            collectionList.add(c);
-        }
-        return collectionList;
-    }
-
     /**
      * Returns total collection archived items
      *
+     * @param context          DSpace Context
      * @param collection       Collection
      * @return                 total collection archived items
-     * @throws ItemCountException
      */
     @Override
-    public int countArchivedItems(Collection collection) throws ItemCountException {
-        return ItemCounter.getInstance().getCount(collection);
+    public int countArchivedItems(Context context, Collection collection) {
+        return itemCounter.getCount(context, collection);
     }
 }

dspace-api/src/main/java/org/dspace/content/Community.java

@@ -25,7 +25,6 @@ import javax.persistence.Table;
 import javax.persistence.Transient;
 
 import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.dspace.browse.ItemCountException;
 import org.dspace.content.comparator.NameAscendingComparator;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.content.service.CommunityService;
@@ -123,6 +122,9 @@ public class Community extends DSpaceObject implements DSpaceObjectLegacySupport
 
     void setLogo(Bitstream logo) {
         this.logo = logo;
+        if (logo != null) {
+            logo.setCommunity(this);
+        }
         setModified();
     }
 
@@ -264,17 +266,4 @@ public class Community extends DSpaceObject implements DSpaceObjectLegacySupport
         }
         return communityService;
     }
-
-    /**
-     * return count of the community items
-     *
-     * @return int
-     */
-    public int countArchivedItems() {
-        try {
-            return communityService.countArchivedItems(this);
-        } catch (ItemCountException e) {
-            throw new RuntimeException(e);
-        }
-    }
 }

dspace-api/src/main/java/org/dspace/content/CommunityServiceImpl.java

@@ -24,7 +24,6 @@ import org.dspace.authorize.AuthorizeConfiguration;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.authorize.ResourcePolicy;
 import org.dspace.authorize.service.AuthorizeService;
-import org.dspace.browse.ItemCountException;
 import org.dspace.browse.ItemCounter;
 import org.dspace.content.dao.CommunityDAO;
 import org.dspace.content.service.BitstreamService;
@@ -78,6 +77,8 @@ public class CommunityServiceImpl extends DSpaceObjectServiceImpl<Community> imp
     protected IdentifierService identifierService;
     @Autowired(required = true)
     protected SubscribeService subscribeService;
+    @Autowired
+    protected ItemCounter itemCounter;
 
     protected CommunityServiceImpl() {
         super();
@@ -694,10 +695,15 @@ public class CommunityServiceImpl extends DSpaceObjectServiceImpl<Community> imp
 
     @Override
     public Community findByIdOrLegacyId(Context context, String id) throws SQLException {
-        if (StringUtils.isNumeric(id)) {
-            return findByLegacyId(context, Integer.parseInt(id));
-        } else {
-            return find(context, UUID.fromString(id));
+        try {
+            if (StringUtils.isNumeric(id)) {
+                return findByLegacyId(context, Integer.parseInt(id));
+            } else {
+                return find(context, UUID.fromString(id));
+            }
+        } catch (IllegalArgumentException e) {
+            // Not a valid legacy ID or valid UUID
+            return null;
         }
     }
 
@@ -714,12 +720,12 @@ public class CommunityServiceImpl extends DSpaceObjectServiceImpl<Community> imp
     /**
      * Returns total community archived items
      *
+     * @param context         DSpace context
      * @param community       Community
      * @return                total community archived items
-     * @throws ItemCountException
      */
     @Override
-    public int countArchivedItems(Community community) throws ItemCountException {
-        return ItemCounter.getInstance().getCount(community);
+    public int countArchivedItems(Context context, Community community) {
+        return itemCounter.getCount(context, community);
     }
 }

dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java

@@ -187,11 +187,11 @@ public abstract class DSpaceObjectServiceImpl<T extends DSpaceObject> implements
                                            String authority) {
         List<MetadataValue> metadata = getMetadata(dso, schema, element, qualifier, lang);
         List<MetadataValue> result = new ArrayList<>(metadata);
-        if (!authority.equals(Item.ANY)) {
+        if (!Item.ANY.equals(authority)) {
             Iterator<MetadataValue> iterator = result.iterator();
             while (iterator.hasNext()) {
                 MetadataValue metadataValue = iterator.next();
-                if (!authority.equals(metadataValue.getAuthority())) {
+                if (!StringUtils.equals(authority, metadataValue.getAuthority())) {
                     iterator.remove();
                 }
             }
@@ -242,10 +242,31 @@ public abstract class DSpaceObjectServiceImpl<T extends DSpaceObject> implements
 
     }
 
+    /**
+     * Add metadata value(s) to a MetadataField of a DSpace Object
+     * @param context current DSpace context
+     * @param dso DSpaceObject to modify
+     * @param metadataField MetadataField to add values to
+     * @param lang Language code to add
+     * @param values One or more metadata values to add
+     * @param authorities One or more authorities to add
+     * @param confidences One or more confidences to add (for authorities)
+     * @param placeSupplier Supplier of "place" for new metadata values
+     * @return List of newly added metadata values
+     * @throws SQLException if database error occurs
+     * @throws IllegalArgumentException for an empty list of values
+     */
     public List<MetadataValue> addMetadata(Context context, T dso, MetadataField metadataField, String lang,
             List<String> values, List<String> authorities, List<Integer> confidences, Supplier<Integer> placeSupplier)
                     throws SQLException {
 
+        // Throw an error if we are attempting to add empty values
+        if (values == null || values.isEmpty()) {
+            throw new IllegalArgumentException("Cannot add empty values to a new metadata field " +
+                                                   metadataField.toString() + " on object with uuid = " +
+                                                   dso.getID().toString() + " and type = " + getTypeText(dso));
+        }
+
         boolean authorityControlled = metadataAuthorityService.isAuthorityControlled(metadataField);
         boolean authorityRequired = metadataAuthorityService.isAuthorityRequired(metadataField);
         List<MetadataValue> newMetadata = new ArrayList<>();
@@ -314,20 +335,26 @@ public abstract class DSpaceObjectServiceImpl<T extends DSpaceObject> implements
     @Override
     public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language,
                             String value, String authority, int confidence) throws SQLException {
-        return addMetadata(context, dso, metadataField, language, Arrays.asList(value), Arrays.asList(authority),
-                    Arrays.asList(confidence)).get(0);
+        List<MetadataValue> metadataValues =
+            addMetadata(context, dso, metadataField, language, Arrays.asList(value), Arrays.asList(authority),
+                        Arrays.asList(confidence));
+        return CollectionUtils.isNotEmpty(metadataValues) ? metadataValues.get(0) : null;
     }
 
     @Override
     public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier,
                              String lang, String value) throws SQLException {
-        return addMetadata(context, dso, schema, element, qualifier, lang, Arrays.asList(value)).get(0);
+        List<MetadataValue> metadataValues =
+            addMetadata(context, dso, schema, element, qualifier, lang, Arrays.asList(value));
+        return CollectionUtils.isNotEmpty(metadataValues) ? metadataValues.get(0) : null;
     }
 
     @Override
     public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, String value)
         throws SQLException {
-        return addMetadata(context, dso, metadataField, language, Arrays.asList(value)).get(0);
+        List<MetadataValue> metadataValues =
+            addMetadata(context, dso, metadataField, language, Arrays.asList(value));
+        return CollectionUtils.isNotEmpty(metadataValues) ? metadataValues.get(0) : null;
     }
 
     @Override
@@ -482,7 +509,7 @@ public abstract class DSpaceObjectServiceImpl<T extends DSpaceObject> implements
         MetadataField metadataField = metadataValue.getMetadataField();
         MetadataSchema metadataSchema = metadataField.getMetadataSchema();
         // We will attempt to disprove a match - if we can't we have a match
-        if (!element.equals(Item.ANY) && !element.equals(metadataField.getElement())) {
+        if (!Item.ANY.equals(element) && !StringUtils.equals(element, metadataField.getElement())) {
             // Elements do not match, no wildcard
             return false;
         }
@@ -493,9 +520,9 @@ public abstract class DSpaceObjectServiceImpl<T extends DSpaceObject> implements
                 // Value is qualified, so no match
                 return false;
             }
-        } else if (!qualifier.equals(Item.ANY)) {
+        } else if (!Item.ANY.equals(qualifier)) {
             // Not a wildcard, so qualifier must match exactly
-            if (!qualifier.equals(metadataField.getQualifier())) {
+            if (!StringUtils.equals(qualifier, metadataField.getQualifier())) {
                 return false;
             }
         }
@@ -506,15 +533,15 @@ public abstract class DSpaceObjectServiceImpl<T extends DSpaceObject> implements
                 // Value is qualified, so no match
                 return false;
             }
-        } else if (!language.equals(Item.ANY)) {
+        } else if (!Item.ANY.equals(language)) {
             // Not a wildcard, so language must match exactly
-            if (!language.equals(metadataValue.getLanguage())) {
+            if (!StringUtils.equals(language, metadataValue.getLanguage())) {
                 return false;
             }
         }
 
-        if (!schema.equals(Item.ANY)) {
-            if (metadataSchema != null && !metadataSchema.getName().equals(schema)) {
+        if (!Item.ANY.equals(schema)) {
+            if (!StringUtils.equals(schema, metadataSchema.getName())) {
                 // The namespace doesn't match
                 return false;
             }

dspace-api/src/main/java/org/dspace/content/EntityServiceImpl.java

@@ -64,7 +64,7 @@ public class EntityServiceImpl implements EntityService {
         List<Relationship> fullList = entity.getRelationships();
         List<Relationship> listToReturn = new LinkedList<>();
         for (Relationship relationship : fullList) {
-            if (relationship.getLeftItem().getID() == entity.getItem().getID()) {
+            if (relationship.getLeftItem().getID().equals(entity.getItem().getID())) {
                 listToReturn.add(relationship);
             }
         }
@@ -76,7 +76,7 @@ public class EntityServiceImpl implements EntityService {
         List<Relationship> fullList = entity.getRelationships();
         List<Relationship> listToReturn = new LinkedList<>();
         for (Relationship relationship : fullList) {
-            if (relationship.getRightItem().getID() == entity.getItem().getID()) {
+            if (relationship.getRightItem().getID().equals(entity.getItem().getID())) {
                 listToReturn.add(relationship);
             }
         }

dspace-api/src/main/java/org/dspace/content/EntityTypeServiceImpl.java

@@ -30,6 +30,7 @@ import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.discovery.SolrSearchCore;
 import org.dspace.discovery.indexobject.IndexableCollection;
+import org.dspace.eperson.EPerson;
 import org.dspace.eperson.Group;
 import org.dspace.eperson.service.GroupService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -124,31 +125,40 @@ public class EntityTypeServiceImpl implements EntityTypeService {
     public List<String> getSubmitAuthorizedTypes(Context context)
             throws SQLException, SolrServerException, IOException {
         List<String> types = new ArrayList<>();
-        StringBuilder query = new StringBuilder();
-        org.dspace.eperson.EPerson currentUser = context.getCurrentUser();
+        StringBuilder query = null;
+        EPerson currentUser = context.getCurrentUser();
         if (!authorizeService.isAdmin(context)) {
             String userId = "";
             if (currentUser != null) {
                 userId = currentUser.getID().toString();
+                query = new StringBuilder();
+                query.append("submit:(e").append(userId);
             }
-            query.append("submit:(e").append(userId);
+
             Set<Group> groups = groupService.allMemberGroupsSet(context, currentUser);
             for (Group group : groups) {
-                query.append(" OR g").append(group.getID());
+                if (query == null) {
+                    query = new StringBuilder();
+                    query.append("submit:(g");
+                } else {
+                    query.append(" OR g");
+                }
+                query.append(group.getID());
             }
             query.append(")");
-        } else {
-            query.append("*:*");
         }
 
-        SolrQuery sQuery = new SolrQuery(query.toString());
+        SolrQuery sQuery = new SolrQuery("*:*");
+        if (query != null) {
+            sQuery.addFilterQuery(query.toString());
+        }
         sQuery.addFilterQuery("search.resourcetype:" + IndexableCollection.TYPE);
         sQuery.setRows(0);
         sQuery.addFacetField("search.entitytype");
         sQuery.setFacetMinCount(1);
         sQuery.setFacetLimit(Integer.MAX_VALUE);
         sQuery.setFacetSort(FacetParams.FACET_SORT_INDEX);
-        QueryResponse qResp = solrSearchCore.getSolr().query(sQuery);
+        QueryResponse qResp = solrSearchCore.getSolr().query(sQuery, solrSearchCore.REQUEST_METHOD);
         FacetField facetField = qResp.getFacetField("search.entitytype");
         if (Objects.nonNull(facetField)) {
             for (Count c : facetField.getValues()) {

dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java

@@ -18,6 +18,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -65,6 +66,7 @@ import org.dspace.event.Event;
 import org.dspace.harvest.HarvestedItem;
 import org.dspace.harvest.service.HarvestedItemService;
 import org.dspace.identifier.DOI;
+import org.dspace.identifier.DOIIdentifierProvider;
 import org.dspace.identifier.IdentifierException;
 import org.dspace.identifier.service.DOIService;
 import org.dspace.identifier.service.IdentifierService;
@@ -78,6 +80,9 @@ import org.dspace.orcid.service.OrcidSynchronizationService;
 import org.dspace.orcid.service.OrcidTokenService;
 import org.dspace.profile.service.ResearcherProfileService;
 import org.dspace.services.ConfigurationService;
+import org.dspace.versioning.Version;
+import org.dspace.versioning.VersionHistory;
+import org.dspace.versioning.service.VersionHistoryService;
 import org.dspace.versioning.service.VersioningService;
 import org.dspace.workflow.WorkflowItemService;
 import org.dspace.workflow.factory.WorkflowServiceFactory;
@@ -170,6 +175,9 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl<Item> implements It
     @Autowired(required = true)
     protected SubscribeService subscribeService;
 
+    @Autowired
+    private VersionHistoryService versionHistoryService;
+
     protected ItemServiceImpl() {
         super();
     }
@@ -276,6 +284,55 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl<Item> implements It
         }
     }
 
+    @Override
+    public void populateWithTemplateItemMetadata(Context context, Collection collection, boolean template, Item item)
+        throws SQLException {
+
+        Item templateItem = collection.getTemplateItem();
+
+        Optional<MetadataValue> colEntityType = getDSpaceEntityType(collection);
+        Optional<MetadataValue> templateItemEntityType = getDSpaceEntityType(templateItem);
+
+        if (template && colEntityType.isPresent() && templateItemEntityType.isPresent() &&
+            !StringUtils.equals(colEntityType.get().getValue(), templateItemEntityType.get().getValue())) {
+            throw new IllegalStateException("The template item has entity type : (" +
+                templateItemEntityType.get().getValue() + ") different than collection entity type : " +
+                colEntityType.get().getValue());
+        }
+
+        if (template && colEntityType.isPresent() && templateItemEntityType.isEmpty()) {
+            MetadataValue original = colEntityType.get();
+            MetadataField metadataField = original.getMetadataField();
+            MetadataSchema metadataSchema = metadataField.getMetadataSchema();
+            // NOTE: dspace.entity.type = <blank> does not make sense
+            //       the collection entity type is by default blank when a collection is first created
+            if (StringUtils.isNotBlank(original.getValue())) {
+                addMetadata(context, item, metadataSchema.getName(), metadataField.getElement(),
+                    metadataField.getQualifier(), original.getLanguage(), original.getValue());
+            }
+        }
+
+        if (template && (templateItem != null)) {
+            List<MetadataValue> md = getMetadata(templateItem, Item.ANY, Item.ANY, Item.ANY, Item.ANY);
+
+            for (MetadataValue aMd : md) {
+                MetadataField metadataField = aMd.getMetadataField();
+                MetadataSchema metadataSchema = metadataField.getMetadataSchema();
+                addMetadata(context, item, metadataSchema.getName(), metadataField.getElement(),
+                    metadataField.getQualifier(), aMd.getLanguage(), aMd.getValue());
+            }
+        }
+    }
+
+    private Optional<MetadataValue> getDSpaceEntityType(DSpaceObject dSpaceObject) {
+        return Objects.nonNull(dSpaceObject) ? dSpaceObject.getMetadata()
+            .stream()
+            .filter(x -> x.getMetadataField().toString('.')
+                .equalsIgnoreCase("dspace.entity.type"))
+            .findFirst()
+            : Optional.empty();
+    }
+
     @Override
     public Iterator<Item> findAll(Context context) throws SQLException {
         return itemDAO.findAll(context, true);
@@ -798,6 +855,7 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl<Item> implements It
         DOI doi = doiService.findDOIByDSpaceObject(context, item);
         if (doi != null) {
             doi.setDSpaceObject(null);
+            doi.setStatus(DOIIdentifierProvider.TO_BE_DELETED);
         }
 
         // remove version attached to the item
@@ -1571,13 +1629,13 @@ prevent the generation of resource policy entry values with null dspace_object a
 
     @Override
     public int countItems(Context context, Collection collection) throws SQLException {
-        return itemDAO.countItems(context, collection, true, false);
+        return itemDAO.countItems(context, collection, true, false, true);
     }
 
     @Override
     public int countAllItems(Context context, Collection collection) throws SQLException {
-        return itemDAO.countItems(context, collection, true, false) + itemDAO.countItems(context, collection,
-                                                                                         false, true);
+        return itemDAO.countItems(context, collection, true, false, true) + itemDAO.countItems(context, collection,
+                                                                                         false, true, true);
     }
 
     @Override
@@ -1586,7 +1644,7 @@ prevent the generation of resource policy entry values with null dspace_object a
         List<Collection> collections = communityService.getAllCollections(context, community);
 
         // Now, lets count unique items across that list of collections
-        return itemDAO.countItems(context, collections, true, false);
+        return itemDAO.countItems(context, collections, true, false, true);
     }
 
     @Override
@@ -1595,8 +1653,8 @@ prevent the generation of resource policy entry values with null dspace_object a
         List<Collection> collections = communityService.getAllCollections(context, community);
 
         // Now, lets count unique items across that list of collections
-        return itemDAO.countItems(context, collections, true, false) + itemDAO.countItems(context, collections,
-                                                                                          false, true);
+        return itemDAO.countItems(context, collections, true, false, true) + itemDAO.countItems(context, collections,
+                                                                                          false, false, true);
     }
 
     @Override
@@ -1609,10 +1667,15 @@ prevent the generation of resource policy entry values with null dspace_object a
 
     @Override
     public Item findByIdOrLegacyId(Context context, String id) throws SQLException {
-        if (StringUtils.isNumeric(id)) {
-            return findByLegacyId(context, Integer.parseInt(id));
-        } else {
-            return find(context, UUID.fromString(id));
+        try {
+            if (StringUtils.isNumeric(id)) {
+                return findByLegacyId(context, Integer.parseInt(id));
+            } else {
+                return find(context, UUID.fromString(id));
+            }
+        } catch (IllegalArgumentException e) {
+            // Not a valid legacy ID or valid UUID
+            return null;
         }
     }
 
@@ -1635,19 +1698,19 @@ prevent the generation of resource policy entry values with null dspace_object a
     @Override
     public int countNotArchivedItems(Context context) throws SQLException {
         // return count of items not in archive and also not withdrawn
-        return itemDAO.countItems(context, false, false);
+        return itemDAO.countItems(context, false, false, true);
     }
 
     @Override
     public int countArchivedItems(Context context) throws SQLException {
         // return count of items in archive and also not withdrawn
-        return itemDAO.countItems(context, true, false);
+        return itemDAO.countItems(context, true, false, true);
     }
 
     @Override
     public int countWithdrawnItems(Context context) throws SQLException {
         // return count of items that are not in archive and withdrawn
-        return itemDAO.countItems(context, false, true);
+        return itemDAO.countItems(context, false, true, true );
     }
 
     @Override
@@ -1733,7 +1796,7 @@ prevent the generation of resource policy entry values with null dspace_object a
                 //Retrieve the applicable relationship
                 Relationship rs = relationshipService.find(context,
                         ((RelationshipMetadataValue) rr).getRelationshipId());
-                if (rs.getLeftItem() == dso) {
+                if (rs.getLeftItem().equals(dso)) {
                     rs.setLeftPlace(place);
                 } else {
                     rs.setRightPlace(place);
@@ -1865,4 +1928,40 @@ prevent the generation of resource policy entry values with null dspace_object a
         }
     }
 
+    @Override
+    public boolean isLatestVersion(Context context, Item item) throws SQLException {
+
+        VersionHistory history = versionHistoryService.findByItem(context, item);
+        if (history == null) {
+            // not all items have a version history
+            // if an item does not have a version history, it is by definition the latest
+            // version
+            return true;
+        }
+
+        // start with the very latest version of the given item (may still be in
+        // workspace)
+        Version latestVersion = versionHistoryService.getLatestVersion(context, history);
+
+        // find the latest version of the given item that is archived
+        while (latestVersion != null && !latestVersion.getItem().isArchived()) {
+            latestVersion = versionHistoryService.getPrevious(context, history, latestVersion);
+        }
+
+        // could not find an archived version of the given item
+        if (latestVersion == null) {
+            // this scenario should never happen, but let's err on the side of showing too
+            // many items vs. to little
+            // (see discovery.xml, a lot of discovery configs filter out all items that are
+            // not the latest version)
+            return true;
+        }
+
+        // sanity check
+        assert latestVersion.getItem().isArchived();
+
+        return item.equals(latestVersion.getItem());
+
+    }
+
 }

dspace-api/src/main/java/org/dspace/content/MetadataDSpaceCsvExportServiceImpl.java

@@ -23,6 +23,7 @@ import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.handle.factory.HandleServiceFactory;
 import org.dspace.scripts.handler.DSpaceRunnableHandler;
+import org.dspace.services.ConfigurationService;
 import org.springframework.beans.factory.annotation.Autowired;
 
 /**
@@ -36,6 +37,11 @@ public class MetadataDSpaceCsvExportServiceImpl implements MetadataDSpaceCsvExpo
     @Autowired
     private DSpaceObjectUtils dSpaceObjectUtils;
 
+    @Autowired
+    private ConfigurationService configurationService;
+
+    private int csxExportLimit = -1;
+
     @Override
     public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String identifier,
                                   DSpaceRunnableHandler handler) throws Exception {
@@ -43,7 +49,7 @@ public class MetadataDSpaceCsvExportServiceImpl implements MetadataDSpaceCsvExpo
 
         if (exportAllItems) {
             handler.logInfo("Exporting whole repository WARNING: May take some time!");
-            toExport = itemService.findAll(context);
+            toExport = itemService.findAll(context, getCsvExportLimit(), 0);
         } else {
             DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService()
                 .resolveToObject(context, identifier);
@@ -63,7 +69,7 @@ public class MetadataDSpaceCsvExportServiceImpl implements MetadataDSpaceCsvExpo
             } else if (dso.getType() == Constants.COLLECTION) {
                 handler.logInfo("Exporting collection '" + dso.getName() + "' (" + identifier + ")");
                 Collection collection = (Collection) dso;
-                toExport = itemService.findByCollection(context, collection);
+                toExport = itemService.findByCollection(context, collection, getCsvExportLimit(), 0);
             } else if (dso.getType() == Constants.COMMUNITY) {
                 handler.logInfo("Exporting community '" + dso.getName() + "' (" + identifier + ")");
                 toExport = buildFromCommunity(context, (Community) dso);
@@ -74,18 +80,21 @@ public class MetadataDSpaceCsvExportServiceImpl implements MetadataDSpaceCsvExpo
             }
         }
 
-        DSpaceCSV csv = this.export(context, toExport, exportAllMetadata);
+        DSpaceCSV csv = this.export(context, toExport, exportAllMetadata, handler);
         return csv;
     }
 
     @Override
-    public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception {
+    public DSpaceCSV export(Context context, Iterator<Item> toExport,
+                            boolean exportAll, DSpaceRunnableHandler handler) throws Exception {
         Context.Mode originalMode = context.getCurrentMode();
         context.setMode(Context.Mode.READ_ONLY);
 
-        // Process each item
+        // Process each item until we reach the limit
+        int itemExportLimit = getCsvExportLimit();
         DSpaceCSV csv = new DSpaceCSV(exportAll);
-        while (toExport.hasNext()) {
+
+        for (int itemsAdded = 0; toExport.hasNext() && itemsAdded < itemExportLimit; itemsAdded++) {
             Item item = toExport.next();
             csv.addItem(item);
             context.uncacheEntity(item);
@@ -97,8 +106,9 @@ public class MetadataDSpaceCsvExportServiceImpl implements MetadataDSpaceCsvExpo
     }
 
     @Override
-    public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception {
-        return export(context, buildFromCommunity(context, community), exportAll);
+    public DSpaceCSV export(Context context, Community community,
+                            boolean exportAll, DSpaceRunnableHandler handler) throws Exception {
+        return export(context, buildFromCommunity(context, community), exportAll, handler);
     }
 
     /**
@@ -117,21 +127,30 @@ public class MetadataDSpaceCsvExportServiceImpl implements MetadataDSpaceCsvExpo
         // Add all the collections
         List<Collection> collections = community.getCollections();
         for (Collection collection : collections) {
-            Iterator<Item> items = itemService.findByCollection(context, collection);
-            while (items.hasNext()) {
+            // Never obtain more items than the configured limit
+            Iterator<Item> items = itemService.findByCollection(context, collection, getCsvExportLimit(), 0);
+            while (result.size() < getCsvExportLimit() && items.hasNext()) {
                 result.add(items.next());
             }
         }
 
-    // Add all the sub-communities
+        // Add all the sub-communities
         List<Community> communities = community.getSubcommunities();
         for (Community subCommunity : communities) {
             Iterator<Item> items = buildFromCommunity(context, subCommunity);
-            while (items.hasNext()) {
+            while (result.size() < getCsvExportLimit() && items.hasNext()) {
                 result.add(items.next());
             }
         }
 
         return result.iterator();
     }
+
+    @Override
+    public int getCsvExportLimit() {
+        if (csxExportLimit == -1) {
+            csxExportLimit = configurationService.getIntProperty("bulkedit.export.max.items", 500);
+        }
+        return csxExportLimit;
+    }
 }

dspace-api/src/main/java/org/dspace/content/MetadataValue.java

@@ -21,6 +21,7 @@ import javax.persistence.SequenceGenerator;
 import javax.persistence.Table;
 import javax.persistence.Transient;
 
+import org.apache.commons.lang3.StringUtils;
 import org.dspace.core.Context;
 import org.dspace.core.ReloadableEntity;
 import org.hibernate.annotations.Type;
@@ -143,6 +144,9 @@ public class MetadataValue implements ReloadableEntity<Integer> {
      * @param language new language
      */
     public void setLanguage(String language) {
+        if (StringUtils.equals(language, Item.ANY)) {
+            language = null;
+        }
         this.language = language;
     }
 

dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java

@@ -12,11 +12,8 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
 import java.util.UUID;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.app.util.DCInputsReaderException;
 import org.dspace.app.util.Util;
@@ -96,11 +93,18 @@ public class WorkspaceItemServiceImpl implements WorkspaceItemService {
     @Override
     public WorkspaceItem create(Context context, Collection collection, boolean template)
             throws AuthorizeException, SQLException {
-        return create(context, collection, null, template);
+        return create(context, collection, null, template, false);
     }
 
     @Override
-    public WorkspaceItem create(Context context, Collection collection, UUID uuid, boolean template)
+    public WorkspaceItem create(Context context, Collection collection, boolean template, boolean isNewVersion)
+            throws AuthorizeException, SQLException {
+        return create(context, collection, null, template, isNewVersion);
+    }
+
+    @Override
+    public WorkspaceItem create(Context context, Collection collection, UUID uuid, boolean template,
+                                boolean isNewVersion)
         throws AuthorizeException, SQLException {
         // Check the user has permission to ADD to the collection
         authorizeService.authorizeAction(context, collection, Constants.ADD);
@@ -134,48 +138,16 @@ public class WorkspaceItemServiceImpl implements WorkspaceItemService {
             .addPolicy(context, item, Constants.DELETE, item.getSubmitter(), ResourcePolicy.TYPE_SUBMISSION);
 
         // Copy template if appropriate
-        Item templateItem = collection.getTemplateItem();
-
-        Optional<MetadataValue> colEntityType = getDSpaceEntityType(collection);
-        Optional<MetadataValue> templateItemEntityType = getDSpaceEntityType(templateItem);
-
-        if (template && colEntityType.isPresent() && templateItemEntityType.isPresent() &&
-                !StringUtils.equals(colEntityType.get().getValue(), templateItemEntityType.get().getValue())) {
-            throw new IllegalStateException("The template item has entity type : (" +
-                      templateItemEntityType.get().getValue() + ") different than collection entity type : " +
-                      colEntityType.get().getValue());
-        }
-
-        if (template && colEntityType.isPresent() && templateItemEntityType.isEmpty()) {
-            MetadataValue original = colEntityType.get();
-            MetadataField metadataField = original.getMetadataField();
-            MetadataSchema metadataSchema = metadataField.getMetadataSchema();
-            // NOTE: dspace.entity.type = <blank> does not make sense
-            //       the collection entity type is by default blank when a collection is first created
-            if (StringUtils.isNotBlank(original.getValue())) {
-                itemService.addMetadata(context, item, metadataSchema.getName(), metadataField.getElement(),
-                                        metadataField.getQualifier(), original.getLanguage(), original.getValue());
-            }
-        }
-
-        if (template && (templateItem != null)) {
-            List<MetadataValue> md = itemService.getMetadata(templateItem, Item.ANY, Item.ANY, Item.ANY, Item.ANY);
-
-            for (MetadataValue aMd : md) {
-                MetadataField metadataField = aMd.getMetadataField();
-                MetadataSchema metadataSchema = metadataField.getMetadataSchema();
-                itemService.addMetadata(context, item, metadataSchema.getName(), metadataField.getElement(),
-                                        metadataField.getQualifier(), aMd.getLanguage(),
-                                        aMd.getValue());
-            }
-        }
+        itemService.populateWithTemplateItemMetadata(context, collection, template, item);
 
         itemService.update(context, item);
 
         // If configured, register identifiers (eg handle, DOI) now. This is typically used with the Show Identifiers
         // submission step which previews minted handles and DOIs during the submission process. Default: false
+        // Additional check needed: if we are creating a new version of an existing item we skip the identifier
+        // generation here, as this will be performed when the new version is created in VersioningServiceImpl
         if (DSpaceServicesFactory.getInstance().getConfigurationService()
-                .getBooleanProperty("identifiers.submission.register", false)) {
+                .getBooleanProperty("identifiers.submission.register", false) && !isNewVersion) {
             try {
                 // Get map of filters to use for identifier types, while the item is in progress
                 Map<Class<? extends Identifier>, Filter> filters = FilterUtils.getIdentifierFilters(true);
@@ -204,17 +176,16 @@ public class WorkspaceItemServiceImpl implements WorkspaceItemService {
         return workspaceItem;
     }
 
-    private Optional<MetadataValue> getDSpaceEntityType(DSpaceObject dSpaceObject) {
-        return Objects.nonNull(dSpaceObject) ? dSpaceObject.getMetadata()
-                                                           .stream()
-                                                           .filter(x -> x.getMetadataField().toString('.')
-                                                                         .equalsIgnoreCase("dspace.entity.type"))
-                                                           .findFirst()
-                                             : Optional.empty();
-    }
-
     @Override
     public WorkspaceItem create(Context c, WorkflowItem workflowItem) throws SQLException, AuthorizeException {
+        WorkspaceItem potentialDuplicate = findByItem(c, workflowItem.getItem());
+        if (potentialDuplicate != null) {
+            throw new IllegalArgumentException(String.format(
+                "A workspace item referring to item %s already exists (%d)",
+                workflowItem.getItem().getID(),
+                potentialDuplicate.getID()
+            ));
+        }
         WorkspaceItem workspaceItem = workspaceItemDAO.create(c, new WorkspaceItem());
         workspaceItem.setItem(workflowItem.getItem());
         workspaceItem.setCollection(workflowItem.getCollection());

dspace-api/src/main/java/org/dspace/content/authority/ChoiceAuthorityServiceImpl.java

@@ -242,7 +242,7 @@ public final class ChoiceAuthorityServiceImpl implements ChoiceAuthorityService
             // check if it is the requested collection
             Map<String, ChoiceAuthority> controllerFormDef = controllerFormDefinitions.get(fieldKey);
             SubmissionConfig submissionConfig = submissionConfigService
-                    .getSubmissionConfigByCollection(collection.getHandle());
+                    .getSubmissionConfigByCollection(collection);
             String submissionName = submissionConfig.getSubmissionName();
             // check if the requested collection has a submission definition that use an authority for the metadata
             if (controllerFormDef.containsKey(submissionName)) {
@@ -495,7 +495,7 @@ public final class ChoiceAuthorityServiceImpl implements ChoiceAuthorityService
             try {
                 configReaderService = SubmissionServiceFactory.getInstance().getSubmissionConfigService();
                 SubmissionConfig submissionName = configReaderService
-                        .getSubmissionConfigByCollection(collection.getHandle());
+                        .getSubmissionConfigByCollection(collection);
                 ma = controllerFormDefinitions.get(fieldKey).get(submissionName.getSubmissionName());
             } catch (SubmissionConfigReaderException e) {
                 // the system is in an illegal state as the submission definition is not valid
@@ -577,7 +577,7 @@ public final class ChoiceAuthorityServiceImpl implements ChoiceAuthorityService
                             .collect(Collectors.toList()));
                 }
                 DiscoverySearchFilterFacet matchingFacet = null;
-                for (DiscoverySearchFilterFacet facetConfig : searchConfigurationService.getAllFacetsConfig()) {
+                for (DiscoverySearchFilterFacet facetConfig : searchConfigurationService.getAllUniqueFacetsConfig()) {
                     boolean coversAllFieldsFromVocab = true;
                     for (String fieldFromVocab: metadataFields) {
                         boolean coversFieldFromVocab = false;

dspace-api/src/main/java/org/dspace/content/authority/DCInputAuthority.java

@@ -156,7 +156,8 @@ public class DCInputAuthority extends SelfNamedPlugin implements ChoiceAuthority
         int found = 0;
         List<Choice> v = new ArrayList<Choice>();
         for (int i = 0; i < valuesLocale.length; ++i) {
-            if (query == null || StringUtils.containsIgnoreCase(valuesLocale[i], query)) {
+            // In a DCInputAuthority context, a user will want to query the labels, not the values
+            if (query == null || StringUtils.containsIgnoreCase(labelsLocale[i], query)) {
                 if (found >= start && v.size() < limit) {
                     v.add(new Choice(null, valuesLocale[i], labelsLocale[i]));
                     if (valuesLocale[i].equalsIgnoreCase(query)) {

dspace-api/src/main/java/org/dspace/content/authority/DSpaceControlledVocabulary.java

@@ -34,35 +34,37 @@ import org.xml.sax.InputSource;
  * from {@code ${dspace.dir}/config/controlled-vocabularies/*.xml} and turns
  * them into autocompleting authorities.
  *
- * Configuration: This MUST be configured as a self-named plugin, e.g.: {@code
- * plugin.selfnamed.org.dspace.content.authority.ChoiceAuthority = \
+ * <p>Configuration: This MUST be configured as a self-named plugin, e.g.: {@code
+ * plugin.selfnamed.org.dspace.content.authority.ChoiceAuthority =
  * org.dspace.content.authority.DSpaceControlledVocabulary
  * }
  *
- * It AUTOMATICALLY configures a plugin instance for each XML file in the
+ * <p>It AUTOMATICALLY configures a plugin instance for each XML file in the
  * controlled vocabularies directory. The name of the plugin is the basename of
  * the file; e.g., {@code ${dspace.dir}/config/controlled-vocabularies/nsi.xml}
  * would generate a plugin called "nsi".
  *
- * Each configured plugin comes with three configuration options: {@code
- * vocabulary.plugin._plugin_.hierarchy.store = <true|false>
- * # Store entire hierarchy along with selected value. Default: TRUE
- * vocabulary.plugin._plugin_.hierarchy.suggest =
- * <true|false>  # Display entire hierarchy in the suggestion list.  Default: TRUE
- * vocabulary.plugin._plugin_.delimiter = "<string>"
- * # Delimiter to use when building hierarchy strings. Default: "::"
- * }
+ * <p>Each configured plugin comes with three configuration options:
+ * <ul>
+ *  <li>{@code vocabulary.plugin._plugin_.hierarchy.store = <true|false>}
+ * # Store entire hierarchy along with selected value. Default: TRUE</li>
+ *  <li>{@code vocabulary.plugin._plugin_.hierarchy.suggest =
+ * <true|false>  # Display entire hierarchy in the suggestion list.  Default: TRUE}</li>
+ *  <li>{@code vocabulary.plugin._plugin_.delimiter = "<string>"
+ * # Delimiter to use when building hierarchy strings. Default: "::"}</li>
+ * </ul>
  *
  * @author Michael B. Klein
  */
 
 public class DSpaceControlledVocabulary extends SelfNamedPlugin implements HierarchicalAuthority {
 
-    private static Logger log = org.apache.logging.log4j.LogManager.getLogger(DSpaceControlledVocabulary.class);
+    private static final Logger log = org.apache.logging.log4j.LogManager.getLogger();
     protected static String xpathTemplate = "//node[contains(translate(@label,'ABCDEFGHIJKLMNOPQRSTUVWXYZ'," +
-        "'abcdefghijklmnopqrstuvwxyz'),'%s')]";
-    protected static String idTemplate = "//node[@id = '%s']";
-    protected static String labelTemplate = "//node[@label = '%s']";
+        "'abcdefghijklmnopqrstuvwxyz'),%s)]";
+    protected static String idTemplate = "//node[@id = %s]";
+    protected static String idTemplateQuoted = "//node[@id = '%s']";
+    protected static String labelTemplate = "//node[@label = %s]";
     protected static String idParentTemplate = "//node[@id = '%s']/parent::isComposedBy/parent::node";
     protected static String rootTemplate = "/node";
     protected static String pluginNames[] = null;
@@ -102,10 +104,11 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
                 }
             }
             String vocabulariesPath = DSpaceServicesFactory.getInstance().getConfigurationService()
-                                                           .getProperty(
-                                                               "dspace.dir") + "/config/controlled-vocabularies/";
+                                                           .getProperty("dspace.dir") +
+                                                           File.separator + "config" +
+                                                           File.separator + "controlled-vocabularies";
             String[] xmlFiles = (new File(vocabulariesPath)).list(new xmlFilter());
-            List<String> names = new ArrayList<String>();
+            List<String> names = new ArrayList<>();
             for (String filename : xmlFiles) {
                 names.add((new File(filename)).getName().replace(".xml", ""));
             }
@@ -120,7 +123,8 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
 
             log.info("Initializing " + this.getClass().getName());
             vocabularyName = this.getPluginInstanceName();
-            String vocabulariesPath = config.getProperty("dspace.dir") + "/config/controlled-vocabularies/";
+            String vocabulariesPath = config.getProperty("dspace.dir") + File.separator + "config" +
+                File.separator + "controlled-vocabularies" + File.separator;
             String configurationPrefix = "vocabulary.plugin." + vocabularyName;
             storeHierarchy = config.getBooleanProperty(configurationPrefix + ".hierarchy.store", storeHierarchy);
             suggestHierarchy = config.getBooleanProperty(configurationPrefix + ".hierarchy.suggest", suggestHierarchy);
@@ -160,14 +164,23 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
     public Choices getMatches(String text, int start, int limit, String locale) {
         init();
         log.debug("Getting matches for '" + text + "'");
-        String xpathExpression = "";
         String[] textHierarchy = text.split(hierarchyDelimiter, -1);
+        StringBuilder xpathExpressionBuilder = new StringBuilder();
         for (int i = 0; i < textHierarchy.length; i++) {
-            xpathExpression += String.format(xpathTemplate, textHierarchy[i].replaceAll("'", "&apos;").toLowerCase());
+            xpathExpressionBuilder.append(String.format(xpathTemplate, "$var" + i));
         }
+        String xpathExpression = xpathExpressionBuilder.toString();
         XPath xpath = XPathFactory.newInstance().newXPath();
-        int total = 0;
-        List<Choice> choices = new ArrayList<Choice>();
+        xpath.setXPathVariableResolver(variableName -> {
+            String varName = variableName.getLocalPart();
+            if (varName.startsWith("var")) {
+                int index = Integer.parseInt(varName.substring(3));
+                return textHierarchy[index].toLowerCase();
+            }
+            throw new IllegalArgumentException("Unexpected variable: " + varName);
+        });
+        int total;
+        List<Choice> choices;
         try {
             NodeList results = (NodeList) xpath.evaluate(xpathExpression, vocabulary, XPathConstants.NODESET);
             total = results.getLength();
@@ -183,14 +196,23 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
     @Override
     public Choices getBestMatch(String text, String locale) {
         init();
-        log.debug("Getting best matches for '" + text + "'");
-        String xpathExpression = "";
+        log.debug("Getting best matches for {}'", text);
         String[] textHierarchy = text.split(hierarchyDelimiter, -1);
+        StringBuilder xpathExpressionBuilder = new StringBuilder();
         for (int i = 0; i < textHierarchy.length; i++) {
-            xpathExpression += String.format(labelTemplate, textHierarchy[i].replaceAll("'", "&apos;"));
+            xpathExpressionBuilder.append(String.format(labelTemplate, "$var" + i));
         }
+        String xpathExpression = xpathExpressionBuilder.toString();
         XPath xpath = XPathFactory.newInstance().newXPath();
-        List<Choice> choices = new ArrayList<Choice>();
+        xpath.setXPathVariableResolver(variableName -> {
+            String varName = variableName.getLocalPart();
+            if (varName.startsWith("var")) {
+                int index = Integer.parseInt(varName.substring(3));
+                return textHierarchy[index];
+            }
+            throw new IllegalArgumentException("Unexpected variable: " + varName);
+        });
+        List<Choice> choices;
         try {
             NodeList results = (NodeList) xpath.evaluate(xpathExpression, vocabulary, XPathConstants.NODESET);
             choices = getChoicesFromNodeList(results, 0, 1);
@@ -224,6 +246,7 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
 
     @Override
     public boolean isHierarchical() {
+        init();
         return true;
     }
 
@@ -237,7 +260,7 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
     @Override
     public Choices getChoicesByParent(String authorityName, String parentId, int start, int limit, String locale) {
         init();
-        String xpathExpression = String.format(idTemplate, parentId);
+        String xpathExpression = String.format(idTemplateQuoted, parentId);
         return getChoicesByXpath(xpathExpression, start, limit);
     }
 
@@ -256,19 +279,17 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
 
     @Override
     public Integer getPreloadLevel() {
+        init();
         return preloadLevel;
     }
 
     private boolean isRootElement(Node node) {
-        if (node != null && node.getOwnerDocument().getDocumentElement().equals(node)) {
-            return true;
-        }
-        return false;
+        return node != null && node.getOwnerDocument().getDocumentElement().equals(node);
     }
 
     private Node getNode(String key) throws XPathExpressionException {
         init();
-        String xpathExpression = String.format(idTemplate, key);
+        String xpathExpression = String.format(idTemplateQuoted, key);
         Node node = getNodeFromXPath(xpathExpression);
         return node;
     }
@@ -280,7 +301,7 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
     }
 
     private List<Choice> getChoicesFromNodeList(NodeList results, int start, int limit) {
-        List<Choice> choices = new ArrayList<Choice>();
+        List<Choice> choices = new ArrayList<>();
         for (int i = 0; i < results.getLength(); i++) {
             if (i < start) {
                 continue;
@@ -299,14 +320,14 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
 
     private Map<String, String> addOtherInformation(String parentCurr, String noteCurr,
             List<String> childrenCurr, String authorityCurr) {
-        Map<String, String> extras = new HashMap<String, String>();
+        Map<String, String> extras = new HashMap<>();
         if (StringUtils.isNotBlank(parentCurr)) {
             extras.put("parent", parentCurr);
         }
         if (StringUtils.isNotBlank(noteCurr)) {
             extras.put("note", noteCurr);
         }
-        if (childrenCurr.size() > 0) {
+        if (!childrenCurr.isEmpty()) {
             extras.put("hasChildren", "true");
         } else {
             extras.put("hasChildren", "false");
@@ -364,7 +385,7 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
     }
 
     private List<String> getChildren(Node node) {
-        List<String> children = new ArrayList<String>();
+        List<String> children = new ArrayList<>();
         NodeList childNodes = node.getChildNodes();
         for (int ci = 0; ci < childNodes.getLength(); ci++) {
             Node firstChild = childNodes.item(ci);
@@ -387,7 +408,7 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
     private boolean isSelectable(Node node) {
         Node selectableAttr = node.getAttributes().getNamedItem("selectable");
         if (null != selectableAttr) {
-            return Boolean.valueOf(selectableAttr.getNodeValue());
+            return Boolean.parseBoolean(selectableAttr.getNodeValue());
         } else { // Default is true
             return true;
         }
@@ -414,7 +435,7 @@ public class DSpaceControlledVocabulary extends SelfNamedPlugin implements Hiera
     }
 
     private Choices getChoicesByXpath(String xpathExpression, int start, int limit) {
-        List<Choice> choices = new ArrayList<Choice>();
+        List<Choice> choices = new ArrayList<>();
         XPath xpath = XPathFactory.newInstance().newXPath();
         try {
             Node parentNode = (Node) xpath.evaluate(xpathExpression, vocabulary, XPathConstants.NODE);

dspace-api/src/main/java/org/dspace/content/crosswalk/CreativeCommonsRDFStreamDisseminationCrosswalk.java

@@ -59,7 +59,6 @@ public class CreativeCommonsRDFStreamDisseminationCrosswalk
             Bitstream cc = creativeCommonsService.getLicenseRdfBitstream((Item) dso);
             if (cc != null) {
                 Utils.copy(bitstreamService.retrieve(context, cc), out);
-                out.close();
             }
         }
     }

dspace-api/src/main/java/org/dspace/content/crosswalk/CreativeCommonsTextStreamDisseminationCrosswalk.java

@@ -65,7 +65,6 @@ public class CreativeCommonsTextStreamDisseminationCrosswalk
             Bitstream cc = creativeCommonsService.getLicenseTextBitstream((Item) dso);
             if (cc != null) {
                 Utils.copy(bitstreamService.retrieve(context, cc), out);
-                out.close();
             }
         }
     }

dspace-api/src/main/java/org/dspace/content/crosswalk/LicenseStreamDisseminationCrosswalk.java

@@ -57,7 +57,6 @@ public class LicenseStreamDisseminationCrosswalk
 
             if (licenseBs != null) {
                 Utils.copy(bitstreamService.retrieve(context, licenseBs), out);
-                out.close();
             }
         }
     }

dspace-api/src/main/java/org/dspace/content/crosswalk/METSDisseminationCrosswalk.java

@@ -14,6 +14,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang3.ArrayUtils;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.packager.PackageDisseminator;
@@ -129,7 +130,7 @@ public class METSDisseminationCrosswalk
 
             try {
                 //Return just the root Element of the METS file
-                SAXBuilder builder = new SAXBuilder();
+                SAXBuilder builder = XMLUtils.getSAXBuilder();
                 Document metsDocument = builder.build(tempFile);
                 return metsDocument.getRootElement();
             } catch (JDOMException je) {

dspace-api/src/main/java/org/dspace/content/crosswalk/METSRightsCrosswalk.java

@@ -432,31 +432,7 @@ public class METSRightsCrosswalk
                     //get what class of context this is
                     String contextClass = element.getAttributeValue("CONTEXTCLASS");
 
-                    ResourcePolicy rp = resourcePolicyService.create(context);
-                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
-
-                    // get reference to the <Permissions> element
-                    // Note: we are assuming here that there will only ever be ONE <Permissions>
-                    //  element. Currently there are no known use cases for multiple.
-                    Element permsElement = element.getChild("Permissions", METSRights_NS);
-                    if (permsElement == null) {
-                        log.error("No <Permissions> element was found. Skipping this <Context> element.");
-                        continue;
-                    }
-
-                    if (element.getAttributeValue("rpName") != null) {
-                        rp.setRpName(element.getAttributeValue("rpName"));
-                    }
-                    try {
-                        if (element.getAttributeValue("start-date") != null) {
-                            rp.setStartDate(sdf.parse(element.getAttributeValue("start-date")));
-                        }
-                        if (element.getAttributeValue("end-date") != null) {
-                            rp.setEndDate(sdf.parse(element.getAttributeValue("end-date")));
-                        }
-                    } catch (ParseException ex) {
-                        log.error("Failed to parse embargo date. The date needs to be in the format 'yyyy-MM-dd'.", ex);
-                    }
+                    ResourcePolicy rp = null;
 
                     //Check if this permission pertains to Anonymous users
                     if (ANONYMOUS_CONTEXTCLASS.equals(contextClass)) {
@@ -464,22 +440,23 @@ public class METSRightsCrosswalk
                         Group anonGroup = groupService.findByName(context, Group.ANONYMOUS);
                         if (anonGroup == null) {
                             throw new CrosswalkInternalException(
-                                "The DSpace database has not been properly initialized.  The Anonymous Group is " +
-                                    "missing from the database.");
+                                    "The DSpace database has not been properly initialized.  The Anonymous Group is " +
+                                            "missing from the database.");
                         }
 
-                        rp.setGroup(anonGroup);
+                        rp = resourcePolicyService.create(context, null, anonGroup);
                     } else if (ADMIN_CONTEXTCLASS.equals(contextClass)) {
                         // else if this permission declaration pertains to Administrators
                         // get DSpace Administrator group
                         Group adminGroup = groupService.findByName(context, Group.ADMIN);
                         if (adminGroup == null) {
                             throw new CrosswalkInternalException(
-                                "The DSpace database has not been properly initialized.  The Administrator Group is " +
-                                    "missing from the database.");
+                                    "The DSpace database has not been properly initialized.  " +
+                                            "The Administrator Group is " +
+                                            "missing from the database.");
                         }
 
-                        rp.setGroup(adminGroup);
+                        rp = resourcePolicyService.create(context, null, adminGroup);
                     } else if (GROUP_CONTEXTCLASS.equals(contextClass)) {
                         // else if this permission pertains to another DSpace group
                         try {
@@ -498,18 +475,17 @@ public class METSRightsCrosswalk
                             //if not found, throw an error -- user should restore group from the SITE AIP
                             if (group == null) {
                                 throw new CrosswalkInternalException("Cannot restore Group permissions on object ("
-                                                                         + "type=" + Constants.typeText[dso
-                                    .getType()] + ", "
-                                                                         + "handle=" + dso.getHandle() + ", "
-                                                                         + "ID=" + dso.getID()
-                                                                         + "). The Group named '" + groupName + "' is" +
-                                                                         " missing from DSpace. "
-                                                                         + "Please restore this group using the SITE " +
-                                                                         "AIP, or recreate it.");
+                                                                 + "type=" + Constants.typeText[dso.getType()] + ", "
+                                                                 + "handle=" + dso.getHandle() + ", "
+                                                                 + "ID=" + dso.getID()
+                                                                 + "). The Group named '" + groupName + "' is" +
+                                                                 " missing from DSpace. "
+                                                                 + "Please restore this group using the SITE " +
+                                                                 "AIP, or recreate it.");
                             }
 
                             //assign group to policy
-                            rp.setGroup(group);
+                            rp = resourcePolicyService.create(context, null, group);
                         } catch (PackageException pe) {
                             //A PackageException will only be thrown if translateDefaultGroupName() fails
                             //We'll just wrap it as a CrosswalkException and throw it upwards
@@ -535,25 +511,52 @@ public class METSRightsCrosswalk
                         //if not found, throw an error -- user should restore person from the SITE AIP
                         if (person == null) {
                             throw new CrosswalkInternalException("Cannot restore Person permissions on object ("
-                                                                     + "type=" + Constants.typeText[dso
-                                .getType()] + ", "
-                                                                     + "handle=" + dso.getHandle() + ", "
-                                                                     + "ID=" + dso.getID()
-                                                                     + "). The Person with email/netid '" +
-                                                                     personEmail + "' is missing from DSpace. "
-                                                                     + "Please restore this Person object using the " +
-                                                                     "SITE AIP, or recreate it.");
+                                                                 + "type=" + Constants.typeText[dso.getType()] + ", "
+                                                                 + "handle=" + dso.getHandle() + ", "
+                                                                 + "ID=" + dso.getID()
+                                                                 + "). The Person with email/netid '" +
+                                                                 personEmail + "' is missing from DSpace. "
+                                                                 + "Please restore this Person object using the " +
+                                                                 "SITE AIP, or recreate it.");
                         }
 
-                        //assign person to the policy
-                        rp.setEPerson(person);
+                        //create rp with the person
+                        rp = resourcePolicyService.create(context, person, null);
                     } else {
                         log.error("Unrecognized CONTEXTCLASS:  " + contextClass);
                     }
+                    if (rp != null) {
+                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
 
-                    //set permissions on policy add to list of policies
-                    rp.setAction(parsePermissions(permsElement));
-                    policies.add(rp);
+                        // get reference to the <Permissions> element
+                        // Note: we are assuming here that there will only ever be ONE <Permissions>
+                        //  element. Currently there are no known use cases for multiple.
+                        Element permsElement = element.getChild("Permissions", METSRights_NS);
+                        if (permsElement == null) {
+                            log.error("No <Permissions> element was found. Skipping this <Context> element.");
+                            continue;
+                        }
+
+                        if (element.getAttributeValue("rpName") != null) {
+                            rp.setRpName(element.getAttributeValue("rpName"));
+                        }
+                        try {
+                            if (element.getAttributeValue("start-date") != null) {
+                                rp.setStartDate(sdf.parse(element.getAttributeValue("start-date")));
+                            }
+                            if (element.getAttributeValue("end-date") != null) {
+                                rp.setEndDate(sdf.parse(element.getAttributeValue("end-date")));
+                            }
+                        } catch (ParseException ex) {
+                            log.error("Failed to parse embargo date. The date needs to be in the format 'yyyy-MM-dd'.",
+                                      ex);
+                        }
+
+                        //set permissions and type on policy and add to list of policies
+                        rp.setAction(parsePermissions(permsElement));
+                        rp.setRpType(ResourcePolicy.TYPE_CUSTOM);
+                        policies.add(rp);
+                    }
                 } //end if "Context" element
             } //end for loop
 

dspace-api/src/main/java/org/dspace/content/crosswalk/MODSDisseminationCrosswalk.java

@@ -22,6 +22,7 @@ import java.util.Properties;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.Collection;
 import org.dspace.content.Community;
@@ -144,7 +145,7 @@ public class MODSDisseminationCrosswalk extends SelfNamedPlugin
         MODS_NS.getURI() + " " + MODS_XSD;
 
     private static final XMLOutputter outputUgly = new XMLOutputter();
-    private static final SAXBuilder builder = new SAXBuilder();
+    private static final SAXBuilder builder = XMLUtils.getSAXBuilder();
 
     private Map<String, modsTriple> modsMap = null;
 

dspace-api/src/main/java/org/dspace/content/crosswalk/PREMISCrosswalk.java

@@ -20,9 +20,7 @@ import org.apache.logging.log4j.Logger;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.Bitstream;
 import org.dspace.content.BitstreamFormat;
-import org.dspace.content.Bundle;
 import org.dspace.content.DSpaceObject;
-import org.dspace.content.Item;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.content.service.BitstreamFormatService;
 import org.dspace.content.service.BitstreamService;
@@ -224,29 +222,17 @@ public class PREMISCrosswalk
         //  c. made-up name based on sequence ID and extension.
         String sid = String.valueOf(bitstream.getSequenceID());
         String baseUrl = configurationService.getProperty("dspace.ui.url");
-        String handle = null;
-        // get handle of parent Item of this bitstream, if there is one:
-        List<Bundle> bn = bitstream.getBundles();
-        if (bn.size() > 0) {
-            List<Item> bi = bn.get(0).getItems();
-            if (bi.size() > 0) {
-                handle = bi.get(0).getHandle();
-            }
-        }
         // get or make up name for bitstream:
         String bsName = bitstream.getName();
         if (bsName == null) {
             List<String> ext = bitstream.getFormat(context).getExtensions();
             bsName = "bitstream_" + sid + (ext.size() > 0 ? ext.get(0) : "");
         }
-        if (handle != null && baseUrl != null) {
+        if (baseUrl != null) {
             oiv.setText(baseUrl
-                            + "/bitstream/"
-                            + URLEncoder.encode(handle, "UTF-8")
-                            + "/"
-                            + sid
-                            + "/"
-                            + URLEncoder.encode(bsName, "UTF-8"));
+                + "/bitstreams/"
+                + bitstream.getID()
+                + "/download");
         } else {
             oiv.setText(URLEncoder.encode(bsName, "UTF-8"));
         }

dspace-api/src/main/java/org/dspace/content/crosswalk/QDCCrosswalk.java

@@ -22,6 +22,7 @@ import java.util.Properties;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.Item;
@@ -125,7 +126,7 @@ public class QDCCrosswalk extends SelfNamedPlugin
     // XML schemaLocation fragment for this crosswalk, from config.
     private String schemaLocation = null;
 
-    private static final SAXBuilder builder = new SAXBuilder();
+    private static final SAXBuilder builder = XMLUtils.getSAXBuilder();
 
     protected ItemService itemService = ContentServiceFactory.getInstance().getItemService();
 

dspace-api/src/main/java/org/dspace/content/crosswalk/RoleCrosswalk.java

@@ -13,6 +13,7 @@ import java.io.IOException;
 import java.sql.SQLException;
 import java.util.List;
 
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.packager.PackageDisseminator;
@@ -208,7 +209,7 @@ public class RoleCrosswalk
 
             try {
                 //Try to parse our XML results (which were disseminated by the Packager)
-                SAXBuilder builder = new SAXBuilder();
+                SAXBuilder builder = XMLUtils.getSAXBuilder();
                 Document xmlDocument = builder.build(tempFile);
                 //If XML parsed successfully, return root element of doc
                 if (xmlDocument != null && xmlDocument.hasRootElement()) {

dspace-api/src/main/java/org/dspace/content/crosswalk/SubscriptionDsoMetadataForEmailCompose.java

@@ -49,7 +49,7 @@ public class SubscriptionDsoMetadataForEmailCompose implements StreamDisseminati
             for (String actualMetadata : metadata) {
                 String[] splitted = actualMetadata.split("\\.");
                 String qualifier = null;
-                if (splitted.length == 1) {
+                if (splitted.length == 3) {
                     qualifier = splitted[2];
                 }
                 var metadataValue = itemService.getMetadataFirstValue(item, splitted[0], splitted[1], qualifier, ANY);

dspace-api/src/main/java/org/dspace/content/crosswalk/XSLTIngestionCrosswalk.java

@@ -18,6 +18,7 @@ import javax.xml.transform.TransformerException;
 
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.Logger;
+import org.dspace.app.util.XMLUtils;
 import org.dspace.authorize.AuthorizeException;
 import org.dspace.content.Collection;
 import org.dspace.content.Community;
@@ -297,7 +298,7 @@ public class XSLTIngestionCrosswalk
                 "Failed to initialize transformer, probably error loading stylesheet.");
         }
 
-        SAXBuilder builder = new SAXBuilder();
+        SAXBuilder builder = XMLUtils.getSAXBuilder();
         Document inDoc = builder.build(new FileInputStream(argv[i + 1]));
         XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat());
         List dimList;

dspace-api/src/main/java/org/dspace/content/dao/ItemDAO.java

@@ -128,7 +128,8 @@ public interface ItemDAO extends DSpaceObjectLegacySupportDAO<Item> {
      * @return item count
      * @throws SQLException if database error
      */
-    public int countItems(Context context, Collection collection, boolean includeArchived, boolean includeWithdrawn)
+    int countItems(Context context, Collection collection, boolean includeArchived, boolean includeWithdrawn,
+                   boolean discoverable)
         throws SQLException;
 
     /**
@@ -144,8 +145,8 @@ public interface ItemDAO extends DSpaceObjectLegacySupportDAO<Item> {
      * @return item count
      * @throws SQLException if database error
      */
-    public int countItems(Context context, List<Collection> collections, boolean includeArchived,
-                          boolean includeWithdrawn) throws SQLException;
+    int countItems(Context context, List<Collection> collections, boolean includeArchived,
+                          boolean includeWithdrawn, boolean discoverable) throws SQLException;
 
     /**
      * Get all Items installed or withdrawn, discoverable, and modified since a Date.
@@ -180,7 +181,8 @@ public interface ItemDAO extends DSpaceObjectLegacySupportDAO<Item> {
      * @return count of items
      * @throws SQLException if database error
      */
-    int countItems(Context context, boolean includeArchived, boolean includeWithdrawn) throws SQLException;
+    int countItems(Context context, boolean includeArchived, boolean includeWithdrawn,
+                   boolean discoverable) throws SQLException;
 
     /**
      * Count number of items from the specified submitter based on specific status flags
@@ -192,7 +194,8 @@ public interface ItemDAO extends DSpaceObjectLegacySupportDAO<Item> {
      * @return count of items
      * @throws SQLException if database error
      */
-    public int countItems(Context context, EPerson submitter, boolean includeArchived, boolean includeWithdrawn)
+    int countItems(Context context, EPerson submitter, boolean includeArchived, boolean includeWithdrawn,
+                   boolean discoverable)
         throws SQLException;
 
 }

dspace-api/src/main/java/org/dspace/content/dao/impl/BitstreamDAOImpl.java

@@ -12,6 +12,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import javax.persistence.Query;
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
@@ -26,6 +27,7 @@ import org.dspace.content.dao.BitstreamDAO;
 import org.dspace.core.AbstractHibernateDSODAO;
 import org.dspace.core.Constants;
 import org.dspace.core.Context;
+import org.dspace.core.UUIDIterator;
 
 /**
  * Hibernate implementation of the Database Access Object interface class for the Bitstream object.
@@ -76,7 +78,7 @@ public class BitstreamDAOImpl extends AbstractHibernateDSODAO<Bitstream> impleme
 
     @Override
     public Iterator<Bitstream> findByCommunity(Context context, Community community) throws SQLException {
-        Query query = createQuery(context, "select b from Bitstream b " +
+        Query query = createQuery(context, "select b.id from Bitstream b " +
             "join b.bundles bitBundles " +
             "join bitBundles.items item " +
             "join item.collections itemColl " +
@@ -84,40 +86,45 @@ public class BitstreamDAOImpl extends AbstractHibernateDSODAO<Bitstream> impleme
             "WHERE :community IN community");
 
         query.setParameter("community", community);
-
-        return iterate(query);
+        @SuppressWarnings("unchecked")
+        List<UUID> uuids = query.getResultList();
+        return new UUIDIterator<Bitstream>(context, uuids, Bitstream.class, this);
     }
 
     @Override
     public Iterator<Bitstream> findByCollection(Context context, Collection collection) throws SQLException {
-        Query query = createQuery(context, "select b from Bitstream b " +
+        Query query = createQuery(context, "select b.id from Bitstream b " +
             "join b.bundles bitBundles " +
             "join bitBundles.items item " +
             "join item.collections c " +
             "WHERE :collection IN c");
 
         query.setParameter("collection", collection);
-
-        return iterate(query);
+        @SuppressWarnings("unchecked")
+        List<UUID> uuids = query.getResultList();
+        return new UUIDIterator<Bitstream>(context, uuids, Bitstream.class, this);
     }
 
     @Override
     public Iterator<Bitstream> findByItem(Context context, Item item) throws SQLException {
-        Query query = createQuery(context, "select b from Bitstream b " +
+        Query query = createQuery(context, "select b.id from Bitstream b " +
             "join b.bundles bitBundles " +
             "join bitBundles.items item " +
             "WHERE :item IN item");
 
         query.setParameter("item", item);
-
-        return iterate(query);
+        @SuppressWarnings("unchecked")
+        List<UUID> uuids = query.getResultList();
+        return new UUIDIterator<Bitstream>(context, uuids, Bitstream.class, this);
     }
 
     @Override
     public Iterator<Bitstream> findByStoreNumber(Context context, Integer storeNumber) throws SQLException {
-        Query query = createQuery(context, "select b from Bitstream b where b.storeNumber = :storeNumber");
+        Query query = createQuery(context, "select b.id from Bitstream b where b.storeNumber = :storeNumber");
         query.setParameter("storeNumber", storeNumber);
-        return iterate(query);
+        @SuppressWarnings("unchecked")
+        List<UUID> uuids = query.getResultList();
+        return new UUIDIterator<Bitstream>(context, uuids, Bitstream.class, this);
     }
 
     @Override
@@ -145,7 +152,7 @@ public class BitstreamDAOImpl extends AbstractHibernateDSODAO<Bitstream> impleme
     @Override
     public int countWithNoPolicy(Context context) throws SQLException {
         Query query = createQuery(context,
-                                  "SELECT count(bit.id) from Bitstream bit where bit.deleted<>true and bit.id not in" +
+                                  "SELECT count(bit.id) from Bitstream bit where bit.deleted<>true and bit not in" +
                                       " (select res.dSpaceObject from ResourcePolicy res where res.resourceTypeId = " +
                                       ":typeId )");
         query.setParameter("typeId", Constants.BITSTREAM);