hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into w2p-92917_Fix-trusted-IP-ranges_Contribute_PR

Browse Source
This commit is contained in:
Yura Bondarenko
2022-07-12 09:40:09 +02:00
parent 01d5bf4f4a fb691719a3
commit 93a1ef4809
1320 changed files with 102131 additions and 11772 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
.github/workflows/build.yml vendored
View File

@@ -11,40 +11,45 @@ jobs:
runs-on: ubuntu-latest
env:
# Give Maven 1GB of memory to work with
# Suppress all Maven "downloading" messages in Travis logs (see https://stackoverflow.com/a/35653426)
# This also slightly speeds builds, as there is less logging
MAVEN_OPTS: "-Xmx1024M -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
MAVEN_OPTS: "-Xmx1024M"
strategy:
# Create a matrix of two separate configurations for Unit vs Integration Tests
# This will ensure those tasks are run in parallel
# Also specify version of Java to use (this can allow us to optionally run tests on multiple JDKs in future)
matrix:
include:
# NOTE: Unit Tests include deprecated REST API v6 (as it has unit tests)
# - surefire.rerunFailingTestsCount => try again for flakey tests, and keep track of/report on number of retries
- type: "Unit Tests"
mvnflags: "-DskipUnitTests=false -Pdspace-rest"
java: 11
mvnflags: "-DskipUnitTests=false -Pdspace-rest -Dsurefire.rerunFailingTestsCount=2"
resultsdir: "**/target/surefire-reports/**"
# NOTE: ITs skip all code validation checks, as they are already done by Unit Test job.
# - enforcer.skip => Skip maven-enforcer-plugin rules
# - checkstyle.skip => Skip all checkstyle checks by maven-checkstyle-plugin
# - license.skip => Skip all license header checks by license-maven-plugin
# - xml.skip => Skip all XML/XSLT validation by xml-maven-plugin
# - failsafe.rerunFailingTestsCount => try again for flakey tests, and keep track of/report on number of retries
- type: "Integration Tests"
mvnflags: "-DskipIntegrationTests=false -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true"
java: 11
mvnflags: "-DskipIntegrationTests=false -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true -Dfailsafe.rerunFailingTestsCount=2"
resultsdir: "**/target/failsafe-reports/**"
# Do NOT exit immediately if one matrix job fails
# This ensures ITs continue running even if Unit Tests fail, or visa versa
fail-fast: false
name: Run ${{ matrix.type }}
# These are the actual CI steps to perform per job
steps:
# https://github.com/actions/checkout
- name: Checkout codebase
uses: actions/checkout@v1
uses: actions/checkout@v2
# https://github.com/actions/setup-java
- name: Install JDK 11
uses: actions/setup-java@v1
- name: Install JDK ${{ matrix.java }}
uses: actions/setup-java@v2
with:
java-version: 11
java-version: ${{ matrix.java }}
distribution: 'temurin'
# https://github.com/actions/cache
- name: Cache Maven dependencies
@@ -60,7 +65,7 @@ jobs:
- name: Run Maven ${{ matrix.type }}
env:
TEST_FLAGS: ${{ matrix.mvnflags }}
run: mvn install -B -V -P-assembly -Pcoverage-report $TEST_FLAGS
run: mvn --no-transfer-progress -V install -P-assembly -Pcoverage-report $TEST_FLAGS
# If previous step failed, save results of tests to downloadable artifact for this job
# (This artifact is downloadable at the bottom of any job's summary page)
@@ -70,8 +75,7 @@ jobs:
with:
name: ${{ matrix.type }} results
path: ${{ matrix.resultsdir }}
retention-days: 7
# https://github.com/codecov/codecov-action
- name: Upload coverage to Codecov.io
uses: codecov/codecov-action@v1
uses: codecov/codecov-action@v2

169
.github/workflows/docker.yml vendored Normal file
View File

@@ -0,0 +1,169 @@
# DSpace Docker image build for hub.docker.com
name: Docker images
# Run this Build for all pushes to 'main' or maintenance branches, or tagged releases.
# Also run for PRs to ensure PR doesn't break Docker build process
on:
push:
branches:
- main
- 'dspace-**'
tags:
- 'dspace-**'
pull_request:
jobs:
docker:
# Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace'
if: github.repository == 'dspace/dspace'
runs-on: ubuntu-latest
env:
# Define tags to use for Docker images based on Git tags/branches (for docker/metadata-action)
# For a new commit on default branch (main), use the literal tag 'dspace-7_x' on Docker image.
# For a new commit on other branches, use the branch name as the tag for Docker image.
# For a new tag, copy that tag name as the tag for Docker image.
IMAGE_TAGS: |
type=raw,value=dspace-7_x,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
type=ref,event=branch,enable=${{ !endsWith(github.ref, github.event.repository.default_branch) }}
type=ref,event=tag
# Define default tag "flavor" for docker/metadata-action per
# https://github.com/docker/metadata-action#flavor-input
# We turn off 'latest' tag by default.
TAGS_FLAVOR: |
latest=false
# Architectures / Platforms for which we will build Docker images
# If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work.
# If this is NOT a PR (e.g. a tag or merge commit), also build for ARM64. NOTE: The ARM64 build takes MUCH
# longer (around 45mins or so) which is why we only run it when pushing a new Docker image.
PLATFORMS: linux/amd64${{ github.event_name != 'pull_request' && ', linux/arm64' || '' }}
steps:
# https://github.com/actions/checkout
- name: Checkout codebase
uses: actions/checkout@v2
# https://github.com/docker/setup-buildx-action
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v1
# https://github.com/docker/setup-qemu-action
- name: Set up QEMU emulation to build for multiple architectures
uses: docker/setup-qemu-action@v2
# https://github.com/docker/login-action
- name: Login to DockerHub
# Only login if not a PR, as PRs only trigger a Docker build and not a push
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
####################################################
# Build/Push the 'dspace/dspace-dependencies' image
####################################################
# https://github.com/docker/metadata-action
# Get Metadata for docker_build_deps step below
- name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-dependencies' image
id: meta_build_deps
uses: docker/metadata-action@v3
with:
images: dspace/dspace-dependencies
tags: ${{ env.IMAGE_TAGS }}
flavor: ${{ env.TAGS_FLAVOR }}
# https://github.com/docker/build-push-action
- name: Build and push 'dspace-dependencies' image
id: docker_build_deps
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile.dependencies
platforms: ${{ env.PLATFORMS }}
# For pull requests, we run the Docker build (to ensure no PR changes break the build),
# but we ONLY do an image push to DockerHub if it's NOT a PR
push: ${{ github.event_name != 'pull_request' }}
# Use tags / labels provided by 'docker/metadata-action' above
tags: ${{ steps.meta_build_deps.outputs.tags }}
labels: ${{ steps.meta_build_deps.outputs.labels }}
#######################################
# Build/Push the 'dspace/dspace' image
#######################################
# Get Metadata for docker_build step below
- name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace' image
id: meta_build
uses: docker/metadata-action@v3
with:
images: dspace/dspace
tags: ${{ env.IMAGE_TAGS }}
flavor: ${{ env.TAGS_FLAVOR }}
- name: Build and push 'dspace' image
id: docker_build
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
platforms: ${{ env.PLATFORMS }}
# For pull requests, we run the Docker build (to ensure no PR changes break the build),
# but we ONLY do an image push to DockerHub if it's NOT a PR
push: ${{ github.event_name != 'pull_request' }}
# Use tags / labels provided by 'docker/metadata-action' above
tags: ${{ steps.meta_build.outputs.tags }}
labels: ${{ steps.meta_build.outputs.labels }}
#####################################################
# Build/Push the 'dspace/dspace' image ('-test' tag)
#####################################################
# Get Metadata for docker_build_test step below
- name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-test' image
id: meta_build_test
uses: docker/metadata-action@v3
with:
images: dspace/dspace
tags: ${{ env.IMAGE_TAGS }}
# As this is a test/development image, its tags are all suffixed with "-test". Otherwise, it uses the same
# tagging logic as the primary 'dspace/dspace' image above.
flavor: ${{ env.TAGS_FLAVOR }}
suffix=-test
- name: Build and push 'dspace-test' image
id: docker_build_test
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile.test
platforms: ${{ env.PLATFORMS }}
# For pull requests, we run the Docker build (to ensure no PR changes break the build),
# but we ONLY do an image push to DockerHub if it's NOT a PR
push: ${{ github.event_name != 'pull_request' }}
# Use tags / labels provided by 'docker/metadata-action' above
tags: ${{ steps.meta_build_test.outputs.tags }}
labels: ${{ steps.meta_build_test.outputs.labels }}
###########################################
# Build/Push the 'dspace/dspace-cli' image
###########################################
# Get Metadata for docker_build_test step below
- name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-cli' image
id: meta_build_cli
uses: docker/metadata-action@v3
with:
images: dspace/dspace-cli
tags: ${{ env.IMAGE_TAGS }}
flavor: ${{ env.TAGS_FLAVOR }}
- name: Build and push 'dspace-cli' image
id: docker_build_cli
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile.cli
platforms: ${{ env.PLATFORMS }}
# For pull requests, we run the Docker build (to ensure no PR changes break the build),
# but we ONLY do an image push to DockerHub if it's NOT a PR
push: ${{ github.event_name != 'pull_request' }}
# Use tags / labels provided by 'docker/metadata-action' above
tags: ${{ steps.meta_build_cli.outputs.tags }}
labels: ${{ steps.meta_build_cli.outputs.labels }}

4
.gitignore vendored
View File

@@ -42,3 +42,7 @@ nb-configuration.xml
##Ignore JRebel project configuration
rebel.xml
## Ignore jenv configuration
.java-version

48
Dockerfile
View File

@@ -1,63 +1,67 @@
# This image will be published as dspace/dspace
# See https://github.com/DSpace/DSpace/tree/main/dspace/src/main/docker for usage details
#
# This version is JDK11 compatible
# - tomcat:8-jdk11
# - ANT 1.10.7
# - maven:3-jdk-11 (see dspace-dependencies)
# - note: default tag for branch: dspace/dspace: dspace/dspace:dspace-7_x
# This Dockerfile uses JDK11 by default, but has also been tested with JDK17.
# To build with JDK17, use "--build-arg JDK_VERSION=17"
ARG JDK_VERSION=11
# Step 1 - Run Maven Build
FROM dspace/dspace-dependencies:dspace-7_x as build
ARG TARGET_DIR=dspace-installer
WORKDIR /app
# The dspace-install directory will be written to /install
# The dspace-installer directory will be written to /install
RUN mkdir /install \
&& chown -Rv dspace: /install \
&& chown -Rv dspace: /app
USER dspace
# Copy the DSpace source code into the workdir (excluding .dockerignore contents)
# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents)
ADD --chown=dspace . /app/
COPY dspace/src/main/docker/local.cfg /app/local.cfg
# Build DSpace (note: this build doesn't include the optional, deprecated "dspace-rest" webapp)
# Copy the dspace-install directory to /install. Clean up the build to keep the docker image small
RUN mvn package && \
# Copy the dspace-installer directory to /install. Clean up the build to keep the docker image small
RUN mvn --no-transfer-progress package && \
mv /app/dspace/target/${TARGET_DIR}/* /install && \
mvn clean
# Step 2 - Run Ant Deploy
FROM tomcat:8-jdk11 as ant_build
FROM openjdk:${JDK_VERSION}-slim as ant_build
ARG TARGET_DIR=dspace-installer
# COPY the /install directory from 'build' container to /dspace-src in this container
COPY --from=build /install /dspace-src
WORKDIR /dspace-src
# Create the initial install deployment using ANT
ENV ANT_VERSION 1.10.7
ENV ANT_VERSION 1.10.12
ENV ANT_HOME /tmp/ant-$ANT_VERSION
ENV PATH $ANT_HOME/bin:$PATH
# Need wget to install ant
RUN apt-get update \
&& apt-get install -y --no-install-recommends wget \
&& apt-get purge -y --auto-remove \
&& rm -rf /var/lib/apt/lists/*
# Download and install 'ant'
RUN mkdir $ANT_HOME && \
wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
# Run necessary 'ant' deploy scripts
RUN ant init_installation update_configs update_code update_webapps
# Step 3 - Run tomcat
# Create a new tomcat image that does not retain the the build directory contents
FROM tomcat:8-jdk11
FROM tomcat:9-jdk${JDK_VERSION}
# NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration.
ENV DSPACE_INSTALL=/dspace
# Copy the /dspace directory from 'ant_build' containger to /dspace in this container
COPY --from=ant_build /dspace $DSPACE_INSTALL
# Expose Tomcat port and AJP port
EXPOSE 8080 8009
# Give java extra memory (2GB)
ENV JAVA_OPTS=-Xmx2000m
# Run the "server" webapp off the /server path (e.g. http://localhost:8080/server/)
# Link the DSpace 'server' webapp into Tomcat's webapps directory.
# This ensures that when we start Tomcat, it runs from /server path (e.g. http://localhost:8080/server/)
RUN ln -s $DSPACE_INSTALL/webapps/server /usr/local/tomcat/webapps/server
# If you wish to run "server" webapp off the ROOT path, then comment out the above RUN, and uncomment the below RUN.
# You also MUST update the URL in dspace/src/main/docker/local.cfg
# You also MUST update the 'dspace.server.url' configuration to match.
# Please note that server webapp should only run on one path at a time.
#RUN mv /usr/local/tomcat/webapps/ROOT /usr/local/tomcat/webapps/ROOT.bk && \
# ln -s $DSPACE_INSTALL/webapps/server /usr/local/tomcat/webapps/ROOT

43
Dockerfile.cli
View File

@@ -1,53 +1,54 @@
# This image will be published as dspace/dspace-cli
# See https://github.com/DSpace/DSpace/tree/main/dspace/src/main/docker for usage details
#
# This version is JDK11 compatible
# - openjdk:11
# - ANT 1.10.7
# - maven:3-jdk-11 (see dspace-dependencies)
# - note: default tag for branch: dspace/dspace-cli: dspace/dspace-cli:dspace-7_x
# This Dockerfile uses JDK11 by default, but has also been tested with JDK17.
# To build with JDK17, use "--build-arg JDK_VERSION=17"
ARG JDK_VERSION=11
# Step 1 - Run Maven Build
FROM dspace/dspace-dependencies:dspace-7_x as build
ARG TARGET_DIR=dspace-installer
WORKDIR /app
# The dspace-install directory will be written to /install
# The dspace-installer directory will be written to /install
RUN mkdir /install \
&& chown -Rv dspace: /install \
&& chown -Rv dspace: /app
USER dspace
# Copy the DSpace source code into the workdir (excluding .dockerignore contents)
# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents)
ADD --chown=dspace . /app/
COPY dspace/src/main/docker/local.cfg /app/local.cfg
# Build DSpace. Copy the dspace-install directory to /install. Clean up the build to keep the docker image small
RUN mvn package && \
# Build DSpace. Copy the dspace-installer directory to /install. Clean up the build to keep the docker image small
RUN mvn --no-transfer-progress package && \
mv /app/dspace/target/${TARGET_DIR}/* /install && \
mvn clean
# Step 2 - Run Ant Deploy
FROM openjdk:11 as ant_build
FROM openjdk:${JDK_VERSION}-slim as ant_build
ARG TARGET_DIR=dspace-installer
# COPY the /install directory from 'build' container to /dspace-src in this container
COPY --from=build /install /dspace-src
WORKDIR /dspace-src
# Create the initial install deployment using ANT
ENV ANT_VERSION 1.10.7
ENV ANT_VERSION 1.10.12
ENV ANT_HOME /tmp/ant-$ANT_VERSION
ENV PATH $ANT_HOME/bin:$PATH
# Need wget to install ant
RUN apt-get update \
&& apt-get install -y --no-install-recommends wget \
&& apt-get purge -y --auto-remove \
&& rm -rf /var/lib/apt/lists/*
# Download and install 'ant'
RUN mkdir $ANT_HOME && \
wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
# Run necessary 'ant' deploy scripts
RUN ant init_installation update_configs update_code
# Step 3 - Run jdk
# Create a new tomcat image that does not retain the the build directory contents
FROM openjdk:11
FROM openjdk:${JDK_VERSION}
# NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration.
ENV DSPACE_INSTALL=/dspace
# Copy the /dspace directory from 'ant_build' container to /dspace in this container
COPY --from=ant_build /dspace $DSPACE_INSTALL
# Give java extra memory (1GB)
ENV JAVA_OPTS=-Xmx1000m

29
Dockerfile.dependencies
View File

@@ -1,27 +1,36 @@
# This image will be published as dspace/dspace-dependencies
# The purpose of this image is to make the build for dspace/dspace run faster
#
# This version is JDK11 compatible
# - maven:3-jdk-11
# This Dockerfile uses JDK11 by default, but has also been tested with JDK17.
# To build with JDK17, use "--build-arg JDK_VERSION=17"
ARG JDK_VERSION=11
# Step 1 - Run Maven Build
FROM maven:3-jdk-11 as build
FROM maven:3-openjdk-${JDK_VERSION}-slim as build
ARG TARGET_DIR=dspace-installer
WORKDIR /app
# Create the 'dspace' user account & home directory
RUN useradd dspace \
&& mkdir /home/dspace \
&& mkdir -p /home/dspace \
&& chown -Rv dspace: /home/dspace
RUN chown -Rv dspace: /app
# Need git to support buildnumber-maven-plugin, which lets us know what version of DSpace is being run.
RUN apt-get update \
&& apt-get install -y --no-install-recommends git \
&& apt-get purge -y --auto-remove \
&& rm -rf /var/lib/apt/lists/*
# Switch to dspace user & run below commands as that user
USER dspace
# Copy the DSpace source code into the workdir (excluding .dockerignore contents)
# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents)
ADD --chown=dspace . /app/
COPY dspace/src/main/docker/local.cfg /app/local.cfg
# Trigger the installation of all maven dependencies
RUN mvn package
# Trigger the installation of all maven dependencies (hide download progress messages)
RUN mvn --no-transfer-progress package
# Clear the contents of the /app directory (including all maven builds), so no artifacts remain.
# This ensures when dspace:dspace is built, it will just the Maven local cache (.m2) for dependencies
# This ensures when dspace:dspace is built, it will use the Maven local cache (~/.m2) for dependencies
USER root
RUN rm -rf /app/*

51
Dockerfile.test
View File

@@ -1,77 +1,80 @@
# This image will be published as dspace/dspace
# See https://github.com/DSpace/DSpace/tree/main/dspace/src/main/docker for usage details
#
# This version is JDK11 compatible
# - tomcat:8-jdk11
# - ANT 1.10.7
# - maven:3-jdk-11 (see dspace-dependencies)
# - note: default tag for branch: dspace/dspace: dspace/dspace:dspace-7_x-test
#
# This image is meant for TESTING/DEVELOPMENT ONLY as it deploys the old v6 REST API under HTTP (not HTTPS)
# This Dockerfile uses JDK11 by default, but has also been tested with JDK17.
# To build with JDK17, use "--build-arg JDK_VERSION=17"
ARG JDK_VERSION=11
# Step 1 - Run Maven Build
FROM dspace/dspace-dependencies:dspace-7_x as build
ARG TARGET_DIR=dspace-installer
WORKDIR /app
# The dspace-install directory will be written to /install
# The dspace-installer directory will be written to /install
RUN mkdir /install \
&& chown -Rv dspace: /install \
&& chown -Rv dspace: /app
USER dspace
# Copy the DSpace source code into the workdir (excluding .dockerignore contents)
# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents)
ADD --chown=dspace . /app/
COPY dspace/src/main/docker/local.cfg /app/local.cfg
# Build DSpace (including the optional, deprecated "dspace-rest" webapp)
# Copy the dspace-install directory to /install. Clean up the build to keep the docker image small
RUN mvn package -Pdspace-rest && \
# Build DSpace (INCLUDING the optional, deprecated "dspace-rest" webapp)
# Copy the dspace-installer directory to /install. Clean up the build to keep the docker image small
RUN mvn --no-transfer-progress package -Pdspace-rest && \
mv /app/dspace/target/${TARGET_DIR}/* /install && \
mvn clean
# Step 2 - Run Ant Deploy
FROM tomcat:8-jdk11 as ant_build
FROM openjdk:${JDK_VERSION}-slim as ant_build
ARG TARGET_DIR=dspace-installer
# COPY the /install directory from 'build' container to /dspace-src in this container
COPY --from=build /install /dspace-src
WORKDIR /dspace-src
# Create the initial install deployment using ANT
ENV ANT_VERSION 1.10.7
ENV ANT_VERSION 1.10.12
ENV ANT_HOME /tmp/ant-$ANT_VERSION
ENV PATH $ANT_HOME/bin:$PATH
# Need wget to install ant
RUN apt-get update \
&& apt-get install -y --no-install-recommends wget \
&& apt-get purge -y --auto-remove \
&& rm -rf /var/lib/apt/lists/*
# Download and install 'ant'
RUN mkdir $ANT_HOME && \
wget -qO- "https://archive.apache.org/dist/ant/binaries/apache-ant-$ANT_VERSION-bin.tar.gz" | tar -zx --strip-components=1 -C $ANT_HOME
# Run necessary 'ant' deploy scripts
RUN ant init_installation update_configs update_code update_webapps
# Step 3 - Run tomcat
# Create a new tomcat image that does not retain the the build directory contents
FROM tomcat:8-jdk11
FROM tomcat:9-jdk${JDK_VERSION}
ENV DSPACE_INSTALL=/dspace
ENV TOMCAT_INSTALL=/usr/local/tomcat
# Copy the /dspace directory from 'ant_build' containger to /dspace in this container
COPY --from=ant_build /dspace $DSPACE_INSTALL
# Enable the AJP connector in Tomcat's server.xml
# NOTE: secretRequired="false" should only be used when AJP is NOT accessible from an external network. But, secretRequired="true" isn't supported by mod_proxy_ajp until Apache 2.5
RUN sed -i '/Service name="Catalina".*/a \\n <Connector protocol="AJP/1.3" port="8009" address="0.0.0.0" redirectPort="8443" URIEncoding="UTF-8" secretRequired="false" />' $TOMCAT_INSTALL/conf/server.xml
# Expose Tomcat port and AJP port
EXPOSE 8080 8009
# Give java extra memory (2GB)
ENV JAVA_OPTS=-Xmx2000m
# Run the "server" webapp off the /server path (e.g. http://localhost:8080/server/)
# and the v6.x (deprecated) REST API off the "/rest" path
# Link the DSpace 'server' webapp into Tomcat's webapps directory.
# This ensures that when we start Tomcat, it runs from /server path (e.g. http://localhost:8080/server/)
# Also link the v6.x (deprecated) REST API off the "/rest" path
RUN ln -s $DSPACE_INSTALL/webapps/server /usr/local/tomcat/webapps/server && \
ln -s $DSPACE_INSTALL/webapps/rest /usr/local/tomcat/webapps/rest
# If you wish to run "server" webapp off the ROOT path, then comment out the above RUN, and uncomment the below RUN.
# You also MUST update the URL in dspace/src/main/docker/local.cfg
# You also MUST update the 'dspace.server.url' configuration to match.
# Please note that server webapp should only run on one path at a time.
#RUN mv /usr/local/tomcat/webapps/ROOT /usr/local/tomcat/webapps/ROOT.bk && \
# ln -s $DSPACE_INSTALL/webapps/server /usr/local/tomcat/webapps/ROOT && \
# ln -s $DSPACE_INSTALL/webapps/rest /usr/local/tomcat/webapps/rest
# Overwrite the v6.x (deprecated) REST API's web.xml, so that we can run it on HTTP (defaults to requiring HTTPS)
# WARNING: THIS IS OBVIOUSLY INSECURE. NEVER DO THIS IN PRODUCTION.
COPY dspace/src/main/docker/test/rest_web.xml $DSPACE_INSTALL/webapps/rest/WEB-INF/web.xml
RUN sed -i -e "s|\${dspace.dir}|$DSPACE_INSTALL|" $DSPACE_INSTALL/webapps/rest/WEB-INF/web.xml

20
LICENSE
View File

@@ -1,4 +1,4 @@
DSpace source code BSD License:
BSD 3-Clause License
Copyright (c) 2002-2021, LYRASIS. All rights reserved.
@@ -13,13 +13,12 @@ notice, this list of conditions and the following disclaimer.
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
- Neither the name DuraSpace nor the name of the DSpace Foundation
nor the names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.
- Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
@@ -29,11 +28,4 @@ OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
DAMAGE.
DSpace uses third-party libraries which may be distributed under
different licenses to the above. Information about these licenses
is detailed in the LICENSES_THIRD_PARTY file at the root of the source
tree. You must agree to the terms of these licenses, in addition to
the above DSpace source code license, in order to use this software.
DAMAGE.

0
LICENSE_HEADER → LICENSE.header
View File

810
LICENSES_THIRD_PARTY
View File

File diff suppressed because it is too large Load Diff

10
NOTICE
View File

@@ -1,3 +1,13 @@
Licenses of Third-Party Libraries
=================================
DSpace uses third-party libraries which may be distributed under
different licenses than specified in our LICENSE file. Information
about these licenses is detailed in the LICENSES_THIRD_PARTY file at
the root of the source tree. You must agree to the terms of these
licenses, in addition to the DSpace source code license, in order to
use this software.
Licensing Notices
=================

5
README.md
View File

@@ -35,7 +35,7 @@ Documentation for each release may be viewed online or downloaded via our [Docum
The latest DSpace Installation instructions are available at:
https://wiki.lyrasis.org/display/DSDOC7x/Installing+DSpace
Please be aware that, as a Java web application, DSpace requires a database (PostgreSQL or Oracle)
Please be aware that, as a Java web application, DSpace requires a database (PostgreSQL)
and a servlet container (usually Tomcat) in order to function.
More information about these and all other prerequisites can be found in the Installation instructions above.
@@ -136,3 +136,6 @@ run automatically by [GitHub Actions](https://github.com/DSpace/DSpace/actions?q
DSpace source code is freely available under a standard [BSD 3-Clause license](https://opensource.org/licenses/BSD-3-Clause).
The full license is available in the [LICENSE](LICENSE) file or online at http://www.dspace.org/license/
DSpace uses third-party libraries which may be distributed under different licenses. Those licenses are listed
in the [LICENSES_THIRD_PARTY](LICENSES_THIRD_PARTY) file.

17
docker-compose-cli.yml
View File

@@ -7,10 +7,23 @@ services:
build:
context: .
dockerfile: Dockerfile.cli
#environment:
environment:
# Below syntax may look odd, but it is how to override dspace.cfg settings via env variables.
# See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml
# __P__ => "." (e.g. dspace__P__dir => dspace.dir)
# __D__ => "-" (e.g. google__D__metadata => google-metadata)
# dspace.dir: Must match with Dockerfile's DSPACE_INSTALL directory.
dspace__P__dir: /dspace
# db.url: Ensure we are using the 'dspacedb' image for our database
db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace'
# solr.server: Ensure we are using the 'dspacesolr' image for Solr
solr__P__server: http://dspacesolr:8983/solr
volumes:
- ./dspace/src/main/docker-compose/local.cfg:/dspace/config/local.cfg
# Keep DSpace assetstore directory between reboots
- assetstore:/dspace/assetstore
# Mount local [src]/dspace/config/ to container. This syncs your local configs with container
# NOTE: Environment variables specified above will OVERRIDE any configs in local.cfg or dspace.cfg
- ./dspace/config:/dspace/config
entrypoint: /dspace/bin/dspace
command: help
networks:

34
docker-compose.yml
View File

@@ -4,12 +4,30 @@ networks:
ipam:
config:
# Define a custom subnet for our DSpace network, so that we can easily trust requests from host to container.
# If you customize this value, be sure to customize the 'proxies.trusted.ipranges' in your local.cfg.
# If you customize this value, be sure to customize the 'proxies.trusted.ipranges' env variable below.
- subnet: 172.23.0.0/16
services:
# DSpace (backend) webapp container
dspace:
container_name: dspace
environment:
# Below syntax may look odd, but it is how to override dspace.cfg settings via env variables.
# See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml
# __P__ => "." (e.g. dspace__P__dir => dspace.dir)
# __D__ => "-" (e.g. google__D__metadata => google-metadata)
# dspace.dir: Must match with Dockerfile's DSPACE_INSTALL directory.
dspace__P__dir: /dspace
# Uncomment to set a non-default value for dspace.server.url or dspace.ui.url
# dspace__P__server__P__url: http://localhost:8080/server
# dspace__P__ui__P__url: http://localhost:4000
dspace__P__name: 'DSpace Started with Docker Compose'
# db.url: Ensure we are using the 'dspacedb' image for our database
db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace'
# solr.server: Ensure we are using the 'dspacesolr' image for Solr
solr__P__server: http://dspacesolr:8983/solr
# proxies.trusted.ipranges: This setting is required for a REST API running in Docker to trust requests
# from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above.
proxies__P__trusted__P__ipranges: '172.23.0'
image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}"
build:
context: .
@@ -26,8 +44,11 @@ services:
stdin_open: true
tty: true
volumes:
# Keep DSpace assetstore directory between reboots
- assetstore:/dspace/assetstore
- ./dspace/src/main/docker-compose/local.cfg:/dspace/config/local.cfg
# Mount local [src]/dspace/config/ to container. This syncs your local configs with container
# NOTE: Environment variables specified above will OVERRIDE any configs in local.cfg or dspace.cfg
- ./dspace/config:/dspace/config
# Ensure that the database is ready BEFORE starting tomcat
# 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep
# 2. Then, run database migration to init database tables
@@ -59,7 +80,7 @@ services:
dspacesolr:
container_name: dspacesolr
# Uses official Solr image at https://hub.docker.com/_/solr/
image: solr:8.8
image: solr:8.11-slim
networks:
dspacenet:
ports:
@@ -77,15 +98,22 @@ services:
# Keep Solr data directory between reboots
- solr_data:/var/solr/data
# Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr
# * First, run precreate-core to create the core (if it doesn't yet exist). If exists already, this is a no-op
# * Second, copy updated configs from mounted configsets to this core. If it already existed, this updates core
# to the latest configs. If it's a newly created core, this is a no-op.
entrypoint:
- /bin/bash
- '-c'
- |
init-var-solr
precreate-core authority /opt/solr/server/solr/configsets/authority
cp -r -u /opt/solr/server/solr/configsets/authority/* authority
precreate-core oai /opt/solr/server/solr/configsets/oai
cp -r -u /opt/solr/server/solr/configsets/oai/* oai
precreate-core search /opt/solr/server/solr/configsets/search
cp -r -u /opt/solr/server/solr/configsets/search/* search
precreate-core statistics /opt/solr/server/solr/configsets/statistics
cp -r -u /opt/solr/server/solr/configsets/statistics/* statistics
exec solr -f
volumes:
assetstore:

298
dspace-api/pom.xml
View File

@@ -12,7 +12,7 @@
<parent>
<groupId>org.dspace</groupId>
<artifactId>dspace-parent</artifactId>
<version>7.0</version>
<version>7.4-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
@@ -334,18 +334,47 @@
</profiles>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-ehcache</artifactId>
<artifactId>hibernate-core</artifactId>
<exclusions>
<!-- Newer version pulled in via Jersey below -->
<exclusion>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-jcache</artifactId>
</dependency>
<dependency>
<groupId>org.ehcache</groupId>
<artifactId>ehcache</artifactId>
<version>${ehcache.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-cache
Caching dependencies for sherpa service. -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
<version>${spring-boot.version}</version>
<exclusions>
<!-- Newer version pulled in via Jersey below -->
<exclusion>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>javax.cache</groupId>
<artifactId>cache-api</artifactId>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-jpamodelgen</artifactId>
@@ -358,7 +387,7 @@
<dependency>
<groupId>org.hibernate.javax.persistence</groupId>
<artifactId>hibernate-jpa-2.1-api</artifactId>
<version>1.0.0.Final</version>
<version>1.0.2.Final</version>
</dependency>
<dependency>
@@ -379,7 +408,7 @@
<groupId>org.ow2.asm</groupId>
<artifactId>asm-commons</artifactId>
</exclusion>
<!-- Newer version of Bouncycastle brought in via solr-cell -->
<!-- Newer version of Bouncycastle brought in via Tika -->
<exclusion>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
@@ -481,8 +510,8 @@
<artifactId>commons-validator</artifactId>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
@@ -505,7 +534,7 @@
</dependency>
<dependency>
<groupId>org.jdom</groupId>
<artifactId>jdom</artifactId>
<artifactId>jdom2</artifactId>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
@@ -515,26 +544,21 @@
<groupId>org.apache.pdfbox</groupId>
<artifactId>fontbox</artifactId>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
</dependency>
<dependency>
<groupId>xalan</groupId>
<artifactId>xalan</artifactId>
</dependency>
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
</dependency>
<!-- Codebase at https://github.com/OCLC-Research/oaiharvester2/ -->
<dependency>
<groupId>org.dspace</groupId>
<artifactId>oclc-harvester2</artifactId>
</dependency>
<!-- Xalan is REQUIRED by 'oclc-harvester2' listed above (OAI harvesting fails without it).
Please do NOT use Xalan in DSpace codebase as it is not well maintained. -->
<dependency>
<groupId>xalan</groupId>
<artifactId>xalan</artifactId>
</dependency>
<dependency>
<groupId>org.dspace</groupId>
<artifactId>dspace-services</artifactId>
@@ -558,13 +582,6 @@
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
<exclusions>
<!-- Different version provided by hibernate-ehcache -->
<exclusion>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
@@ -573,9 +590,12 @@
</dependency>
<!-- Used for RSS / ATOM syndication feeds -->
<dependency>
<groupId>org.rometools</groupId>
<groupId>com.rometools</groupId>
<artifactId>rome</artifactId>
</dependency>
<dependency>
<groupId>com.rometools</groupId>
<artifactId>rome-modules</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>org.jbibtex</groupId>
@@ -586,28 +606,22 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
</dependency>
<!-- SolrJ is used to communicate with Solr throughout the dspace-api -->
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr.client.version}</version>
<exclusions>
<!-- Newer Jetty version brought in via Parent POM -->
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-http</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Solr Core is needed for Integration Tests (to run a MockSolrServer) -->
<!-- Solr Core is only needed for Integration Tests (to run a MockSolrServer) -->
<!-- The following Solr / Lucene dependencies also support integration tests -->
<dependency>
<groupId>org.apache.solr</groupId>
@@ -635,39 +649,10 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-cell</artifactId>
<exclusions>
<!-- Newer version brought in by opencsv -->
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
</exclusion>
<!-- Newer Jetty version brought in via Parent POM -->
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-http</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
<!-- Used for full-text indexing with Solr -->
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parsers</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-icu</artifactId>
@@ -683,9 +668,15 @@
<artifactId>lucene-analyzers-stempel</artifactId>
<scope>test</scope>
</dependency>
<!-- Tika is used to extract full text from documents in order to index in Solr -->
<dependency>
<groupId>org.apache.xmlbeans</groupId>
<artifactId>xmlbeans</artifactId>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parsers-standard-package</artifactId>
</dependency>
<dependency>
@@ -709,13 +700,6 @@
<version>1.1.1</version>
</dependency>
<!-- Gson: Java to Json conversion -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@@ -743,7 +727,7 @@
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
<version>6.5.5</version>
<version>8.4.4</version>
</dependency>
<!-- Google Analytics -->
@@ -799,44 +783,6 @@
<artifactId>jaxb-runtime</artifactId>
</dependency>
<!-- Apache Axiom -->
<dependency>
<groupId>org.apache.ws.commons.axiom</groupId>
<artifactId>axiom-impl</artifactId>
<version>${axiom.version}</version>
<exclusions>
<!-- Exclude Geronimo as it is NOT necessary when using javax.activation (which we use)
See: https://ws.apache.org/axiom/userguide/ch04.html#d0e732 -->
<exclusion>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>*</artifactId>
</exclusion>
<!-- Exclude Woodstox, as later version provided by Solr dependencies -->
<exclusion>
<groupId>org.codehaus.woodstox</groupId>
<artifactId>woodstox-core-asl</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.ws.commons.axiom</groupId>
<artifactId>axiom-api</artifactId>
<version>${axiom.version}</version>
<exclusions>
<!-- Exclude Geronimo as it is NOT necessary when using javax.activation (which we use)
See: https://ws.apache.org/axiom/userguide/ch04.html#d0e732 -->
<exclusion>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>*</artifactId>
</exclusion>
<!-- Exclude Woodstox, as later version provided by Solr dependencies -->
<exclusion>
<groupId>org.codehaus.woodstox</groupId>
<artifactId>woodstox-core-asl</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Jersey / JAX-RS client (javax.ws.rs.*) dependencies needed to integrate with external sources/services -->
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
@@ -855,7 +801,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.10.50</version>
<version>1.12.116</version>
</dependency>
<dependency>
@@ -892,6 +838,14 @@
<version>20180130</version>
</dependency>
<!-- Useful for testing command-line tools -->
<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-rules</artifactId>
<version>1.19.0</version>
<scope>test</scope>
</dependency>
<!-- Used for Solr core export/import -->
<dependency>
<groupId>com.opencsv</groupId>
@@ -903,14 +857,11 @@
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
<version>2.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.xmlunit</groupId>
<artifactId>xmlunit-core</artifactId>
<version>2.6.3</version>
<scope>test</scope>
</dependency>
@@ -920,6 +871,95 @@
<version>6.4.0</version>
</dependency>
<!-- required for openaire api integration -->
<dependency>
<groupId>eu.openaire</groupId>
<artifactId>funders-model</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-junit-rule</artifactId>
<version>5.11.2</version>
<scope>test</scope>
<exclusions>
<!-- Exclude snakeyaml to avoid conflicts with: spring-boot-starter-cache -->
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<!-- for mockserver -->
<!-- Solve dependency convergence issues related to
'mockserver-junit-rule' by selecting the versions we want to use. -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>4.1.68.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>4.1.68.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.68.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.68.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.68.Final</version>
</dependency>
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.xmlunit</groupId>
<artifactId>xmlunit-core</artifactId>
<version>2.8.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.java-json-tools</groupId>
<artifactId>json-schema-validator</artifactId>
<version>2.2.14</version>
</dependency>
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<version>2.3.3</version>
</dependency>
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.1.Final</version>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-core</artifactId>
<version>1.6.2</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

30
dspace-api/src/main/java/org/dspace/access/status/AccessStatusHelper.java Normal file
View File

@@ -0,0 +1,30 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.access.status;
import java.sql.SQLException;
import java.util.Date;
import org.dspace.content.Item;
import org.dspace.core.Context;
/**
* Plugin interface for the access status calculation.
*/
public interface AccessStatusHelper {
/**
* Calculate the access status for the item.
*
* @param context the DSpace context
* @param item the item
* @return an access status value
* @throws SQLException An exception that provides information on a database access error or other errors.
*/
public String getAccessStatusFromItem(Context context, Item item, Date threshold)
throws SQLException;
}

66
dspace-api/src/main/java/org/dspace/access/status/AccessStatusServiceImpl.java Normal file
View File

@@ -0,0 +1,66 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.access.status;
import java.sql.SQLException;
import java.util.Date;
import org.dspace.access.status.service.AccessStatusService;
import org.dspace.content.Item;
import org.dspace.core.Context;
import org.dspace.core.service.PluginService;
import org.dspace.services.ConfigurationService;
import org.joda.time.LocalDate;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Implementation for the access status calculation service.
*/
public class AccessStatusServiceImpl implements AccessStatusService {
// Plugin implementation, set from the DSpace configuration by init().
protected AccessStatusHelper helper = null;
protected Date forever_date = null;
@Autowired(required = true)
protected ConfigurationService configurationService;
@Autowired(required = true)
protected PluginService pluginService;
/**
* Initialize the bean (after dependency injection has already taken place).
* Ensures the configurationService is injected, so that we can get the plugin
* and the forever embargo date threshold from the configuration.
* Called by "init-method" in Spring configuration.
*
* @throws Exception on generic exception
*/
public void init() throws Exception {
if (helper == null) {
helper = (AccessStatusHelper) pluginService.getSinglePlugin(AccessStatusHelper.class);
if (helper == null) {
throw new IllegalStateException("The AccessStatusHelper plugin was not defined in "
+ "DSpace configuration.");
}
// Defines the embargo forever date threshold for the access status.
// Look at EmbargoService.FOREVER for some improvements?
int year = configurationService.getIntProperty("access.status.embargo.forever.year");
int month = configurationService.getIntProperty("access.status.embargo.forever.month");
int day = configurationService.getIntProperty("access.status.embargo.forever.day");
forever_date = new LocalDate(year, month, day).toDate();
}
}
@Override
public String getAccessStatus(Context context, Item item) throws SQLException {
return helper.getAccessStatusFromItem(context, item, forever_date);
}
}

159
dspace-api/src/main/java/org/dspace/access/status/DefaultAccessStatusHelper.java Normal file
View File

@@ -0,0 +1,159 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.access.status;
import java.sql.SQLException;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.dspace.authorize.ResourcePolicy;
import org.dspace.authorize.factory.AuthorizeServiceFactory;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.authorize.service.ResourcePolicyService;
import org.dspace.content.Bitstream;
import org.dspace.content.Bundle;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.eperson.Group;
/**
* Default plugin implementation of the access status helper.
* The getAccessStatusFromItem method provides a simple logic to
* calculate the access status of an item based on the policies of
* the primary or the first bitstream in the original bundle.
* Users can override this method for enhanced functionality.
*/
public class DefaultAccessStatusHelper implements AccessStatusHelper {
public static final String EMBARGO = "embargo";
public static final String METADATA_ONLY = "metadata.only";
public static final String OPEN_ACCESS = "open.access";
public static final String RESTRICTED = "restricted";
public static final String UNKNOWN = "unknown";
protected ItemService itemService =
ContentServiceFactory.getInstance().getItemService();
protected ResourcePolicyService resourcePolicyService =
AuthorizeServiceFactory.getInstance().getResourcePolicyService();
protected AuthorizeService authorizeService =
AuthorizeServiceFactory.getInstance().getAuthorizeService();
public DefaultAccessStatusHelper() {
super();
}
/**
* Look at the item's policies to determine an access status value.
* It is also considering a date threshold for embargos and restrictions.
*
* If the item is null, simply returns the "unknown" value.
*
* @param context the DSpace context
* @param item the item to embargo
* @param threshold the embargo threshold date
* @return an access status value
*/
@Override
public String getAccessStatusFromItem(Context context, Item item, Date threshold)
throws SQLException {
if (item == null) {
return UNKNOWN;
}
// Consider only the original bundles.
List<Bundle> bundles = item.getBundles(Constants.DEFAULT_BUNDLE_NAME);
// Check for primary bitstreams first.
Bitstream bitstream = bundles.stream()
.map(bundle -> bundle.getPrimaryBitstream())
.filter(Objects::nonNull)
.findFirst()
.orElse(null);
if (bitstream == null) {
// If there is no primary bitstream,
// take the first bitstream in the bundles.
bitstream = bundles.stream()
.map(bundle -> bundle.getBitstreams())
.flatMap(List::stream)
.findFirst()
.orElse(null);
}
return caculateAccessStatusForDso(context, bitstream, threshold);
}
/**
* Look at the DSpace object's policies to determine an access status value.
*
* If the object is null, returns the "metadata.only" value.
* If any policy attached to the object is valid for the anonymous group,
* returns the "open.access" value.
* Otherwise, if the policy start date is before the embargo threshold date,
* returns the "embargo" value.
* Every other cases return the "restricted" value.
*
* @param context the DSpace context
* @param dso the DSpace object
* @param threshold the embargo threshold date
* @return an access status value
*/
private String caculateAccessStatusForDso(Context context, DSpaceObject dso, Date threshold)
throws SQLException {
if (dso == null) {
return METADATA_ONLY;
}
// Only consider read policies.
List<ResourcePolicy> policies = authorizeService
.getPoliciesActionFilter(context, dso, Constants.READ);
int openAccessCount = 0;
int embargoCount = 0;
int restrictedCount = 0;
int unknownCount = 0;
// Looks at all read policies.
for (ResourcePolicy policy : policies) {
boolean isValid = resourcePolicyService.isDateValid(policy);
Group group = policy.getGroup();
// The group must not be null here. However,
// if it is, consider this as an unexpected case.
if (group == null) {
unknownCount++;
} else if (StringUtils.equals(group.getName(), Group.ANONYMOUS)) {
// Only calculate the status for the anonymous group.
if (isValid) {
// If the policy is valid, the anonymous group have access
// to the bitstream.
openAccessCount++;
} else {
Date startDate = policy.getStartDate();
if (startDate != null && !startDate.before(threshold)) {
// If the policy start date have a value and if this value
// is equal or superior to the configured forever date, the
// access status is also restricted.
restrictedCount++;
} else {
// If the current date is not between the policy start date
// and end date, the access status is embargo.
embargoCount++;
}
}
}
}
if (openAccessCount > 0) {
return OPEN_ACCESS;
}
if (embargoCount > 0 && restrictedCount == 0) {
return EMBARGO;
}
if (unknownCount > 0) {
return UNKNOWN;
}
return RESTRICTED;
}
}

25
dspace-api/src/main/java/org/dspace/access/status/factory/AccessStatusServiceFactory.java Normal file
View File

@@ -0,0 +1,25 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.access.status.factory;
import org.dspace.access.status.service.AccessStatusService;
import org.dspace.services.factory.DSpaceServicesFactory;
/**
* Abstract factory to get services for the access status package,
* use AccessStatusServiceFactory.getInstance() to retrieve an implementation.
*/
public abstract class AccessStatusServiceFactory {
public abstract AccessStatusService getAccessStatusService();
public static AccessStatusServiceFactory getInstance() {
return DSpaceServicesFactory.getInstance().getServiceManager()
.getServiceByName("accessStatusServiceFactory", AccessStatusServiceFactory.class);
}
}

26
dspace-api/src/main/java/org/dspace/access/status/factory/AccessStatusServiceFactoryImpl.java Normal file
View File

@@ -0,0 +1,26 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.access.status.factory;
import org.dspace.access.status.service.AccessStatusService;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Factory implementation to get services for the access status package,
* use AccessStatusServiceFactory.getInstance() to retrieve an implementation.
*/
public class AccessStatusServiceFactoryImpl extends AccessStatusServiceFactory {
@Autowired(required = true)
private AccessStatusService accessStatusService;
@Override
public AccessStatusService getAccessStatusService() {
return accessStatusService;
}
}

30
dspace-api/src/main/java/org/dspace/access/status/package-info.java Normal file
View File

@@ -0,0 +1,30 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
/**
* <p>
* Access status allows the users to view the bitstreams availability before
* browsing into the item itself.
* </p>
* <p>
* The access status is calculated through a pluggable class:
* {@link org.dspace.access.status.AccessStatusHelper}.
* The {@link org.dspace.access.status.AccessStatusServiceImpl}
* must be configured to specify this class, as well as a forever embargo date
* threshold year, month and day.
* </p>
* <p>
* See {@link org.dspace.access.status.DefaultAccessStatusHelper} for a simple calculation
* based on the primary or the first bitstream of the original bundle. You can
* supply your own class to implement more complex access statuses.
* </p>
* <p>
* For now, the access status is calculated when the item is shown in a list.
* </p>
*/
package org.dspace.access.status;

46
dspace-api/src/main/java/org/dspace/access/status/service/AccessStatusService.java Normal file
View File

@@ -0,0 +1,46 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.access.status.service;
import java.sql.SQLException;
import org.dspace.content.Item;
import org.dspace.core.Context;
/**
* Public interface to the access status subsystem.
* <p>
* Configuration properties: (with examples)
* {@code
* # values for the forever embargo date threshold
* # This threshold date is used in the default access status helper to dermine if an item is
* # restricted or embargoed based on the start date of the primary (or first) file policies.
* # In this case, if the policy start date is inferior to the threshold date, the status will
* # be embargo, else it will be restricted.
* # You might want to change this threshold based on your needs. For example: some databases
* # doesn't accept a date superior to 31 december 9999.
* access.status.embargo.forever.year = 10000
* access.status.embargo.forever.month = 1
* access.status.embargo.forever.day = 1
* # implementation of access status helper plugin - replace with local implementation if applicable
* # This default access status helper provides an item status based on the policies of the primary
* # bitstream (or first bitstream in the original bundles if no primary file is specified).
* plugin.single.org.dspace.access.status.AccessStatusHelper = org.dspace.access.status.DefaultAccessStatusHelper
* }
*/
public interface AccessStatusService {
/**
* Calculate the access status for an Item while considering the forever embargo date threshold.
*
* @param context the DSpace context
* @param item the item
* @throws SQLException An exception that provides information on a database access error or other errors.
*/
public String getAccessStatus(Context context, Item item) throws SQLException;
}

43
dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java
View File

@@ -11,13 +11,16 @@ import java.io.IOException;
import java.sql.SQLException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.xpath.XPathAPI;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.MetadataField;
import org.dspace.content.MetadataSchema;
@@ -81,7 +84,7 @@ public class MetadataImporter {
* @throws SQLException if database error
* @throws IOException if IO error
* @throws TransformerException if transformer error
* @throws ParserConfigurationException if config error
* @throws ParserConfigurationException if configuration error
* @throws AuthorizeException if authorization error
* @throws SAXException if parser error
* @throws NonUniqueMetadataException if duplicate metadata
@@ -90,8 +93,7 @@ public class MetadataImporter {
public static void main(String[] args)
throws ParseException, SQLException, IOException, TransformerException,
ParserConfigurationException, AuthorizeException, SAXException,
NonUniqueMetadataException, RegistryImportException {
boolean forceUpdate = false;
NonUniqueMetadataException, RegistryImportException, XPathExpressionException {
// create an options object and populate it
CommandLineParser parser = new DefaultParser();
@@ -100,16 +102,14 @@ public class MetadataImporter {
options.addOption("u", "update", false, "update an existing schema");
CommandLine line = parser.parse(options, args);
String file = null;
if (line.hasOption('f')) {
file = line.getOptionValue('f');
String file = line.getOptionValue('f');
boolean forceUpdate = line.hasOption('u');
loadRegistry(file, forceUpdate);
} else {
usage();
System.exit(0);
System.exit(1);
}
forceUpdate = line.hasOption('u');
loadRegistry(file, forceUpdate);
}
/**
@@ -120,15 +120,15 @@ public class MetadataImporter {
* @throws SQLException if database error
* @throws IOException if IO error
* @throws TransformerException if transformer error
* @throws ParserConfigurationException if config error
* @throws ParserConfigurationException if configuration error
* @throws AuthorizeException if authorization error
* @throws SAXException if parser error
* @throws NonUniqueMetadataException if duplicate metadata
* @throws RegistryImportException if import fails
*/
public static void loadRegistry(String file, boolean forceUpdate)
throws SQLException, IOException, TransformerException, ParserConfigurationException,
AuthorizeException, SAXException, NonUniqueMetadataException, RegistryImportException {
throws SQLException, IOException, TransformerException, ParserConfigurationException, AuthorizeException,
SAXException, NonUniqueMetadataException, RegistryImportException, XPathExpressionException {
Context context = null;
try {
@@ -140,7 +140,9 @@ public class MetadataImporter {
Document document = RegistryImporter.loadXML(file);
// Get the nodes corresponding to types
NodeList schemaNodes = XPathAPI.selectNodeList(document, "/dspace-dc-types/dc-schema");
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList schemaNodes = (NodeList) xPath.compile("/dspace-dc-types/dc-schema")
.evaluate(document, XPathConstants.NODESET);
// Add each one as a new format to the registry
for (int i = 0; i < schemaNodes.getLength(); i++) {
@@ -149,7 +151,8 @@ public class MetadataImporter {
}
// Get the nodes corresponding to types
NodeList typeNodes = XPathAPI.selectNodeList(document, "/dspace-dc-types/dc-type");
NodeList typeNodes = (NodeList) xPath.compile("/dspace-dc-types/dc-type")
.evaluate(document, XPathConstants.NODESET);
// Add each one as a new format to the registry
for (int i = 0; i < typeNodes.getLength(); i++) {
@@ -181,8 +184,8 @@ public class MetadataImporter {
* @throws RegistryImportException if import fails
*/
private static void loadSchema(Context context, Node node, boolean updateExisting)
throws SQLException, IOException, TransformerException,
AuthorizeException, NonUniqueMetadataException, RegistryImportException {
throws SQLException, AuthorizeException, NonUniqueMetadataException, RegistryImportException,
XPathExpressionException {
// Get the values
String name = RegistryImporter.getElementData(node, "name");
String namespace = RegistryImporter.getElementData(node, "namespace");
@@ -227,7 +230,7 @@ public class MetadataImporter {
/**
* Process a node in the metadata registry XML file. The node must
* be a "dc-type" node. If the type already exists, then it
* will not be reimported
* will not be re-imported.
*
* @param context DSpace context object
* @param node the node in the DOM tree
@@ -239,8 +242,8 @@ public class MetadataImporter {
* @throws RegistryImportException if import fails
*/
private static void loadType(Context context, Node node)
throws SQLException, IOException, TransformerException,
AuthorizeException, NonUniqueMetadataException, RegistryImportException {
throws SQLException, IOException, AuthorizeException, NonUniqueMetadataException, RegistryImportException,
XPathExpressionException {
// Get the values
String schema = RegistryImporter.getElementData(node, "schema");
String element = RegistryImporter.getElementData(node, "element");

15
dspace-api/src/main/java/org/dspace/administer/RegistryImporter.java
View File

@@ -13,8 +13,11 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.xpath.XPathAPI;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@@ -72,9 +75,10 @@ public class RegistryImporter {
* @throws TransformerException if error
*/
public static String getElementData(Node parentElement, String childName)
throws TransformerException {
throws XPathExpressionException {
// Grab the child node
Node childNode = XPathAPI.selectSingleNode(parentElement, childName);
XPath xPath = XPathFactory.newInstance().newXPath();
Node childNode = (Node) xPath.compile(childName).evaluate(parentElement, XPathConstants.NODE);
if (childNode == null) {
// No child node, so no values
@@ -115,9 +119,10 @@ public class RegistryImporter {
* @throws TransformerException if error
*/
public static String[] getRepeatedElementData(Node parentElement,
String childName) throws TransformerException {
String childName) throws XPathExpressionException {
// Grab the child node
NodeList childNodes = XPathAPI.selectNodeList(parentElement, childName);
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList childNodes = (NodeList) xPath.compile(childName).evaluate(parentElement, XPathConstants.NODESET);
String[] data = new String[childNodes.getLength()];

31
dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java
View File

@@ -16,15 +16,18 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.logging.log4j.Logger;
import org.apache.xpath.XPathAPI;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.BitstreamFormat;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.BitstreamFormatService;
import org.dspace.core.Context;
import org.dspace.core.LogManager;
import org.dspace.core.LogHelper;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@@ -95,7 +98,7 @@ public class RegistryLoader {
System.exit(1);
} catch (Exception e) {
log.fatal(LogManager.getHeader(context, "error_loading_registries",
log.fatal(LogHelper.getHeader(context, "error_loading_registries",
""), e);
System.err.println("Error: \n - " + e.getMessage());
@@ -122,12 +125,13 @@ public class RegistryLoader {
*/
public static void loadBitstreamFormats(Context context, String filename)
throws SQLException, IOException, ParserConfigurationException,
SAXException, TransformerException, AuthorizeException {
SAXException, TransformerException, AuthorizeException, XPathExpressionException {
Document document = loadXML(filename);
// Get the nodes corresponding to formats
NodeList typeNodes = XPathAPI.selectNodeList(document,
"dspace-bitstream-types/bitstream-type");
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList typeNodes = (NodeList) xPath.compile("dspace-bitstream-types/bitstream-type")
.evaluate(document, XPathConstants.NODESET);
// Add each one as a new format to the registry
for (int i = 0; i < typeNodes.getLength(); i++) {
@@ -135,7 +139,7 @@ public class RegistryLoader {
loadFormat(context, n);
}
log.info(LogManager.getHeader(context, "load_bitstream_formats",
log.info(LogHelper.getHeader(context, "load_bitstream_formats",
"number_loaded=" + typeNodes.getLength()));
}
@@ -151,8 +155,7 @@ public class RegistryLoader {
* @throws AuthorizeException if authorization error
*/
private static void loadFormat(Context context, Node node)
throws SQLException, IOException, TransformerException,
AuthorizeException {
throws SQLException, AuthorizeException, XPathExpressionException {
// Get the values
String mimeType = getElementData(node, "mimetype");
String shortDesc = getElementData(node, "short_description");
@@ -231,9 +234,10 @@ public class RegistryLoader {
* @throws TransformerException if transformer error
*/
private static String getElementData(Node parentElement, String childName)
throws TransformerException {
throws XPathExpressionException {
// Grab the child node
Node childNode = XPathAPI.selectSingleNode(parentElement, childName);
XPath xPath = XPathFactory.newInstance().newXPath();
Node childNode = (Node) xPath.compile(childName).evaluate(parentElement, XPathConstants.NODE);
if (childNode == null) {
// No child node, so no values
@@ -274,9 +278,10 @@ public class RegistryLoader {
* @throws TransformerException if transformer error
*/
private static String[] getRepeatedElementData(Node parentElement,
String childName) throws TransformerException {
String childName) throws XPathExpressionException {
// Grab the child node
NodeList childNodes = XPathAPI.selectNodeList(parentElement, childName);
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList childNodes = (NodeList) xPath.compile(childName).evaluate(parentElement, XPathConstants.NODESET);
String[] data = new String[childNodes.getLength()];

64
dspace-api/src/main/java/org/dspace/administer/StructBuilder.java
View File

@@ -30,6 +30,10 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -38,7 +42,6 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.xpath.XPathAPI;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Collection;
import org.dspace.content.Community;
@@ -52,9 +55,9 @@ import org.dspace.content.service.CommunityService;
import org.dspace.core.Context;
import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.eperson.service.EPersonService;
import org.jdom.Element;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;
import org.jdom2.Element;
import org.jdom2.output.Format;
import org.jdom2.output.XMLOutputter;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@@ -137,8 +140,8 @@ public class StructBuilder {
* @throws TransformerException if the input document is invalid.
*/
public static void main(String[] argv)
throws ParserConfigurationException, SQLException,
FileNotFoundException, IOException, TransformerException {
throws ParserConfigurationException, SQLException,
IOException, TransformerException, XPathExpressionException {
// Define command line options.
Options options = new Options();
@@ -240,7 +243,7 @@ public class StructBuilder {
* @throws SQLException
*/
static void importStructure(Context context, InputStream input, OutputStream output)
throws IOException, ParserConfigurationException, SQLException, TransformerException {
throws IOException, ParserConfigurationException, SQLException, TransformerException, XPathExpressionException {
// load the XML
Document document = null;
@@ -258,13 +261,15 @@ public class StructBuilder {
// is properly structured.
try {
validate(document);
} catch (TransformerException ex) {
} catch (XPathExpressionException ex) {
System.err.format("The input document is invalid: %s%n", ex.getMessage());
System.exit(1);
}
// Check for 'identifier' attributes -- possibly output by this class.
NodeList identifierNodes = XPathAPI.selectNodeList(document, "//*[@identifier]");
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList identifierNodes = (NodeList) xPath.compile("//*[@identifier]")
.evaluate(document, XPathConstants.NODESET);
if (identifierNodes.getLength() > 0) {
System.err.println("The input document has 'identifier' attributes, which will be ignored.");
}
@@ -287,7 +292,8 @@ public class StructBuilder {
Element[] elements = new Element[]{};
try {
// get the top level community list
NodeList first = XPathAPI.selectNodeList(document, "/import_structure/community");
NodeList first = (NodeList) xPath.compile("/import_structure/community")
.evaluate(document, XPathConstants.NODESET);
// run the import starting with the top level communities
elements = handleCommunities(context, first, null);
@@ -307,7 +313,7 @@ public class StructBuilder {
}
// finally write the string into the output file.
final org.jdom.Document xmlOutput = new org.jdom.Document(root);
final org.jdom2.Document xmlOutput = new org.jdom2.Document(root);
try {
new XMLOutputter().output(xmlOutput, output);
} catch (IOException e) {
@@ -411,7 +417,7 @@ public class StructBuilder {
}
// Now write the structure out.
org.jdom.Document xmlOutput = new org.jdom.Document(rootElement);
org.jdom2.Document xmlOutput = new org.jdom2.Document(rootElement);
try {
XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat());
outputter.output(xmlOutput, output);
@@ -456,14 +462,16 @@ public class StructBuilder {
* @throws TransformerException if transformer error
*/
private static void validate(org.w3c.dom.Document document)
throws TransformerException {
throws XPathExpressionException {
StringBuilder err = new StringBuilder();
boolean trip = false;
err.append("The following errors were encountered parsing the source XML.\n");
err.append("No changes have been made to the DSpace instance.\n\n");
NodeList first = XPathAPI.selectNodeList(document, "/import_structure/community");
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList first = (NodeList) xPath.compile("/import_structure/community")
.evaluate(document, XPathConstants.NODESET);
if (first.getLength() == 0) {
err.append("-There are no top level communities in the source document.");
System.out.println(err.toString());
@@ -493,14 +501,15 @@ public class StructBuilder {
* no errors.
*/
private static String validateCommunities(NodeList communities, int level)
throws TransformerException {
throws XPathExpressionException {
StringBuilder err = new StringBuilder();
boolean trip = false;
String errs = null;
XPath xPath = XPathFactory.newInstance().newXPath();
for (int i = 0; i < communities.getLength(); i++) {
Node n = communities.item(i);
NodeList name = XPathAPI.selectNodeList(n, "name");
NodeList name = (NodeList) xPath.compile("name").evaluate(n, XPathConstants.NODESET);
if (name.getLength() != 1) {
String pos = Integer.toString(i + 1);
err.append("-The level ").append(level)
@@ -510,7 +519,7 @@ public class StructBuilder {
}
// validate sub communities
NodeList subCommunities = XPathAPI.selectNodeList(n, "community");
NodeList subCommunities = (NodeList) xPath.compile("community").evaluate(n, XPathConstants.NODESET);
String comErrs = validateCommunities(subCommunities, level + 1);
if (comErrs != null) {
err.append(comErrs);
@@ -518,7 +527,7 @@ public class StructBuilder {
}
// validate collections
NodeList collections = XPathAPI.selectNodeList(n, "collection");
NodeList collections = (NodeList) xPath.compile("collection").evaluate(n, XPathConstants.NODESET);
String colErrs = validateCollections(collections, level + 1);
if (colErrs != null) {
err.append(colErrs);
@@ -542,14 +551,15 @@ public class StructBuilder {
* @return the errors to be generated by the calling method, or null if none
*/
private static String validateCollections(NodeList collections, int level)
throws TransformerException {
throws XPathExpressionException {
StringBuilder err = new StringBuilder();
boolean trip = false;
String errs = null;
XPath xPath = XPathFactory.newInstance().newXPath();
for (int i = 0; i < collections.getLength(); i++) {
Node n = collections.item(i);
NodeList name = XPathAPI.selectNodeList(n, "name");
NodeList name = (NodeList) xPath.compile("name").evaluate(n, XPathConstants.NODESET);
if (name.getLength() != 1) {
String pos = Integer.toString(i + 1);
err.append("-The level ").append(level)
@@ -613,8 +623,9 @@ public class StructBuilder {
* created communities (e.g. the handles they have been assigned)
*/
private static Element[] handleCommunities(Context context, NodeList communities, Community parent)
throws TransformerException, SQLException, AuthorizeException {
throws TransformerException, SQLException, AuthorizeException, XPathExpressionException {
Element[] elements = new Element[communities.getLength()];
XPath xPath = XPathFactory.newInstance().newXPath();
for (int i = 0; i < communities.getLength(); i++) {
Community community;
@@ -634,7 +645,7 @@ public class StructBuilder {
// now update the metadata
Node tn = communities.item(i);
for (Map.Entry<String, MetadataFieldName> entry : communityMap.entrySet()) {
NodeList nl = XPathAPI.selectNodeList(tn, entry.getKey());
NodeList nl = (NodeList) xPath.compile(entry.getKey()).evaluate(tn, XPathConstants.NODESET);
if (nl.getLength() == 1) {
communityService.setMetadataSingleValue(context, community,
entry.getValue(), null, getStringValue(nl.item(0)));
@@ -700,11 +711,11 @@ public class StructBuilder {
}
// handle sub communities
NodeList subCommunities = XPathAPI.selectNodeList(tn, "community");
NodeList subCommunities = (NodeList) xPath.compile("community").evaluate(tn, XPathConstants.NODESET);
Element[] subCommunityElements = handleCommunities(context, subCommunities, community);
// handle collections
NodeList collections = XPathAPI.selectNodeList(tn, "collection");
NodeList collections = (NodeList) xPath.compile("collection").evaluate(tn, XPathConstants.NODESET);
Element[] collectionElements = handleCollections(context, collections, community);
int j;
@@ -731,8 +742,9 @@ public class StructBuilder {
* created collections (e.g. the handle)
*/
private static Element[] handleCollections(Context context, NodeList collections, Community parent)
throws TransformerException, SQLException, AuthorizeException {
throws SQLException, AuthorizeException, XPathExpressionException {
Element[] elements = new Element[collections.getLength()];
XPath xPath = XPathFactory.newInstance().newXPath();
for (int i = 0; i < collections.getLength(); i++) {
Element element = new Element("collection");
@@ -745,7 +757,7 @@ public class StructBuilder {
// import the rest of the metadata
Node tn = collections.item(i);
for (Map.Entry<String, MetadataFieldName> entry : collectionMap.entrySet()) {
NodeList nl = XPathAPI.selectNodeList(tn, entry.getKey());
NodeList nl = (NodeList) xPath.compile(entry.getKey()).evaluate(tn, XPathConstants.NODESET);
if (nl.getLength() == 1) {
collectionService.setMetadataSingleValue(context, collection,
entry.getValue(), null, getStringValue(nl.item(0)));

42
dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java
View File

@@ -138,7 +138,7 @@ public class DSpaceCSV implements Serializable {
/**
* Create a new instance, reading the lines in from file
*
* @param inputStream the inputstream to read from
* @param inputStream the input stream to read from
* @param c The DSpace Context
* @throws Exception thrown if there is an error reading or processing the file
*/
@@ -159,7 +159,7 @@ public class DSpaceCSV implements Serializable {
columnCounter++;
// Remove surrounding quotes if there are any
if ((element.startsWith("\"")) && (element.endsWith("\""))) {
if (element.startsWith("\"") && element.endsWith("\"")) {
element = element.substring(1, element.length() - 1);
}
@@ -337,15 +337,15 @@ public class DSpaceCSV implements Serializable {
/**
* Set the value separator for multiple values stored in one csv value.
*
* Is set in bulkedit.cfg as valueseparator
* Is set in {@code bulkedit.cfg} as {@code valueseparator}.
*
* If not set, defaults to double pipe '||'
* If not set, defaults to double pipe '||'.
*/
private void setValueSeparator() {
// Get the value separator
valueSeparator = DSpaceServicesFactory.getInstance().getConfigurationService()
.getProperty("bulkedit.valueseparator");
if ((valueSeparator != null) && (!"".equals(valueSeparator.trim()))) {
if ((valueSeparator != null) && !valueSeparator.trim().isEmpty()) {
valueSeparator = valueSeparator.trim();
} else {
valueSeparator = "||";
@@ -360,7 +360,7 @@ public class DSpaceCSV implements Serializable {
/**
* Set the field separator use to separate fields in the csv.
*
* Is set in bulkedit.cfg as fieldseparator
* Is set in {@code bulkedit.cfg} as {@code fieldseparator}.
*
* If not set, defaults to comma ','.
*
@@ -371,7 +371,7 @@ public class DSpaceCSV implements Serializable {
// Get the value separator
fieldSeparator = DSpaceServicesFactory.getInstance().getConfigurationService()
.getProperty("bulkedit.fieldseparator");
if ((fieldSeparator != null) && (!"".equals(fieldSeparator.trim()))) {
if ((fieldSeparator != null) && !fieldSeparator.trim().isEmpty()) {
fieldSeparator = fieldSeparator.trim();
if ("tab".equals(fieldSeparator)) {
fieldSeparator = "\t";
@@ -395,15 +395,15 @@ public class DSpaceCSV implements Serializable {
/**
* Set the authority separator for value with authority data.
*
* Is set in dspace.cfg as bulkedit.authorityseparator
* Is set in {@code dspace.cfg} as {@code bulkedit.authorityseparator}.
*
* If not set, defaults to double colon '::'
* If not set, defaults to double colon '::'.
*/
private void setAuthoritySeparator() {
// Get the value separator
authoritySeparator = DSpaceServicesFactory.getInstance().getConfigurationService()
.getProperty("bulkedit.authorityseparator");
if ((authoritySeparator != null) && (!"".equals(authoritySeparator.trim()))) {
if ((authoritySeparator != null) && !authoritySeparator.trim().isEmpty()) {
authoritySeparator = authoritySeparator.trim();
} else {
authoritySeparator = "::";
@@ -508,7 +508,7 @@ public class DSpaceCSV implements Serializable {
int i = 0;
for (String part : bits) {
int bitcounter = part.length() - part.replaceAll("\"", "").length();
if ((part.startsWith("\"")) && ((!part.endsWith("\"")) || ((bitcounter & 1) == 1))) {
if (part.startsWith("\"") && (!part.endsWith("\"") || ((bitcounter & 1) == 1))) {
found = true;
String add = bits.get(i) + fieldSeparator + bits.get(i + 1);
bits.remove(i);
@@ -524,7 +524,7 @@ public class DSpaceCSV implements Serializable {
// Deal with quotes around the elements
int i = 0;
for (String part : bits) {
if ((part.startsWith("\"")) && (part.endsWith("\""))) {
if (part.startsWith("\"") && part.endsWith("\"")) {
part = part.substring(1, part.length() - 1);
bits.set(i, part);
}
@@ -564,7 +564,7 @@ public class DSpaceCSV implements Serializable {
for (String part : bits) {
if (i > 0) {
// Is this a last empty item?
if ((last) && (i == headings.size())) {
if (last && (i == headings.size())) {
part = "";
}
@@ -577,7 +577,7 @@ public class DSpaceCSV implements Serializable {
csvLine.add(headings.get(i - 1), null);
String[] elements = part.split(escapedValueSeparator);
for (String element : elements) {
if ((element != null) && (!"".equals(element))) {
if ((element != null) && !element.isEmpty()) {
csvLine.add(headings.get(i - 1), element);
}
}
@@ -629,18 +629,18 @@ public class DSpaceCSV implements Serializable {
public InputStream getInputStream() {
StringBuilder stringBuilder = new StringBuilder();
for (String csvLine : getCSVLinesAsStringArray()) {
stringBuilder.append(csvLine + "\n");
stringBuilder.append(csvLine).append("\n");
}
return IOUtils.toInputStream(stringBuilder.toString(), StandardCharsets.UTF_8);
}
/**
* Is it Ok to export this value? When exportAll is set to false, we don't export
* Is it okay to export this value? When exportAll is set to false, we don't export
* some of the metadata elements.
*
* The list can be configured via the key ignore-on-export in bulkedit.cfg
* The list can be configured via the key ignore-on-export in {@code bulkedit.cfg}.
*
* @param md The Metadatum to examine
* @param md The MetadataField to examine
* @return Whether or not it is OK to export this element
*/
protected boolean okToExport(MetadataField md) {
@@ -649,12 +649,8 @@ public class DSpaceCSV implements Serializable {
if (md.getQualifier() != null) {
key += "." + md.getQualifier();
}
if (ignore.get(key) != null) {
return false;
}
// Must be OK, so don't ignore
return true;
return ignore.get(key) == null;
}
/**

2
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataDeletionScriptConfiguration.java
View File

@@ -41,10 +41,8 @@ public class MetadataDeletionScriptConfiguration<T extends MetadataDeletion> ext
Options options = new Options();
options.addOption("m", "metadata", true, "metadata field name");
options.getOption("m").setType(String.class);
options.addOption("l", "list", false, "lists the metadata fields that can be deleted");
options.getOption("l").setType(boolean.class);
super.options = options;
}

3
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportScriptConfiguration.java
View File

@@ -54,12 +54,9 @@ public class MetadataExportScriptConfiguration<T extends MetadataExport> extends
Options options = new Options();
options.addOption("i", "id", true, "ID or handle of thing to export (item, collection, or community)");
options.getOption("i").setType(String.class);
options.addOption("a", "all", false,
"include all metadata fields that are not normally changed (e.g. provenance)");
options.getOption("a").setType(boolean.class);
options.addOption("h", "help", false, "help");
options.getOption("h").setType(boolean.class);
super.options = options;

170
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearch.java Normal file
View File

@@ -0,0 +1,170 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.bulkedit;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import org.apache.commons.cli.ParseException;
import org.dspace.content.Item;
import org.dspace.content.MetadataDSpaceCsvExportServiceImpl;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.CollectionService;
import org.dspace.content.service.CommunityService;
import org.dspace.content.service.MetadataDSpaceCsvExportService;
import org.dspace.core.Context;
import org.dspace.discovery.DiscoverQuery;
import org.dspace.discovery.IndexableObject;
import org.dspace.discovery.SearchService;
import org.dspace.discovery.SearchUtils;
import org.dspace.discovery.configuration.DiscoveryConfiguration;
import org.dspace.discovery.configuration.DiscoveryConfigurationService;
import org.dspace.discovery.indexobject.IndexableCollection;
import org.dspace.discovery.indexobject.IndexableCommunity;
import org.dspace.discovery.utils.DiscoverQueryBuilder;
import org.dspace.discovery.utils.parameter.QueryBuilderSearchFilter;
import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.eperson.service.EPersonService;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.sort.SortOption;
import org.dspace.utils.DSpace;
/**
* Metadata exporter to allow the batch export of metadata from a discovery search into a file
*
*/
public class MetadataExportSearch extends DSpaceRunnable<MetadataExportSearchScriptConfiguration> {
private static final String EXPORT_CSV = "exportCSV";
private boolean help = false;
private String identifier;
private String discoveryConfigName;
private String[] filterQueryStrings;
private boolean hasScope = false;
private String query;
private SearchService searchService;
private MetadataDSpaceCsvExportService metadataDSpaceCsvExportService;
private EPersonService ePersonService;
private DiscoveryConfigurationService discoveryConfigurationService;
private CommunityService communityService;
private CollectionService collectionService;
private DiscoverQueryBuilder queryBuilder;
@Override
public MetadataExportSearchScriptConfiguration getScriptConfiguration() {
return new DSpace().getServiceManager()
.getServiceByName("metadata-export-search", MetadataExportSearchScriptConfiguration.class);
}
@Override
public void setup() throws ParseException {
searchService = SearchUtils.getSearchService();
metadataDSpaceCsvExportService = new DSpace().getServiceManager()
.getServiceByName(
MetadataDSpaceCsvExportServiceImpl.class.getCanonicalName(),
MetadataDSpaceCsvExportService.class
);
ePersonService = EPersonServiceFactory.getInstance().getEPersonService();
discoveryConfigurationService = SearchUtils.getConfigurationService();
communityService = ContentServiceFactory.getInstance().getCommunityService();
collectionService = ContentServiceFactory.getInstance().getCollectionService();
queryBuilder = SearchUtils.getQueryBuilder();
if (commandLine.hasOption('h')) {
help = true;
return;
}
if (commandLine.hasOption('q')) {
query = commandLine.getOptionValue('q');
}
if (commandLine.hasOption('s')) {
hasScope = true;
identifier = commandLine.getOptionValue('s');
}
if (commandLine.hasOption('c')) {
discoveryConfigName = commandLine.getOptionValue('c');
}
if (commandLine.hasOption('f')) {
filterQueryStrings = commandLine.getOptionValues('f');
}
}
@Override
public void internalRun() throws Exception {
if (help) {
loghelpinfo();
printHelp();
return;
}
handler.logDebug("starting search export");
IndexableObject dso = null;
Context context = new Context();
context.setCurrentUser(ePersonService.find(context, this.getEpersonIdentifier()));
if (hasScope) {
dso = resolveScope(context, identifier);
}
DiscoveryConfiguration discoveryConfiguration =
discoveryConfigurationService.getDiscoveryConfiguration(discoveryConfigName);
List<QueryBuilderSearchFilter> queryBuilderSearchFilters = new ArrayList<>();
handler.logDebug("processing filter queries");
if (filterQueryStrings != null) {
for (String filterQueryString: filterQueryStrings) {
String field = filterQueryString.split(",", 2)[0];
String operator = filterQueryString.split("(,|=)", 3)[1];
String value = filterQueryString.split("=", 2)[1];
QueryBuilderSearchFilter queryBuilderSearchFilter =
new QueryBuilderSearchFilter(field, operator, value);
queryBuilderSearchFilters.add(queryBuilderSearchFilter);
}
}
handler.logDebug("building query");
DiscoverQuery discoverQuery =
queryBuilder.buildQuery(context, dso, discoveryConfiguration, query, queryBuilderSearchFilters,
"Item", 10, Long.getLong("0"), null, SortOption.DESCENDING);
handler.logDebug("creating iterator");
Iterator<Item> itemIterator = searchService.iteratorSearch(context, dso, discoverQuery);
handler.logDebug("creating dspacecsv");
DSpaceCSV dSpaceCSV = metadataDSpaceCsvExportService.export(context, itemIterator, true);
handler.logDebug("writing to file " + getFileNameOrExportFile());
handler.writeFilestream(context, getFileNameOrExportFile(), dSpaceCSV.getInputStream(), EXPORT_CSV);
context.restoreAuthSystemState();
context.complete();
}
protected void loghelpinfo() {
handler.logInfo("metadata-export");
}
protected String getFileNameOrExportFile() {
return "metadataExportSearch.csv";
}
public IndexableObject resolveScope(Context context, String id) throws SQLException {
UUID uuid = UUID.fromString(id);
IndexableObject scopeObj = new IndexableCommunity(communityService.find(context, uuid));
if (scopeObj.getIndexedObject() == null) {
scopeObj = new IndexableCollection(collectionService.find(context, uuid));
}
return scopeObj;
}
}

20
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearchCli.java Normal file
View File

@@ -0,0 +1,20 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.bulkedit;
/**
* The cli version of the {@link MetadataExportSearch} script
*/
public class MetadataExportSearchCli extends MetadataExportSearch {
@Override
protected String getFileNameOrExportFile() {
return commandLine.getOptionValue('n');
}
}

26
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearchCliScriptConfiguration.java Normal file
View File

@@ -0,0 +1,26 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.bulkedit;
import org.apache.commons.cli.Options;
/**
* This is the CLI version of the {@link MetadataExportSearchScriptConfiguration} class that handles the
* configuration for the {@link MetadataExportSearchCli} script
*/
public class MetadataExportSearchCliScriptConfiguration
extends MetadataExportSearchScriptConfiguration<MetadataExportSearchCli> {
@Override
public Options getOptions() {
Options options = super.getOptions();
options.addOption("n", "filename", true, "the filename to export to");
return super.getOptions();
}
}

62
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearchScriptConfiguration.java Normal file
View File

@@ -0,0 +1,62 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.bulkedit;
import org.apache.commons.cli.Options;
import org.dspace.core.Context;
import org.dspace.scripts.configuration.ScriptConfiguration;
/**
* The {@link ScriptConfiguration} for the {@link MetadataExportSearch} script
*/
public class MetadataExportSearchScriptConfiguration<T extends MetadataExportSearch> extends ScriptConfiguration<T> {
private Class<T> dspaceRunnableclass;
@Override
public Class<T> getDspaceRunnableClass() {
return dspaceRunnableclass;
}
@Override
public void setDspaceRunnableClass(Class<T> dspaceRunnableClass) {
this.dspaceRunnableclass = dspaceRunnableClass;
}
@Override
public boolean isAllowedToExecute(Context context) {
return true;
}
@Override
public Options getOptions() {
if (options == null) {
Options options = new Options();
options.addOption("q", "query", true,
"The discovery search string to will be used to match records. Not URL encoded");
options.getOption("q").setType(String.class);
options.addOption("s", "scope", true,
"UUID of a specific DSpace container (site, community or collection) to which the search has to be " +
"limited");
options.getOption("s").setType(String.class);
options.addOption("c", "configuration", true,
"The name of a Discovery configuration that should be used by this search");
options.getOption("c").setType(String.class);
options.addOption("f", "filter", true,
"Advanced search filter that has to be used to filter the result set, with syntax `<:filter-name>," +
"<:filter-operator>=<:filter-value>`. Not URL encoded. For example `author," +
"authority=5df05073-3be7-410d-8166-e254369e4166` or `title,contains=sample text`");
options.getOption("f").setType(String.class);
options.addOption("h", "help", false, "help");
super.options = options;
}
return options;
}
}

51
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java
View File

@@ -25,6 +25,7 @@ import javax.annotation.Nullable;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.app.util.RelationshipUtils;
import org.dspace.authority.AuthorityValue;
import org.dspace.authority.factory.AuthorityServiceFactory;
import org.dspace.authority.service.AuthorityValueService;
@@ -53,7 +54,7 @@ import org.dspace.content.service.RelationshipTypeService;
import org.dspace.content.service.WorkspaceItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.LogManager;
import org.dspace.core.LogHelper;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.handle.factory.HandleServiceFactory;
@@ -640,7 +641,7 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
all += part + ",";
}
all = all.substring(0, all.length());
log.debug(LogManager.getHeader(c, "metadata_import",
log.debug(LogHelper.getHeader(c, "metadata_import",
"item_id=" + item.getID() + ",fromCSV=" + all));
// Don't compare collections or actions or rowNames
@@ -677,7 +678,7 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
qualifier = qualifier.substring(0, qualifier.indexOf('['));
}
}
log.debug(LogManager.getHeader(c, "metadata_import",
log.debug(LogHelper.getHeader(c, "metadata_import",
"item_id=" + item.getID() + ",fromCSV=" + all +
",looking_for_schema=" + schema +
",looking_for_element=" + element +
@@ -697,7 +698,7 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
.getConfidence() : Choices.CF_ACCEPTED);
}
i++;
log.debug(LogManager.getHeader(c, "metadata_import",
log.debug(LogHelper.getHeader(c, "metadata_import",
"item_id=" + item.getID() + ",fromCSV=" + all +
",found=" + dcv.getValue()));
}
@@ -748,7 +749,7 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
// column "dc.contributor.author" so don't remove it
if ((value != null) && (!"".equals(value)) && (!contains(value, fromCSV)) && fromAuthority == null) {
// Remove it
log.debug(LogManager.getHeader(c, "metadata_import",
log.debug(LogHelper.getHeader(c, "metadata_import",
"item_id=" + item.getID() + ",fromCSV=" + all +
",removing_schema=" + schema +
",removing_element=" + element +
@@ -924,11 +925,10 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
rightItem = item;
}
// Create the relationship
int leftPlace = relationshipService.findNextLeftPlaceByLeftItem(c, leftItem);
int rightPlace = relationshipService.findNextRightPlaceByRightItem(c, rightItem);
Relationship persistedRelationship = relationshipService.create(c, leftItem, rightItem,
foundRelationshipType, leftPlace, rightPlace);
// Create the relationship, appending to the end
Relationship persistedRelationship = relationshipService.create(
c, leftItem, rightItem, foundRelationshipType, -1, -1
);
relationshipService.update(c, persistedRelationship);
}
@@ -1793,36 +1793,7 @@ public class MetadataImport extends DSpaceRunnable<MetadataImportScriptConfigura
*/
private RelationshipType matchRelationshipType(List<RelationshipType> relTypes,
String targetType, String originType, String originTypeName) {
RelationshipType foundRelationshipType = null;
if (originTypeName.split("\\.").length > 1) {
originTypeName = originTypeName.split("\\.")[1];
}
for (RelationshipType relationshipType : relTypes) {
// Is origin type leftward or righward
boolean isLeft = false;
if (relationshipType.getLeftType().getLabel().equalsIgnoreCase(originType)) {
isLeft = true;
}
if (isLeft) {
// Validate typeName reference
if (!relationshipType.getLeftwardType().equalsIgnoreCase(originTypeName)) {
continue;
}
if (relationshipType.getLeftType().getLabel().equalsIgnoreCase(originType) &&
relationshipType.getRightType().getLabel().equalsIgnoreCase(targetType)) {
foundRelationshipType = relationshipType;
}
} else {
if (!relationshipType.getRightwardType().equalsIgnoreCase(originTypeName)) {
continue;
}
if (relationshipType.getLeftType().getLabel().equalsIgnoreCase(targetType) &&
relationshipType.getRightType().getLabel().equalsIgnoreCase(originType)) {
foundRelationshipType = relationshipType;
}
}
}
return foundRelationshipType;
return RelationshipUtils.matchRelationshipType(relTypes, targetType, originType, originTypeName);
}
}

1
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportCliScriptConfiguration.java
View File

@@ -19,7 +19,6 @@ public class MetadataImportCliScriptConfiguration extends MetadataImportScriptCo
public Options getOptions() {
Options options = super.getOptions();
options.addOption("e", "email", true, "email address or user id of user (required if adding new items)");
options.getOption("e").setType(String.class);
options.getOption("e").setRequired(true);
super.options = options;
return options;

6
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportScriptConfiguration.java
View File

@@ -59,20 +59,14 @@ public class MetadataImportScriptConfiguration<T extends MetadataImport> extends
options.getOption("f").setRequired(true);
options.addOption("s", "silent", false,
"silent operation - doesn't request confirmation of changes USE WITH CAUTION");
options.getOption("s").setType(boolean.class);
options.addOption("w", "workflow", false, "workflow - when adding new items, use collection workflow");
options.getOption("w").setType(boolean.class);
options.addOption("n", "notify", false,
"notify - when adding new items using a workflow, send notification emails");
options.getOption("n").setType(boolean.class);
options.addOption("v", "validate-only", false,
"validate - just validate the csv, don't run the import");
options.getOption("v").setType(boolean.class);
options.addOption("t", "template", false,
"template - when adding new items, use the collection template (if it exists)");
options.getOption("t").setType(boolean.class);
options.addOption("h", "help", false, "help");
options.getOption("h").setType(boolean.class);
super.options = options;
}

32
dspace-api/src/main/java/org/dspace/app/exception/ResourceAlreadyExistsException.java Normal file
View File

@@ -0,0 +1,32 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.exception;
/**
* This class provides an exception to be used when trying to save a resource
* that already exists.
*
* @author Luca Giamminonni (luca.giamminonni at 4science.it)
*
*/
public class ResourceAlreadyExistsException extends RuntimeException {
private static final long serialVersionUID = 1L;
/**
* Create a ResourceAlreadyExistsException with a message and the already
* existing resource.
*
* @param message the error message
*/
public ResourceAlreadyExistsException(String message) {
super(message);
}
}

375
dspace-api/src/main/java/org/dspace/app/harvest/Harvest.java
View File

@@ -13,11 +13,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Collection;
import org.dspace.content.DSpaceObject;
@@ -36,224 +33,223 @@ import org.dspace.harvest.HarvestingException;
import org.dspace.harvest.OAIHarvester;
import org.dspace.harvest.factory.HarvestServiceFactory;
import org.dspace.harvest.service.HarvestedCollectionService;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.utils.DSpace;
/**
* Test class for harvested collections.
*
* @author Alexey Maslov
*/
public class Harvest {
private static Context context;
public class Harvest extends DSpaceRunnable<HarvestScriptConfiguration> {
private static final HarvestedCollectionService harvestedCollectionService =
HarvestServiceFactory.getInstance().getHarvestedCollectionService();
private static final EPersonService ePersonService = EPersonServiceFactory.getInstance().getEPersonService();
private static final CollectionService collectionService =
ContentServiceFactory.getInstance().getCollectionService();
private HarvestedCollectionService harvestedCollectionService;
protected EPersonService ePersonService;
private CollectionService collectionService;
public static void main(String[] argv) throws Exception {
// create an options object and populate it
CommandLineParser parser = new DefaultParser();
private boolean help;
private String command = null;
private String collection = null;
private String oaiSource = null;
private String oaiSetID = null;
private String metadataKey = null;
private int harvestType = 0;
Options options = new Options();
options.addOption("p", "purge", false, "delete all items in the collection");
options.addOption("r", "run", false, "run the standard harvest procedure");
options.addOption("g", "ping", false, "test the OAI server and set");
options.addOption("o", "once", false, "run the harvest procedure with specified parameters");
options.addOption("s", "setup", false, "Set the collection up for harvesting");
options.addOption("S", "start", false, "start the harvest loop");
options.addOption("R", "reset", false, "reset harvest status on all collections");
options.addOption("P", "purge", false, "purge all harvestable collections");
protected Context context;
options.addOption("e", "eperson", true,
"eperson");
options.addOption("c", "collection", true,
"harvesting collection (handle or id)");
options.addOption("t", "type", true,
"type of harvesting (0 for none)");
options.addOption("a", "address", true,
"address of the OAI-PMH server");
options.addOption("i", "oai_set_id", true,
"id of the PMH set representing the harvested collection");
options.addOption("m", "metadata_format", true,
"the name of the desired metadata format for harvesting, resolved to namespace and " +
"crosswalk in dspace.cfg");
public HarvestScriptConfiguration getScriptConfiguration() {
return new DSpace().getServiceManager()
.getServiceByName("harvest", HarvestScriptConfiguration.class);
}
options.addOption("h", "help", false, "help");
public void setup() throws ParseException {
harvestedCollectionService =
HarvestServiceFactory.getInstance().getHarvestedCollectionService();
ePersonService = EPersonServiceFactory.getInstance().getEPersonService();
collectionService =
ContentServiceFactory.getInstance().getCollectionService();
CommandLine line = parser.parse(options, argv);
assignCurrentUserInContext();
String command = null;
String eperson = null;
String collection = null;
String oaiSource = null;
String oaiSetID = null;
String metadataKey = null;
int harvestType = 0;
if (line.hasOption('h')) {
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("Harvest\n", options);
System.out.println("\nPING OAI server: Harvest -g -a oai_source -i oai_set_id");
System.out.println(
"RUNONCE harvest with arbitrary options: Harvest -o -e eperson -c collection -t harvest_type -a " +
"oai_source -i oai_set_id -m metadata_format");
System.out.println(
"SETUP a collection for harvesting: Harvest -s -c collection -t harvest_type -a oai_source -i " +
"oai_set_id -m metadata_format");
System.out.println("RUN harvest once: Harvest -r -e eperson -c collection");
System.out.println("START harvest scheduler: Harvest -S");
System.out.println("RESET all harvest status: Harvest -R");
System.out.println("PURGE a collection of items and settings: Harvest -p -e eperson -c collection");
System.out.println("PURGE all harvestable collections: Harvest -P -e eperson");
help = commandLine.hasOption('h');
System.exit(0);
}
if (line.hasOption('s')) {
if (commandLine.hasOption('s')) {
command = "config";
}
if (line.hasOption('p')) {
if (commandLine.hasOption('p')) {
command = "purge";
}
if (line.hasOption('r')) {
if (commandLine.hasOption('r')) {
command = "run";
}
if (line.hasOption('g')) {
if (commandLine.hasOption('g')) {
command = "ping";
}
if (line.hasOption('o')) {
command = "runOnce";
}
if (line.hasOption('S')) {
if (commandLine.hasOption('S')) {
command = "start";
}
if (line.hasOption('R')) {
if (commandLine.hasOption('R')) {
command = "reset";
}
if (line.hasOption('P')) {
if (commandLine.hasOption('P')) {
command = "purgeAll";
}
if (line.hasOption('e')) {
eperson = line.getOptionValue('e');
if (commandLine.hasOption('o')) {
command = "reimport";
}
if (line.hasOption('c')) {
collection = line.getOptionValue('c');
if (commandLine.hasOption('c')) {
collection = commandLine.getOptionValue('c');
}
if (line.hasOption('t')) {
harvestType = Integer.parseInt(line.getOptionValue('t'));
if (commandLine.hasOption('t')) {
harvestType = Integer.parseInt(commandLine.getOptionValue('t'));
} else {
harvestType = 0;
}
if (line.hasOption('a')) {
oaiSource = line.getOptionValue('a');
if (commandLine.hasOption('a')) {
oaiSource = commandLine.getOptionValue('a');
}
if (line.hasOption('i')) {
oaiSetID = line.getOptionValue('i');
if (commandLine.hasOption('i')) {
oaiSetID = commandLine.getOptionValue('i');
}
if (line.hasOption('m')) {
metadataKey = line.getOptionValue('m');
if (commandLine.hasOption('m')) {
metadataKey = commandLine.getOptionValue('m');
}
}
/**
* This method will assign the currentUser to the {@link Context} variable which is also created in this method.
* The instance of the method in this class will fetch the EPersonIdentifier from this class, this identifier
* was given to this class upon instantiation, it'll then be used to find the {@link EPerson} associated with it
* and this {@link EPerson} will be set as the currentUser of the created {@link Context}
* @throws ParseException If something went wrong with the retrieval of the EPerson Identifier
*/
protected void assignCurrentUserInContext() throws ParseException {
UUID currentUserUuid = this.getEpersonIdentifier();
try {
this.context = new Context(Context.Mode.BATCH_EDIT);
EPerson eperson = ePersonService.find(context, currentUserUuid);
if (eperson == null) {
super.handler.logError("EPerson not found: " + currentUserUuid);
throw new IllegalArgumentException("Unable to find a user with uuid: " + currentUserUuid);
}
this.context.setCurrentUser(eperson);
} catch (SQLException e) {
handler.handleException("Something went wrong trying to fetch eperson for uuid: " + currentUserUuid, e);
}
}
public void internalRun() throws Exception {
if (help) {
printHelp();
handler.logInfo("PING OAI server: Harvest -g -a oai_source -i oai_set_id");
handler.logInfo(
"SETUP a collection for harvesting: Harvest -s -c collection -t harvest_type -a oai_source -i " +
"oai_set_id -m metadata_format");
handler.logInfo("RUN harvest once: Harvest -r -e eperson -c collection");
handler.logInfo("START harvest scheduler: Harvest -S");
handler.logInfo("RESET all harvest status: Harvest -R");
handler.logInfo("PURGE a collection of items and settings: Harvest -p -e eperson -c collection");
handler.logInfo("PURGE all harvestable collections: Harvest -P -e eperson");
return;
}
// Instantiate our class
Harvest harvester = new Harvest();
harvester.context = new Context(Context.Mode.BATCH_EDIT);
// Check our options
if (command == null) {
System.out
.println("Error - no parameters specified (run with -h flag for details)");
System.exit(1);
if (StringUtils.isBlank(command)) {
handler.logError("No parameters specified (run with -h flag for details)");
throw new UnsupportedOperationException("No command specified");
} else if ("run".equals(command)) {
// Run a single harvest cycle on a collection using saved settings.
if (collection == null || eperson == null) {
System.out
.println("Error - a target collection and eperson must be provided");
System.out.println(" (run with -h flag for details)");
System.exit(1);
if (collection == null || context.getCurrentUser() == null) {
handler.logError("A target collection and eperson must be provided (run with -h flag for details)");
throw new UnsupportedOperationException("A target collection and eperson must be provided");
}
harvester.runHarvest(collection, eperson);
runHarvest(context, collection);
} else if ("start".equals(command)) {
// start the harvest loop
startHarvester();
} else if ("reset".equals(command)) {
// reset harvesting status
resetHarvesting();
resetHarvesting(context);
} else if ("purgeAll".equals(command)) {
// purge all collections that are set up for harvesting (obviously for testing purposes only)
if (eperson == null) {
System.out
.println("Error - an eperson must be provided");
System.out.println(" (run with -h flag for details)");
System.exit(1);
if (context.getCurrentUser() == null) {
handler.logError("An eperson must be provided (run with -h flag for details)");
throw new UnsupportedOperationException("An eperson must be provided");
}
List<HarvestedCollection> harvestedCollections = harvestedCollectionService.findAll(context);
for (HarvestedCollection harvestedCollection : harvestedCollections) {
System.out.println(
"Purging the following collections (deleting items and resetting harvest status): " +
harvestedCollection
.getCollection().getID().toString());
harvester.purgeCollection(harvestedCollection.getCollection().getID().toString(), eperson);
handler.logInfo(
"Purging the following collections (deleting items and resetting harvest status): " +
harvestedCollection
.getCollection().getID().toString());
purgeCollection(context, harvestedCollection.getCollection().getID().toString());
}
context.complete();
} else if ("purge".equals(command)) {
// Delete all items in a collection. Useful for testing fresh harvests.
if (collection == null || eperson == null) {
System.out
.println("Error - a target collection and eperson must be provided");
System.out.println(" (run with -h flag for details)");
System.exit(1);
if (collection == null || context.getCurrentUser() == null) {
handler.logError("A target collection and eperson must be provided (run with -h flag for details)");
throw new UnsupportedOperationException("A target collection and eperson must be provided");
}
harvester.purgeCollection(collection, eperson);
purgeCollection(context, collection);
context.complete();
} else if ("reimport".equals(command)) {
// Delete all items in a collection. Useful for testing fresh harvests.
if (collection == null || context.getCurrentUser() == null) {
handler.logError("A target collection and eperson must be provided (run with -h flag for details)");
throw new UnsupportedOperationException("A target collection and eperson must be provided");
}
purgeCollection(context, collection);
runHarvest(context, collection);
context.complete();
//TODO: implement this... remove all items and remember to unset "last-harvested" settings
} else if ("config".equals(command)) {
// Configure a collection with the three main settings
if (collection == null) {
System.out.println("Error - a target collection must be provided");
System.out.println(" (run with -h flag for details)");
System.exit(1);
handler.logError("A target collection must be provided (run with -h flag for details)");
throw new UnsupportedOperationException("A target collection must be provided");
}
if (oaiSource == null || oaiSetID == null) {
System.out.println("Error - both the OAI server address and OAI set id must be specified");
System.out.println(" (run with -h flag for details)");
System.exit(1);
handler.logError(
"Both the OAI server address and OAI set id must be specified (run with -h flag for details)");
throw new UnsupportedOperationException("Both the OAI server address and OAI set id must be specified");
}
if (metadataKey == null) {
System.out
.println("Error - a metadata key (commonly the prefix) must be specified for this collection");
System.out.println(" (run with -h flag for details)");
System.exit(1);
handler.logError(
"A metadata key (commonly the prefix) must be specified for this collection (run with -h flag" +
" for details)");
throw new UnsupportedOperationException(
"A metadata key (commonly the prefix) must be specified for this collection");
}
harvester.configureCollection(collection, harvestType, oaiSource, oaiSetID, metadataKey);
configureCollection(context, collection, harvestType, oaiSource, oaiSetID, metadataKey);
} else if ("ping".equals(command)) {
if (oaiSource == null || oaiSetID == null) {
System.out.println("Error - both the OAI server address and OAI set id must be specified");
System.out.println(" (run with -h flag for details)");
System.exit(1);
handler.logError(
"Both the OAI server address and OAI set id must be specified (run with -h flag for details)");
throw new UnsupportedOperationException("Both the OAI server address and OAI set id must be specified");
}
pingResponder(oaiSource, oaiSetID, metadataKey);
} else {
handler.logError(
"Your command '" + command + "' was not recognized properly (run with -h flag for details)");
throw new UnsupportedOperationException("Your command '" + command + "' was not recognized properly");
}
}
/*
* Resolve the ID into a collection and check to see if its harvesting options are set. If so, return
* the collection, if not, bail out.
*/
private Collection resolveCollection(String collectionID) {
private Collection resolveCollection(Context context, String collectionID) {
DSpaceObject dso;
Collection targetCollection = null;
@@ -273,14 +269,14 @@ public class Harvest {
}
} else {
// not a handle, try and treat it as an collection database UUID
System.out.println("Looking up by UUID: " + collectionID + ", " + "in context: " + context);
handler.logInfo("Looking up by UUID: " + collectionID + ", " + "in context: " + context);
targetCollection = collectionService.find(context, UUID.fromString(collectionID));
}
}
// was the collection valid?
if (targetCollection == null) {
System.out.println("Cannot resolve " + collectionID + " to collection");
System.exit(1);
handler.logError("Cannot resolve " + collectionID + " to collection");
throw new UnsupportedOperationException("Cannot resolve " + collectionID + " to collection");
}
} catch (SQLException se) {
se.printStackTrace();
@@ -290,12 +286,12 @@ public class Harvest {
}
private void configureCollection(String collectionID, int type, String oaiSource, String oaiSetId,
private void configureCollection(Context context, String collectionID, int type, String oaiSource, String oaiSetId,
String mdConfigId) {
System.out.println("Running: configure collection");
handler.logInfo("Running: configure collection");
Collection collection = resolveCollection(collectionID);
System.out.println(collection.getID());
Collection collection = resolveCollection(context, collectionID);
handler.logInfo(String.valueOf(collection.getID()));
try {
HarvestedCollection hc = harvestedCollectionService.find(context, collection);
@@ -310,9 +306,8 @@ public class Harvest {
context.restoreAuthSystemState();
context.complete();
} catch (Exception e) {
System.out.println("Changes could not be committed");
e.printStackTrace();
System.exit(1);
handler.logError("Changes could not be committed");
handler.handleException(e);
} finally {
if (context != null) {
context.restoreAuthSystemState();
@@ -323,18 +318,15 @@ public class Harvest {
/**
* Purges a collection of all harvest-related data and settings. All items in the collection will be deleted.
* @param collectionID
*
* @param collectionID
* @param email
*/
private void purgeCollection(String collectionID, String email) {
System.out.println(
"Purging collection of all items and resetting last_harvested and harvest_message: " + collectionID);
Collection collection = resolveCollection(collectionID);
private void purgeCollection(Context context, String collectionID) {
handler.logInfo(
"Purging collection of all items and resetting last_harvested and harvest_message: " + collectionID);
Collection collection = resolveCollection(context, collectionID);
try {
EPerson eperson = ePersonService.findByEmail(context, email);
context.setCurrentUser(eperson);
context.turnOffAuthorisationSystem();
ItemService itemService = ContentServiceFactory.getInstance().getItemService();
@@ -343,7 +335,7 @@ public class Harvest {
while (it.hasNext()) {
i++;
Item item = it.next();
System.out.println("Deleting: " + item.getHandle());
handler.logInfo("Deleting: " + item.getHandle());
collectionService.removeItem(context, collection, item);
context.uncacheEntity(item);// Dispatch events every 50 items
if (i % 50 == 0) {
@@ -363,9 +355,8 @@ public class Harvest {
context.restoreAuthSystemState();
context.dispatchEvents();
} catch (Exception e) {
System.out.println("Changes could not be committed");
e.printStackTrace();
System.exit(1);
handler.logError("Changes could not be committed");
handler.handleException(e);
} finally {
context.restoreAuthSystemState();
}
@@ -375,46 +366,42 @@ public class Harvest {
/**
* Run a single harvest cycle on the specified collection under the authorization of the supplied EPerson
*/
private void runHarvest(String collectionID, String email) {
System.out.println("Running: a harvest cycle on " + collectionID);
private void runHarvest(Context context, String collectionID) {
handler.logInfo("Running: a harvest cycle on " + collectionID);
System.out.print("Initializing the harvester... ");
handler.logInfo("Initializing the harvester... ");
OAIHarvester harvester = null;
try {
Collection collection = resolveCollection(collectionID);
Collection collection = resolveCollection(context, collectionID);
HarvestedCollection hc = harvestedCollectionService.find(context, collection);
harvester = new OAIHarvester(context, collection, hc);
System.out.println("success. ");
handler.logInfo("Initialized the harvester successfully");
} catch (HarvestingException hex) {
System.out.print("failed. ");
System.out.println(hex.getMessage());
handler.logError("Initializing the harvester failed.");
throw new IllegalStateException("Unable to harvest", hex);
} catch (SQLException se) {
System.out.print("failed. ");
System.out.println(se.getMessage());
handler.logError("Initializing the harvester failed.");
throw new IllegalStateException("Unable to access database", se);
}
try {
// Harvest will not work for an anonymous user
EPerson eperson = ePersonService.findByEmail(context, email);
System.out.println("Harvest started... ");
context.setCurrentUser(eperson);
handler.logInfo("Harvest started... ");
harvester.runHarvest();
context.complete();
} catch (SQLException | AuthorizeException | IOException e) {
throw new IllegalStateException("Failed to run harvester", e);
}
System.out.println("Harvest complete. ");
handler.logInfo("Harvest complete. ");
}
/**
* Resets harvest_status and harvest_start_time flags for all collections that have a row in the
* harvested_collections table
*/
private static void resetHarvesting() {
System.out.print("Resetting harvest status flag on all collections... ");
private void resetHarvesting(Context context) {
handler.logInfo("Resetting harvest status flag on all collections... ");
try {
List<HarvestedCollection> harvestedCollections = harvestedCollectionService.findAll(context);
@@ -424,21 +411,21 @@ public class Harvest {
harvestedCollection.setHarvestStatus(HarvestedCollection.STATUS_READY);
harvestedCollectionService.update(context, harvestedCollection);
}
System.out.println("success. ");
handler.logInfo("Reset harvest status flag successfully");
} catch (Exception ex) {
System.out.println("failed. ");
ex.printStackTrace();
handler.logError("Resetting harvest status flag failed");
handler.handleException(ex);
}
}
/**
* Starts up the harvest scheduler. Terminating this process will stop the scheduler.
*/
private static void startHarvester() {
private void startHarvester() {
try {
System.out.print("Starting harvest loop... ");
handler.logInfo("Starting harvest loop... ");
HarvestServiceFactory.getInstance().getHarvestSchedulingService().startNewScheduler();
System.out.println("running. ");
handler.logInfo("running. ");
} catch (Exception ex) {
ex.printStackTrace();
}
@@ -451,29 +438,31 @@ public class Harvest {
* @param set name of an item set.
* @param metadataFormat local prefix name, or null for "dc".
*/
private static void pingResponder(String server, String set, String metadataFormat) {
private void pingResponder(String server, String set, String metadataFormat) {
List<String> errors;
System.out.print("Testing basic PMH access: ");
handler.logInfo("Testing basic PMH access: ");
errors = harvestedCollectionService.verifyOAIharvester(server, set,
(null != metadataFormat) ? metadataFormat : "dc", false);
(null != metadataFormat) ? metadataFormat : "dc", false);
if (errors.isEmpty()) {
System.out.println("OK");
handler.logInfo("OK");
} else {
for (String error : errors) {
System.err.println(error);
handler.logError(error);
}
}
System.out.print("Testing ORE support: ");
handler.logInfo("Testing ORE support: ");
errors = harvestedCollectionService.verifyOAIharvester(server, set,
(null != metadataFormat) ? metadataFormat : "dc", true);
(null != metadataFormat) ? metadataFormat : "dc", true);
if (errors.isEmpty()) {
System.out.println("OK");
handler.logInfo("OK");
} else {
for (String error : errors) {
System.err.println(error);
handler.logError(error);
}
}
}
}

45
dspace-api/src/main/java/org/dspace/app/harvest/HarvestCli.java Normal file
View File

@@ -0,0 +1,45 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.harvest;
import java.sql.SQLException;
import org.apache.commons.cli.ParseException;
import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
public class HarvestCli extends Harvest {
/**
* This is the overridden instance of the {@link Harvest#assignCurrentUserInContext()} method in the parent class
* {@link Harvest}.
* This is done so that the CLI version of the Script is able to retrieve its currentUser from the -e flag given
* with the parameters of the Script.
*
* @throws ParseException If the e flag was not given to the parameters when calling the script
*/
@Override
protected void assignCurrentUserInContext() throws ParseException {
if (this.commandLine.hasOption('e')) {
String ePersonEmail = this.commandLine.getOptionValue('e');
this.context = new Context(Context.Mode.BATCH_EDIT);
try {
EPerson ePerson = ePersonService.findByEmail(this.context, ePersonEmail);
if (ePerson == null) {
super.handler.logError("EPerson not found: " + ePersonEmail);
throw new IllegalArgumentException("Unable to find a user with email: " + ePersonEmail);
}
this.context.setCurrentUser(ePerson);
} catch (SQLException e) {
throw new IllegalArgumentException("SQLException trying to find user with email: " + ePersonEmail);
}
}
}
}

22
dspace-api/src/main/java/org/dspace/app/harvest/HarvestCliScriptConfiguration.java Normal file
View File

@@ -0,0 +1,22 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.harvest;
import org.apache.commons.cli.Options;
public class HarvestCliScriptConfiguration extends HarvestScriptConfiguration {
public Options getOptions() {
Options options = super.getOptions();
options.addOption("e", "eperson", true,
"eperson");
return options;
}
}

70
dspace-api/src/main/java/org/dspace/app/harvest/HarvestScriptConfiguration.java Normal file
View File

@@ -0,0 +1,70 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.harvest;
import java.sql.SQLException;
import org.apache.commons.cli.Options;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.core.Context;
import org.dspace.scripts.configuration.ScriptConfiguration;
import org.springframework.beans.factory.annotation.Autowired;
public class HarvestScriptConfiguration<T extends Harvest> extends ScriptConfiguration<T> {
@Autowired
private AuthorizeService authorizeService;
private Class<T> dspaceRunnableClass;
@Override
public Class<T> getDspaceRunnableClass() {
return dspaceRunnableClass;
}
@Override
public void setDspaceRunnableClass(Class<T> dspaceRunnableClass) {
this.dspaceRunnableClass = dspaceRunnableClass;
}
public boolean isAllowedToExecute(final Context context) {
try {
return authorizeService.isAdmin(context);
} catch (SQLException e) {
throw new RuntimeException("SQLException occurred when checking if the current user is an admin", e);
}
}
public Options getOptions() {
Options options = new Options();
options.addOption("p", "purge", false, "delete all items in the collection");
options.addOption("r", "run", false, "run the standard harvest procedure");
options.addOption("g", "ping", false, "test the OAI server and set");
options.addOption("s", "setup", false, "Set the collection up for harvesting");
options.addOption("S", "start", false, "start the harvest loop");
options.addOption("R", "reset", false, "reset harvest status on all collections");
options.addOption("P", "purgeCollections", false, "purge all harvestable collections");
options.addOption("o", "reimport", false, "reimport all items in the collection, " +
"this is equivalent to -p -r, purging all items in a collection and reimporting them");
options.addOption("c", "collection", true,
"harvesting collection (handle or id)");
options.addOption("t", "type", true,
"type of harvesting (0 for none)");
options.addOption("a", "address", true,
"address of the OAI-PMH server");
options.addOption("i", "oai_set_id", true,
"id of the PMH set representing the harvested collection");
options.addOption("m", "metadata_format", true,
"the name of the desired metadata format for harvesting, resolved to namespace and " +
"crosswalk in dspace.cfg");
options.addOption("h", "help", false, "help");
return options;
}
}

29
dspace-api/src/main/java/org/dspace/app/itemexport/ItemExportServiceImpl.java
View File

@@ -16,6 +16,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -51,7 +52,7 @@ import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.Email;
import org.dspace.core.I18nUtil;
import org.dspace.core.LogManager;
import org.dspace.core.LogHelper;
import org.dspace.core.Utils;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.service.EPersonService;
@@ -129,7 +130,7 @@ public class ItemExportServiceImpl implements ItemExportService {
while (i.hasNext()) {
if (SUBDIR_LIMIT > 0 && ++counter == SUBDIR_LIMIT) {
subdir = Integer.valueOf(subDirSuffix++).toString();
subdir = Integer.toString(subDirSuffix++);
fullPath = destDirName + File.separatorChar + subdir;
counter = 0;
@@ -191,7 +192,7 @@ public class ItemExportServiceImpl implements ItemExportService {
*/
protected void writeMetadata(Context c, Item i, File destDir, boolean migrate)
throws Exception {
Set<String> schemas = new HashSet<String>();
Set<String> schemas = new HashSet<>();
List<MetadataValue> dcValues = itemService.getMetadata(i, Item.ANY, Item.ANY, Item.ANY, Item.ANY);
for (MetadataValue metadataValue : dcValues) {
schemas.add(metadataValue.getMetadataField().getMetadataSchema().getName());
@@ -267,7 +268,7 @@ public class ItemExportServiceImpl implements ItemExportService {
+ Utils.addEntities(dcv.getValue()) + "</dcvalue>\n")
.getBytes("UTF-8");
if ((!migrate) ||
if (!migrate ||
(migrate && !(
("date".equals(metadataField.getElement()) && "issued".equals(qualifier)) ||
("date".equals(metadataField.getElement()) && "accessioned".equals(qualifier)) ||
@@ -292,10 +293,10 @@ public class ItemExportServiceImpl implements ItemExportService {
}
// When migrating, only keep date.issued if it is different to date.accessioned
if ((migrate) &&
if (migrate &&
(dateIssued != null) &&
(dateAccessioned != null) &&
(!dateIssued.equals(dateAccessioned))) {
!dateIssued.equals(dateAccessioned)) {
utf8 = (" <dcvalue element=\"date\" "
+ "qualifier=\"issued\">"
+ Utils.addEntities(dateIssued) + "</dcvalue>\n")
@@ -330,7 +331,7 @@ public class ItemExportServiceImpl implements ItemExportService {
File outFile = new File(destDir, filename);
if (outFile.createNewFile()) {
PrintWriter out = new PrintWriter(new FileWriter(outFile));
PrintWriter out = new PrintWriter(new FileWriter(outFile, StandardCharsets.UTF_8));
out.println(i.getHandle());
@@ -360,7 +361,7 @@ public class ItemExportServiceImpl implements ItemExportService {
File outFile = new File(destDir, "contents");
if (outFile.createNewFile()) {
PrintWriter out = new PrintWriter(new FileWriter(outFile));
PrintWriter out = new PrintWriter(new FileWriter(outFile, StandardCharsets.UTF_8));
List<Bundle> bundles = i.getBundles();
@@ -474,7 +475,7 @@ public class ItemExportServiceImpl implements ItemExportService {
public void createDownloadableExport(DSpaceObject dso,
Context context, boolean migrate) throws Exception {
EPerson eperson = context.getCurrentUser();
ArrayList<DSpaceObject> list = new ArrayList<DSpaceObject>(1);
ArrayList<DSpaceObject> list = new ArrayList<>(1);
list.add(dso);
processDownloadableExport(list, context, eperson == null ? null
: eperson.getEmail(), migrate);
@@ -491,7 +492,7 @@ public class ItemExportServiceImpl implements ItemExportService {
@Override
public void createDownloadableExport(DSpaceObject dso,
Context context, String additionalEmail, boolean migrate) throws Exception {
ArrayList<DSpaceObject> list = new ArrayList<DSpaceObject>(1);
ArrayList<DSpaceObject> list = new ArrayList<>(1);
list.add(dso);
processDownloadableExport(list, context, additionalEmail, migrate);
}
@@ -652,7 +653,7 @@ public class ItemExportServiceImpl implements ItemExportService {
while (iter.hasNext()) {
String keyName = iter.next();
List<UUID> uuids = itemsMap.get(keyName);
List<Item> items = new ArrayList<Item>();
List<Item> items = new ArrayList<>();
for (UUID uuid : uuids) {
items.add(itemService.find(context, uuid));
}
@@ -876,7 +877,7 @@ public class ItemExportServiceImpl implements ItemExportService {
.getIntProperty("org.dspace.app.itemexport.life.span.hours");
Calendar now = Calendar.getInstance();
now.setTime(new Date());
now.add(Calendar.HOUR, (-hours));
now.add(Calendar.HOUR, -hours);
File downloadDir = new File(getExportDownloadDirectory(eperson));
if (downloadDir.exists()) {
File[] files = downloadDir.listFiles();
@@ -896,7 +897,7 @@ public class ItemExportServiceImpl implements ItemExportService {
int hours = configurationService.getIntProperty("org.dspace.app.itemexport.life.span.hours");
Calendar now = Calendar.getInstance();
now.setTime(new Date());
now.add(Calendar.HOUR, (-hours));
now.add(Calendar.HOUR, -hours);
File downloadDir = new File(configurationService.getProperty("org.dspace.app.itemexport.download.dir"));
if (downloadDir.exists()) {
// Get a list of all the sub-directories, potentially one for each ePerson.
@@ -936,7 +937,7 @@ public class ItemExportServiceImpl implements ItemExportService {
email.send();
} catch (Exception e) {
log.warn(LogManager.getHeader(context, "emailSuccessMessage", "cannot notify user of export"), e);
log.warn(LogHelper.getHeader(context, "emailSuccessMessage", "cannot notify user of export"), e);
}
}

82
dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportCLITool.java
View File

@@ -76,7 +76,6 @@ public class ItemImportCLITool {
options.addOption("r", "replace", false, "replace items in mapfile");
options.addOption("d", "delete", false,
"delete items listed in mapfile");
options.addOption("i", "inputtype", true, "input type in case of BTE import");
options.addOption("s", "source", true, "source of items (directory)");
options.addOption("z", "zip", true, "name of zip file");
options.addOption("c", "collection", true,
@@ -100,7 +99,6 @@ public class ItemImportCLITool {
CommandLine line = parser.parse(options, argv);
String command = null; // add replace remove, etc
String bteInputType = null; //ris, endnote, tsv, csv, bibtex
String sourcedir = null;
String mapfile = null;
String eperson = null; // db ID or email
@@ -144,14 +142,6 @@ public class ItemImportCLITool {
command = "delete";
}
if (line.hasOption('b')) {
command = "add-bte";
}
if (line.hasOption('i')) {
bteInputType = line.getOptionValue('i');
}
if (line.hasOption('w')) {
useWorkflow = true;
if (line.hasOption('n')) {
@@ -235,37 +225,6 @@ public class ItemImportCLITool {
System.out.println("No collections given. Assuming 'collections' file inside item directory");
commandLineCollections = false;
}
} else if ("add-bte".equals(command)) {
//Source dir can be null, the user can specify the parameters for his loader in the Spring XML
// configuration file
if (mapfile == null) {
System.out
.println("Error - a map file to hold importing results must be specified");
System.out.println(" (run with -h flag for details)");
System.exit(1);
}
if (eperson == null) {
System.out
.println("Error - an eperson to do the importing must be specified");
System.out.println(" (run with -h flag for details)");
System.exit(1);
}
if (collections == null) {
System.out.println("No collections given. Assuming 'collections' file inside item directory");
commandLineCollections = false;
}
if (bteInputType == null) {
System.out
.println(
"Error - an input type (tsv, csv, ris, endnote, bibtex or any other type you have " +
"specified in BTE Spring XML configuration file) must be specified");
System.out.println(" (run with -h flag for details)");
System.exit(1);
}
} else if ("delete".equals(command)) {
if (eperson == null) {
System.out
@@ -280,9 +239,9 @@ public class ItemImportCLITool {
}
// can only resume for adds
if (isResume && !"add".equals(command) && !"add-bte".equals(command)) {
if (isResume && !"add".equals(command)) {
System.out
.println("Error - resume option only works with the --add or the --add-bte commands");
.println("Error - resume option only works with the --add command");
System.exit(1);
}
@@ -337,29 +296,36 @@ public class ItemImportCLITool {
// validate each collection arg to see if it's a real collection
for (int i = 0; i < collections.length; i++) {
// is the ID a handle?
if (collections[i].indexOf('/') != -1) {
// string has a / so it must be a handle - try and resolve
// it
mycollections.add((Collection) handleService
.resolveToObject(c, collections[i]));
// resolved, now make sure it's a collection
if ((mycollections.get(i) == null)
|| (mycollections.get(i).getType() != Constants.COLLECTION)) {
mycollections.set(i, null);
Collection resolved = null;
if (collections[i] != null) {
// is the ID a handle?
if (collections[i].indexOf('/') != -1) {
// string has a / so it must be a handle - try and resolve
// it
resolved = ((Collection) handleService
.resolveToObject(c, collections[i]));
} else {
// not a handle, try and treat it as an integer collection database ID
resolved = collectionService.find(c, UUID.fromString(collections[i]));
}
} else if (collections[i] != null) {
// not a handle, try and treat it as an integer collection database ID
mycollections.set(i, collectionService.find(c, UUID.fromString(collections[i])));
}
// was the collection valid?
if (mycollections.get(i) == null) {
if ((resolved == null)
|| (resolved.getType() != Constants.COLLECTION)) {
throw new IllegalArgumentException("Cannot resolve "
+ collections[i] + " to collection");
}
// add resolved collection to list
mycollections.add(resolved);
// print progress info
String owningPrefix = "";
@@ -368,7 +334,7 @@ public class ItemImportCLITool {
}
System.out.println(owningPrefix + " Collection: "
+ mycollections.get(i).getName());
+ resolved.getName());
}
} // end of validating collections

558
dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportServiceImpl.java
View File

@@ -7,6 +7,13 @@
*/
package org.dspace.app.itemimport;
import static org.dspace.iiif.util.IIIFSharedUtils.METADATA_IIIF_HEIGHT_QUALIFIER;
import static org.dspace.iiif.util.IIIFSharedUtils.METADATA_IIIF_IMAGE_ELEMENT;
import static org.dspace.iiif.util.IIIFSharedUtils.METADATA_IIIF_LABEL_ELEMENT;
import static org.dspace.iiif.util.IIIFSharedUtils.METADATA_IIIF_SCHEMA;
import static org.dspace.iiif.util.IIIFSharedUtils.METADATA_IIIF_TOC_ELEMENT;
import static org.dspace.iiif.util.IIIFSharedUtils.METADATA_IIIF_WIDTH_QUALIFIER;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
@@ -44,6 +51,10 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.collections4.ComparatorUtils;
import org.apache.commons.io.FileDeleteStrategy;
@@ -52,9 +63,9 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.logging.log4j.Logger;
import org.apache.xpath.XPathAPI;
import org.dspace.app.itemimport.service.ItemImportService;
import org.dspace.app.util.LocalSchemaFilenameFilter;
import org.dspace.app.util.RelationshipUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.ResourcePolicy;
import org.dspace.authorize.service.AuthorizeService;
@@ -68,6 +79,9 @@ import org.dspace.content.Item;
import org.dspace.content.MetadataField;
import org.dspace.content.MetadataSchema;
import org.dspace.content.MetadataSchemaEnum;
import org.dspace.content.MetadataValue;
import org.dspace.content.Relationship;
import org.dspace.content.RelationshipType;
import org.dspace.content.WorkspaceItem;
import org.dspace.content.service.BitstreamFormatService;
import org.dspace.content.service.BitstreamService;
@@ -77,12 +91,15 @@ import org.dspace.content.service.InstallItemService;
import org.dspace.content.service.ItemService;
import org.dspace.content.service.MetadataFieldService;
import org.dspace.content.service.MetadataSchemaService;
import org.dspace.content.service.MetadataValueService;
import org.dspace.content.service.RelationshipService;
import org.dspace.content.service.RelationshipTypeService;
import org.dspace.content.service.WorkspaceItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.Email;
import org.dspace.core.I18nUtil;
import org.dspace.core.LogManager;
import org.dspace.core.LogHelper;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.Group;
import org.dspace.eperson.service.EPersonService;
@@ -151,6 +168,12 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
protected WorkflowService workflowService;
@Autowired(required = true)
protected ConfigurationService configurationService;
@Autowired(required = true)
protected RelationshipService relationshipService;
@Autowired(required = true)
protected RelationshipTypeService relationshipTypeService;
@Autowired(required = true)
protected MetadataValueService metadataValueService;
protected String tempWorkDir;
@@ -160,6 +183,9 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
protected boolean useWorkflowSendEmail = false;
protected boolean isQuiet = false;
//remember which folder item was imported from
Map<String, Item> itemFolderMap = null;
@Override
public void afterPropertiesSet() throws Exception {
tempWorkDir = configurationService.getProperty("org.dspace.app.batchitemimport.work.dir");
@@ -211,10 +237,13 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
// create the mapfile
File outFile = null;
PrintWriter mapOut = null;
try {
Map<String, String> skipItems = new HashMap<>(); // set of items to skip if in 'resume'
// mode
itemFolderMap = new HashMap<>();
System.out.println("Adding items from directory: " + sourceDir);
log.debug("Adding items from directory: " + sourceDir);
System.out.println("Generating mapfile: " + mapFile);
@@ -255,6 +284,12 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
for (int i = 0; i < dircontents.length; i++) {
if (skipItems.containsKey(dircontents[i])) {
System.out.println("Skipping import of " + dircontents[i]);
//we still need the item in the map for relationship linking
String skippedHandle = skipItems.get(dircontents[i]);
Item skippedItem = (Item) handleService.resolveToObject(c, skippedHandle);
itemFolderMap.put(dircontents[i], skippedItem);
} else {
List<Collection> clist;
if (directoryFileCollections) {
@@ -274,12 +309,19 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
} else {
clist = mycollections;
}
Item item = addItem(c, clist, sourceDir, dircontents[i], mapOut, template);
itemFolderMap.put(dircontents[i], item);
c.uncacheEntity(item);
System.out.println(i + " " + dircontents[i]);
}
}
//now that all items are imported, iterate again to link relationships
addRelationships(c, sourceDir);
} finally {
if (mapOut != null) {
mapOut.flush();
@@ -288,6 +330,274 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
}
}
/**
* Add relationships from a 'relationships' manifest file.
*
* @param c Context
* @param sourceDir The parent import source directory
* @throws Exception
*/
protected void addRelationships(Context c, String sourceDir) throws Exception {
for (Map.Entry<String, Item> itemEntry : itemFolderMap.entrySet()) {
String folderName = itemEntry.getKey();
String path = sourceDir + File.separatorChar + folderName;
Item item = itemEntry.getValue();
//look for a 'relationship' manifest
Map<String, List<String>> relationships = processRelationshipFile(path, "relationships");
if (!relationships.isEmpty()) {
for (Map.Entry<String, List<String>> relEntry : relationships.entrySet()) {
String relationshipType = relEntry.getKey();
List<String> identifierList = relEntry.getValue();
for (String itemIdentifier : identifierList) {
if (isTest) {
System.out.println("\tAdding relationship (type: " + relationshipType +
") from " + folderName + " to " + itemIdentifier);
continue;
}
//find referenced item
Item relationItem = resolveRelatedItem(c, itemIdentifier);
if (null == relationItem) {
throw new Exception("Could not find item for " + itemIdentifier);
}
//get entity type of entity and item
String itemEntityType = getEntityType(item);
String relatedEntityType = getEntityType(relationItem);
//find matching relationship type
List<RelationshipType> relTypes = relationshipTypeService.findByLeftwardOrRightwardTypeName(
c, relationshipType);
RelationshipType foundRelationshipType = RelationshipUtils.matchRelationshipType(
relTypes, relatedEntityType, itemEntityType, relationshipType);
if (foundRelationshipType == null) {
throw new Exception("No Relationship type found for:\n" +
"Target type: " + relatedEntityType + "\n" +
"Origin referer type: " + itemEntityType + "\n" +
"with typeName: " + relationshipType
);
}
boolean left = false;
if (foundRelationshipType.getLeftwardType().equalsIgnoreCase(relationshipType)) {
left = true;
}
// Placeholder items for relation placing
Item leftItem = null;
Item rightItem = null;
if (left) {
leftItem = item;
rightItem = relationItem;
} else {
leftItem = relationItem;
rightItem = item;
}
// Create the relationship
Relationship persistedRelationship =
relationshipService.create(c, leftItem, rightItem, foundRelationshipType, -1, -1);
// relationshipService.update(c, persistedRelationship);
System.out.println("\tAdded relationship (type: " + relationshipType + ") from " +
leftItem.getHandle() + " to " + rightItem.getHandle());
}
}
}
}
}
/**
* Get the item's entity type from meta.
*
* @param item
* @return
*/
protected String getEntityType(Item item) throws Exception {
return itemService.getMetadata(item, "dspace", "entity", "type", Item.ANY).get(0).getValue();
}
/**
* Read the relationship manifest file.
*
* Each line in the file contains a relationship type id and an item identifier in the following format:
*
* relation.<relation_key> <handle|uuid|folderName:import_item_folder|schema.element[.qualifier]:value>
*
* The input_item_folder should refer the folder name of another item in this import batch.
*
* @param path The main import folder path.
* @param filename The name of the manifest file to check ('relationships')
* @return Map of found relationships
* @throws Exception
*/
protected Map<String, List<String>> processRelationshipFile(String path, String filename) throws Exception {
File file = new File(path + File.separatorChar + filename);
Map<String, List<String>> result = new HashMap<>();
if (file.exists()) {
System.out.println("\tProcessing relationships file: " + filename);
BufferedReader br = null;
try {
br = new BufferedReader(new FileReader(file));
String line = null;
while ((line = br.readLine()) != null) {
line = line.trim();
if ("".equals(line)) {
continue;
}
String relationshipType = null;
String itemIdentifier = null;
StringTokenizer st = new StringTokenizer(line);
if (st.hasMoreTokens()) {
relationshipType = st.nextToken();
if (relationshipType.split("\\.").length > 1) {
relationshipType = relationshipType.split("\\.")[1];
}
} else {
throw new Exception("Bad mapfile line:\n" + line);
}
if (st.hasMoreTokens()) {
itemIdentifier = st.nextToken("").trim();
} else {
throw new Exception("Bad mapfile line:\n" + line);
}
if (!result.containsKey(relationshipType)) {
result.put(relationshipType, new ArrayList<>());
}
result.get(relationshipType).add(itemIdentifier);
}
} catch (FileNotFoundException e) {
System.out.println("\tNo relationships file found.");
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {
System.out.println("Non-critical problem releasing resources.");
}
}
}
}
return result;
}
/**
* Resolve an item identifier referred to in the relationships manifest file.
*
* The import item map will be checked first to see if the identifier refers to an item folder
* that was just imported. Next it will try to find the item by handle or UUID, or by a unique
* meta value.
*
* @param c Context
* @param itemIdentifier The identifier string found in the import manifest (handle, uuid, or import subfolder)
* @return Item if found, or null.
* @throws Exception
*/
protected Item resolveRelatedItem(Context c, String itemIdentifier) throws Exception {
if (itemIdentifier.contains(":")) {
if (itemIdentifier.startsWith("folderName:") || itemIdentifier.startsWith("rowName:")) {
//identifier refers to a folder name in this import
int i = itemIdentifier.indexOf(":");
String folderName = itemIdentifier.substring(i + 1);
if (itemFolderMap.containsKey(folderName)) {
return itemFolderMap.get(folderName);
}
} else {
//lookup by meta value
int i = itemIdentifier.indexOf(":");
String metaKey = itemIdentifier.substring(0, i);
String metaValue = itemIdentifier.substring(i + 1);
return findItemByMetaValue(c, metaKey, metaValue);
}
} else if (itemIdentifier.indexOf('/') != -1) {
//resolve by handle
return (Item) handleService.resolveToObject(c, itemIdentifier);
} else {
//try to resolve by UUID
return itemService.findByIdOrLegacyId(c, itemIdentifier);
}
return null;
}
/**
* Lookup an item by a (unique) meta value.
*
* @param metaKey
* @param metaValue
* @return Item
* @throws Exception if single item not found.
*/
protected Item findItemByMetaValue(Context c, String metaKey, String metaValue) throws Exception {
Item item = null;
String mf[] = metaKey.split("\\.");
if (mf.length < 2) {
throw new Exception("Bad metadata field in reference: '" + metaKey +
"' (expected syntax is schema.element[.qualifier])");
}
String schema = mf[0];
String element = mf[1];
String qualifier = mf.length == 2 ? null : mf[2];
try {
MetadataField mfo = metadataFieldService.findByElement(c, schema, element, qualifier);
Iterator<MetadataValue> mdv = metadataValueService.findByFieldAndValue(c, mfo, metaValue);
if (mdv.hasNext()) {
MetadataValue mdvVal = mdv.next();
UUID uuid = mdvVal.getDSpaceObject().getID();
if (mdv.hasNext()) {
throw new Exception("Ambiguous reference; multiple matches in db: " + metaKey);
}
item = itemService.find(c, uuid);
}
} catch (SQLException e) {
throw new Exception("Error looking up item by metadata reference: " + metaKey, e);
}
if (item == null) {
throw new Exception("Item not found by metadata reference: " + metaKey);
}
return item;
}
@Override
public void replaceItems(Context c, List<Collection> mycollections,
String sourceDir, String mapFile, boolean template) throws Exception {
@@ -554,7 +864,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
// Load all metadata schemas into the item.
protected void loadMetadata(Context c, Item myitem, String path)
throws SQLException, IOException, ParserConfigurationException,
SAXException, TransformerException, AuthorizeException {
SAXException, TransformerException, AuthorizeException, XPathExpressionException {
// Load the dublin core metadata
loadDublinCore(c, myitem, path + "dublin_core.xml");
@@ -568,14 +878,15 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
protected void loadDublinCore(Context c, Item myitem, String filename)
throws SQLException, IOException, ParserConfigurationException,
SAXException, TransformerException, AuthorizeException {
SAXException, TransformerException, AuthorizeException, XPathExpressionException {
Document document = loadXML(filename);
// Get the schema, for backward compatibility we will default to the
// dublin core schema if the schema name is not available in the import
// file
String schema;
NodeList metadata = XPathAPI.selectNodeList(document, "/dublin_core");
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList metadata = (NodeList) xPath.compile("/dublin_core").evaluate(document, XPathConstants.NODESET);
Node schemaAttr = metadata.item(0).getAttributes().getNamedItem(
"schema");
if (schemaAttr == null) {
@@ -585,8 +896,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
}
// Get the nodes corresponding to formats
NodeList dcNodes = XPathAPI.selectNodeList(document,
"/dublin_core/dcvalue");
NodeList dcNodes = (NodeList) xPath.compile("/dublin_core/dcvalue").evaluate(document, XPathConstants.NODESET);
if (!isQuiet) {
System.out.println("\tLoading dublin core from " + filename);
@@ -827,7 +1137,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
if (iAssetstore == -1 || sFilePath == null) {
System.out.println("\tERROR: invalid contents file line");
System.out.println("\t\tSkipping line: "
+ sRegistrationLine);
+ sRegistrationLine);
continue;
}
@@ -851,9 +1161,9 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
registerBitstream(c, i, iAssetstore, sFilePath, sBundle, sDescription);
System.out.println("\tRegistering Bitstream: " + sFilePath
+ "\tAssetstore: " + iAssetstore
+ "\tBundle: " + sBundle
+ "\tDescription: " + sDescription);
+ "\tAssetstore: " + iAssetstore
+ "\tBundle: " + sBundle
+ "\tDescription: " + sDescription);
continue; // process next line in contents file
}
@@ -870,6 +1180,59 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
boolean bundleExists = false;
boolean permissionsExist = false;
boolean descriptionExists = false;
boolean labelExists = false;
boolean heightExists = false;
boolean widthExists = false;
boolean tocExists = false;
// look for label
String labelMarker = "\tiiif-label";
int lMarkerIndex = line.indexOf(labelMarker);
int lEndIndex = 0;
if (lMarkerIndex > 0) {
lEndIndex = line.indexOf("\t", lMarkerIndex + 1);
if (lEndIndex == -1) {
lEndIndex = line.length();
}
labelExists = true;
}
// look for height
String heightMarker = "\tiiif-height";
int hMarkerIndex = line.indexOf(heightMarker);
int hEndIndex = 0;
if (hMarkerIndex > 0) {
hEndIndex = line.indexOf("\t", hMarkerIndex + 1);
if (hEndIndex == -1) {
hEndIndex = line.length();
}
heightExists = true;
}
// look for width
String widthMarker = "\tiiif-width";
int wMarkerIndex = line.indexOf(widthMarker);
int wEndIndex = 0;
if (wMarkerIndex > 0) {
wEndIndex = line.indexOf("\t", wMarkerIndex + 1);
if (wEndIndex == -1) {
wEndIndex = line.length();
}
widthExists = true;
}
// look for toc
String tocMarker = "\tiiif-toc";
int tMarkerIndex = line.indexOf(tocMarker);
int tEndIndex = 0;
if (tMarkerIndex > 0) {
tEndIndex = line.indexOf("\t", tMarkerIndex + 1);
if (tEndIndex == -1) {
tEndIndex = line.length();
}
tocExists = true;
}
// look for a bundle name
String bundleMarker = "\tbundle:";
@@ -918,18 +1281,20 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
if (bundleExists) {
String bundleName = line.substring(bMarkerIndex
+ bundleMarker.length(), bEndIndex).trim();
+ bundleMarker.length(), bEndIndex).trim();
processContentFileEntry(c, i, path, bitstreamName, bundleName, primary);
System.out.println("\tBitstream: " + bitstreamName +
"\tBundle: " + bundleName +
primaryStr);
"\tBundle: " + bundleName +
primaryStr);
} else {
processContentFileEntry(c, i, path, bitstreamName, null, primary);
System.out.println("\tBitstream: " + bitstreamName + primaryStr);
}
if (permissionsExist || descriptionExists) {
if (permissionsExist || descriptionExists || labelExists || heightExists
|| widthExists || tocExists) {
System.out.println("Gathering options.");
String extraInfo = bitstreamName;
if (permissionsExist) {
@@ -942,6 +1307,26 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
+ line.substring(dMarkerIndex, dEndIndex);
}
if (labelExists) {
extraInfo = extraInfo
+ line.substring(lMarkerIndex, lEndIndex);
}
if (heightExists) {
extraInfo = extraInfo
+ line.substring(hMarkerIndex, hEndIndex);
}
if (widthExists) {
extraInfo = extraInfo
+ line.substring(wMarkerIndex, wEndIndex);
}
if (tocExists) {
extraInfo = extraInfo
+ line.substring(tMarkerIndex, tEndIndex);
}
options.add(extraInfo);
}
}
@@ -1123,11 +1508,16 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
*/
protected void processOptions(Context c, Item myItem, List<String> options)
throws SQLException, AuthorizeException {
System.out.println("Processing options.");
for (String line : options) {
System.out.println("\tprocessing " + line);
boolean permissionsExist = false;
boolean descriptionExists = false;
boolean labelExists = false;
boolean heightExists = false;
boolean widthExists = false;
boolean tocExists = false;
String permissionsMarker = "\tpermissions:";
int pMarkerIndex = line.indexOf(permissionsMarker);
@@ -1151,6 +1541,56 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
descriptionExists = true;
}
// look for label
String labelMarker = "\tiiif-label:";
int lMarkerIndex = line.indexOf(labelMarker);
int lEndIndex = 0;
if (lMarkerIndex > 0) {
lEndIndex = line.indexOf("\t", lMarkerIndex + 1);
if (lEndIndex == -1) {
lEndIndex = line.length();
}
labelExists = true;
}
// look for height
String heightMarker = "\tiiif-height:";
int hMarkerIndex = line.indexOf(heightMarker);
int hEndIndex = 0;
if (hMarkerIndex > 0) {
hEndIndex = line.indexOf("\t", hMarkerIndex + 1);
if (hEndIndex == -1) {
hEndIndex = line.length();
}
heightExists = true;
}
// look for width
String widthMarker = "\tiiif-width:";
int wMarkerIndex = line.indexOf(widthMarker);
int wEndIndex = 0;
if (wMarkerIndex > 0) {
wEndIndex = line.indexOf("\t", wMarkerIndex + 1);
if (wEndIndex == -1) {
wEndIndex = line.length();
}
widthExists = true;
}
// look for toc
String tocMarker = "\tiiif-toc:";
int tMarkerIndex = line.indexOf(tocMarker);
int tEndIndex = 0;
if (tMarkerIndex > 0) {
tEndIndex = line.indexOf("\t", tMarkerIndex + 1);
if (tEndIndex == -1) {
tEndIndex = line.length();
}
tocExists = true;
}
int bsEndIndex = line.indexOf("\t");
String bitstreamName = line.substring(0, bsEndIndex);
@@ -1159,7 +1599,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
Group myGroup = null;
if (permissionsExist) {
String thisPermission = line.substring(pMarkerIndex
+ permissionsMarker.length(), pEndIndex);
+ permissionsMarker.length(), pEndIndex);
// get permission type ("read" or "write")
int pTypeIndex = thisPermission.indexOf('-');
@@ -1176,7 +1616,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
}
groupName = thisPermission.substring(groupIndex + 1,
groupEndIndex);
groupEndIndex);
if (thisPermission.toLowerCase().charAt(pTypeIndex + 1) == 'r') {
actionID = Constants.READ;
@@ -1188,7 +1628,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
myGroup = groupService.findByName(c, groupName);
} catch (SQLException sqle) {
System.out.println("SQL Exception finding group name: "
+ groupName);
+ groupName);
// do nothing, will check for null group later
}
}
@@ -1196,12 +1636,42 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
String thisDescription = "";
if (descriptionExists) {
thisDescription = line.substring(
dMarkerIndex + descriptionMarker.length(), dEndIndex)
dMarkerIndex + descriptionMarker.length(), dEndIndex)
.trim();
}
String thisLabel = "";
if (labelExists) {
thisLabel = line.substring(
lMarkerIndex + labelMarker.length(), lEndIndex)
.trim();
}
String thisHeight = "";
if (heightExists) {
thisHeight = line.substring(
hMarkerIndex + heightMarker.length(), hEndIndex)
.trim();
}
String thisWidth = "";
if (widthExists) {
thisWidth = line.substring(
wMarkerIndex + widthMarker.length(), wEndIndex)
.trim();
}
String thisToc = "";
if (tocExists) {
thisToc = line.substring(
tMarkerIndex + tocMarker.length(), tEndIndex)
.trim();
}
Bitstream bs = null;
boolean notfound = true;
boolean updateRequired = false;
if (!isTest) {
// find bitstream
List<Bitstream> bitstreams = itemService.getNonInternalBitstreams(c, myItem);
@@ -1216,26 +1686,65 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
if (notfound && !isTest) {
// this should never happen
System.out.println("\tdefault permissions set for "
+ bitstreamName);
+ bitstreamName);
} else if (!isTest) {
if (permissionsExist) {
if (myGroup == null) {
System.out.println("\t" + groupName
+ " not found, permissions set to default");
+ " not found, permissions set to default");
} else if (actionID == -1) {
System.out
.println("\tinvalid permissions flag, permissions set to default");
} else {
System.out.println("\tSetting special permissions for "
+ bitstreamName);
+ bitstreamName);
setPermission(c, myGroup, actionID, bs);
}
}
if (descriptionExists) {
System.out.println("\tSetting description for "
+ bitstreamName);
+ bitstreamName);
bs.setDescription(c, thisDescription);
updateRequired = true;
}
if (labelExists) {
MetadataField metadataField = metadataFieldService
.findByElement(c, METADATA_IIIF_SCHEMA, METADATA_IIIF_LABEL_ELEMENT, null);
System.out.println("\tSetting label to " + thisLabel + " in element "
+ metadataField.getElement() + " on " + bitstreamName);
bitstreamService.addMetadata(c, bs, metadataField, null, thisLabel);
updateRequired = true;
}
if (heightExists) {
MetadataField metadataField = metadataFieldService
.findByElement(c, METADATA_IIIF_SCHEMA, METADATA_IIIF_IMAGE_ELEMENT,
METADATA_IIIF_HEIGHT_QUALIFIER);
System.out.println("\tSetting height to " + thisHeight + " in element "
+ metadataField.getElement() + " on " + bitstreamName);
bitstreamService.addMetadata(c, bs, metadataField, null, thisHeight);
updateRequired = true;
}
if (widthExists) {
MetadataField metadataField = metadataFieldService
.findByElement(c, METADATA_IIIF_SCHEMA, METADATA_IIIF_IMAGE_ELEMENT,
METADATA_IIIF_WIDTH_QUALIFIER);
System.out.println("\tSetting width to " + thisWidth + " in element "
+ metadataField.getElement() + " on " + bitstreamName);
bitstreamService.addMetadata(c, bs, metadataField, null, thisWidth);
updateRequired = true;
}
if (tocExists) {
MetadataField metadataField = metadataFieldService
.findByElement(c, METADATA_IIIF_SCHEMA, METADATA_IIIF_TOC_ELEMENT, null);
System.out.println("\tSetting toc to " + thisToc + " in element "
+ metadataField.getElement() + " on " + bitstreamName);
bitstreamService.addMetadata(c, bs, metadataField, null, thisToc);
updateRequired = true;
}
if (updateRequired) {
bitstreamService.update(c, bs);
}
}
@@ -1689,7 +2198,7 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
email.send();
} catch (Exception e) {
log.warn(LogManager.getHeader(context, "emailSuccessMessage", "cannot notify user of import"), e);
log.warn(LogHelper.getHeader(context, "emailSuccessMessage", "cannot notify user of import"), e);
}
}
@@ -1823,4 +2332,5 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
public void setQuiet(boolean isQuiet) {
this.isQuiet = isQuiet;
}
}

4
dspace-api/src/main/java/org/dspace/app/itemimport/service/ItemImportService.java
View File

@@ -105,7 +105,7 @@ public interface ItemImportService {
String inputType, Context context, boolean template) throws Exception;
/**
* Since the BTE batch import is done in a new thread we are unable to communicate
* If a batch import is done in a new thread we are unable to communicate
* with calling method about success or failure. We accomplish this
* communication with email instead. Send a success email once the batch
* import is complete
@@ -119,7 +119,7 @@ public interface ItemImportService {
String fileName) throws MessagingException;
/**
* Since the BTE batch import is done in a new thread we are unable to communicate
* If a batch import is done in a new thread we are unable to communicate
* with calling method about success or failure. We accomplis this
* communication with email instead. Send an error email if the batch
* import fails

12
dspace-api/src/main/java/org/dspace/app/itemmarking/ItemMarkingAvailabilityBitstreamStrategy.java
View File

@@ -11,6 +11,8 @@ import java.io.UnsupportedEncodingException;
import java.sql.SQLException;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.app.util.Util;
import org.dspace.content.Bitstream;
import org.dspace.content.Bundle;
@@ -34,8 +36,9 @@ public class ItemMarkingAvailabilityBitstreamStrategy implements ItemMarkingExtr
@Autowired(required = true)
protected ItemService itemService;
public ItemMarkingAvailabilityBitstreamStrategy() {
private static final Logger LOG = LogManager.getLogger();
public ItemMarkingAvailabilityBitstreamStrategy() {
}
@Override
@@ -43,14 +46,14 @@ public class ItemMarkingAvailabilityBitstreamStrategy implements ItemMarkingExtr
throws SQLException {
List<Bundle> bundles = itemService.getBundles(item, "ORIGINAL");
if (bundles.size() == 0) {
if (bundles.isEmpty()) {
ItemMarkingInfo markInfo = new ItemMarkingInfo();
markInfo.setImageName(nonAvailableImageName);
return markInfo;
} else {
Bundle originalBundle = bundles.iterator().next();
if (originalBundle.getBitstreams().size() == 0) {
if (originalBundle.getBitstreams().isEmpty()) {
ItemMarkingInfo markInfo = new ItemMarkingInfo();
markInfo.setImageName(nonAvailableImageName);
@@ -72,8 +75,7 @@ public class ItemMarkingAvailabilityBitstreamStrategy implements ItemMarkingExtr
try {
bsLink = bsLink + Util.encodeBitstreamName(bitstream.getName(), Constants.DEFAULT_ENCODING);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
LOG.warn("DSpace uses an unsupported encoding", e);
}
signInfo.setLink(bsLink);

1
dspace-api/src/main/java/org/dspace/app/itemupdate/ContentsEntry.java
View File

@@ -105,6 +105,7 @@ public class ContentsEntry {
return new ContentsEntry(arp[0], arp[1], actionId, groupName, arp[3]);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder(filename);
if (bundlename != null) {

1
dspace-api/src/main/java/org/dspace/app/itemupdate/DtoMetadata.java
View File

@@ -120,6 +120,7 @@ class DtoMetadata {
return true;
}
@Override
public String toString() {
String s = "\tSchema: " + schema + " Element: " + element;
if (qualifier != null) {

5
dspace-api/src/main/java/org/dspace/app/itemupdate/ItemArchive.java
View File

@@ -17,6 +17,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -55,7 +56,7 @@ public class ItemArchive {
protected Transformer transformer = null;
protected List<DtoMetadata> dtomList = null;
protected List<DtoMetadata> undoDtomList = new ArrayList<DtoMetadata>();
protected List<DtoMetadata> undoDtomList = new ArrayList<>();
protected List<UUID> undoAddContents = new ArrayList<>(); // for undo of add
@@ -325,7 +326,7 @@ public class ItemArchive {
PrintWriter pw = null;
try {
File f = new File(dir, ItemUpdate.DELETE_CONTENTS_FILE);
pw = new PrintWriter(new BufferedWriter(new FileWriter(f)));
pw = new PrintWriter(new BufferedWriter(new FileWriter(f, StandardCharsets.UTF_8)));
for (UUID i : undoAddContents) {
pw.println(i);
}

19
dspace-api/src/main/java/org/dspace/app/itemupdate/MetadataUtilities.java
View File

@@ -27,10 +27,12 @@ import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.xpath.XPathAPI;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Item;
import org.dspace.content.MetadataField;
import org.dspace.content.MetadataSchema;
@@ -170,24 +172,21 @@ public class MetadataUtilities {
* @param docBuilder DocumentBuilder
* @param is - InputStream of dublin_core.xml
* @return list of DtoMetadata representing the metadata fields relating to an Item
* @throws SQLException if database error
* @throws IOException if IO error
* @throws ParserConfigurationException if parser config error
* @throws SAXException if XML error
* @throws TransformerException if transformer error
* @throws AuthorizeException if authorization error
*/
public static List<DtoMetadata> loadDublinCore(DocumentBuilder docBuilder, InputStream is)
throws SQLException, IOException, ParserConfigurationException,
SAXException, TransformerException, AuthorizeException {
throws IOException, XPathExpressionException, SAXException {
Document document = docBuilder.parse(is);
List<DtoMetadata> dtomList = new ArrayList<DtoMetadata>();
// Get the schema, for backward compatibility we will default to the
// dublin core schema if the schema name is not available in the import file
String schema = null;
NodeList metadata = XPathAPI.selectNodeList(document, "/dublin_core");
String schema;
XPath xPath = XPathFactory.newInstance().newXPath();
NodeList metadata = (NodeList) xPath.compile("/dublin_core").evaluate(document, XPathConstants.NODESET);
Node schemaAttr = metadata.item(0).getAttributes().getNamedItem("schema");
if (schemaAttr == null) {
schema = MetadataSchemaEnum.DC.getName();
@@ -196,7 +195,7 @@ public class MetadataUtilities {
}
// Get the nodes corresponding to formats
NodeList dcNodes = XPathAPI.selectNodeList(document, "/dublin_core/dcvalue");
NodeList dcNodes = (NodeList) xPath.compile("/dublin_core/dcvalue").evaluate(document, XPathConstants.NODESET);
for (int i = 0; i < dcNodes.getLength(); i++) {
Node n = dcNodes.item(i);

2
dspace-api/src/main/java/org/dspace/app/launcher/CommandRunner.java
View File

@@ -16,7 +16,7 @@ import java.io.StreamTokenizer;
import java.util.ArrayList;
import java.util.List;
import org.jdom.Document;
import org.jdom2.Document;
/**
* @author mwood

6
dspace-api/src/main/java/org/dspace/app/launcher/ScriptLauncher.java
View File

@@ -29,9 +29,9 @@ import org.dspace.scripts.service.ScriptService;
import org.dspace.servicemanager.DSpaceKernelImpl;
import org.dspace.servicemanager.DSpaceKernelInit;
import org.dspace.services.RequestService;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.input.SAXBuilder;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.input.SAXBuilder;
/**
* A DSpace script launcher.

99
dspace-api/src/main/java/org/dspace/app/mediafilter/ExcelFilter.java
View File

@@ -1,99 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.mediafilter;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.Logger;
import org.apache.poi.POITextExtractor;
import org.apache.poi.extractor.ExtractorFactory;
import org.apache.poi.hssf.extractor.ExcelExtractor;
import org.apache.poi.xssf.extractor.XSSFExcelExtractor;
import org.dspace.content.Item;
/*
* ExcelFilter
*
* Entries you must add to dspace.cfg:
*
* filter.plugins = blah, \
* Excel Text Extractor
*
* plugin.named.org.dspace.app.mediafilter.FormatFilter = \
* blah = blah, \
* org.dspace.app.mediafilter.ExcelFilter = Excel Text Extractor
*
* #Configure each filter's input Formats
* filter.org.dspace.app.mediafilter.ExcelFilter.inputFormats = Microsoft Excel, Microsoft Excel XML
*
*/
public class ExcelFilter extends MediaFilter {
private static Logger log = org.apache.logging.log4j.LogManager.getLogger(ExcelFilter.class);
public String getFilteredName(String oldFilename) {
return oldFilename + ".txt";
}
/**
* @return String bundle name
*/
public String getBundleName() {
return "TEXT";
}
/**
* @return String bitstream format
*/
public String getFormatString() {
return "Text";
}
/**
* @return String description
*/
public String getDescription() {
return "Extracted text";
}
/**
* @param item item
* @param source source input stream
* @param verbose verbose mode
* @return InputStream the resulting input stream
* @throws Exception if error
*/
@Override
public InputStream getDestinationStream(Item item, InputStream source, boolean verbose)
throws Exception {
String extractedText = null;
try {
POITextExtractor theExtractor = ExtractorFactory.createExtractor(source);
if (theExtractor instanceof ExcelExtractor) {
// for xls file
extractedText = (theExtractor).getText();
} else if (theExtractor instanceof XSSFExcelExtractor) {
// for xlsx file
extractedText = (theExtractor).getText();
}
} catch (Exception e) {
log.error("Error filtering bitstream: " + e.getMessage(), e);
throw e;
}
if (extractedText != null) {
// generate an input stream with the extracted text
return IOUtils.toInputStream(extractedText, StandardCharsets.UTF_8);
}
return null;
}
}

81
dspace-api/src/main/java/org/dspace/app/mediafilter/HTMLFilter.java
View File

@@ -1,81 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.mediafilter;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import javax.swing.text.Document;
import javax.swing.text.html.HTMLEditorKit;
import org.dspace.content.Item;
/*
*
* to do: helpful error messages - can't find mediafilter.cfg - can't
* instantiate filter - bitstream format doesn't exist
*
*/
public class HTMLFilter extends MediaFilter {
@Override
public String getFilteredName(String oldFilename) {
return oldFilename + ".txt";
}
/**
* @return String bundle name
*/
@Override
public String getBundleName() {
return "TEXT";
}
/**
* @return String bitstreamformat
*/
@Override
public String getFormatString() {
return "Text";
}
/**
* @return String description
*/
@Override
public String getDescription() {
return "Extracted text";
}
/**
* @param currentItem item
* @param source source input stream
* @param verbose verbose mode
* @return InputStream the resulting input stream
* @throws Exception if error
*/
@Override
public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose)
throws Exception {
// try and read the document - set to ignore character set directive,
// assuming that the input stream is already set properly (I hope)
HTMLEditorKit kit = new HTMLEditorKit();
Document doc = kit.createDefaultDocument();
doc.putProperty("IgnoreCharsetDirective", Boolean.TRUE);
kit.read(source, doc, 0);
String extractedText = doc.getText(0, doc.getLength());
// generate an input stream with the extracted text
byte[] textBytes = extractedText.getBytes();
ByteArrayInputStream bais = new ByteArrayInputStream(textBytes);
return bais; // will this work? or will the byte array be out of scope?
}
}

196
dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterCLITool.java → dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java
View File

@@ -14,13 +14,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.ArrayUtils;
import org.dspace.app.mediafilter.factory.MediaFilterServiceFactory;
import org.dspace.app.mediafilter.service.MediaFilterService;
@@ -33,7 +27,9 @@ import org.dspace.core.Context;
import org.dspace.core.SelfNamedPlugin;
import org.dspace.core.factory.CoreServiceFactory;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.utils.DSpace;
/**
* MediaFilterManager is the class that invokes the media/format filters over the
@@ -44,7 +40,7 @@ import org.dspace.services.factory.DSpaceServicesFactory;
* scope to a community, collection or item; and -m [max] limits processing to a
* maximum number of items.
*/
public class MediaFilterCLITool {
public class MediaFilterScript extends DSpaceRunnable<MediaFilterScriptConfiguration> {
//key (in dspace.cfg) which lists all enabled filters by name
private static final String MEDIA_FILTER_PLUGINS_KEY = "filter.plugins";
@@ -55,127 +51,78 @@ public class MediaFilterCLITool {
//suffix (in dspace.cfg) for input formats supported by each filter
private static final String INPUT_FORMATS_SUFFIX = "inputFormats";
/**
* Default constructor
*/
private MediaFilterCLITool() { }
private boolean help;
private boolean isVerbose = false;
private boolean isQuiet = false;
private boolean isForce = false; // default to not forced
private String identifier = null; // object scope limiter
private int max2Process = Integer.MAX_VALUE;
private String[] filterNames;
private String[] skipIds = null;
private Map<String, List<String>> filterFormats = new HashMap<>();
public MediaFilterScriptConfiguration getScriptConfiguration() {
return new DSpace().getServiceManager()
.getServiceByName("filter-media", MediaFilterScriptConfiguration.class);
}
public void setup() throws ParseException {
public static void main(String[] argv) throws Exception {
// set headless for non-gui workstations
System.setProperty("java.awt.headless", "true");
// create an options object and populate it
CommandLineParser parser = new DefaultParser();
int status = 0;
help = commandLine.hasOption('h');
Options options = new Options();
options.addOption("v", "verbose", false,
"print all extracted text and other details to STDOUT");
options.addOption("q", "quiet", false,
"do not print anything except in the event of errors.");
options.addOption("f", "force", false,
"force all bitstreams to be processed");
options.addOption("i", "identifier", true,
"ONLY process bitstreams belonging to identifier");
options.addOption("m", "maximum", true,
"process no more than maximum items");
options.addOption("h", "help", false, "help");
//create a "plugin" option (to specify specific MediaFilter plugins to run)
Option pluginOption = Option.builder("p")
.longOpt("plugins")
.hasArg()
.hasArgs()
.valueSeparator(',')
.desc(
"ONLY run the specified Media Filter plugin(s)\n" +
"listed from '" + MEDIA_FILTER_PLUGINS_KEY + "' in dspace.cfg.\n" +
"Separate multiple with a comma (,)\n" +
"(e.g. MediaFilterManager -p \n\"Word Text Extractor\",\"PDF Text Extractor\")")
.build();
options.addOption(pluginOption);
//create a "skip" option (to specify communities/collections/items to skip)
Option skipOption = Option.builder("s")
.longOpt("skip")
.hasArg()
.hasArgs()
.valueSeparator(',')
.desc(
"SKIP the bitstreams belonging to identifier\n" +
"Separate multiple identifiers with a comma (,)\n" +
"(e.g. MediaFilterManager -s \n 123456789/34,123456789/323)")
.build();
options.addOption(skipOption);
boolean isVerbose = false;
boolean isQuiet = false;
boolean isForce = false; // default to not forced
String identifier = null; // object scope limiter
int max2Process = Integer.MAX_VALUE;
Map<String, List<String>> filterFormats = new HashMap<>();
CommandLine line = null;
try {
line = parser.parse(options, argv);
} catch (MissingArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("MediaFilterManager\n", options);
System.exit(1);
}
if (line.hasOption('h')) {
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("MediaFilterManager\n", options);
System.exit(0);
}
if (line.hasOption('v')) {
if (commandLine.hasOption('v')) {
isVerbose = true;
}
isQuiet = line.hasOption('q');
isQuiet = commandLine.hasOption('q');
if (line.hasOption('f')) {
if (commandLine.hasOption('f')) {
isForce = true;
}
if (line.hasOption('i')) {
identifier = line.getOptionValue('i');
if (commandLine.hasOption('i')) {
identifier = commandLine.getOptionValue('i');
}
if (line.hasOption('m')) {
max2Process = Integer.parseInt(line.getOptionValue('m'));
if (commandLine.hasOption('m')) {
max2Process = Integer.parseInt(commandLine.getOptionValue('m'));
if (max2Process <= 1) {
System.out.println("Invalid maximum value '" +
line.getOptionValue('m') + "' - ignoring");
handler.logWarning("Invalid maximum value '" +
commandLine.getOptionValue('m') + "' - ignoring");
max2Process = Integer.MAX_VALUE;
}
}
String filterNames[] = null;
if (line.hasOption('p')) {
if (commandLine.hasOption('p')) {
//specified which media filter plugins we are using
filterNames = line.getOptionValues('p');
if (filterNames == null || filterNames.length == 0) { //display error, since no plugins specified
System.err.println("\nERROR: -p (-plugin) option requires at least one plugin to be specified.\n" +
"(e.g. MediaFilterManager -p \"Word Text Extractor\",\"PDF Text Extractor\")\n");
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("MediaFilterManager\n", options);
System.exit(1);
}
filterNames = commandLine.getOptionValues('p');
} else {
//retrieve list of all enabled media filter plugins!
filterNames = DSpaceServicesFactory.getInstance().getConfigurationService()
.getArrayProperty(MEDIA_FILTER_PLUGINS_KEY);
}
//save to a global skip list
if (commandLine.hasOption('s')) {
//specified which identifiers to skip when processing
skipIds = commandLine.getOptionValues('s');
}
}
public void internalRun() throws Exception {
if (help) {
printHelp();
return;
}
MediaFilterService mediaFilterService = MediaFilterServiceFactory.getInstance().getMediaFilterService();
mediaFilterService.setLogHandler(handler);
mediaFilterService.setForce(isForce);
mediaFilterService.setQuiet(isQuiet);
mediaFilterService.setVerbose(isVerbose);
@@ -184,16 +131,17 @@ public class MediaFilterCLITool {
//initialize an array of our enabled filters
List<FormatFilter> filterList = new ArrayList<>();
//set up each filter
for (int i = 0; i < filterNames.length; i++) {
//get filter of this name & add to list of filters
FormatFilter filter = (FormatFilter) CoreServiceFactory.getInstance().getPluginService()
.getNamedPlugin(FormatFilter.class, filterNames[i]);
if (filter == null) {
System.err.println(
"\nERROR: Unknown MediaFilter specified (either from command-line or in dspace.cfg): '" +
filterNames[i] + "'");
System.exit(1);
handler.handleException("ERROR: Unknown MediaFilter specified (either from command-line or in " +
"dspace.cfg): '" + filterNames[i] + "'");
handler.logError("ERROR: Unknown MediaFilter specified (either from command-line or in " +
"dspace.cfg): '" + filterNames[i] + "'");
} else {
filterList.add(filter);
@@ -218,10 +166,10 @@ public class MediaFilterCLITool {
//For other MediaFilters, format of key is:
// filter.<class-name>.inputFormats
String[] formats =
DSpaceServicesFactory.getInstance().getConfigurationService().getArrayProperty(
FILTER_PREFIX + "." + filterClassName +
(pluginName != null ? "." + pluginName : "") +
"." + INPUT_FORMATS_SUFFIX);
DSpaceServicesFactory.getInstance().getConfigurationService().getArrayProperty(
FILTER_PREFIX + "." + filterClassName +
(pluginName != null ? "." + pluginName : "") +
"." + INPUT_FORMATS_SUFFIX);
//add to internal map of filters to supported formats
if (ArrayUtils.isNotEmpty(formats)) {
@@ -230,8 +178,8 @@ public class MediaFilterCLITool {
//For other MediaFilters, map key is just:
// <class-name>
filterFormats.put(filterClassName +
(pluginName != null ? MediaFilterService.FILTER_PLUGIN_SEPARATOR +
pluginName : ""),
(pluginName != null ? MediaFilterService.FILTER_PLUGIN_SEPARATOR +
pluginName : ""),
Arrays.asList(formats));
}
} //end if filter!=null
@@ -239,11 +187,11 @@ public class MediaFilterCLITool {
//If verbose, print out loaded mediafilter info
if (isVerbose) {
System.out.println("The following MediaFilters are enabled: ");
handler.logInfo("The following MediaFilters are enabled: ");
Iterator<String> i = filterFormats.keySet().iterator();
while (i.hasNext()) {
String filterName = i.next();
System.out.println("Full Filter Name: " + filterName);
handler.logInfo("Full Filter Name: " + filterName);
String pluginName = null;
if (filterName.contains(MediaFilterService.FILTER_PLUGIN_SEPARATOR)) {
String[] fields = filterName.split(MediaFilterService.FILTER_PLUGIN_SEPARATOR);
@@ -251,8 +199,7 @@ public class MediaFilterCLITool {
pluginName = fields[1];
}
System.out.println(filterName +
(pluginName != null ? " (Plugin: " + pluginName + ")" : ""));
handler.logInfo(filterName + (pluginName != null ? " (Plugin: " + pluginName + ")" : ""));
}
}
@@ -262,20 +209,8 @@ public class MediaFilterCLITool {
//Retrieve list of identifiers to skip (if any)
String skipIds[] = null;
if (line.hasOption('s')) {
//specified which identifiers to skip when processing
skipIds = line.getOptionValues('s');
if (skipIds == null || skipIds.length == 0) { //display error, since no identifiers specified to skip
System.err.println("\nERROR: -s (-skip) option requires at least one identifier to SKIP.\n" +
"Make sure to separate multiple identifiers with a comma!\n" +
"(e.g. MediaFilterManager -s 123456789/34,123456789/323)\n");
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("MediaFilterManager\n", options);
System.exit(0);
}
if (skipIds != null && skipIds.length > 0) {
//save to a global skip list
mediaFilterService.setSkipList(Arrays.asList(skipIds));
}
@@ -296,7 +231,7 @@ public class MediaFilterCLITool {
DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService().resolveToObject(c, identifier);
if (dso == null) {
throw new IllegalArgumentException("Cannot resolve "
+ identifier + " to a DSpace object");
+ identifier + " to a DSpace object");
}
switch (dso.getType()) {
@@ -317,12 +252,11 @@ public class MediaFilterCLITool {
c.complete();
c = null;
} catch (Exception e) {
status = 1;
handler.handleException(e);
} finally {
if (c != null) {
c.abort();
}
}
System.exit(status);
}
}

87
dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java Normal file
View File

@@ -0,0 +1,87 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.mediafilter;
import java.sql.SQLException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.core.Context;
import org.dspace.scripts.configuration.ScriptConfiguration;
import org.springframework.beans.factory.annotation.Autowired;
public class MediaFilterScriptConfiguration<T extends MediaFilterScript> extends ScriptConfiguration<T> {
@Autowired
private AuthorizeService authorizeService;
private Class<T> dspaceRunnableClass;
private static final String MEDIA_FILTER_PLUGINS_KEY = "filter.plugins";
@Override
public Class<T> getDspaceRunnableClass() {
return dspaceRunnableClass;
}
@Override
public void setDspaceRunnableClass(Class<T> dspaceRunnableClass) {
this.dspaceRunnableClass = dspaceRunnableClass;
}
@Override
public boolean isAllowedToExecute(final Context context) {
try {
return authorizeService.isAdmin(context);
} catch (SQLException e) {
throw new RuntimeException("SQLException occurred when checking if the current user is an admin", e);
}
}
@Override
public Options getOptions() {
Options options = new Options();
options.addOption("v", "verbose", false, "print all extracted text and other details to STDOUT");
options.addOption("q", "quiet", false, "do not print anything except in the event of errors.");
options.addOption("f", "force", false, "force all bitstreams to be processed");
options.addOption("i", "identifier", true, "ONLY process bitstreams belonging to identifier");
options.addOption("m", "maximum", true, "process no more than maximum items");
options.addOption("h", "help", false, "help");
Option pluginOption = Option.builder("p")
.longOpt("plugins")
.hasArg()
.hasArgs()
.valueSeparator(',')
.desc(
"ONLY run the specified Media Filter plugin(s)\n" +
"listed from '" + MEDIA_FILTER_PLUGINS_KEY + "' in dspace.cfg.\n" +
"Separate multiple with a comma (,)\n" +
"(e.g. MediaFilterManager -p \n\"Word Text Extractor\",\"PDF Text" +
" Extractor\")")
.build();
options.addOption(pluginOption);
Option skipOption = Option.builder("s")
.longOpt("skip")
.hasArg()
.hasArgs()
.valueSeparator(',')
.desc(
"SKIP the bitstreams belonging to identifier\n" +
"Separate multiple identifiers with a comma (,)\n" +
"(e.g. MediaFilterManager -s \n 123456789/34,123456789/323)")
.build();
options.addOption(skipOption);
return options;
}
}

62
dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java
View File

@@ -34,6 +34,7 @@ import org.dspace.core.Context;
import org.dspace.core.SelfNamedPlugin;
import org.dspace.eperson.Group;
import org.dspace.eperson.service.GroupService;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.dspace.services.ConfigurationService;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
@@ -67,6 +68,8 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
@Autowired(required = true)
protected ConfigurationService configurationService;
protected DSpaceRunnableHandler handler;
protected int max2Process = Integer.MAX_VALUE; // maximum number items to process
protected int processed = 0; // number items processed
@@ -225,16 +228,16 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
int assetstore = myBitstream.getStoreNumber();
// Printout helpful information to find the errored bitstream.
System.out.println("ERROR filtering, skipping bitstream:\n");
System.out.println("\tItem Handle: " + handle);
StringBuilder sb = new StringBuilder("ERROR filtering, skipping bitstream:\n");
sb.append("\tItem Handle: ").append(handle);
for (Bundle bundle : bundles) {
System.out.println("\tBundle Name: " + bundle.getName());
sb.append("\tBundle Name: ").append(bundle.getName());
}
System.out.println("\tFile Size: " + size);
System.out.println("\tChecksum: " + checksum);
System.out.println("\tAsset Store: " + assetstore);
System.out.println(e);
e.printStackTrace();
sb.append("\tFile Size: ").append(size);
sb.append("\tChecksum: ").append(checksum);
sb.append("\tAsset Store: ").append(assetstore);
logError(sb.toString());
logError(e.getMessage(), e);
}
} else if (filterClass instanceof SelfRegisterInputFormats) {
// Filter implements self registration, so check to see if it should be applied
@@ -287,7 +290,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
filtered = true;
}
} catch (Exception e) {
System.out.println("ERROR filtering, skipping bitstream #"
logError("ERROR filtering, skipping bitstream #"
+ myBitstream.getID() + " " + e);
e.printStackTrace();
}
@@ -332,7 +335,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
// if exists and overwrite = false, exit
if (!overWrite && (existingBitstream != null)) {
if (!isQuiet) {
System.out.println("SKIPPED: bitstream " + source.getID()
logInfo("SKIPPED: bitstream " + source.getID()
+ " (item: " + item.getHandle() + ") because '" + newName + "' already exists");
}
@@ -340,11 +343,11 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
}
if (isVerbose) {
System.out.println("PROCESSING: bitstream " + source.getID()
logInfo("PROCESSING: bitstream " + source.getID()
+ " (item: " + item.getHandle() + ")");
}
System.out.println("File: " + newName);
logInfo("File: " + newName);
// start filtering of the bitstream, using try with resource to close all InputStreams properly
try (
@@ -356,7 +359,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
) {
if (destStream == null) {
if (!isQuiet) {
System.out.println("SKIPPED: bitstream " + source.getID()
logInfo("SKIPPED: bitstream " + source.getID()
+ " (item: " + item.getHandle() + ") because filtering was unsuccessful");
}
return false;
@@ -402,7 +405,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
formatFilter.postProcessBitstream(context, item, b);
} catch (OutOfMemoryError oome) {
System.out.println("!!! OutOfMemoryError !!!");
logError("!!! OutOfMemoryError !!!");
}
// fixme - set date?
@@ -412,7 +415,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
}
if (!isQuiet) {
System.out.println("FILTERED: bitstream " + source.getID()
logInfo("FILTERED: bitstream " + source.getID()
+ " (item: " + item.getHandle() + ") and created '" + newName + "'");
}
@@ -428,7 +431,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
public boolean inSkipList(String identifier) {
if (skipList != null && skipList.contains(identifier)) {
if (!isQuiet) {
System.out.println("SKIP-LIST: skipped bitstreams within identifier " + identifier);
logInfo("SKIP-LIST: skipped bitstreams within identifier " + identifier);
}
return true;
} else {
@@ -436,6 +439,28 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
}
}
private void logInfo(String message) {
if (handler != null) {
handler.logInfo(message);
} else {
System.out.println(message);
}
}
private void logError(String message) {
if (handler != null) {
handler.logError(message);
} else {
System.out.println(message);
}
}
private void logError(String message, Exception e) {
if (handler != null) {
handler.logError(message, e);
} else {
System.out.println(message);
}
}
@Override
public void setVerbose(boolean isVerbose) {
this.isVerbose = isVerbose;
@@ -470,4 +495,9 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
public void setFilterFormats(Map<String, List<String>> filterFormats) {
this.filterFormats = filterFormats;
}
@Override
public void setLogHandler(DSpaceRunnableHandler handler) {
this.handler = handler;
}
}

137
dspace-api/src/main/java/org/dspace/app/mediafilter/PDFFilter.java
View File

@@ -1,137 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.mediafilter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import org.apache.logging.log4j.Logger;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException;
import org.apache.pdfbox.text.PDFTextStripper;
import org.dspace.content.Item;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
/*
*
* to do: helpful error messages - can't find mediafilter.cfg - can't
* instantiate filter - bitstream format doesn't exist
*
*/
public class PDFFilter extends MediaFilter {
private static Logger log = org.apache.logging.log4j.LogManager.getLogger(PDFFilter.class);
@Override
public String getFilteredName(String oldFilename) {
return oldFilename + ".txt";
}
/**
* @return String bundle name
*/
@Override
public String getBundleName() {
return "TEXT";
}
/**
* @return String bitstreamformat
*/
@Override
public String getFormatString() {
return "Text";
}
/**
* @return String description
*/
@Override
public String getDescription() {
return "Extracted text";
}
/**
* @param currentItem item
* @param source source input stream
* @param verbose verbose mode
* @return InputStream the resulting input stream
* @throws Exception if error
*/
@Override
public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose)
throws Exception {
ConfigurationService configurationService
= DSpaceServicesFactory.getInstance().getConfigurationService();
try {
boolean useTemporaryFile = configurationService.getBooleanProperty("pdffilter.largepdfs", false);
// get input stream from bitstream
// pass to filter, get string back
PDFTextStripper pts = new PDFTextStripper();
pts.setSortByPosition(true);
PDDocument pdfDoc = null;
Writer writer = null;
File tempTextFile = null;
ByteArrayOutputStream byteStream = null;
if (useTemporaryFile) {
tempTextFile = File.createTempFile("dspacepdfextract" + source.hashCode(), ".txt");
tempTextFile.deleteOnExit();
writer = new OutputStreamWriter(new FileOutputStream(tempTextFile));
} else {
byteStream = new ByteArrayOutputStream();
writer = new OutputStreamWriter(byteStream);
}
try {
pdfDoc = PDDocument.load(source);
pts.writeText(pdfDoc, writer);
} catch (InvalidPasswordException ex) {
log.error("PDF is encrypted. Cannot extract text (item: {})",
() -> currentItem.getHandle());
return null;
} finally {
try {
if (pdfDoc != null) {
pdfDoc.close();
}
} catch (Exception e) {
log.error("Error closing PDF file: " + e.getMessage(), e);
}
try {
writer.close();
} catch (Exception e) {
log.error("Error closing temporary extract file: " + e.getMessage(), e);
}
}
if (useTemporaryFile) {
return new FileInputStream(tempTextFile);
} else {
byte[] bytes = byteStream.toByteArray();
return new ByteArrayInputStream(bytes);
}
} catch (OutOfMemoryError oome) {
log.error("Error parsing PDF document " + oome.getMessage(), oome);
if (!configurationService.getBooleanProperty("pdffilter.skiponmemoryexception", false)) {
throw oome;
}
}
return null;
}
}

71
dspace-api/src/main/java/org/dspace/app/mediafilter/PoiWordFilter.java
View File

@@ -1,71 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.mediafilter;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import org.apache.poi.POITextExtractor;
import org.apache.poi.extractor.ExtractorFactory;
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.xmlbeans.XmlException;
import org.dspace.content.Item;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Extract flat text from Microsoft Word documents (.doc, .docx).
*/
public class PoiWordFilter
extends MediaFilter {
private static final Logger LOG = LoggerFactory.getLogger(PoiWordFilter.class);
@Override
public String getFilteredName(String oldFilename) {
return oldFilename + ".txt";
}
@Override
public String getBundleName() {
return "TEXT";
}
@Override
public String getFormatString() {
return "Text";
}
@Override
public String getDescription() {
return "Extracted text";
}
@Override
public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose)
throws Exception {
String text;
try {
// get input stream from bitstream, pass to filter, get string back
POITextExtractor extractor = ExtractorFactory.createExtractor(source);
text = extractor.getText();
} catch (IOException | OpenXML4JException | XmlException e) {
System.err.format("Invalid File Format: %s%n", e.getMessage());
LOG.error("Unable to parse the bitstream: ", e);
throw e;
}
// if verbose flag is set, print out extracted text to STDOUT
if (verbose) {
System.out.println(text);
}
// return the extracted text as a stream.
return new ByteArrayInputStream(text.getBytes());
}
}

113
dspace-api/src/main/java/org/dspace/app/mediafilter/PowerPointFilter.java
View File

@@ -1,113 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.mediafilter;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import org.apache.logging.log4j.Logger;
import org.apache.poi.POITextExtractor;
import org.apache.poi.extractor.ExtractorFactory;
import org.apache.poi.hslf.extractor.PowerPointExtractor;
import org.apache.poi.xslf.extractor.XSLFPowerPointExtractor;
import org.dspace.content.Item;
/*
* TODO: Allow user to configure extraction of only text or only notes
*
*/
public class PowerPointFilter extends MediaFilter {
private static Logger log = org.apache.logging.log4j.LogManager.getLogger(PowerPointFilter.class);
@Override
public String getFilteredName(String oldFilename) {
return oldFilename + ".txt";
}
/**
* @return String bundle name
*/
@Override
public String getBundleName() {
return "TEXT";
}
/**
* @return String bitstream format
*
* TODO: Check that this is correct
*/
@Override
public String getFormatString() {
return "Text";
}
/**
* @return String description
*/
@Override
public String getDescription() {
return "Extracted text";
}
/**
* @param currentItem item
* @param source source input stream
* @param verbose verbose mode
* @return InputStream the resulting input stream
* @throws Exception if error
*/
@Override
public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose)
throws Exception {
try {
String extractedText = null;
new ExtractorFactory();
POITextExtractor pptExtractor = ExtractorFactory
.createExtractor(source);
// PowerPoint XML files and legacy format PowerPoint files
// require different classes and APIs for text extraction
// If this is a PowerPoint XML file, extract accordingly
if (pptExtractor instanceof XSLFPowerPointExtractor) {
// The true method arguments indicate that text from
// the slides and the notes is desired
extractedText = ((XSLFPowerPointExtractor) pptExtractor)
.getText(true, true);
} else if (pptExtractor instanceof PowerPointExtractor) { // Legacy PowerPoint files
extractedText = ((PowerPointExtractor) pptExtractor).getText()
+ " " + ((PowerPointExtractor) pptExtractor).getNotes();
}
if (extractedText != null) {
// if verbose flag is set, print out extracted text
// to STDOUT
if (verbose) {
System.out.println(extractedText);
}
// generate an input stream with the extracted text
byte[] textBytes = extractedText.getBytes();
ByteArrayInputStream bais = new ByteArrayInputStream(textBytes);
return bais;
}
} catch (Exception e) {
log.error("Error filtering bitstream: " + e.getMessage(), e);
throw e;
}
return null;
}
}

183
dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java Normal file
View File

@@ -0,0 +1,183 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.mediafilter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ContentHandlerDecorator;
import org.dspace.content.Item;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.xml.sax.SAXException;
/**
* Text Extraction media filter which uses Apache Tika to extract text from a large number of file formats (including
* all Microsoft formats, PDF, HTML, Text, etc). For a more complete list of file formats supported by Tika see the
* Tika documentation: https://tika.apache.org/2.3.0/formats.html
*/
public class TikaTextExtractionFilter
extends MediaFilter {
private final static Logger log = LogManager.getLogger();
@Override
public String getFilteredName(String oldFilename) {
return oldFilename + ".txt";
}
@Override
public String getBundleName() {
return "TEXT";
}
@Override
public String getFormatString() {
return "Text";
}
@Override
public String getDescription() {
return "Extracted text";
}
@Override
public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose)
throws Exception {
ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();
boolean useTemporaryFile = configurationService.getBooleanProperty("textextractor.use-temp-file", false);
if (useTemporaryFile) {
// Extract text out of source file using a temp file, returning results as InputStream
return extractUsingTempFile(source, verbose);
}
// Not using temporary file. We'll use Tika's default in-memory parsing.
// Get maximum characters to extract. Default is 100,000 chars, which is also Tika's default setting.
String extractedText;
int maxChars = configurationService.getIntProperty("textextractor.max-chars", 100000);
try {
// Use Tika to extract text from input. Tika will automatically detect the file type.
Tika tika = new Tika();
tika.setMaxStringLength(maxChars); // Tell Tika the maximum number of characters to extract
extractedText = tika.parseToString(source);
} catch (IOException e) {
System.err.format("Unable to extract text from bitstream in Item %s%n", currentItem.getID().toString());
e.printStackTrace();
log.error("Unable to extract text from bitstream in Item {}", currentItem.getID().toString(), e);
throw e;
} catch (OutOfMemoryError oe) {
System.err.format("OutOfMemoryError occurred when extracting text from bitstream in Item %s. " +
"You may wish to enable 'textextractor.use-temp-file'.%n", currentItem.getID().toString());
oe.printStackTrace();
log.error("OutOfMemoryError occurred when extracting text from bitstream in Item {}. " +
"You may wish to enable 'textextractor.use-temp-file'.", currentItem.getID().toString(), oe);
throw oe;
}
if (StringUtils.isNotEmpty(extractedText)) {
// if verbose flag is set, print out extracted text to STDOUT
if (verbose) {
System.out.println("(Verbose mode) Extracted text:");
System.out.println(extractedText);
}
// return the extracted text as a UTF-8 stream.
return new ByteArrayInputStream(extractedText.getBytes(StandardCharsets.UTF_8));
}
return null;
}
/**
* Extracts the text out of a given source InputStream, using a temporary file. This decreases the amount of memory
* necessary for text extraction, but can be slower as it requires writing extracted text to a temporary file.
* @param source source InputStream
* @param verbose verbose mode enabled/disabled
* @return InputStream for temporary file containing extracted text
* @throws IOException
* @throws SAXException
* @throws TikaException
*/
private InputStream extractUsingTempFile(InputStream source, boolean verbose)
throws IOException, TikaException, SAXException {
File tempExtractedTextFile = File.createTempFile("dspacetextextract" + source.hashCode(), ".txt");
if (verbose) {
System.out.println("(Verbose mode) Extracted text was written to temporary file at " +
tempExtractedTextFile.getAbsolutePath());
} else {
tempExtractedTextFile.deleteOnExit();
}
// Open temp file for writing
try (FileWriter writer = new FileWriter(tempExtractedTextFile, StandardCharsets.UTF_8)) {
// Initialize a custom ContentHandlerDecorator which is a BodyContentHandler.
// This mimics the behavior of Tika().parseToString(), which only extracts text from the body of the file.
// This custom Handler writes any extracted text to the temp file.
ContentHandlerDecorator handler = new BodyContentHandler(new ContentHandlerDecorator() {
/**
* Write all extracted characters directly to the temp file.
*/
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
try {
writer.append(new String(ch), start, length);
} catch (IOException e) {
String errorMsg = String.format("Could not append to temporary file at %s " +
"when performing text extraction",
tempExtractedTextFile.getAbsolutePath());
log.error(errorMsg, e);
throw new SAXException(errorMsg, e);
}
}
/**
* Write all ignorable whitespace directly to the temp file.
* This mimics the behaviour of Tika().parseToString() which extracts ignorableWhitespace characters
* (like blank lines, indentations, etc.), so that we get the same extracted text either way.
*/
@Override
public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
try {
writer.append(new String(ch), start, length);
} catch (IOException e) {
String errorMsg = String.format("Could not append to temporary file at %s " +
"when performing text extraction",
tempExtractedTextFile.getAbsolutePath());
log.error(errorMsg, e);
throw new SAXException(errorMsg, e);
}
}
});
AutoDetectParser parser = new AutoDetectParser();
Metadata metadata = new Metadata();
// parse our source InputStream using the above custom handler
parser.parse(source, handler, metadata);
}
// At this point, all extracted text is written to our temp file. So, return a FileInputStream for that file
return new FileInputStream(tempExtractedTextFile);
}
}

7
dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java
View File

@@ -16,6 +16,7 @@ import org.dspace.content.Collection;
import org.dspace.content.Community;
import org.dspace.content.Item;
import org.dspace.core.Context;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
/**
* MediaFilterManager is the class that invokes the media/format filters over the
@@ -124,4 +125,10 @@ public interface MediaFilterService {
public void setSkipList(List<String> skipList);
public void setFilterFormats(Map<String, List<String>> filterFormats);
/**
* Set the log handler used in the DSpace scripts and processes framework
* @param handler
*/
public void setLogHandler(DSpaceRunnableHandler handler);
}

7
dspace-api/src/main/java/org/dspace/app/requestitem/RequestItem.java
View File

@@ -33,7 +33,6 @@ import org.dspace.core.ReloadableEntity;
@Table(name = "requestitem")
public class RequestItem implements ReloadableEntity<Integer> {
@Id
@Column(name = "requestitem_id")
@GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "requestitem_seq")
@@ -54,8 +53,6 @@ public class RequestItem implements ReloadableEntity<Integer> {
@Column(name = "request_name", length = 64)
private String reqName;
// @Column(name = "request_message")
// @Lob
@Column(name = "request_message", columnDefinition = "text")
private String reqMessage;
@@ -82,8 +79,8 @@ public class RequestItem implements ReloadableEntity<Integer> {
/**
* Protected constructor, create object using:
* {@link org.dspace.app.requestitem.service.RequestItemService#createRequest(Context, Bitstream, Item,
* boolean, String, String, String)}
* {@link org.dspace.app.requestitem.service.RequestItemService#createRequest(
* Context, Bitstream, Item, boolean, String, String, String)}
*/
protected RequestItem() {
}

228
dspace-api/src/main/java/org/dspace/app/requestitem/RequestItemEmailNotifier.java Normal file
View File

@@ -0,0 +1,228 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.requestitem;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
import javax.mail.MessagingException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.app.requestitem.factory.RequestItemServiceFactory;
import org.dspace.app.requestitem.service.RequestItemService;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.content.Bundle;
import org.dspace.content.Item;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.BitstreamService;
import org.dspace.core.Context;
import org.dspace.core.Email;
import org.dspace.core.I18nUtil;
import org.dspace.core.LogHelper;
import org.dspace.eperson.EPerson;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.handle.service.HandleService;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
/**
* Send item requests and responses by email.
*
* @author Mark H. Wood <mwood@iupui.edu>
*/
public class RequestItemEmailNotifier {
private static final Logger LOG = LogManager.getLogger();
private static final BitstreamService bitstreamService
= ContentServiceFactory.getInstance().getBitstreamService();
private static final ConfigurationService configurationService
= DSpaceServicesFactory.getInstance().getConfigurationService();
private static final HandleService handleService
= HandleServiceFactory.getInstance().getHandleService();
private static final RequestItemService requestItemService
= RequestItemServiceFactory.getInstance().getRequestItemService();
private static final RequestItemAuthorExtractor requestItemAuthorExtractor
= DSpaceServicesFactory.getInstance()
.getServiceManager()
.getServiceByName(null, RequestItemAuthorExtractor.class);
private RequestItemEmailNotifier() {}
/**
* Send the request to the approver(s).
*
* @param context current DSpace session.
* @param ri the request.
* @param responseLink link back to DSpace to send the response.
* @throws IOException passed through.
* @throws SQLException if the message was not sent.
*/
static public void sendRequest(Context context, RequestItem ri, String responseLink)
throws IOException, SQLException {
// Who is making this request?
RequestItemAuthor author = requestItemAuthorExtractor
.getRequestItemAuthor(context, ri.getItem());
String authorEmail = author.getEmail();
String authorName = author.getFullName();
// Build an email to the approver.
Email email = Email.getEmail(I18nUtil.getEmailFilename(context.getCurrentLocale(),
"request_item.author"));
email.addRecipient(authorEmail);
email.setReplyTo(ri.getReqEmail()); // Requester's address
email.addArgument(ri.getReqName()); // {0} Requester's name
email.addArgument(ri.getReqEmail()); // {1} Requester's address
email.addArgument(ri.isAllfiles() // {2} All bitstreams or just one?
? I18nUtil.getMessage("itemRequest.all") : ri.getBitstream().getName());
email.addArgument(handleService.getCanonicalForm(ri.getItem().getHandle()));
email.addArgument(ri.getItem().getName()); // {4} requested item's title
email.addArgument(ri.getReqMessage()); // {5} message from requester
email.addArgument(responseLink); // {6} Link back to DSpace for action
email.addArgument(authorName); // {7} corresponding author name
email.addArgument(authorEmail); // {8} corresponding author email
email.addArgument(configurationService.getProperty("dspace.name"));
email.addArgument(configurationService.getProperty("mail.helpdesk"));
// Send the email.
try {
email.send();
Bitstream bitstream = ri.getBitstream();
String bitstreamID;
if (null == bitstream) {
bitstreamID = "null";
} else {
bitstreamID = ri.getBitstream().getID().toString();
}
LOG.info(LogHelper.getHeader(context,
"sent_email_requestItem",
"submitter_id={},bitstream_id={},requestEmail={}"),
ri.getReqEmail(), bitstreamID, ri.getReqEmail());
} catch (MessagingException e) {
LOG.warn(LogHelper.getHeader(context,
"error_mailing_requestItem", e.getMessage()));
throw new IOException("Request not sent: " + e.getMessage());
}
}
/**
* Send the approver's response back to the requester, with files attached
* if approved.
*
* @param context current DSpace session.
* @param ri the request.
* @param subject email subject header value.
* @param message email body (may be empty).
* @throws IOException if sending failed.
*/
static public void sendResponse(Context context, RequestItem ri, String subject,
String message)
throws IOException {
// Build an email back to the requester.
Email email = new Email();
email.setContent("body", message);
email.setSubject(subject);
email.addRecipient(ri.getReqEmail());
if (ri.isAccept_request()) {
// Attach bitstreams.
try {
if (ri.isAllfiles()) {
Item item = ri.getItem();
List<Bundle> bundles = item.getBundles("ORIGINAL");
for (Bundle bundle : bundles) {
List<Bitstream> bitstreams = bundle.getBitstreams();
for (Bitstream bitstream : bitstreams) {
if (!bitstream.getFormat(context).isInternal() &&
requestItemService.isRestricted(context,
bitstream)) {
email.addAttachment(bitstreamService.retrieve(context,
bitstream), bitstream.getName(),
bitstream.getFormat(context).getMIMEType());
}
}
}
} else {
Bitstream bitstream = ri.getBitstream();
email.addAttachment(bitstreamService.retrieve(context, bitstream),
bitstream.getName(),
bitstream.getFormat(context).getMIMEType());
}
email.send();
} catch (MessagingException | IOException | SQLException | AuthorizeException e) {
LOG.warn(LogHelper.getHeader(context,
"error_mailing_requestItem", e.getMessage()));
throw new IOException("Reply not sent: " + e.getMessage());
}
}
LOG.info(LogHelper.getHeader(context,
"sent_attach_requestItem", "token={}"), ri.getToken());
}
/**
* Send, to a repository administrator, a request to open access to a
* requested object.
*
* @param context current DSpace session
* @param ri the item request that the approver is handling
* @throws IOException if the message body cannot be loaded or the message
* cannot be sent.
*/
static public void requestOpenAccess(Context context, RequestItem ri)
throws IOException {
Email message = Email.getEmail(I18nUtil.getEmailFilename(context.getCurrentLocale(),
"request_item.admin"));
// Which Bitstream(s) requested?
Bitstream bitstream = ri.getBitstream();
String bitstreamName;
if (bitstream != null) {
bitstreamName = bitstream.getName();
} else {
bitstreamName = "all"; // TODO localize
}
// Which Item?
Item item = ri.getItem();
// Fill the message's placeholders.
EPerson approver = context.getCurrentUser();
message.addArgument(bitstreamName); // {0} bitstream name or "all"
message.addArgument(item.getHandle()); // {1} Item handle
message.addArgument(ri.getToken()); // {2} Request token
message.addArgument(approver.getFullName()); // {3} Approver's name
message.addArgument(approver.getEmail()); // {4} Approver's address
// Who gets this message?
String recipient;
EPerson submitter = item.getSubmitter();
if (submitter != null) {
recipient = submitter.getEmail();
} else {
recipient = configurationService.getProperty("mail.helpdesk");
}
if (null == recipient) {
recipient = configurationService.getProperty("mail.admin");
}
message.addRecipient(recipient);
// Send the message.
try {
message.send();
} catch (MessagingException ex) {
LOG.warn(LogHelper.getHeader(context, "error_mailing_requestItem",
ex.getMessage()));
throw new IOException("Open Access request not sent: " + ex.getMessage());
}
}
}

60
dspace-api/src/main/java/org/dspace/app/requestitem/RequestItemServiceImpl.java
View File

@@ -9,37 +9,53 @@ package org.dspace.app.requestitem;
import java.sql.SQLException;
import java.util.Date;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.app.requestitem.dao.RequestItemDAO;
import org.dspace.app.requestitem.service.RequestItemService;
import org.dspace.authorize.ResourcePolicy;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.authorize.service.ResourcePolicyService;
import org.dspace.content.Bitstream;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.LogHelper;
import org.dspace.core.Utils;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Service implementation for the RequestItem object.
* This class is responsible for all business logic calls for the RequestItem object and is autowired by spring.
* This class is responsible for all business logic calls for the RequestItem
* object and is autowired by Spring.
* This class should never be accessed directly.
*
* @author kevinvandevelde at atmire.com
*/
public class RequestItemServiceImpl implements RequestItemService {
private final Logger log = org.apache.logging.log4j.LogManager.getLogger(RequestItemServiceImpl.class);
private final Logger log = LogManager.getLogger();
@Autowired(required = true)
protected RequestItemDAO requestItemDAO;
@Autowired(required = true)
protected AuthorizeService authorizeService;
@Autowired(required = true)
protected ResourcePolicyService resourcePolicyService;
protected RequestItemServiceImpl() {
}
@Override
public String createRequest(Context context, Bitstream bitstream, Item item, boolean allFiles, String reqEmail,
String reqName, String reqMessage) throws SQLException {
public String createRequest(Context context, Bitstream bitstream, Item item,
boolean allFiles, String reqEmail, String reqName, String reqMessage)
throws SQLException {
RequestItem requestItem = requestItemDAO.create(context, new RequestItem());
requestItem.setToken(Utils.generateHexKey());
@@ -53,13 +69,17 @@ public class RequestItemServiceImpl implements RequestItemService {
requestItemDAO.save(context, requestItem);
if (log.isDebugEnabled()) {
log.debug("Created requestitem_token " + requestItem.getID()
+ " with token " + requestItem.getToken() + "\"");
}
log.debug("Created RequestItem with ID {} and token {}",
requestItem::getID, requestItem::getToken);
return requestItem.getToken();
}
@Override
public List<RequestItem> findAll(Context context)
throws SQLException {
return requestItemDAO.findAll(context, RequestItem.class);
}
@Override
public RequestItem findByToken(Context context, String token) {
try {
@@ -78,4 +98,28 @@ public class RequestItemServiceImpl implements RequestItemService {
log.error(e.getMessage());
}
}
@Override
public void delete(Context context, RequestItem requestItem) {
log.debug(LogHelper.getHeader(context, "delete_itemrequest", "request_id={}"),
requestItem.getID());
try {
requestItemDAO.delete(context, requestItem);
} catch (SQLException e) {
log.error(e.getMessage());
}
}
@Override
public boolean isRestricted(Context context, DSpaceObject o)
throws SQLException {
List<ResourcePolicy> policies = authorizeService
.getPoliciesActionFilter(context, o, Constants.READ);
for (ResourcePolicy rp : policies) {
if (resourcePolicyService.isDateValid(rp)) {
return false;
}
}
return true;
}
}

16
dspace-api/src/main/java/org/dspace/app/requestitem/dao/RequestItemDAO.java
View File

@@ -15,13 +15,21 @@ import org.dspace.core.GenericDAO;
/**
* Database Access Object interface class for the RequestItem object.
* The implementation of this class is responsible for all database calls for the RequestItem object and is autowired
* by spring
* This class should only be accessed from a single service and should never be exposed outside of the API
* The implementation of this class is responsible for all database calls for
* the RequestItem object and is autowired by Spring.
* This class should only be accessed from a single service and should never be
* exposed outside of the API.
*
* @author kevinvandevelde at atmire.com
*/
public interface RequestItemDAO extends GenericDAO<RequestItem> {
/**
* Fetch a request named by its unique token (passed in emails).
*
* @param context the current DSpace context.
* @param token uniquely identifies the request.
* @return the found request (or {@code null}?)
* @throws SQLException passed through.
*/
public RequestItem findByToken(Context context, String token) throws SQLException;
}

6
dspace-api/src/main/java/org/dspace/app/requestitem/dao/impl/RequestItemDAOImpl.java
View File

@@ -20,7 +20,7 @@ import org.dspace.core.Context;
/**
* Hibernate implementation of the Database Access Object interface class for the RequestItem object.
* This class is responsible for all database calls for the RequestItem object and is autowired by spring
* This class is responsible for all database calls for the RequestItem object and is autowired by Spring.
* This class should never be accessed directly.
*
* @author kevinvandevelde at atmire.com
@@ -37,8 +37,6 @@ public class RequestItemDAOImpl extends AbstractHibernateDAO<RequestItem> implem
Root<RequestItem> requestItemRoot = criteriaQuery.from(RequestItem.class);
criteriaQuery.select(requestItemRoot);
criteriaQuery.where(criteriaBuilder.equal(requestItemRoot.get(RequestItem_.token), token));
return uniqueResult(context, criteriaQuery, false, RequestItem.class, -1, -1);
return uniqueResult(context, criteriaQuery, false, RequestItem.class);
}
}

21
dspace-api/src/main/java/org/dspace/app/requestitem/package-info.java Normal file
View File

@@ -0,0 +1,21 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
/**
* Feature for conveying a request that materials forbidden to the requester
* by resource policy be made available by other means. The request will be
* e-mailed to a responsible party for consideration and action. Find details
* in the user documentation under the rubric "Request a Copy".
*
* <p>This package includes several "strategy" classes which discover responsible
* parties in various ways. See {@link RequestItemSubmitterStrategy} and the
* classes which extend it. A strategy class must be configured and identified
* as {@link RequestItemAuthorExtractor} for injection into code which requires
* Request a Copy services.
*/
package org.dspace.app.requestitem;

43
dspace-api/src/main/java/org/dspace/app/requestitem/service/RequestItemService.java
View File

@@ -8,16 +8,18 @@
package org.dspace.app.requestitem.service;
import java.sql.SQLException;
import java.util.List;
import org.dspace.app.requestitem.RequestItem;
import org.dspace.content.Bitstream;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.core.Context;
/**
* Service interface class for the RequestItem object.
* The implementation of this class is responsible for all business logic calls for the RequestItem object and is
* autowired by spring
* The implementation of this class is responsible for all business logic calls
* for the RequestItem object and is autowired by Spring.
*
* @author kevinvandevelde at atmire.com
*/
@@ -37,10 +39,27 @@ public interface RequestItemService {
* @return the token of the request item
* @throws SQLException if database error
*/
public String createRequest(Context context, Bitstream bitstream, Item item, boolean allFiles, String reqEmail,
String reqName, String reqMessage)
public String createRequest(Context context, Bitstream bitstream, Item item,
boolean allFiles, String reqEmail, String reqName, String reqMessage)
throws SQLException;
/**
* Fetch all item requests.
*
* @param context current DSpace session.
* @return all item requests.
* @throws java.sql.SQLException passed through.
*/
public List<RequestItem> findAll(Context context)
throws SQLException;
/**
* Retrieve a request by its token.
*
* @param context current DSpace session.
* @param token the token identifying the request.
* @return the matching request, or null if not found.
*/
public RequestItem findByToken(Context context, String token);
/**
@@ -51,5 +70,21 @@ public interface RequestItemService {
*/
public void update(Context context, RequestItem requestItem);
/**
* Remove the record from the database.
*
* @param context current DSpace context.
* @param request record to be removed.
*/
public void delete(Context context, RequestItem request);
/**
* Is there at least one valid READ resource policy for this object?
* @param context current DSpace session.
* @param o the object.
* @return true if a READ policy applies.
* @throws SQLException passed through.
*/
public boolean isRestricted(Context context, DSpaceObject o)
throws SQLException;
}

7
dspace-api/src/main/java/org/dspace/app/sherpa/SHERPAService.java
View File

@@ -31,6 +31,7 @@ import org.dspace.app.sherpa.v2.SHERPAResponse;
import org.dspace.app.sherpa.v2.SHERPAUtils;
import org.dspace.services.ConfigurationService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
/**
* SHERPAService is responsible for making the HTTP call to the SHERPA v2 API
@@ -43,6 +44,7 @@ import org.springframework.beans.factory.annotation.Autowired;
* @author Kim Shepherd
*/
public class SHERPAService {
private CloseableHttpClient client = null;
private int maxNumberOfTries;
@@ -73,6 +75,7 @@ public class SHERPAService {
/**
* Complete initialization of the Bean.
*/
@SuppressWarnings("unused")
@PostConstruct
private void init() {
// Get endoint and API key from configuration
@@ -90,6 +93,7 @@ public class SHERPAService {
* @param query ISSN string to pass in an "issn equals" API query
* @return SHERPAResponse containing an error or journal policies
*/
@Cacheable(key = "#query", cacheNames = "sherpa.searchByJournalISSN")
public SHERPAResponse searchByJournalISSN(String query) {
return performRequest("publication", "issn", "equals", query, 0, 1);
}
@@ -412,4 +416,5 @@ public class SHERPAService {
public void setTimeout(int timeout) {
this.timeout = timeout;
}
}
}

71
dspace-api/src/main/java/org/dspace/app/sherpa/cache/SherpaCacheEvictService.java vendored Normal file
View File

@@ -0,0 +1,71 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.sherpa.cache;
import java.util.Objects;
import java.util.Set;
import org.dspace.app.sherpa.submit.SHERPASubmitService;
import org.dspace.content.Item;
import org.dspace.core.Context;
import org.springframework.cache.CacheManager;
/**
* This service is responsible to deal with the SherpaService cache.
*
* @author Mykhaylo Boychuk (mykhaylo.boychuk at 4science.com)
*/
public class SherpaCacheEvictService {
// The cache that is managed by this service.
static final String CACHE_NAME = "sherpa.searchByJournalISSN";
private CacheManager cacheManager;
private SHERPASubmitService sherpaSubmitService;
/**
* Remove immediately from the cache all the response that are related to a specific item
* extracting the ISSNs from the item
*
* @param context The DSpace context
* @param item an Item
*/
public void evictCacheValues(Context context, Item item) {
Set<String> ISSNs = sherpaSubmitService.getISSNs(context, item);
for (String issn : ISSNs) {
Objects.requireNonNull(cacheManager.getCache(CACHE_NAME)).evictIfPresent(issn);
}
}
/**
* Invalidate immediately the Sherpa cache
*/
public void evictAllCacheValues() {
Objects.requireNonNull(cacheManager.getCache(CACHE_NAME)).invalidate();
}
/**
* Set the reference to the cacheManager
*
* @param cacheManager
*/
public void setCacheManager(CacheManager cacheManager) {
this.cacheManager = cacheManager;
}
/**
* Set the reference to the SherpaSubmitService
*
* @param sherpaSubmitService
*/
public void setSherpaSubmitService(SHERPASubmitService sherpaSubmitService) {
this.sherpaSubmitService = sherpaSubmitService;
}
}

34
dspace-api/src/main/java/org/dspace/app/sherpa/cache/SherpaCacheLogger.java vendored Normal file
View File

@@ -0,0 +1,34 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.sherpa.cache;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ehcache.event.CacheEvent;
import org.ehcache.event.CacheEventListener;
/**
* This is a EHCache listner responsible for logging sherpa cache events. It is
* bound to the sherpa cache via the dspace/config/ehcache.xml file. We need a
* dedicated Logger for each cache as the CacheEvent doesn't include details
* about where the event occur
*
* @author Mykhaylo Boychuk (mykhaylo.boychuk at 4science.com)
*
*/
public class SherpaCacheLogger implements CacheEventListener<Object, Object> {
private static final Logger log = LogManager.getLogger(SherpaCacheLogger.class);
@Override
public void onEvent(CacheEvent<?, ?> cacheEvent) {
log.debug("Sherpa Cache Event Type: {} | Key: {} ",
cacheEvent.getType(), cacheEvent.getKey());
}
}

20
dspace-api/src/main/java/org/dspace/app/sherpa/submit/SHERPASubmitService.java
View File

@@ -9,7 +9,6 @@ package org.dspace.app.sherpa.submit;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
@@ -18,7 +17,7 @@ import org.dspace.app.sherpa.SHERPAService;
import org.dspace.app.sherpa.v2.SHERPAResponse;
import org.dspace.content.Item;
import org.dspace.core.Context;
import org.dspace.core.LogManager;
import org.dspace.core.LogHelper;
/**
* SHERPASubmitService is
@@ -63,19 +62,19 @@ public class SHERPASubmitService {
* issnItemExtractor(s) in the SHERPA spring configuration.
* The ISSNs are not validated with a regular expression or other rules - any values
* extracted will be included in API queries.
* Return the first not empty response from Sherpa
* @see "dspace-dspace-addon-sherpa-configuration-services.xml"
* @param context DSpace context
* @param item DSpace item containing ISSNs to be checked
* @return SHERPA v2 API response (policy data)
*/
public List<SHERPAResponse> searchRelatedJournals(Context context, Item item) {
public SHERPAResponse searchRelatedJournals(Context context, Item item) {
Set<String> issns = getISSNs(context, item);
if (issns == null || issns.size() == 0) {
return null;
} else {
// SHERPA v2 API no longer supports "OR'd" ISSN search, perform individual searches instead
Iterator<String> issnIterator = issns.iterator();
List<SHERPAResponse> responses = new LinkedList<>();
while (issnIterator.hasNext()) {
String issn = issnIterator.next();
SHERPAResponse response = sherpaService.searchByJournalISSN(issn);
@@ -83,14 +82,13 @@ public class SHERPASubmitService {
// Continue with loop
log.warn("Failed to look up SHERPA ROMeO result for ISSN: " + issn
+ ": " + response.getMessage());
return response;
} else if (!response.getJournals().isEmpty()) {
// return this response, if it is not empty
return response;
}
// Store this response, even if it has an error (useful for UI reporting)
responses.add(response);
}
if (responses.isEmpty()) {
responses.add(new SHERPAResponse("SHERPA ROMeO lookup failed"));
}
return responses;
return new SHERPAResponse();
}
}
@@ -115,7 +113,7 @@ public class SHERPASubmitService {
public Set<String> getISSNs(Context context, Item item) {
Set<String> issns = new LinkedHashSet<String>();
if (configuration.getIssnItemExtractors() == null) {
log.warn(LogManager.getHeader(context, "searchRelatedJournals",
log.warn(LogHelper.getHeader(context, "searchRelatedJournals",
"no issnItemExtractors defined"));
return null;
}

45
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPAEmbargo.java Normal file
View File

@@ -0,0 +1,45 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.sherpa.v2;
import java.io.Serializable;
/**
* Model class for the Embargo of SHERPAv2 API (JSON)
*
* @author Mykhaylo Boychuk (mykhaylo.boychuk@4science.com)
*/
public class SHERPAEmbargo implements Serializable {
private static final long serialVersionUID = 6140668058547523656L;
private int amount;
private String units;
public SHERPAEmbargo(int amount, String units) {
this.amount = amount;
this.units = units;
}
public int getAmount() {
return amount;
}
public void setAmount(int amount) {
this.amount = amount;
}
public String getUnits() {
return units;
}
public void setUnits(String units) {
this.units = units;
}
}

3
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPAJournal.java
View File

@@ -7,6 +7,7 @@
*/
package org.dspace.app.sherpa.v2;
import java.io.Serializable;
import java.util.List;
/**
@@ -21,7 +22,7 @@ import java.util.List;
*
* @author Kim Shepherd
*/
public class SHERPAJournal {
public class SHERPAJournal implements Serializable {
private List<String> titles;
private String url;

27
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPAPermittedVersion.java
View File

@@ -7,25 +7,31 @@
*/
package org.dspace.app.sherpa.v2;
import java.io.Serializable;
import java.util.List;
/**
* Plain java representation of a SHERPA Permitted Version object, based on SHERPA API v2 responses.
*
* In a SHERPA search for journal deposit policies, this data is contained within a publisher policy.
* Each permitted version is for a particular article version (eg. submitted, accepted, published) and contains
* Each permitted version is for a particular article version (e.g. submitted, accepted, published) and contains:
*
* A list of general conditions / terms for deposit of this version of work
* A list of allowed locations (eg. institutional repository, personal homepage, non-commercial repository)
* A list of prerequisite conditions for deposit (eg. attribution, linking to published version)
* A list of required licences for the deposited work (eg. CC-BY-NC)
* Embargo requirements, if any
* <ul>
* <li>A list of general conditions / terms for deposit of this version of work</li>
* <li>A list of allowed locations (e.g. institutional repository, personal homepage, non-commercial repository)</li>
* <li>A list of prerequisite conditions for deposit (e.g. attribution, linking to published version)</li>
* <li>A list of required licenses for the deposited work (e.g. CC-BY-NC)</li>
* <li>Embargo requirements, if any</li>
* </ul>
*
* This class also has some helper data for labels, which can be used with i18n when displaying policy information
* This class also has some helper data for labels, which can be used with i18n
* when displaying policy information.
*
* @see SHERPAPublisherPolicy
*/
public class SHERPAPermittedVersion {
public class SHERPAPermittedVersion implements Serializable {
private static final long serialVersionUID = 4992181606327727442L;
// Version (submitted, accepted, published)
private String articleVersion;
@@ -44,11 +50,6 @@ public class SHERPAPermittedVersion {
// Embargo
private SHERPAEmbargo embargo;
protected class SHERPAEmbargo {
String units;
int amount;
}
public String getArticleVersion() {
return articleVersion;
}

4
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPAPublisher.java
View File

@@ -7,6 +7,8 @@
*/
package org.dspace.app.sherpa.v2;
import java.io.Serializable;
/**
* Plain java representation of a SHERPA Publisher object, based on SHERPA API v2 responses.
*
@@ -18,7 +20,7 @@ package org.dspace.app.sherpa.v2;
* @see SHERPAJournal
* @see SHERPAPublisherResponse
*/
public class SHERPAPublisher {
public class SHERPAPublisher implements Serializable {
private String name = null;
private String relationshipType;
private String country;

3
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPAPublisherPolicy.java
View File

@@ -7,6 +7,7 @@
*/
package org.dspace.app.sherpa.v2;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
@@ -22,7 +23,7 @@ import java.util.Map;
* @see SHERPAJournal
* @see SHERPAPermittedVersion
*/
public class SHERPAPublisherPolicy {
public class SHERPAPublisherPolicy implements Serializable {
private int id;
private boolean openAccessPermitted;

7
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPAPublisherResponse.java
View File

@@ -10,7 +10,8 @@ package org.dspace.app.sherpa.v2;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.LinkedList;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.apache.logging.log4j.LogManager;
@@ -74,7 +75,7 @@ public class SHERPAPublisherResponse {
* @param jsonData - the JSON input stream from the API result response body
*/
private void parseJSON(InputStream jsonData) throws IOException {
InputStreamReader streamReader = new InputStreamReader(jsonData);
InputStreamReader streamReader = new InputStreamReader(jsonData, StandardCharsets.UTF_8);
JSONTokener jsonTokener = new JSONTokener(streamReader);
JSONObject httpResponse;
try {
@@ -86,7 +87,7 @@ public class SHERPAPublisherResponse {
// parsing the full journal / policy responses
if (items.length() > 0) {
metadata = new SHERPASystemMetadata();
this.publishers = new LinkedList<>();
this.publishers = new ArrayList<>();
// Iterate search result items
for (int itemIndex = 0; itemIndex < items.length(); itemIndex++) {
SHERPAPublisher sherpaPublisher = new SHERPAPublisher();

37
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPAResponse.java
View File

@@ -10,12 +10,15 @@ package org.dspace.app.sherpa.v2;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
@@ -33,7 +36,10 @@ import org.json.JSONTokener;
* @author Kim Shepherd
*
*/
public class SHERPAResponse {
public class SHERPAResponse implements Serializable {
private static final long serialVersionUID = 2732963970169240597L;
// Is this response to be treated as an error?
private boolean error;
@@ -52,6 +58,9 @@ public class SHERPAResponse {
// SHERPA URI (the human page version of this API response)
private String uri;
@JsonIgnore
private Date retrievalTime = new Date();
// Format enum - currently only JSON is supported
public enum SHERPAFormat {
JSON, XML
@@ -71,6 +80,11 @@ public class SHERPAResponse {
}
}
/**
* Create an empty SHERPAResponse representation
*/
public SHERPAResponse() {}
/**
* Parse the SHERPA v2 API JSON and construct Romeo policy data for display
* This method does not return a value, but rather populates the metadata and journals objects
@@ -78,7 +92,7 @@ public class SHERPAResponse {
* @param jsonData - the JSON input stream from the API result response body
*/
private void parseJSON(InputStream jsonData) throws IOException {
InputStreamReader streamReader = new InputStreamReader(jsonData);
InputStreamReader streamReader = new InputStreamReader(jsonData, StandardCharsets.UTF_8);
JSONTokener jsonTokener = new JSONTokener(streamReader);
JSONObject httpResponse;
try {
@@ -90,10 +104,10 @@ public class SHERPAResponse {
// - however, we only ever want one result since we're passing an "equals ISSN" query
if (items.length() > 0) {
metadata = new SHERPASystemMetadata();
this.journals = new LinkedList<>();
this.journals = new ArrayList<>();
// Iterate search result items
for (int itemIndex = 0; itemIndex < items.length(); itemIndex++) {
List<SHERPAPublisher> sherpaPublishers = new LinkedList<>();
List<SHERPAPublisher> sherpaPublishers = new ArrayList<>();
List<SHERPAPublisherPolicy> policies = new ArrayList<>();
SHERPAPublisher sherpaPublisher = new SHERPAPublisher();
SHERPAJournal sherpaJournal = new SHERPAJournal();
@@ -289,7 +303,7 @@ public class SHERPAResponse {
// Is the item in DOAJ?
if (item.has("listed_in_doaj")) {
sherpaJournal.setInDOAJ(("yes".equals(item.getString("listed_in_doaj"))));
sherpaJournal.setInDOAJ("yes".equals(item.getString("listed_in_doaj")));
}
return sherpaJournal;
@@ -403,7 +417,6 @@ public class SHERPAResponse {
// published = pdfversion
// These strings can be used to construct i18n messages.
String articleVersion = "unknown";
String versionLabel = "Unknown";
// Each 'permitted OA' can actually refer to multiple versions
if (permitted.has("article_version")) {
@@ -480,6 +493,12 @@ public class SHERPAResponse {
}
permittedVersion.setLicenses(sherpaLicenses);
if (permitted.has("embargo")) {
JSONObject embargo = permitted.getJSONObject("embargo");
SHERPAEmbargo SHERPAEmbargo = new SHERPAEmbargo(embargo.getInt("amount"), embargo.getString("units"));
permittedVersion.setEmbargo(SHERPAEmbargo);
}
return permittedVersion;
}
@@ -543,4 +562,8 @@ public class SHERPAResponse {
public SHERPASystemMetadata getMetadata() {
return metadata;
}
public Date getRetrievalTime() {
return retrievalTime;
}
}

4
dspace-api/src/main/java/org/dspace/app/sherpa/v2/SHERPASystemMetadata.java
View File

@@ -7,6 +7,8 @@
*/
package org.dspace.app.sherpa.v2;
import java.io.Serializable;
/**
* Plain java representation of a SHERPA System Metadata object, based on SHERPA API v2 responses.
*
@@ -18,7 +20,7 @@ package org.dspace.app.sherpa.v2;
*
* @author Kim Shepherd
*/
public class SHERPASystemMetadata {
public class SHERPASystemMetadata implements Serializable {
private int id;
private String uri;

6
dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java
View File

@@ -40,7 +40,7 @@ import org.dspace.content.service.CollectionService;
import org.dspace.content.service.CommunityService;
import org.dspace.content.service.ItemService;
import org.dspace.core.Context;
import org.dspace.core.LogManager;
import org.dspace.core.LogHelper;
import org.dspace.discovery.DiscoverQuery;
import org.dspace.discovery.DiscoverResult;
import org.dspace.discovery.SearchService;
@@ -284,7 +284,7 @@ public class GenerateSitemaps {
if (makeHTMLMap) {
int files = html.finish();
log.info(LogManager.getHeader(c, "write_sitemap",
log.info(LogHelper.getHeader(c, "write_sitemap",
"type=html,num_files=" + files + ",communities="
+ comms.size() + ",collections=" + colls.size()
+ ",items=" + itemCount));
@@ -292,7 +292,7 @@ public class GenerateSitemaps {
if (makeSitemapOrg) {
int files = sitemapsOrg.finish();
log.info(LogManager.getHeader(c, "write_sitemap",
log.info(LogHelper.getHeader(c, "write_sitemap",
"type=html,num_files=" + files + ",communities="
+ comms.size() + ",collections=" + colls.size()
+ ",items=" + itemCount));

2
dspace-api/src/main/java/org/dspace/app/sitemap/SitemapsOrgGenerator.java
View File

@@ -86,7 +86,7 @@ public class SitemapsOrgGenerator extends AbstractGenerator {
@Override
public String getURLText(String url, Date lastMod) {
StringBuffer urlText = new StringBuffer();
StringBuilder urlText = new StringBuilder();
urlText.append("<url><loc>").append(url).append("</loc>");
if (lastMod != null) {

175
dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java Normal file
View File

@@ -0,0 +1,175 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.solrdatabaseresync;
import static org.dspace.discovery.indexobject.ItemIndexFactoryImpl.STATUS_FIELD;
import static org.dspace.discovery.indexobject.ItemIndexFactoryImpl.STATUS_FIELD_PREDB;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.cli.ParseException;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.dspace.core.Context;
import org.dspace.discovery.IndexableObject;
import org.dspace.discovery.IndexingService;
import org.dspace.discovery.SearchServiceException;
import org.dspace.discovery.SearchUtils;
import org.dspace.discovery.SolrSearchCore;
import org.dspace.discovery.indexobject.IndexableItem;
import org.dspace.discovery.indexobject.factory.IndexObjectFactoryFactory;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.util.SolrUtils;
import org.dspace.utils.DSpace;
/**
* {@link DSpaceRunnable} implementation to update solr items with "predb" status to either:
* - Delete them from solr if they're not present in the database
* - Remove their status if they're present in the database
*/
public class SolrDatabaseResyncCli extends DSpaceRunnable<SolrDatabaseResyncCliScriptConfiguration> {
/* Log4j logger */
private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(SolrDatabaseResyncCli.class);
public static final String TIME_UNTIL_REINDEX_PROPERTY = "solr-database-resync.time-until-reindex";
private IndexingService indexingService;
private SolrSearchCore solrSearchCore;
private IndexObjectFactoryFactory indexObjectServiceFactory;
private ConfigurationService configurationService;
private int timeUntilReindex = 0;
private String maxTime;
@Override
public SolrDatabaseResyncCliScriptConfiguration getScriptConfiguration() {
return new DSpace().getServiceManager()
.getServiceByName("solr-database-resync", SolrDatabaseResyncCliScriptConfiguration.class);
}
public static void runScheduled() throws Exception {
SolrDatabaseResyncCli script = new SolrDatabaseResyncCli();
script.setup();
script.internalRun();
}
@Override
public void setup() throws ParseException {
indexingService = DSpaceServicesFactory.getInstance().getServiceManager()
.getServiceByName(IndexingService.class.getName(), IndexingService.class);
solrSearchCore = DSpaceServicesFactory.getInstance().getServiceManager()
.getServicesByType(SolrSearchCore.class).get(0);
indexObjectServiceFactory = IndexObjectFactoryFactory.getInstance();
configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();
}
@Override
public void internalRun() throws Exception {
logInfoAndOut("Starting Item resync of Solr and Database...");
timeUntilReindex = getTimeUntilReindex();
maxTime = getMaxTime();
Context context = new Context();
try {
context.turnOffAuthorisationSystem();
performStatusUpdate(context);
} finally {
context.restoreAuthSystemState();
context.complete();
}
}
private void performStatusUpdate(Context context) throws SearchServiceException, SolrServerException, IOException {
SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery(STATUS_FIELD + ":" + STATUS_FIELD_PREDB);
solrQuery.addFilterQuery(SearchUtils.RESOURCE_TYPE_FIELD + ":" + IndexableItem.TYPE);
String dateRangeFilter = SearchUtils.LAST_INDEXED_FIELD + ":[* TO " + maxTime + "]";
logDebugAndOut("Date range filter used; " + dateRangeFilter);
solrQuery.addFilterQuery(dateRangeFilter);
solrQuery.addField(SearchUtils.RESOURCE_ID_FIELD);
solrQuery.addField(SearchUtils.RESOURCE_UNIQUE_ID);
QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD);
if (response != null) {
logInfoAndOut(response.getResults().size() + " items found to process");
for (SolrDocument doc : response.getResults()) {
String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD);
String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID);
logDebugAndOut("Processing item with UUID: " + uuid);
Optional<IndexableObject> indexableObject = Optional.empty();
try {
indexableObject = indexObjectServiceFactory
.getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid);
} catch (SQLException e) {
log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid +
"\" from the database, removing related solr document", e);
}
try {
if (indexableObject.isPresent()) {
logDebugAndOut("Item exists in DB, updating solr document");
updateItem(context, indexableObject.get());
} else {
logDebugAndOut("Item doesn't exist in DB, removing solr document");
removeItem(context, uniqueId);
}
} catch (SQLException | IOException e) {
log.error(e.getMessage(), e);
}
}
}
indexingService.commit();
}
private void updateItem(Context context, IndexableObject indexableObject) throws SolrServerException, IOException {
Map<String,Object> fieldModifier = new HashMap<>(1);
fieldModifier.put("remove", STATUS_FIELD_PREDB);
indexingService.atomicUpdate(context, indexableObject.getUniqueIndexID(), STATUS_FIELD, fieldModifier);
}
private void removeItem(Context context, String uniqueId) throws IOException, SQLException {
indexingService.unIndexContent(context, uniqueId);
}
private String getMaxTime() {
Calendar cal = Calendar.getInstance();
if (timeUntilReindex > 0) {
cal.add(Calendar.MILLISECOND, -timeUntilReindex);
}
return SolrUtils.getDateFormatter().format(cal.getTime());
}
private int getTimeUntilReindex() {
return configurationService.getIntProperty(TIME_UNTIL_REINDEX_PROPERTY, 0);
}
private void logInfoAndOut(String message) {
log.info(message);
System.out.println(message);
}
private void logDebugAndOut(String message) {
log.debug(message);
System.out.println(message);
}
}

42
dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCliScriptConfiguration.java Normal file
View File

@@ -0,0 +1,42 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.solrdatabaseresync;
import org.apache.commons.cli.Options;
import org.dspace.core.Context;
import org.dspace.scripts.configuration.ScriptConfiguration;
/**
* The {@link ScriptConfiguration} for the {@link SolrDatabaseResyncCli} script.
*/
public class SolrDatabaseResyncCliScriptConfiguration extends ScriptConfiguration<SolrDatabaseResyncCli> {
private Class<SolrDatabaseResyncCli> dspaceRunnableClass;
@Override
public Class<SolrDatabaseResyncCli> getDspaceRunnableClass() {
return dspaceRunnableClass;
}
@Override
public void setDspaceRunnableClass(Class<SolrDatabaseResyncCli> dspaceRunnableClass) {
this.dspaceRunnableClass = dspaceRunnableClass;
}
@Override
public boolean isAllowedToExecute(Context context) {
return true;
}
@Override
public Options getOptions() {
if (options == null) {
options = new Options();
}
return options;
}
}

43
dspace-api/src/main/java/org/dspace/app/statistics/CreateStatReport.java
View File

@@ -64,10 +64,6 @@ public class CreateStatReport {
*/
private static Context context;
/**
* the config file from which to configure the analyser
*/
/**
* Default constructor
*/
@@ -170,22 +166,19 @@ public class CreateStatReport {
String myLogDir = null;
String myFileTemplate = null;
String myConfigFile = null;
StringBuffer myOutFile = null;
Date myStartDate = null;
Date myEndDate = null;
boolean myLookUp = false;
Calendar start = new GregorianCalendar(calendar.get(Calendar.YEAR),
calendar.get(Calendar.MONTH),
calendar.getActualMinimum(Calendar.DAY_OF_MONTH));
myStartDate = start.getTime();
Date myStartDate = start.getTime();
Calendar end = new GregorianCalendar(calendar.get(Calendar.YEAR),
calendar.get(Calendar.MONTH),
calendar.getActualMaximum(Calendar.DAY_OF_MONTH));
myEndDate = end.getTime();
Date myEndDate = end.getTime();
myOutFile = new StringBuffer(outputLogDirectory);
StringBuilder myOutFile = new StringBuilder(outputLogDirectory);
myOutFile.append(outputPrefix);
myOutFile.append(calendar.get(Calendar.YEAR));
myOutFile.append("-");
@@ -211,12 +204,11 @@ public class CreateStatReport {
String myLogDir = null;
String myFileTemplate = null;
String myConfigFile = null;
StringBuffer myOutFile = null;
Date myStartDate = null;
Date myEndDate = null;
boolean myLookUp = false;
myOutFile = new StringBuffer(outputLogDirectory);
StringBuilder myOutFile = new StringBuilder(outputLogDirectory);
myOutFile.append(outputPrefix);
myOutFile.append(calendar.get(Calendar.YEAR));
myOutFile.append("-");
@@ -245,9 +237,6 @@ public class CreateStatReport {
String myLogDir = null;
String myFileTemplate = null;
String myConfigFile = null;
StringBuffer myOutFile = null;
Date myStartDate = null;
Date myEndDate = null;
boolean myLookUp = false;
Calendar reportEndDate = new GregorianCalendar(calendar.get(Calendar.YEAR),
@@ -260,14 +249,14 @@ public class CreateStatReport {
Calendar start = new GregorianCalendar(currentMonth.get(Calendar.YEAR),
currentMonth.get(Calendar.MONTH),
currentMonth.getActualMinimum(Calendar.DAY_OF_MONTH));
myStartDate = start.getTime();
Date myStartDate = start.getTime();
Calendar end = new GregorianCalendar(currentMonth.get(Calendar.YEAR),
currentMonth.get(Calendar.MONTH),
currentMonth.getActualMaximum(Calendar.DAY_OF_MONTH));
myEndDate = end.getTime();
Date myEndDate = end.getTime();
myOutFile = new StringBuffer(outputLogDirectory);
StringBuilder myOutFile = new StringBuilder(outputLogDirectory);
myOutFile.append(outputPrefix);
myOutFile.append(currentMonth.get(Calendar.YEAR));
myOutFile.append("-");
@@ -293,11 +282,9 @@ public class CreateStatReport {
String outputPrefix = "report-general-";
String myFormat = "html";
StringBuffer myInput = null;
StringBuffer myOutput = null;
String myMap = null;
myInput = new StringBuffer(outputLogDirectory);
StringBuilder myInput = new StringBuilder(outputLogDirectory);
myInput.append(inputPrefix);
myInput.append(calendar.get(Calendar.YEAR));
myInput.append("-");
@@ -306,7 +293,7 @@ public class CreateStatReport {
myInput.append(calendar.get(Calendar.DAY_OF_MONTH));
myInput.append(outputSuffix);
myOutput = new StringBuffer(outputReportDirectory);
StringBuilder myOutput = new StringBuilder(outputReportDirectory);
myOutput.append(outputPrefix);
myOutput.append(calendar.get(Calendar.YEAR));
myOutput.append("-");
@@ -332,8 +319,6 @@ public class CreateStatReport {
String outputPrefix = "report-";
String myFormat = "html";
StringBuffer myInput = null;
StringBuffer myOutput = null;
String myMap = null;
Calendar reportEndDate = new GregorianCalendar(calendar.get(Calendar.YEAR),
@@ -344,14 +329,14 @@ public class CreateStatReport {
while (currentMonth.before(reportEndDate)) {
myInput = new StringBuffer(outputLogDirectory);
StringBuilder myInput = new StringBuilder(outputLogDirectory);
myInput.append(inputPrefix);
myInput.append(currentMonth.get(Calendar.YEAR));
myInput.append("-");
myInput.append(currentMonth.get(Calendar.MONTH) + 1);
myInput.append(outputSuffix);
myOutput = new StringBuffer(outputReportDirectory);
StringBuilder myOutput = new StringBuilder(outputReportDirectory);
myOutput.append(outputPrefix);
myOutput.append(currentMonth.get(Calendar.YEAR));
myOutput.append("-");
@@ -376,18 +361,16 @@ public class CreateStatReport {
String outputPrefix = "report-";
String myFormat = "html";
StringBuffer myInput = null;
StringBuffer myOutput = null;
String myMap = null;
myInput = new StringBuffer(outputLogDirectory);
StringBuilder myInput = new StringBuilder(outputLogDirectory);
myInput.append(inputPrefix);
myInput.append(calendar.get(Calendar.YEAR));
myInput.append("-");
myInput.append(calendar.get(Calendar.MONTH) + 1);
myInput.append(outputSuffix);
myOutput = new StringBuffer(outputReportDirectory);
StringBuilder myOutput = new StringBuilder(outputReportDirectory);
myOutput.append(outputPrefix);
myOutput.append(calendar.get(Calendar.YEAR));
myOutput.append("-");

12
dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java
View File

@@ -31,7 +31,7 @@ import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.dspace.core.Context;
import org.dspace.core.LogManager;
import org.dspace.core.LogHelper;
import org.dspace.core.Utils;
import org.dspace.discovery.DiscoverQuery;
import org.dspace.discovery.SearchServiceException;
@@ -1144,17 +1144,17 @@ public class LogAnalyser {
if (match.matches()) {
// set up a new log line object
LogLine logLine = new LogLine(parseDate(match.group(1).trim()),
LogManager.unescapeLogField(match.group(2)).trim(),
LogManager.unescapeLogField(match.group(3)).trim(),
LogManager.unescapeLogField(match.group(4)).trim(),
LogManager.unescapeLogField(match.group(5)).trim());
LogHelper.unescapeLogField(match.group(2)).trim(),
LogHelper.unescapeLogField(match.group(3)).trim(),
LogHelper.unescapeLogField(match.group(4)).trim(),
LogHelper.unescapeLogField(match.group(5)).trim());
return logLine;
} else {
match = validBase.matcher(line);
if (match.matches()) {
LogLine logLine = new LogLine(parseDate(match.group(1).trim()),
LogManager.unescapeLogField(match.group(2)).trim(),
LogHelper.unescapeLogField(match.group(2)).trim(),
null,
null,
null

58
dspace-api/src/main/java/org/dspace/app/util/CacheSnooper.java
View File

@@ -1,58 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.util;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import org.dspace.core.Context;
import org.dspace.servicemanager.DSpaceKernelImpl;
import org.dspace.servicemanager.DSpaceKernelInit;
import org.dspace.services.CachingService;
/**
* List all EhCache CacheManager and Cache instances.
*
* <p>This is a debugging tool, not used in the daily operation of DSpace.
* Just run it from the installed instance using
* {@code bin/dspace dsrun org.dspace.app.util.CacheSnooper}
* to check that the cache configuration is what you expect it to be,
* given your configuration.
*
* <p>This was created to prove a specific cache configuration patch,
* but I leave it here in the hope that it may be useful to others.
*
* @author Mark H. Wood <mwood@iupui.edu>
*/
public class CacheSnooper {
private CacheSnooper() { }
public static void main(String[] argv) {
// Ensure that the DSpace kernel is started.
DSpaceKernelImpl kernel = DSpaceKernelInit.getKernel(null);
// Ensure that the services cache manager is started.
CachingService serviceCaches = kernel.getServiceManager()
.getServiceByName(null, CachingService.class);
// Ensure that the database layer is started.
Context ctx = new Context();
// List those caches!
for (CacheManager manager : CacheManager.ALL_CACHE_MANAGERS) {
System.out.format("CacheManager: %s%n", manager);
for (String cacheName : manager.getCacheNames()) {
Cache cache = manager.getCache(cacheName);
System.out.format(" Cache: '%s'; maxHeap: %d; maxDisk: %d%n",
cacheName,
cache.getCacheConfiguration().getMaxEntriesLocalHeap(),
cache.getCacheConfiguration().getMaxEntriesLocalDisk());
}
}
}
}

30
dspace-api/src/main/java/org/dspace/app/util/Configuration.java
View File

@@ -37,6 +37,7 @@ public class Configuration {
* <li>{@code --property name} prints the value of the DSpace configuration
* property {@code name} to the standard output.</li>
* <li>{@code --raw} suppresses parameter substitution in the output.</li>
* <li>{@code --first} print only the first of multiple values.</li>
* <li>{@code --help} describes these options.</li>
* </ul>
* If the property does not exist, nothing is written.
@@ -51,6 +52,8 @@ public class Configuration {
"optional name of the module in which 'property' exists");
options.addOption("r", "raw", false,
"do not do property substitution on the value");
options.addOption("f", "first", false,
"display only the first value of an array property");
options.addOption("?", "Get help");
options.addOption("h", "help", false, "Get help");
@@ -90,19 +93,36 @@ public class Configuration {
propNameBuilder.append(cmd.getOptionValue('p'));
String propName = propNameBuilder.toString();
// Print the property's value, if it exists
// Print the property's value(s), if it exists
ConfigurationService cfg = DSpaceServicesFactory.getInstance().getConfigurationService();
if (!cfg.hasProperty(propName)) {
System.out.println();
} else {
String val;
if (cmd.hasOption('r')) {
val = cfg.getPropertyValue(propName).toString();
// Print "raw" values (without property substitutions)
Object rawValue = cfg.getPropertyValue(propName);
if (rawValue.getClass().isArray()) {
for (Object value : (Object[]) rawValue) {
System.out.println(value.toString());
if (cmd.hasOption('f')) {
break; // If --first print only one value
}
}
} else { // Not an array
System.out.println(rawValue.toString());
}
} else {
val = cfg.getProperty(propName);
// Print values with property substitutions
String[] values = cfg.getArrayProperty(propName);
for (String value : values) {
System.out.println(value);
if (cmd.hasOption('f')) {
break; // If --first print only one value
}
}
}
System.out.println(val);
}
System.exit(0);
}
}

9
dspace-api/src/main/java/org/dspace/app/util/DCInput.java
View File

@@ -561,6 +561,15 @@ public class DCInput {
return true;
}
/**
* Get the type bind list for use in determining whether
* to display this field in angular dynamic form building
* @return list of bound types
*/
public List<String> getTypeBindList() {
return typeBind;
}
/**
* Verify whether the current field contains an entity relationship
* This also implies a relationship type is defined for this field

47
dspace-api/src/main/java/org/dspace/app/util/DCInputSet.java
View File

@@ -7,6 +7,7 @@
*/
package org.dspace.app.util;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -176,4 +177,50 @@ public class DCInputSet {
return true;
}
/**
* Iterate DC input rows and populate a list of all allowed field names in this submission configuration.
* This is important because an input can be configured repeatedly in a form (for example it could be required
* for type Book, and allowed but not required for type Article).
* If the field is allowed for this document type it'll never be stripped from metadata on validation.
*
* This can be more efficient than isFieldPresent to avoid looping the input set with each check.
*
* @param documentTypeValue Document type eg. Article, Book
* @return ArrayList of field names to use in validation
*/
public List<String> populateAllowedFieldNames(String documentTypeValue) {
List<String> allowedFieldNames = new ArrayList<>();
// Before iterating each input for validation, run through all inputs + fields and populate a lookup
// map with inputs for this type. Because an input can be configured repeatedly in a form (for example
// it could be required for type Book, and allowed but not required for type Article), allowed=true will
// always take precedence
for (DCInput[] row : inputs) {
for (DCInput input : row) {
if (input.isQualdropValue()) {
List<Object> inputPairs = input.getPairs();
//starting from the second element of the list and skipping one every time because the display
// values are also in the list and before the stored values.
for (int i = 1; i < inputPairs.size(); i += 2) {
String fullFieldname = input.getFieldName() + "." + inputPairs.get(i);
if (input.isAllowedFor(documentTypeValue)) {
if (!allowedFieldNames.contains(fullFieldname)) {
allowedFieldNames.add(fullFieldname);
}
// For the purposes of qualdrop, we have to add the field name without the qualifier
// too, or a required qualdrop will get confused and incorrectly reject a value
if (!allowedFieldNames.contains(input.getFieldName())) {
allowedFieldNames.add(input.getFieldName());
}
}
}
} else {
if (input.isAllowedFor(documentTypeValue) && !allowedFieldNames.contains(input.getFieldName())) {
allowedFieldNames.add(input.getFieldName());
}
}
}
}
return allowedFieldNames;
}
}

4
dspace-api/src/main/java/org/dspace/app/util/GoogleBitstreamComparator.java
View File

@@ -86,8 +86,10 @@ public class GoogleBitstreamComparator implements Comparator<Bitstream> {
if (priority1 > priority2) {
return 1;
} else if (priority1 == priority2) {
if (b1.getSizeBytes() <= b2.getSizeBytes()) {
if (b1.getSizeBytes() < b2.getSizeBytes()) {
return 1;
} else if (b1.getSizeBytes() == b2.getSizeBytes()) {
return 0;
} else {
return -1;
}

2
dspace-api/src/main/java/org/dspace/app/util/GoogleMetadata.java
View File

@@ -42,7 +42,7 @@ import org.dspace.core.Context;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.jdom.Element;
import org.jdom2.Element;
/**
* Configuration and mapping for Google Scholar output metadata

13
dspace-api/src/main/java/org/dspace/app/util/InitializeEntities.java
View File

@@ -10,8 +10,6 @@ package org.dspace.app.util;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
@@ -30,7 +28,6 @@ import org.dspace.content.EntityType;
import org.dspace.content.RelationshipType;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.EntityTypeService;
import org.dspace.content.service.RelationshipService;
import org.dspace.content.service.RelationshipTypeService;
import org.dspace.core.Context;
import org.w3c.dom.Document;
@@ -40,22 +37,20 @@ import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
/**
* This script is used to initialize the database with a set of relationshiptypes that are written
* This script is used to initialize the database with a set of relationship types that are written
* in an xml file that is given to this script.
* This XML file needs to have a proper XML structure and needs to define the variables of the RelationshipType object
* This XML file needs to have a proper XML structure and needs to define the variables of the RelationshipType object.
*/
public class InitializeEntities {
private final static Logger log = LogManager.getLogger();
private final RelationshipTypeService relationshipTypeService;
private final RelationshipService relationshipService;
private final EntityTypeService entityTypeService;
private InitializeEntities() {
relationshipTypeService = ContentServiceFactory.getInstance().getRelationshipTypeService();
relationshipService = ContentServiceFactory.getInstance().getRelationshipService();
entityTypeService = ContentServiceFactory.getInstance().getEntityTypeService();
}
@@ -111,14 +106,12 @@ public class InitializeEntities {
try {
File fXmlFile = new File(fileLocation);
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = null;
dBuilder = dbFactory.newDocumentBuilder();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
Document doc = dBuilder.parse(fXmlFile);
doc.getDocumentElement().normalize();
NodeList nList = doc.getElementsByTagName("type");
List<RelationshipType> relationshipTypes = new LinkedList<>();
for (int i = 0; i < nList.getLength(); i++) {
Node nNode = nList.item(i);

24
dspace-api/src/main/java/org/dspace/app/util/OpenSearchServiceImpl.java
View File

@@ -16,10 +16,10 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import com.sun.syndication.feed.module.opensearch.OpenSearchModule;
import com.sun.syndication.feed.module.opensearch.entity.OSQuery;
import com.sun.syndication.feed.module.opensearch.impl.OpenSearchModuleImpl;
import com.sun.syndication.io.FeedException;
import com.rometools.modules.opensearch.OpenSearchModule;
import com.rometools.modules.opensearch.entity.OSQuery;
import com.rometools.modules.opensearch.impl.OpenSearchModuleImpl;
import com.rometools.rome.io.FeedException;
import org.apache.logging.log4j.Logger;
import org.dspace.app.util.service.OpenSearchService;
import org.dspace.content.DSpaceObject;
@@ -29,11 +29,11 @@ import org.dspace.discovery.IndexableObject;
import org.dspace.handle.service.HandleService;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
import org.jdom.output.DOMOutputter;
import org.jdom.output.XMLOutputter;
import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.jdom2.Namespace;
import org.jdom2.output.DOMOutputter;
import org.jdom2.output.XMLOutputter;
import org.springframework.beans.factory.annotation.Autowired;
import org.w3c.dom.Document;
@@ -192,7 +192,7 @@ public class OpenSearchServiceImpl implements OpenSearchService {
* @param scope - null for the entire repository, or a collection/community handle
* @return Service Document
*/
protected org.jdom.Document getServiceDocument(String scope) {
protected org.jdom2.Document getServiceDocument(String scope) {
ConfigurationService config = DSpaceServicesFactory.getInstance().getConfigurationService();
Namespace ns = Namespace.getNamespace(osNs);
@@ -245,7 +245,7 @@ public class OpenSearchServiceImpl implements OpenSearchService {
url.setAttribute("template", template.toString());
root.addContent(url);
}
return new org.jdom.Document(root);
return new org.jdom2.Document(root);
}
/**
@@ -255,7 +255,7 @@ public class OpenSearchServiceImpl implements OpenSearchService {
* @return W3C Document object
* @throws IOException if IO error
*/
protected Document jDomToW3(org.jdom.Document jdomDoc) throws IOException {
protected Document jDomToW3(org.jdom2.Document jdomDoc) throws IOException {
DOMOutputter domOut = new DOMOutputter();
try {
return domOut.output(jdomDoc);

69
dspace-api/src/main/java/org/dspace/app/util/RelationshipUtils.java Normal file
View File

@@ -0,0 +1,69 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.util;
import java.util.List;
import org.dspace.content.RelationshipType;
public class RelationshipUtils {
private RelationshipUtils() {
}
/**
* Matches two Entity types to a Relationship Type from a set of Relationship Types.
*
* Given a list of Relationship Types, this method will find a Relationship Type that
* is configured between the originType and the targetType, with the matching originTypeName.
* It will match a relationship between these two entities in either direction (eg leftward
* or rightward).
*
* Example: originType = Author, targetType = Publication, originTypeName = isAuthorOfPublication.
*
* @param relTypes set of Relationship Types in which to find a match.
* @param targetType entity type of target (eg. Publication).
* @param originType entity type of origin referer (eg. Author).
* @param originTypeName the name of the relationship (eg. isAuthorOfPublication)
* @return null or matched Relationship Type.
*/
public static RelationshipType matchRelationshipType(List<RelationshipType> relTypes, String targetType,
String originType, String originTypeName) {
RelationshipType foundRelationshipType = null;
if (originTypeName.split("\\.").length > 1) {
originTypeName = originTypeName.split("\\.")[1];
}
for (RelationshipType relationshipType : relTypes) {
// Is origin type leftward or righward
boolean isLeft = false;
if (relationshipType.getLeftType().getLabel().equalsIgnoreCase(originType)) {
isLeft = true;
}
if (isLeft) {
// Validate typeName reference
if (!relationshipType.getLeftwardType().equalsIgnoreCase(originTypeName)) {
continue;
}
if (relationshipType.getLeftType().getLabel().equalsIgnoreCase(originType)
&& relationshipType.getRightType().getLabel().equalsIgnoreCase(targetType)) {
foundRelationshipType = relationshipType;
}
} else {
if (!relationshipType.getRightwardType().equalsIgnoreCase(originTypeName)) {
continue;
}
if (relationshipType.getLeftType().getLabel().equalsIgnoreCase(targetType)
&& relationshipType.getRightType().getLabel().equalsIgnoreCase(originType)) {
foundRelationshipType = relationshipType;
}
}
}
return foundRelationshipType;
}
}

1
dspace-api/src/main/java/org/dspace/app/util/SubmissionStepConfig.java
View File

@@ -30,6 +30,7 @@ public class SubmissionStepConfig implements Serializable {
public static final String INPUT_FORM_STEP_NAME = "submission-form";
public static final String UPLOAD_STEP_NAME = "upload";
public static final String ACCESS_CONDITION_STEP_NAME = "accessCondition";
/*
* The identifier for the Select Collection step

49
dspace-api/src/main/java/org/dspace/app/util/SyndicationFeed.java
View File

@@ -15,26 +15,26 @@ import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import com.sun.syndication.feed.module.DCModule;
import com.sun.syndication.feed.module.DCModuleImpl;
import com.sun.syndication.feed.module.Module;
import com.sun.syndication.feed.module.itunes.EntryInformation;
import com.sun.syndication.feed.module.itunes.EntryInformationImpl;
import com.sun.syndication.feed.module.itunes.types.Duration;
import com.sun.syndication.feed.synd.SyndContent;
import com.sun.syndication.feed.synd.SyndContentImpl;
import com.sun.syndication.feed.synd.SyndEnclosure;
import com.sun.syndication.feed.synd.SyndEnclosureImpl;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndEntryImpl;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.feed.synd.SyndFeedImpl;
import com.sun.syndication.feed.synd.SyndImage;
import com.sun.syndication.feed.synd.SyndImageImpl;
import com.sun.syndication.feed.synd.SyndPerson;
import com.sun.syndication.feed.synd.SyndPersonImpl;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedOutput;
import com.rometools.modules.itunes.EntryInformation;
import com.rometools.modules.itunes.EntryInformationImpl;
import com.rometools.modules.itunes.types.Duration;
import com.rometools.rome.feed.module.DCModule;
import com.rometools.rome.feed.module.DCModuleImpl;
import com.rometools.rome.feed.module.Module;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndFeedImpl;
import com.rometools.rome.feed.synd.SyndImage;
import com.rometools.rome.feed.synd.SyndImageImpl;
import com.rometools.rome.feed.synd.SyndPerson;
import com.rometools.rome.feed.synd.SyndPersonImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedOutput;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
@@ -193,13 +193,11 @@ public class SyndicationFeed {
String defaultTitle = null;
boolean podcastFeed = false;
this.request = request;
// dso is null for the whole site, or a search without scope
if (dso == null) {
defaultTitle = configurationService.getProperty("dspace.name");
feed.setDescription(localize(labels, MSG_FEED_DESCRIPTION));
objectURL = resolveURL(request, null);
logoURL = configurationService.getProperty("webui.feed.logo.url");
} else {
Bitstream logo = null;
if (dso instanceof IndexableCollection) {
@@ -329,7 +327,8 @@ public class SyndicationFeed {
dcDescriptionField != null) {
DCModule dc = new DCModuleImpl();
if (dcCreatorField != null) {
List<MetadataValue> dcAuthors = itemService.getMetadataByMetadataString(item, dcCreatorField);
List<MetadataValue> dcAuthors = itemService
.getMetadataByMetadataString(item, dcCreatorField);
if (dcAuthors.size() > 0) {
List<String> creators = new ArrayList<>();
for (MetadataValue author : dcAuthors) {
@@ -345,7 +344,8 @@ public class SyndicationFeed {
}
}
if (dcDescriptionField != null) {
List<MetadataValue> v = itemService.getMetadataByMetadataString(item, dcDescriptionField);
List<MetadataValue> v = itemService
.getMetadataByMetadataString(item, dcDescriptionField);
if (v.size() > 0) {
StringBuilder descs = new StringBuilder();
for (MetadataValue d : v) {
@@ -376,6 +376,7 @@ public class SyndicationFeed {
enc.setLength(bit.getSizeBytes());
enc.setUrl(urlOfBitstream(request, bit));
enclosures.add(enc);
}
}
}

14
dspace-api/src/main/java/org/dspace/app/util/Util.java
View File

@@ -38,13 +38,12 @@ import org.dspace.core.Utils;
*
* @author Robert Tansley
* @author Mark Diggory
* @version $Revision$
*/
public class Util {
// cache for source version result
private static String sourceVersion = null;
private static Logger log = org.apache.logging.log4j.LogManager.getLogger(Util.class);
private static final Logger log = org.apache.logging.log4j.LogManager.getLogger();
/**
* Default constructor. Must be protected as org.dspace.xmlworkflow.WorkflowUtils extends it
@@ -60,7 +59,7 @@ public class Util {
* spaces
*/
public static String nonBreakSpace(String s) {
StringBuffer newString = new StringBuffer();
StringBuilder newString = new StringBuilder();
for (int i = 0; i < s.length(); i++) {
char ch = s.charAt(i);
@@ -99,7 +98,7 @@ public class Util {
return "";
}
StringBuffer out = new StringBuffer();
StringBuilder out = new StringBuilder();
final String[] pctEncoding = {"%00", "%01", "%02", "%03", "%04",
"%05", "%06", "%07", "%08", "%09", "%0a", "%0b", "%0c", "%0d",
@@ -263,7 +262,7 @@ public class Util {
return null;
}
List<UUID> return_values = new ArrayList<UUID>(request_values.length);
List<UUID> return_values = new ArrayList<>(request_values.length);
for (String s : request_values) {
try {
@@ -402,7 +401,7 @@ public class Util {
Item item, List<MetadataValue> values, String schema, String element,
String qualifier, Locale locale) throws SQLException,
DCInputsReaderException {
List<String> toReturn = new ArrayList<String>();
List<String> toReturn = new ArrayList<>();
DCInput myInputs = null;
boolean myInputsFound = false;
String formFileName = I18nUtil.getInputFormsFileName(locale);
@@ -478,8 +477,9 @@ public class Util {
}
/**
* Split a list in an array of i sub-lists uniformly sized
* Split a list in an array of i sub-lists uniformly sized.
*
* @param <T> type of objects in the list.
* @param idsList the list to split
* @param i the number of sublists to return
*

1
dspace-api/src/main/java/org/dspace/app/util/WebApp.java
View File

@@ -58,6 +58,7 @@ public class WebApp implements ReloadableEntity<Integer> {
}
@Override
public Integer getID() {
return id;
}

Some files were not shown because too many files have changed in this diff Show More