diff --git a/.dockerignore b/.dockerignore index 0e42960dc9..7d3bdc2b4b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,5 @@ dspace/modules/*/target/ Dockerfile.* dspace/src/main/docker/dspace-postgres-pgcrypto dspace/src/main/docker/dspace-postgres-pgcrypto-curl -dspace/src/main/docker/solr dspace/src/main/docker/README.md dspace/src/main/docker-compose/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52714a8ba2..99c9efe019 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,6 +79,39 @@ jobs: name: ${{ matrix.type }} results path: ${{ matrix.resultsdir }} - # https://github.com/codecov/codecov-action + # Upload code coverage report to artifact, so that it can be shared with the 'codecov' job (see below) + - name: Upload code coverage report to Artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.type }} coverage report + path: 'dspace/target/site/jacoco-aggregate/jacoco.xml' + retention-days: 14 + + # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test + # job above. This is necessary because Codecov uploads seem to randomly fail at times. + # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 + codecov: + # Must run after 'tests' job above + needs: tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Download artifacts from previous 'tests' job + - name: Download coverage artifacts + uses: actions/download-artifact@v3 + + # Now attempt upload to Codecov using its action. + # NOTE: We use a retry action to retry the Codecov upload if it fails the first time. + # + # Retry action: https://github.com/marketplace/actions/retry-action + # Codecov action: https://github.com/codecov/codecov-action - name: Upload coverage to Codecov.io - uses: codecov/codecov-action@v3 + uses: Wandalen/wretry.action@v1.0.36 + with: + action: codecov/codecov-action@v3 + # Try upload 5 times max + attempt_limit: 5 + # Run again in 30 seconds + attempt_delay: 30000 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 64e12f01aa..971954a5e1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -170,3 +170,86 @@ jobs: # Use tags / labels provided by 'docker/metadata-action' above tags: ${{ steps.meta_build_cli.outputs.tags }} labels: ${{ steps.meta_build_cli.outputs.labels }} + + ########################################### + # Build/Push the 'dspace/dspace-solr' image + ########################################### + # Get Metadata for docker_build_solr step below + - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-solr' image + id: meta_build_solr + uses: docker/metadata-action@v4 + with: + images: dspace/dspace-solr + tags: ${{ env.IMAGE_TAGS }} + flavor: ${{ env.TAGS_FLAVOR }} + + - name: Build and push 'dspace-solr' image + id: docker_build_solr + uses: docker/build-push-action@v3 + with: + context: . + file: ./dspace/src/main/docker/dspace-solr/Dockerfile + platforms: ${{ env.PLATFORMS }} + # For pull requests, we run the Docker build (to ensure no PR changes break the build), + # but we ONLY do an image push to DockerHub if it's NOT a PR + push: ${{ github.event_name != 'pull_request' }} + # Use tags / labels provided by 'docker/metadata-action' above + tags: ${{ steps.meta_build_solr.outputs.tags }} + labels: ${{ steps.meta_build_solr.outputs.labels }} + + ########################################################### + # Build/Push the 'dspace/dspace-postgres-pgcrypto' image + ########################################################### + # Get Metadata for docker_build_postgres step below + - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-postgres-pgcrypto' image + id: meta_build_postgres + uses: docker/metadata-action@v4 + with: + images: dspace/dspace-postgres-pgcrypto + tags: ${{ env.IMAGE_TAGS }} + flavor: ${{ env.TAGS_FLAVOR }} + + - name: Build and push 'dspace-postgres-pgcrypto' image + id: docker_build_postgres + uses: docker/build-push-action@v3 + with: + # Must build out of subdirectory to have access to install script for pgcrypto + context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ + dockerfile: Dockerfile + platforms: ${{ env.PLATFORMS }} + # For pull requests, we run the Docker build (to ensure no PR changes break the build), + # but we ONLY do an image push to DockerHub if it's NOT a PR + push: ${{ github.event_name != 'pull_request' }} + # Use tags / labels provided by 'docker/metadata-action' above + tags: ${{ steps.meta_build_postgres.outputs.tags }} + labels: ${{ steps.meta_build_postgres.outputs.labels }} + + ########################################################### + # Build/Push the 'dspace/dspace-postgres-pgcrypto' image ('-loadsql' tag) + ########################################################### + # Get Metadata for docker_build_postgres_loadsql step below + - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-postgres-pgcrypto-loadsql' image + id: meta_build_postgres_loadsql + uses: docker/metadata-action@v4 + with: + images: dspace/dspace-postgres-pgcrypto + tags: ${{ env.IMAGE_TAGS }} + # Suffix all tags with "-loadsql". Otherwise, it uses the same + # tagging logic as the primary 'dspace/dspace-postgres-pgcrypto' image above. + flavor: ${{ env.TAGS_FLAVOR }} + suffix=-loadsql + + - name: Build and push 'dspace-postgres-pgcrypto-loadsql' image + id: docker_build_postgres_loadsql + uses: docker/build-push-action@v3 + with: + # Must build out of subdirectory to have access to install script for pgcrypto + context: ./dspace/src/main/docker/dspace-postgres-pgcrypto-curl/ + dockerfile: Dockerfile + platforms: ${{ env.PLATFORMS }} + # For pull requests, we run the Docker build (to ensure no PR changes break the build), + # but we ONLY do an image push to DockerHub if it's NOT a PR + push: ${{ github.event_name != 'pull_request' }} + # Use tags / labels provided by 'docker/metadata-action' above + tags: ${{ steps.meta_build_postgres_loadsql.outputs.tags }} + labels: ${{ steps.meta_build_postgres_loadsql.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/issue_opened.yml b/.github/workflows/issue_opened.yml index 5d7c1c30f7..b4436dca3a 100644 --- a/.github/workflows/issue_opened.yml +++ b/.github/workflows/issue_opened.yml @@ -16,7 +16,7 @@ jobs: # Only add to project board if issue is flagged as "needs triage" or has no labels # NOTE: By default we flag new issues as "needs triage" in our issue template if: (contains(github.event.issue.labels.*.name, 'needs triage') || join(github.event.issue.labels.*.name) == '') - uses: actions/add-to-project@v0.3.0 + uses: actions/add-to-project@v0.5.0 # Note, the authentication token below is an ORG level Secret. # It must be created/recreated manually via a personal access token with admin:org, project, public_repo permissions # See: https://docs.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token#permissions-for-the-github_token diff --git a/.github/workflows/label_merge_conflicts.yml b/.github/workflows/label_merge_conflicts.yml index d71d244c2b..cc0c7099f4 100644 --- a/.github/workflows/label_merge_conflicts.yml +++ b/.github/workflows/label_merge_conflicts.yml @@ -23,7 +23,7 @@ jobs: steps: # See: https://github.com/prince-chrismc/label-merge-conflicts-action - name: Auto-label PRs with merge conflicts - uses: prince-chrismc/label-merge-conflicts-action@v2 + uses: prince-chrismc/label-merge-conflicts-action@v3 # Add "merge conflict" label if a merge conflict is detected. Remove it when resolved. # Note, the authentication token is created automatically # See: https://docs.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token diff --git a/Dockerfile b/Dockerfile index 444a1bcf0b..f1ff6adf5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ ARG TARGET_DIR=dspace-installer COPY --from=build /install /dspace-src WORKDIR /dspace-src # Create the initial install deployment using ANT -ENV ANT_VERSION 1.10.12 +ENV ANT_VERSION 1.10.13 ENV ANT_HOME /tmp/ant-$ANT_VERSION ENV PATH $ANT_HOME/bin:$PATH # Need wget to install ant diff --git a/Dockerfile.cli b/Dockerfile.cli index 76e559fc83..6878d8f8db 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -30,7 +30,7 @@ ARG TARGET_DIR=dspace-installer COPY --from=build /install /dspace-src WORKDIR /dspace-src # Create the initial install deployment using ANT -ENV ANT_VERSION 1.10.12 +ENV ANT_VERSION 1.10.13 ENV ANT_HOME /tmp/ant-$ANT_VERSION ENV PATH $ANT_HOME/bin:$PATH # Need wget to install ant diff --git a/docker-compose.yml b/docker-compose.yml index 6008b873ae..36ba6af2c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,13 +62,17 @@ services: while (! /dev/null 2>&1; do sleep 1; done; /dspace/bin/dspace database migrate catalina.sh run - # DSpace database container + # DSpace PostgreSQL database container dspacedb: container_name: dspacedb + # Uses a custom Postgres image with pgcrypto installed + image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7_x}" + build: + # Must build out of subdirectory to have access to install script for pgcrypto + context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ environment: PGDATA: /pgdata - # Uses a custom Postgres image with pgcrypto installed - image: dspace/dspace-postgres-pgcrypto + POSTGRES_PASSWORD: dspace networks: dspacenet: ports: @@ -77,12 +81,17 @@ services: stdin_open: true tty: true volumes: + # Keep Postgres data directory between reboots - pgdata:/pgdata # DSpace Solr container dspacesolr: container_name: dspacesolr - # Uses official Solr image at https://hub.docker.com/_/solr/ - image: solr:8.11-slim + image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}" + build: + context: . + dockerfile: ./dspace/src/main/docker/dspace-solr/Dockerfile + args: + SOLR_VERSION: "${SOLR_VER:-8.11}" networks: dspacenet: ports: @@ -92,30 +101,25 @@ services: tty: true working_dir: /var/solr/data volumes: - # Mount our local Solr core configs so that they are available as Solr configsets on container - - ./dspace/solr/authority:/opt/solr/server/solr/configsets/authority - - ./dspace/solr/oai:/opt/solr/server/solr/configsets/oai - - ./dspace/solr/search:/opt/solr/server/solr/configsets/search - - ./dspace/solr/statistics:/opt/solr/server/solr/configsets/statistics # Keep Solr data directory between reboots - solr_data:/var/solr/data - # Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr + # Initialize all DSpace Solr cores then start Solr: # * First, run precreate-core to create the core (if it doesn't yet exist). If exists already, this is a no-op - # * Second, copy updated configs from mounted configsets to this core. If it already existed, this updates core - # to the latest configs. If it's a newly created core, this is a no-op. + # * Second, copy configsets to this core: + # Updates to Solr configs require the container to be rebuilt/restarted: `docker compose -p d7 up -d --build dspacesolr` entrypoint: - /bin/bash - '-c' - | init-var-solr precreate-core authority /opt/solr/server/solr/configsets/authority - cp -r -u /opt/solr/server/solr/configsets/authority/* authority + cp -r /opt/solr/server/solr/configsets/authority/* authority precreate-core oai /opt/solr/server/solr/configsets/oai - cp -r -u /opt/solr/server/solr/configsets/oai/* oai + cp -r /opt/solr/server/solr/configsets/oai/* oai precreate-core search /opt/solr/server/solr/configsets/search - cp -r -u /opt/solr/server/solr/configsets/search/* search + cp -r /opt/solr/server/solr/configsets/search/* search precreate-core statistics /opt/solr/server/solr/configsets/statistics - cp -r -u /opt/solr/server/solr/configsets/statistics/* statistics + cp -r /opt/solr/server/solr/configsets/statistics/* statistics exec solr -f volumes: assetstore: diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index d33387859f..c1811a9d63 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -776,7 +776,7 @@ org.json json - 20180130 + 20230227 diff --git a/dspace-api/src/main/java/org/dspace/app/util/SyndicationFeed.java b/dspace-api/src/main/java/org/dspace/app/util/SyndicationFeed.java index 8f155b6330..c1402499c4 100644 --- a/dspace-api/src/main/java/org/dspace/app/util/SyndicationFeed.java +++ b/dspace-api/src/main/java/org/dspace/app/util/SyndicationFeed.java @@ -51,6 +51,7 @@ import org.dspace.content.service.CollectionService; import org.dspace.content.service.CommunityService; import org.dspace.content.service.ItemService; import org.dspace.core.Context; +import org.dspace.core.I18nUtil; import org.dspace.discovery.IndexableObject; import org.dspace.discovery.indexobject.IndexableCollection; import org.dspace.discovery.indexobject.IndexableCommunity; @@ -91,6 +92,7 @@ public class SyndicationFeed { // default DC fields for entry protected String defaultTitleField = "dc.title"; + protected String defaultDescriptionField = "dc.description"; protected String defaultAuthorField = "dc.contributor.author"; protected String defaultDateField = "dc.date.issued"; private static final String[] defaultDescriptionFields = @@ -196,15 +198,15 @@ public class SyndicationFeed { // dso is null for the whole site, or a search without scope if (dso == null) { defaultTitle = configurationService.getProperty("dspace.name"); - feed.setDescription(localize(labels, MSG_FEED_DESCRIPTION)); + defaultDescriptionField = localize(labels, MSG_FEED_DESCRIPTION); objectURL = resolveURL(request, null); } else { Bitstream logo = null; if (dso instanceof IndexableCollection) { Collection col = ((IndexableCollection) dso).getIndexedObject(); defaultTitle = col.getName(); - feed.setDescription(collectionService.getMetadataFirstValue(col, - CollectionService.MD_SHORT_DESCRIPTION, Item.ANY)); + defaultDescriptionField = collectionService.getMetadataFirstValue(col, + CollectionService.MD_SHORT_DESCRIPTION, Item.ANY); logo = col.getLogo(); String cols = configurationService.getProperty("webui.feed.podcast.collections"); if (cols != null && cols.length() > 1 && cols.contains(col.getHandle())) { @@ -214,8 +216,8 @@ public class SyndicationFeed { } else if (dso instanceof IndexableCommunity) { Community comm = ((IndexableCommunity) dso).getIndexedObject(); defaultTitle = comm.getName(); - feed.setDescription(communityService.getMetadataFirstValue(comm, - CommunityService.MD_SHORT_DESCRIPTION, Item.ANY)); + defaultDescriptionField = communityService.getMetadataFirstValue(comm, + CommunityService.MD_SHORT_DESCRIPTION, Item.ANY); logo = comm.getLogo(); String comms = configurationService.getProperty("webui.feed.podcast.communities"); if (comms != null && comms.length() > 1 && comms.contains(comm.getHandle())) { @@ -230,6 +232,12 @@ public class SyndicationFeed { } feed.setTitle(labels.containsKey(MSG_FEED_TITLE) ? localize(labels, MSG_FEED_TITLE) : defaultTitle); + + if (defaultDescriptionField == null || defaultDescriptionField == "") { + defaultDescriptionField = I18nUtil.getMessage("org.dspace.app.util.SyndicationFeed.no-description"); + } + + feed.setDescription(defaultDescriptionField); feed.setLink(objectURL); feed.setPublishedDate(new Date()); feed.setUri(objectURL); diff --git a/dspace-api/src/main/java/org/dspace/authenticate/IPAuthentication.java b/dspace-api/src/main/java/org/dspace/authenticate/IPAuthentication.java index 9c37fcee47..3b23660344 100644 --- a/dspace-api/src/main/java/org/dspace/authenticate/IPAuthentication.java +++ b/dspace-api/src/main/java/org/dspace/authenticate/IPAuthentication.java @@ -52,11 +52,6 @@ public class IPAuthentication implements AuthenticationMethod { */ private static Logger log = org.apache.logging.log4j.LogManager.getLogger(IPAuthentication.class); - /** - * Whether to look for x-forwarded headers for logging IP addresses - */ - protected static Boolean useProxies; - /** * All the IP matchers */ @@ -250,7 +245,7 @@ public class IPAuthentication implements AuthenticationMethod { log.debug(LogHelper.getHeader(context, "authenticated", "special_groups=" + gsb.toString() - + " (by IP=" + addr + ", useProxies=" + useProxies.toString() + ")" + + " (by IP=" + addr + ")" )); } diff --git a/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java index 071bf3972f..cc89cea33a 100644 --- a/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java @@ -332,8 +332,8 @@ public class BitstreamServiceImpl extends DSpaceObjectServiceImpl imp } @Override - public List findDeletedBitstreams(Context context) throws SQLException { - return bitstreamDAO.findDeletedBitstreams(context); + public List findDeletedBitstreams(Context context, int limit, int offset) throws SQLException { + return bitstreamDAO.findDeletedBitstreams(context, limit, offset); } @Override diff --git a/dspace-api/src/main/java/org/dspace/content/DSpaceObject.java b/dspace-api/src/main/java/org/dspace/content/DSpaceObject.java index 1ac88241f4..59217a109f 100644 --- a/dspace-api/src/main/java/org/dspace/content/DSpaceObject.java +++ b/dspace-api/src/main/java/org/dspace/content/DSpaceObject.java @@ -48,6 +48,12 @@ public abstract class DSpaceObject implements Serializable, ReloadableEntity metadata = new ArrayList<>(); @@ -116,7 +122,7 @@ public abstract class DSpaceObject implements Serializable, ReloadableEntity handle) { diff --git a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java index 24778824bf..2119959073 100644 --- a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java @@ -126,6 +126,11 @@ public abstract class DSpaceObjectServiceImpl implements } } + // Sort the metadataValues if they have been modified, + // is used to preserve the default order. + if (dso.isMetadataModified()) { + values.sort(MetadataValueComparators.defaultComparator); + } // Create an array of matching values return values; } @@ -542,7 +547,7 @@ public abstract class DSpaceObjectServiceImpl implements int add = 4 - tokens.length; if (add > 0) { - tokens = (String[]) ArrayUtils.addAll(tokens, new String[add]); + tokens = ArrayUtils.addAll(tokens, new String[add]); } return tokens; @@ -603,21 +608,18 @@ public abstract class DSpaceObjectServiceImpl implements //If two places are the same then the MetadataValue instance will be placed before the //RelationshipMetadataValue instance. //This is done to ensure that the order is correct. - metadataValues.sort(new Comparator() { - @Override - public int compare(MetadataValue o1, MetadataValue o2) { - int compare = o1.getPlace() - o2.getPlace(); - if (compare == 0) { - if (o1 instanceof RelationshipMetadataValue && o2 instanceof RelationshipMetadataValue) { - return compare; - } else if (o1 instanceof RelationshipMetadataValue) { - return 1; - } else if (o2 instanceof RelationshipMetadataValue) { - return -1; - } + metadataValues.sort((o1, o2) -> { + int compare = o1.getPlace() - o2.getPlace(); + if (compare == 0) { + if (o1 instanceof RelationshipMetadataValue && o2 instanceof RelationshipMetadataValue) { + return compare; + } else if (o1 instanceof RelationshipMetadataValue) { + return 1; + } else if (o2 instanceof RelationshipMetadataValue) { + return -1; } - return compare; } + return compare; }); for (MetadataValue metadataValue : metadataValues) { //Retrieve & store the place for each metadata value @@ -634,7 +636,7 @@ public abstract class DSpaceObjectServiceImpl implements String authority = metadataValue.getAuthority(); String relationshipId = StringUtils.split(authority, "::")[1]; Relationship relationship = relationshipService.find(context, Integer.parseInt(relationshipId)); - if (relationship.getLeftItem().equals((Item) dso)) { + if (relationship.getLeftItem().equals(dso)) { relationship.setLeftPlace(mvPlace); } else { relationship.setRightPlace(mvPlace); diff --git a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java index de5826fa01..a290cb0d99 100644 --- a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java @@ -12,7 +12,6 @@ import java.io.InputStream; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.Date; import java.util.Iterator; import java.util.LinkedList; @@ -288,9 +287,10 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl implements It return itemDAO.findAll(context, true, true); } + @Override public Iterator findAllRegularItems(Context context) throws SQLException { return itemDAO.findAllRegularItems(context); - }; + } @Override public Iterator findBySubmitter(Context context, EPerson eperson) throws SQLException { @@ -1054,7 +1054,7 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl implements It List linkedCollections = item.getCollections(); List notLinkedCollections = new ArrayList<>(allCollections.size() - linkedCollections.size()); - if ((allCollections.size() - linkedCollections.size()) == 0) { + if (allCollections.size() - linkedCollections.size() == 0) { return notLinkedCollections; } for (Collection collection : allCollections) { @@ -1149,6 +1149,7 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl implements It * @return true if the item is an inprogress submission, i.e. a WorkspaceItem or WorkflowItem * @throws SQLException An exception that provides information on a database access error or other errors. */ + @Override public boolean isInProgressSubmission(Context context, Item item) throws SQLException { return workspaceItemService.findByItem(context, item) != null || workflowItemService.findByItem(context, item) != null; @@ -1179,8 +1180,8 @@ prevent the generation of resource policy entry values with null dspace_object a if (!authorizeService .isAnIdenticalPolicyAlreadyInPlace(context, dso, defaultPolicy.getGroup(), Constants.READ, defaultPolicy.getID()) && - ((!appendMode && this.isNotAlreadyACustomRPOfThisTypeOnDSO(context, dso)) || - (appendMode && this.shouldBeAppended(context, dso, defaultPolicy)))) { + (!appendMode && this.isNotAlreadyACustomRPOfThisTypeOnDSO(context, dso) || + appendMode && this.shouldBeAppended(context, dso, defaultPolicy))) { ResourcePolicy newPolicy = resourcePolicyService.clone(context, defaultPolicy); newPolicy.setdSpaceObject(dso); newPolicy.setAction(Constants.READ); @@ -1222,7 +1223,7 @@ prevent the generation of resource policy entry values with null dspace_object a * Check if the provided default policy should be appended or not to the final * item. If an item has at least one custom READ policy any anonymous READ * policy with empty start/end date should be skipped - * + * * @param context DSpace context * @param dso DSpace object to check for custom read RP * @param defaultPolicy The policy to check @@ -1611,7 +1612,7 @@ prevent the generation of resource policy entry values with null dspace_object a fullMetadataValueList.addAll(relationshipMetadataService.getRelationshipMetadata(item, true)); fullMetadataValueList.addAll(dbMetadataValues); - item.setCachedMetadata(sortMetadataValueList(fullMetadataValueList)); + item.setCachedMetadata(MetadataValueComparators.sort(fullMetadataValueList)); } log.debug("Called getMetadata for " + item.getID() + " based on cache"); @@ -1653,28 +1654,6 @@ prevent the generation of resource policy entry values with null dspace_object a } } - /** - * This method will sort the List of MetadataValue objects based on the MetadataSchema, MetadataField Element, - * MetadataField Qualifier and MetadataField Place in that order. - * @param listToReturn The list to be sorted - * @return The list sorted on those criteria - */ - private List sortMetadataValueList(List listToReturn) { - Comparator comparator = Comparator.comparing( - metadataValue -> metadataValue.getMetadataField().getMetadataSchema().getName(), - Comparator.nullsFirst(Comparator.naturalOrder())); - comparator = comparator.thenComparing(metadataValue -> metadataValue.getMetadataField().getElement(), - Comparator.nullsFirst(Comparator.naturalOrder())); - comparator = comparator.thenComparing(metadataValue -> metadataValue.getMetadataField().getQualifier(), - Comparator.nullsFirst(Comparator.naturalOrder())); - comparator = comparator.thenComparing(metadataValue -> metadataValue.getPlace(), - Comparator.nullsFirst(Comparator.naturalOrder())); - - Stream metadataValueStream = listToReturn.stream().sorted(comparator); - listToReturn = metadataValueStream.collect(Collectors.toList()); - return listToReturn; - } - @Override public MetadataValue addMetadata(Context context, Item dso, String schema, String element, String qualifier, String lang, String value, String authority, int confidence, int place) throws SQLException { diff --git a/dspace-api/src/main/java/org/dspace/content/MetadataValue.java b/dspace-api/src/main/java/org/dspace/content/MetadataValue.java index 9ff3cb9ec2..c3deaacd80 100644 --- a/dspace-api/src/main/java/org/dspace/content/MetadataValue.java +++ b/dspace-api/src/main/java/org/dspace/content/MetadataValue.java @@ -19,6 +19,7 @@ import javax.persistence.Lob; import javax.persistence.ManyToOne; import javax.persistence.SequenceGenerator; import javax.persistence.Table; +import javax.persistence.Transient; import org.dspace.core.Context; import org.dspace.core.ReloadableEntity; @@ -171,6 +172,14 @@ public class MetadataValue implements ReloadableEntity { this.metadataField = metadataField; } + /** + * @return {@code MetadataField#getID()} + */ + @Transient + protected Integer getMetadataFieldId() { + return getMetadataField().getID(); + } + /** * Get the metadata value. * diff --git a/dspace-api/src/main/java/org/dspace/content/MetadataValueComparators.java b/dspace-api/src/main/java/org/dspace/content/MetadataValueComparators.java new file mode 100644 index 0000000000..306258f36a --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/MetadataValueComparators.java @@ -0,0 +1,51 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * This class contains only static members that can be used + * to sort list of {@link MetadataValue} + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + * + */ +public final class MetadataValueComparators { + + private MetadataValueComparators() {} + + /** + * This is the default comparator that mimics the ordering + * applied by the standard {@code @OrderBy} annotation inside + * {@link DSpaceObject#getMetadata()} + */ + public static final Comparator defaultComparator = + Comparator.comparing(MetadataValue::getMetadataFieldId) + .thenComparing( + MetadataValue::getPlace, + Comparator.nullsFirst(Comparator.naturalOrder()) + ); + + /** + * This method creates a new {@code List} ordered by the + * {@code MetadataComparators#defaultComparator}. + * + * @param metadataValues + * @return {@code List} ordered copy list using stream. + */ + public static final List sort(List metadataValues) { + return metadataValues + .stream() + .sorted(MetadataValueComparators.defaultComparator) + .collect(Collectors.toList()); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/content/dao/BitstreamDAO.java b/dspace-api/src/main/java/org/dspace/content/dao/BitstreamDAO.java index c1ef923131..0d7afaa3cd 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/BitstreamDAO.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/BitstreamDAO.java @@ -29,7 +29,7 @@ public interface BitstreamDAO extends DSpaceObjectLegacySupportDAO { public Iterator findAll(Context context, int limit, int offset) throws SQLException; - public List findDeletedBitstreams(Context context) throws SQLException; + public List findDeletedBitstreams(Context context, int limit, int offset) throws SQLException; public List findDuplicateInternalIdentifier(Context context, Bitstream bitstream) throws SQLException; diff --git a/dspace-api/src/main/java/org/dspace/content/dao/impl/BitstreamDAOImpl.java b/dspace-api/src/main/java/org/dspace/content/dao/impl/BitstreamDAOImpl.java index 02e3509c31..d6d77fe7f0 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/impl/BitstreamDAOImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/impl/BitstreamDAOImpl.java @@ -41,13 +41,14 @@ public class BitstreamDAOImpl extends AbstractHibernateDSODAO impleme } @Override - public List findDeletedBitstreams(Context context) throws SQLException { + public List findDeletedBitstreams(Context context, int limit, int offset) throws SQLException { CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context); CriteriaQuery criteriaQuery = getCriteriaQuery(criteriaBuilder, Bitstream.class); Root bitstreamRoot = criteriaQuery.from(Bitstream.class); criteriaQuery.select(bitstreamRoot); + criteriaQuery.orderBy(criteriaBuilder.desc(bitstreamRoot.get(Bitstream_.ID))); criteriaQuery.where(criteriaBuilder.equal(bitstreamRoot.get(Bitstream_.deleted), true)); - return list(context, criteriaQuery, false, Bitstream.class, -1, -1); + return list(context, criteriaQuery, false, Bitstream.class, limit, offset); } diff --git a/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java b/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java index 4621c95e7c..8effabf284 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java @@ -183,7 +183,7 @@ public interface BitstreamService extends DSpaceObjectService, DSpace * @return a list of all bitstreams that have been "deleted" * @throws SQLException if database error */ - public List findDeletedBitstreams(Context context) throws SQLException; + public List findDeletedBitstreams(Context context, int limit, int offset) throws SQLException; /** diff --git a/dspace-api/src/main/java/org/dspace/event/Consumer.java b/dspace-api/src/main/java/org/dspace/event/Consumer.java index 1a8b16e98a..f56efcc7ba 100644 --- a/dspace-api/src/main/java/org/dspace/event/Consumer.java +++ b/dspace-api/src/main/java/org/dspace/event/Consumer.java @@ -10,18 +10,16 @@ package org.dspace.event; import org.dspace.core.Context; /** - * Interface for content event consumers. Note that the consumer cannot tell if - * it is invoked synchronously or asynchronously; the consumer interface and - * sequence of calls is the same for both. Asynchronous consumers may see more - * consume() calls between the start and end of the event stream, if they are - * invoked asynchronously, once in a long time period, rather than synchronously - * after every Context.commit(). - * - * @version $Revision$ + * Interface for content event consumers. Note that the consumer cannot tell + * if it is invoked synchronously or asynchronously; the consumer interface + * and sequence of calls is the same for both. Asynchronous consumers may see + * more consume() calls between the start and end of the event stream, if they + * are invoked asynchronously, once in a long time period, rather than + * synchronously after every Context.commit(). */ public interface Consumer { /** - * Initialize - allocate any resources required to operate. This may include + * Allocate any resources required to operate. This may include * initializing any pooled JMS resources. Called ONCE when created by the * dispatcher pool. This should be used to set up expensive resources that * will remain for the lifetime of the consumer. @@ -31,12 +29,17 @@ public interface Consumer { public void initialize() throws Exception; /** - * Consume an event; events may get filtered at the dispatcher level, hiding - * it from the consumer. This behavior is based on the dispatcher/consumer - * configuration. Should include logic to initialize any resources required - * for a batch of events. + * Consume an event. Events may be filtered by a dispatcher, hiding them + * from the consumer. This behavior is based on the dispatcher/consumer + * configuration. Should include logic to initialize any resources + * required for a batch of events. * - * @param ctx the execution context object + *

This method must not commit the context. Committing causes + * re-dispatch of the event queue, which can result in infinite recursion + * leading to memory exhaustion as seen in + * {@link https://github.com/DSpace/DSpace/pull/8756}. + * + * @param ctx the current DSpace session * @param event the content event * @throws Exception if error */ diff --git a/dspace-api/src/main/java/org/dspace/event/package-info.java b/dspace-api/src/main/java/org/dspace/event/package-info.java new file mode 100644 index 0000000000..544dfb271a --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/event/package-info.java @@ -0,0 +1,20 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +/** + * Actions which alter DSpace model objects can queue {@link Event}s, which + * are presented to {@link Consumer}s by a {@link Dispatcher}. A pool of + * {@code Dispatcher}s is managed by an {@link service.EventService}, guided + * by configuration properties {@code event.dispatcher.*}. + * + *

One must be careful not to commit the current DSpace {@code Context} + * during event dispatch. {@code commit()} triggers event dispatching, and + * doing this during event dispatch can lead to infinite recursion and + * memory exhaustion. + */ + +package org.dspace.event; diff --git a/dspace-api/src/main/java/org/dspace/identifier/VersionedHandleIdentifierProviderWithCanonicalHandles.java b/dspace-api/src/main/java/org/dspace/identifier/VersionedHandleIdentifierProviderWithCanonicalHandles.java index 7705fd2b57..0fac326ca1 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/VersionedHandleIdentifierProviderWithCanonicalHandles.java +++ b/dspace-api/src/main/java/org/dspace/identifier/VersionedHandleIdentifierProviderWithCanonicalHandles.java @@ -306,6 +306,7 @@ public class VersionedHandleIdentifierProviderWithCanonicalHandles extends Ident public DSpaceObject resolve(Context context, String identifier, String... attributes) { // We can do nothing with this, return null try { + identifier = handleService.parseHandle(identifier); return handleService.resolveToObject(context, identifier); } catch (IllegalStateException | SQLException e) { log.error(LogHelper.getHeader(context, "Error while resolving handle to item", "handle: " + identifier), @@ -426,6 +427,19 @@ public class VersionedHandleIdentifierProviderWithCanonicalHandles extends Ident } } + DSpaceObject itemWithCanonicalHandle = handleService.resolveToObject(context, canonical); + if (itemWithCanonicalHandle != null) { + if (itemWithCanonicalHandle.getID() != previous.getItem().getID()) { + log.warn("The previous version's item (" + previous.getItem().getID() + + ") does not match with the item containing handle " + canonical + + " (" + itemWithCanonicalHandle.getID() + ")"); + } + // Move the original handle from whatever item it's on to the newest version + handleService.modifyHandleDSpaceObject(context, canonical, dso); + } else { + handleService.createHandle(context, dso, canonical); + } + // add a new Identifier for this item: 12345/100.x String idNew = canonical + DOT + version.getVersionNumber(); //Make sure we don't have an old handle hanging around (if our previous version was deleted in the workspace) diff --git a/dspace-api/src/main/java/org/dspace/identifier/doi/DOIConsumer.java b/dspace-api/src/main/java/org/dspace/identifier/doi/DOIConsumer.java index 1961ce8274..33ef058e16 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/doi/DOIConsumer.java +++ b/dspace-api/src/main/java/org/dspace/identifier/doi/DOIConsumer.java @@ -141,7 +141,6 @@ public class DOIConsumer implements Consumer { + item.getID() + " and DOI " + doi + ".", ex); } } - ctx.commit(); } } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/liveimportclient/service/LiveImportClientImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/liveimportclient/service/LiveImportClientImpl.java index 81a6631127..1a8a7a7861 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/liveimportclient/service/LiveImportClientImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/liveimportclient/service/LiveImportClientImpl.java @@ -60,7 +60,8 @@ public class LiveImportClientImpl implements LiveImportClient { requestConfigBuilder.setConnectionRequestTimeout(timeout); RequestConfig defaultRequestConfig = requestConfigBuilder.build(); - method = new HttpGet(buildUrl(URL, params.get(URI_PARAMETERS))); + String uri = buildUrl(URL, params.get(URI_PARAMETERS)); + method = new HttpGet(uri); method.setConfig(defaultRequestConfig); Map headerParams = params.get(HEADER_PARAMETERS); @@ -71,7 +72,9 @@ public class LiveImportClientImpl implements LiveImportClient { } configureProxy(method, defaultRequestConfig); - + if (log.isDebugEnabled()) { + log.debug("Performing GET request to \"" + uri + "\"..."); + } HttpResponse httpResponse = httpClient.execute(method); if (isNotSuccessfull(httpResponse)) { throw new RuntimeException("The request failed with: " + getStatusCode(httpResponse) + " code, reason= " @@ -98,7 +101,8 @@ public class LiveImportClientImpl implements LiveImportClient { Builder requestConfigBuilder = RequestConfig.custom(); RequestConfig defaultRequestConfig = requestConfigBuilder.build(); - method = new HttpPost(buildUrl(URL, params.get(URI_PARAMETERS))); + String uri = buildUrl(URL, params.get(URI_PARAMETERS)); + method = new HttpPost(uri); method.setConfig(defaultRequestConfig); if (StringUtils.isNotBlank(entry)) { method.setEntity(new StringEntity(entry)); @@ -106,7 +110,9 @@ public class LiveImportClientImpl implements LiveImportClient { setHeaderParams(method, params); configureProxy(method, defaultRequestConfig); - + if (log.isDebugEnabled()) { + log.debug("Performing POST request to \"" + uri + "\"..." ); + } HttpResponse httpResponse = httpClient.execute(method); if (isNotSuccessfull(httpResponse)) { throw new RuntimeException(); @@ -185,4 +191,4 @@ public class LiveImportClientImpl implements LiveImportClient { this.httpClient = httpClient; } -} \ No newline at end of file +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/metadatamapping/contributor/PubmedDateMetadatumContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/metadatamapping/contributor/PubmedDateMetadatumContributor.java index ba23167553..6536026058 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/metadatamapping/contributor/PubmedDateMetadatumContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/metadatamapping/contributor/PubmedDateMetadatumContributor.java @@ -121,12 +121,14 @@ public class PubmedDateMetadatumContributor implements MetadataContributor int j = 0; // Use the first dcDate that has been formatted (Config should go from most specific to most lenient) - while (j < dateFormatsToAttempt.size() && dcDate == null) { + while (j < dateFormatsToAttempt.size()) { String dateFormat = dateFormatsToAttempt.get(j); try { SimpleDateFormat formatter = new SimpleDateFormat(dateFormat); Date date = formatter.parse(dateString); dcDate = new DCDate(date); + values.add(metadataFieldMapping.toDCValue(field, formatter.format(date))); + break; } catch (ParseException e) { // Multiple dateformats can be configured, we don't want to print the entire stacktrace every // time one of those formats fails. @@ -136,9 +138,7 @@ public class PubmedDateMetadatumContributor implements MetadataContributor } j++; } - if (dcDate != null) { - values.add(metadataFieldMapping.toDCValue(field, dcDate.toString())); - } else { + if (dcDate == null) { log.info( "Failed parsing " + dateString + ", check " + "the configured dataformats in config/spring/api/pubmed-integration.xml"); diff --git a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java index b30ea22ca4..933d6b1446 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java @@ -292,7 +292,14 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat int countAttempt = 0; while (StringUtils.isBlank(response) && countAttempt <= attempt) { countAttempt++; + + long time = System.currentTimeMillis() - lastRequest; + if ((time) < interRequestTime) { + Thread.sleep(interRequestTime - time); + } + response = liveImportClient.executeHttpGetRequest(1000, uriBuilder.toString(), params); + lastRequest = System.currentTimeMillis(); } if (StringUtils.isBlank(response)) { @@ -316,7 +323,13 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat countAttempt = 0; while (StringUtils.isBlank(response2) && countAttempt <= attempt) { countAttempt++; + long time = System.currentTimeMillis() - lastRequest; + if ((time) < interRequestTime) { + Thread.sleep(interRequestTime - time); + } response2 = liveImportClient.executeHttpGetRequest(1000, uriBuilder2.toString(), params2); + + lastRequest = System.currentTimeMillis(); } if (StringUtils.isBlank(response2)) { @@ -418,7 +431,13 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat int countAttempt = 0; while (StringUtils.isBlank(response) && countAttempt <= attempt) { countAttempt++; + long time = System.currentTimeMillis() - lastRequest; + if ((time) < interRequestTime) { + Thread.sleep(interRequestTime - time); + } + response = liveImportClient.executeHttpGetRequest(1000, uriBuilder.toString(), params); + lastRequest = System.currentTimeMillis(); } if (StringUtils.isBlank(response)) { @@ -441,7 +460,12 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat countAttempt = 0; while (StringUtils.isBlank(response2) && countAttempt <= attempt) { countAttempt++; + long time = System.currentTimeMillis() - lastRequest; + if ((time) < interRequestTime) { + Thread.sleep(interRequestTime - time); + } response2 = liveImportClient.executeHttpGetRequest(1000, uriBuilder2.toString(), params2); + lastRequest = System.currentTimeMillis(); } if (StringUtils.isBlank(response2)) { @@ -501,4 +525,4 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat this.urlSearch = urlSearch; } -} \ No newline at end of file +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractRemoteMetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractRemoteMetadataSource.java index 38632a1a2b..29801433e3 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractRemoteMetadataSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractRemoteMetadataSource.java @@ -183,6 +183,7 @@ public abstract class AbstractRemoteMetadataSource { log.warn("Error in trying operation " + operationId + " " + retry + " " + warning + ", retrying !", e); } finally { + this.lastRequest = System.currentTimeMillis(); lock.unlock(); } @@ -262,5 +263,7 @@ public abstract class AbstractRemoteMetadataSource { */ public abstract void init() throws Exception; - + public void setInterRequestTime(final long interRequestTime) { + this.interRequestTime = interRequestTime; + } } diff --git a/dspace-api/src/main/java/org/dspace/orcid/OrcidHistory.java b/dspace-api/src/main/java/org/dspace/orcid/OrcidHistory.java index 33edea112e..a567c6e7a7 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/OrcidHistory.java +++ b/dspace-api/src/main/java/org/dspace/orcid/OrcidHistory.java @@ -79,6 +79,8 @@ public class OrcidHistory implements ReloadableEntity { /** * A description of the synchronized resource. */ + @Lob + @Type(type = "org.dspace.storage.rdbms.hibernate.DatabaseAwareLobType") @Column(name = "description") private String description; diff --git a/dspace-api/src/main/java/org/dspace/orcid/OrcidQueue.java b/dspace-api/src/main/java/org/dspace/orcid/OrcidQueue.java index 4794e89008..9261f14eea 100644 --- a/dspace-api/src/main/java/org/dspace/orcid/OrcidQueue.java +++ b/dspace-api/src/main/java/org/dspace/orcid/OrcidQueue.java @@ -64,6 +64,8 @@ public class OrcidQueue implements ReloadableEntity { /** * A description of the resource to be synchronized. */ + @Lob + @Type(type = "org.dspace.storage.rdbms.hibernate.DatabaseAwareLobType") @Column(name = "description") private String description; diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java index b8a1a2e96a..977b5b7b32 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java @@ -17,6 +17,7 @@ import java.util.Map; import java.util.UUID; import javax.annotation.Nullable; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -224,25 +225,62 @@ public class BitstreamStorageServiceImpl implements BitstreamStorageService, Ini @Override public void cleanup(boolean deleteDbRecords, boolean verbose) throws SQLException, IOException, AuthorizeException { Context context = new Context(Context.Mode.BATCH_EDIT); - int commitCounter = 0; + + int offset = 0; + int limit = 100; + + int cleanedBitstreamCount = 0; + + int deletedBitstreamCount = bitstreamService.countDeletedBitstreams(context); + System.out.println("Found " + deletedBitstreamCount + " deleted bistream to cleanup"); try { context.turnOffAuthorisationSystem(); - List storage = bitstreamService.findDeletedBitstreams(context); - for (Bitstream bitstream : storage) { - UUID bid = bitstream.getID(); - Map wantedMetadata = new HashMap(); - wantedMetadata.put("size_bytes", null); - wantedMetadata.put("modified", null); - Map receivedMetadata = this.getStore(bitstream.getStoreNumber()).about(bitstream, wantedMetadata); + while (cleanedBitstreamCount < deletedBitstreamCount) { + + List storage = bitstreamService.findDeletedBitstreams(context, limit, offset); + + if (CollectionUtils.isEmpty(storage)) { + break; + } + + for (Bitstream bitstream : storage) { + UUID bid = bitstream.getID(); + Map wantedMetadata = new HashMap(); + wantedMetadata.put("size_bytes", null); + wantedMetadata.put("modified", null); + Map receivedMetadata = this.getStore(bitstream.getStoreNumber()).about(bitstream, wantedMetadata); - // Make sure entries which do not exist are removed - if (MapUtils.isEmpty(receivedMetadata)) { - log.debug("bitstore.about is empty, so file is not present"); + // Make sure entries which do not exist are removed + if (MapUtils.isEmpty(receivedMetadata)) { + log.debug("bitstore.about is empty, so file is not present"); + if (deleteDbRecords) { + log.debug("deleting record"); + if (verbose) { + System.out.println(" - Deleting bitstream information (ID: " + bid + ")"); + } + checksumHistoryService.deleteByBitstream(context, bitstream); + if (verbose) { + System.out.println(" - Deleting bitstream record from database (ID: " + bid + ")"); + } + bitstreamService.expunge(context, bitstream); + } + context.uncacheEntity(bitstream); + continue; + } + + // This is a small chance that this is a file which is + // being stored -- get it next time. + if (isRecent(Long.valueOf(receivedMetadata.get("modified").toString()))) { + log.debug("file is recent"); + context.uncacheEntity(bitstream); + continue; + } + if (deleteDbRecords) { - log.debug("deleting record"); + log.debug("deleting db record"); if (verbose) { System.out.println(" - Deleting bitstream information (ID: " + bid + ")"); } @@ -252,64 +290,42 @@ public class BitstreamStorageServiceImpl implements BitstreamStorageService, Ini } bitstreamService.expunge(context, bitstream); } + + if (isRegisteredBitstream(bitstream.getInternalId())) { + context.uncacheEntity(bitstream); + continue; // do not delete registered bitstreams + } + + + // Since versioning allows for multiple bitstreams, check if the internal + // identifier isn't used on + // another place + if (bitstreamService.findDuplicateInternalIdentifier(context, bitstream).isEmpty()) { + this.getStore(bitstream.getStoreNumber()).remove(bitstream); + + String message = ("Deleted bitstreamID " + bid + ", internalID " + bitstream.getInternalId()); + if (log.isDebugEnabled()) { + log.debug(message); + } + if (verbose) { + System.out.println(message); + } + } + context.uncacheEntity(bitstream); - continue; } - // This is a small chance that this is a file which is - // being stored -- get it next time. - if (isRecent(Long.valueOf(receivedMetadata.get("modified").toString()))) { - log.debug("file is recent"); - context.uncacheEntity(bitstream); - continue; + // Commit actual changes to DB after dispatch events + System.out.print("Performing incremental commit to the database..."); + context.commit(); + System.out.println(" Incremental commit done!"); + + cleanedBitstreamCount = cleanedBitstreamCount + storage.size(); + + if (!deleteDbRecords) { + offset = offset + limit; } - if (deleteDbRecords) { - log.debug("deleting db record"); - if (verbose) { - System.out.println(" - Deleting bitstream information (ID: " + bid + ")"); - } - checksumHistoryService.deleteByBitstream(context, bitstream); - if (verbose) { - System.out.println(" - Deleting bitstream record from database (ID: " + bid + ")"); - } - bitstreamService.expunge(context, bitstream); - } - - if (isRegisteredBitstream(bitstream.getInternalId())) { - context.uncacheEntity(bitstream); - continue; // do not delete registered bitstreams - } - - - // Since versioning allows for multiple bitstreams, check if the internal identifier isn't used on - // another place - if (bitstreamService.findDuplicateInternalIdentifier(context, bitstream).isEmpty()) { - this.getStore(bitstream.getStoreNumber()).remove(bitstream); - - String message = ("Deleted bitstreamID " + bid + ", internalID " + bitstream.getInternalId()); - if (log.isDebugEnabled()) { - log.debug(message); - } - if (verbose) { - System.out.println(message); - } - } - - // Make sure to commit our outstanding work every 100 - // iterations. Otherwise you risk losing the entire transaction - // if we hit an exception, which isn't useful at all for large - // amounts of bitstreams. - commitCounter++; - if (commitCounter % 100 == 0) { - context.dispatchEvents(); - // Commit actual changes to DB after dispatch events - System.out.print("Performing incremental commit to the database..."); - context.commit(); - System.out.println(" Incremental commit done!"); - } - - context.uncacheEntity(bitstream); } System.out.print("Committing changes to the database..."); diff --git a/dspace-api/src/main/resources/Messages.properties b/dspace-api/src/main/resources/Messages.properties index c478e4e69b..9be443f5ea 100644 --- a/dspace-api/src/main/resources/Messages.properties +++ b/dspace-api/src/main/resources/Messages.properties @@ -51,6 +51,7 @@ metadata.bitstream.iiif-virtual.bytes = File size metadata.bitstream.iiif-virtual.checksum = Checksum org.dspace.app.itemexport.no-result = The DSpaceObject that you specified has no items. +org.dspace.app.util.SyndicationFeed.no-description = No Description org.dspace.checker.ResultsLogger.bitstream-format = Bitstream format org.dspace.checker.ResultsLogger.bitstream-found = Bitstream found org.dspace.checker.ResultsLogger.bitstream-id = Bitstream ID diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql new file mode 100644 index 0000000000..7641eb9fc2 --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql @@ -0,0 +1,10 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +ALTER TABLE orcid_history ALTER COLUMN description SET DATA TYPE CLOB; +ALTER TABLE orcid_queue ALTER COLUMN description SET DATA TYPE CLOB; diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/oracle/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/oracle/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql new file mode 100644 index 0000000000..509e0a2869 --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/oracle/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql @@ -0,0 +1,10 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +ALTER TABLE orcid_history MODIFY (description CLOB); +ALTER TABLE orcid_queue MODIFY (description CLOB); diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql new file mode 100644 index 0000000000..ae0e414e44 --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2023.03.29__orcid_queue_and_history_descriptions_to_text_type.sql @@ -0,0 +1,10 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +ALTER TABLE orcid_history ALTER COLUMN description TYPE TEXT; +ALTER TABLE orcid_queue ALTER COLUMN description TYPE TEXT; diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index b8f02c46ff..6b0ef3e9b9 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -57,6 +57,7 @@ + xml @@ -199,4 +200,4 @@ - \ No newline at end of file + diff --git a/dspace-api/src/main/resources/spring/spring-dspace-core-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-core-services.xml index 87bfcbc86c..3ce641d99c 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-core-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-core-services.xml @@ -13,15 +13,6 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> - - - - @@ -31,12 +22,6 @@ - - - - diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/identifier-service.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/identifier-service.xml index 206b801d08..8f7cc297d7 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/identifier-service.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/identifier-service.xml @@ -19,7 +19,18 @@ + scope="singleton"> + + + + + + + + + + + - - 9.4.48.v20220622 + 9.4.51.v20230217 2.17.1 2.0.27 1.18.0