diff --git a/dspace/etc/database_schema.sql b/dspace/etc/database_schema.sql index 498e471fcd..cfe496dcbc 100644 --- a/dspace/etc/database_schema.sql +++ b/dspace/etc/database_schema.sql @@ -191,7 +191,7 @@ CREATE TABLE Item submitter_id INTEGER REFERENCES EPerson(eperson_id), in_archive BOOL, withdrawn BOOL, - withdrawal_date VARCHAR(21) + last_modified TIMESTAMP ); ------------------------------------------------------- diff --git a/dspace/jsp/admin/edit-item-form.jsp b/dspace/jsp/admin/edit-item-form.jsp index 53769f6983..71e9269009 100644 --- a/dspace/jsp/admin/edit-item-form.jsp +++ b/dspace/jsp/admin/edit-item-form.jsp @@ -72,8 +72,6 @@ String handle = (String) request.getAttribute("handle"); Collection[] collections = (Collection[]) request.getAttribute("collections"); DCType[] dcTypes = (DCType[]) request.getAttribute("dc.types"); - - DCDate withdrawalDate = item.getWithdrawalDate(); %> Item internal ID: <%= item.getID() %> - + <% - if (withdrawalDate == null) + if (!item.isWithdrawn()) { %>
@@ -126,6 +124,9 @@ Handle: <%= (handle == null ? "None" : handle) %> + + Last modified: + In Collections: @@ -153,11 +154,10 @@ <% - if (withdrawalDate != null) + if (item.isWithdrawn()) { %> -

This item was withdrawn from DSpace on -

+

This item was withdrawn from DSpace

<% } %> diff --git a/dspace/src/org/dspace/administer/DCType.java b/dspace/src/org/dspace/administer/DCType.java index c7c908bb96..7a0d2a5ec2 100644 --- a/dspace/src/org/dspace/administer/DCType.java +++ b/dspace/src/org/dspace/administer/DCType.java @@ -74,7 +74,16 @@ public class DCType /** The row in the table representing this type */ private TableRow typeRow; + /** + * For quick access in the system, an array of the elements in the + * registry + */ + private static String[] elements = null; + + /** The qualifiers in the system */ + private static String[] qualifiers = null; + /** * Class constructor for creating a BitstreamFormat object * based on the contents of a DB table row. @@ -88,7 +97,67 @@ public class DCType typeRow = row; } + + /** + * Utility method for quick access to an element and qualifier + * given the type ID. + * + * @param context context, in case DC types need to be read in from DB + * @param id the DC type ID + * @return a two-String array, string 0 is the element, + * string 1 is the qualifier + */ + public static String[] quickFind(Context context, int id) + throws SQLException + { + if (elements == null) + { + loadDC(context); + } + + // We use the ID to get the element and qualifier out of + // the relevant array. But should 'sanity check' first. + if (id > elements.length || id < 0) + { + return null; + } + + String[] result = new String[2]; + result[0] = elements[id]; + result[1] = qualifiers[id]; + return result; + } + + + /** + * Load in the DC type registry for quick access via quickFind. + * + * @param context DSpace context object + */ + public static void loadDC(Context context) throws SQLException + { + // First get the highest ID so we know how big to make the array + TableRow row = DatabaseManager.querySingle(context, + "SELECT MAX(dc_type_id) AS foo FROM dctyperegistry"); + int numberFields = row.getIntColumn("foo") + 1; + elements = new String[numberFields]; + qualifiers = new String[numberFields]; + + // Grab rows from DB + TableRowIterator tri = DatabaseManager.query(context, + "SELECT * FROM dctyperegistry"); + + while (tri.hasNext()) + { + row = tri.next(); + int id = row.getIntColumn("dc_type_id"); + elements[id] = row.getStringColumn("element"); + qualifiers[id] = row.getStringColumn("qualifier"); + } + } + + /** * Get a bitstream format from the database. * diff --git a/dspace/src/org/dspace/administer/Upgrade101To11.java b/dspace/src/org/dspace/administer/Upgrade101To11.java new file mode 100644 index 0000000000..1abd5afcdf --- /dev/null +++ b/dspace/src/org/dspace/administer/Upgrade101To11.java @@ -0,0 +1,130 @@ +/* + * Upgrade101To11.java + * + * Version: $Revision$ + * + * Date: $Date$ + * + * Copyright (c) 2002, Hewlett-Packard Company and Massachusetts + * Institute of Technology. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * - Neither the name of the Hewlett-Packard Company nor the name of the + * Massachusetts Institute of Technology nor the names of their + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +package org.dspace.administer; + +import org.dspace.content.DCDate; +import org.dspace.core.Context; +import org.dspace.storage.rdbms.DatabaseManager; +import org.dspace.storage.rdbms.TableRow; +import org.dspace.storage.rdbms.TableRowIterator; + + +/** + * A command-line tool for performing necessary tweaks in the database + * for the new last_modified column in the item table. + * + * @author Robert Tansley + * @version $Revision$ + */ +public class Upgrade101To11 +{ + /** + * For invoking via the command line + * + * @param argv command-line arguments + */ + public static void main(String argv[]) + { + Context context = null; + + try + { + context = new Context(); + + // Deal with withdrawn items first. + // last_modified takes the value of the deletion date + TableRowIterator tri = DatabaseManager.query( + context, + "item", + "SELECT * FROM item WHERE withdrawal_date IS NOT NULL"); + + while (tri.hasNext()) + { + TableRow row = tri.next(); + DCDate d = new DCDate(row.getStringColumn("withdrawal_date")); + row.setColumn("last_modified", d.toDate()); + DatabaseManager.update(context, row); + } + + + // Next, update those items with a date.available + tri = DatabaseManager.query( + context, + "SELECT item.item_id, dcvalue.text_value FROM item, dctyperegistry, dcvalue WHERE item.item_id=dcvalue.item_id AND dcvalue.dc_type_id=dctyperegistry.dc_type_id AND dctyperegistry.element LIKE 'date' AND dctyperegistry.qualifier LIKE 'available'"); + + while (tri.hasNext()) + { + TableRow resultRow = tri.next(); + DCDate d = new DCDate(resultRow.getStringColumn("text_value")); + + // Can't update the row, have to do a separate query + TableRow itemRow = DatabaseManager.find(context, + "item", + resultRow.getIntColumn("item_id")); + itemRow.setColumn("last_modified", d.toDate()); + DatabaseManager.update(context, itemRow); + } + + // Finally, for all items that have no date.available or withdrawal + // date, set the update time to now! + DatabaseManager.updateQuery(context, + "UPDATE item SET last_modified=now() WHERE last_modified IS NULL"); + + context.complete(); + + System.out.println("Last modified dates set"); + + System.exit(0); + } + catch (Exception e) + { + System.err.println("Exception occurred:" + e); + e.printStackTrace(); + + if (context != null) + { + context.abort(); + } + + System.exit(1); + } + } +} diff --git a/dspace/src/org/dspace/app/oai/DSpaceOAICatalog.java b/dspace/src/org/dspace/app/oai/DSpaceOAICatalog.java index 2dd58c8bc3..3aa49e426a 100644 --- a/dspace/src/org/dspace/app/oai/DSpaceOAICatalog.java +++ b/dspace/src/org/dspace/app/oai/DSpaceOAICatalog.java @@ -47,6 +47,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.StringTokenizer; import java.util.Vector; import org.apache.log4j.Logger; @@ -91,7 +92,9 @@ public class DSpaceOAICatalog extends AbstractCatalog "http://www.openarchives.org/OAI/2.0/oai_dc/ " + "http://www.openarchives.org/OAI/2.0/oai_dc.xsd"; - + /** Maximum number of records returned by one request */ + private final int MAX_RECORDS = 100; + public DSpaceOAICatalog(Properties properties) { // Don't need to do anything @@ -113,6 +116,10 @@ public class DSpaceOAICatalog extends AbstractCatalog public Vector getSchemaLocations(String identifier) throws OAIInternalServerError, IdDoesNotExistException, NoMetadataFormatsException { + log.info(LogManager.getHeader(null, + "oai_request", + "verb=getSchemaLocations,identifier=" + (identifier == null ? "null" : identifier))); + HarvestedItemInfo itemInfo = null; Context context = null; @@ -191,6 +198,13 @@ public class DSpaceOAICatalog extends AbstractCatalog throws OAIInternalServerError, NoSetHierarchyException, NoItemsMatchException, CannotDisseminateFormatException { + log.info(LogManager.getHeader(null, + "oai_request", + "verb=listIdentifiers,from=" + (from == null ? "null" : from) + + ",until=" + (until == null ? "null" : until) + + ",set=" + (set == null ? "null" : set) + + ",metadataPrefix=" + (metadataPrefix == null ? "null" : metadataPrefix))); + // We can produce oai_dc and simple DC for all items, so just return IDs Context context = null; @@ -205,11 +219,15 @@ public class DSpaceOAICatalog extends AbstractCatalog // Get the relevant OAIItemInfo objects to make headers DSpaceObject scope = resolveSet(context, set); List itemInfos = Harvest.harvest(context, scope, from, until, + 0, 0, // Everything for now false, true, true); // No Item objects, but we need to know containers and withdrawn items if (itemInfos.size() == 0) { + log.info(LogManager.getHeader(null, + "oai_error", + "no_items_match")); throw new NoItemsMatchException(); } @@ -269,7 +287,7 @@ public class DSpaceOAICatalog extends AbstractCatalog throw new BadResumptionTokenException(); } - + /** * Retrieve the specified metadata for the specified identifier * @@ -285,6 +303,11 @@ public class DSpaceOAICatalog extends AbstractCatalog throws OAIInternalServerError, CannotDisseminateFormatException, IdDoesNotExistException { + log.info(LogManager.getHeader(null, + "oai_request", + "verb=getRecord,identifier=" + (identifier == null ? "null" : identifier) + + ",metadataPrefix=" + (metadataPrefix == null ? "null" : metadataPrefix))); + Context context = null; HarvestedItemInfo itemInfo = null; @@ -322,10 +345,16 @@ public class DSpaceOAICatalog extends AbstractCatalog if (itemInfo == null) { + log.info(LogManager.getHeader(null, + "oai_error", + "id_does_not_exist")); throw new IdDoesNotExistException(identifier); } else if ((schemaURL = getCrosswalks().getSchemaURL(metadataPrefix)) == null) { + log.info(LogManager.getHeader(null, + "oai_error", + "cannot_disseminate_format")); throw new CannotDisseminateFormatException(metadataPrefix); } @@ -357,13 +386,104 @@ public class DSpaceOAICatalog extends AbstractCatalog public Map listRecords(String from, String until, String set, String metadataPrefix) throws OAIInternalServerError, NoSetHierarchyException, CannotDisseminateFormatException, NoItemsMatchException + { + log.info(LogManager.getHeader(null, + "oai_request", + "verb=listRecords,from=" + (from == null ? "null" : from) + + ",until=" + (until == null ? "null" : until) + + ",set=" + (set == null ? "null" : set) + + ",metadataPrefix=" + (metadataPrefix == null ? "null" : metadataPrefix))); + + Map m = doRecordHarvest(from, until, set, metadataPrefix, 0); + + // Null means bad metadata prefix was bad + if (m == null) + { + log.info(LogManager.getHeader(null, + "oai_error", + "cannot_disseminate_format")); + throw new CannotDisseminateFormatException(metadataPrefix); + } + + // If there were zero results, return the appropriate error + Iterator i = (Iterator) m.get("records"); + + if (i == null || !i.hasNext()) + { + log.info(LogManager.getHeader(null, + "oai_error", + "no_items_match")); + throw new NoItemsMatchException(); + } + + return m; + } + + + /** + * Retrieve the next set of records associated with the resumptionToken + * + * @param resumptionToken implementation-dependent format taken from the + * previous listRecords() Map result. + * @return a Map object containing entries for "headers" and "identifiers" Iterators + * (both containing Strings) as well as an optional "resumptionMap" Map. + * @exception OAIInternalServerError signals an http status code 500 problem + * @exception BadResumptionTokenException the value of the resumptionToken argument + * is invalid or expired. + */ + public Map listRecords(String resumptionToken) + throws BadResumptionTokenException, OAIInternalServerError + { + log.info(LogManager.getHeader(null, + "oai_request", + "verb=listRecords,resumptionToken=" + resumptionToken)); + /* + * FIXME: This may return zero records if the previous harvest + * returned a number of records that's an exact multiple of + * MAX_RECORDS. I hope that's OK. + */ + Object[] params = decodeResumptionToken(resumptionToken); + Integer offset = (Integer) params[4]; + + Map m = doRecordHarvest((String) params[0], (String) params[1], + (String) params[2], (String) params[3], offset.intValue()); + + // null result means bad prefix, which means bad resumption token + if (m == null) + { + log.info(LogManager.getHeader(null, + "oai_error", + "bad_resumption_token")); + throw new BadResumptionTokenException(); + } + + return m; + } + + + /** + * Method to do the actual harvest of records + * + * @param from OAI 'from' parameter + * @param until OAI 'until' parameter + * @param set OAI 'set' parameter + * @param metadataPrefix OAI 'metadataPrefix' parameter + * @param offset where to start this harvest + * + * @return the Map for listRecords to return, or null if the metadataPrefix + * is invalid + */ + private Map doRecordHarvest(String from, String until, String set, + String metadataPrefix, int offset) + throws OAIInternalServerError { Context context = null; String schemaURL = getCrosswalks().getSchemaURL(metadataPrefix); + Map results = new HashMap(); if (schemaURL == null) { - throw new CannotDisseminateFormatException(metadataPrefix); + return null; } // List to put results in @@ -376,12 +496,8 @@ public class DSpaceOAICatalog extends AbstractCatalog // Get the relevant HarvestedItemInfo objects to make headers DSpaceObject scope = resolveSet(context, set); List itemInfos = Harvest.harvest(context, scope, from, until, - true, true, true); // Need items, containers + withdrawals - - if (itemInfos.size() == 0) - { - throw new NoItemsMatchException(); - } + offset, MAX_RECORDS, // Limit amount returned from one request + true, true, true); // Need items, containers + withdrawals // Build list of XML records from item info objects Iterator i = itemInfos.iterator(); @@ -390,16 +506,52 @@ public class DSpaceOAICatalog extends AbstractCatalog { HarvestedItemInfo itemInfo = (HarvestedItemInfo) i.next(); - /* - * FIXME: I've a feeling a "CannotDisseminateFormatException" - * should be caught and discarded here - it's OK if some - * records in the requested date range don't have the - * requested metadata format available - */ - String recordXML = getRecordFactory().create( - itemInfo, schemaURL, metadataPrefix); + try + { + String recordXML = getRecordFactory().create( + itemInfo, schemaURL, metadataPrefix); + records.add(recordXML); + } + catch (CannotDisseminateFormatException cdfe) + { + /* + * FIXME: I've a feeling a "CannotDisseminateFormatException" + * should be discarded here - it's OK if some + * records in the requested date range don't have the + * requested metadata format available. I'll just log it for + * now. + */ + if (log.isDebugEnabled()) + log.debug(LogManager.getHeader(context, + "oai_warning", "Couldn't disseminate " + + metadataPrefix + " for " + itemInfo.handle)); + } + } + + // Put results in form needed to return + results.put("records", records.iterator()); - records.add(recordXML); + log.info(LogManager.getHeader(context, + "oai_harvest", + "results=" + records.size())); + + // If we have MAX_RECORDS records, we need to provide a resumption + // token + if (records.size() >= MAX_RECORDS) + { + String resumptionToken = makeResumptionToken( + from, until, set, metadataPrefix, offset + MAX_RECORDS); + + if (log.isDebugEnabled()) + { + log.debug(LogManager.getHeader(context, + "made_resumption_token", + "token=" + resumptionToken)); + } + + results.put("resumptionMap", + getResumptionMap(resumptionToken)); + //results.put("resumptionToken", resumptionToken); } } catch (SQLException se) @@ -418,33 +570,10 @@ public class DSpaceOAICatalog extends AbstractCatalog context.abort(); } } - - // Put results in form needed to return - Map results = new HashMap(); - results.put("records", records.iterator()); + return results; } - - - /** - * Retrieve the next set of records associated with the resumptionToken - * - * @param resumptionToken implementation-dependent format taken from the - * previous listRecords() Map result. - * @return a Map object containing entries for "headers" and "identifiers" Iterators - * (both containing Strings) as well as an optional "resumptionMap" Map. - * @exception OAIInternalServerError signals an http status code 500 problem - * @exception BadResumptionTokenException the value of the resumptionToken argument - * is invalid or expired. - */ - public Map listRecords(String resumptionToken) - throws BadResumptionTokenException, OAIInternalServerError - { - // Resumption tokens not yet supported - throw new BadResumptionTokenException(); - } - /** * Retrieve a list of sets that satisfy the specified criteria * @@ -456,6 +585,9 @@ public class DSpaceOAICatalog extends AbstractCatalog public Map listSets() throws NoSetHierarchyException, OAIInternalServerError { + log.info(LogManager.getHeader(null, + "oai_request", + "verb=listSets")); Context context = null; // List to put results in @@ -582,4 +714,102 @@ public class DSpaceOAICatalog extends AbstractCatalog return Community.find(context, Integer.parseInt(set)); } } + + + /** + * Create a resumption token. The relevant parameters for the harvest + * are put in a + * + * @param from OAI 'from' parameter + * @param until OAI 'until' parameter + * @param set OAI 'set' parameter + * @param prefix OAI 'metadataPrefix' parameter + * @param offset where to start the next harvest + * + * @return the appropriate resumption token + */ + private String makeResumptionToken(String from, String until, String set, + String prefix, int offset) + { + StringBuffer token = new StringBuffer(); + if (from != null) + { + token.append(from); + } + token.append("/"); + if (until != null) + { + token.append(until); + } + token.append("/"); + if (set != null) + { + token.append(set); + } + token.append("/"); + if (prefix != null) + { + token.append(prefix); + } + token.append("/"); + token.append(String.valueOf(offset)); + + return (token.toString()); + } + + + /** + * Get the information out of a resumption token + * + * @param token the resumption token + * @return a 5-long array of Objects; 4 Strings (from, until, set, prefix) + * and an Integer (the offset) + */ + private Object[] decodeResumptionToken(String token) + throws BadResumptionTokenException + { + Object[] obj = new Object[5]; + StringTokenizer st = new StringTokenizer(token, "/", true); + + // Extract from, until, set, prefix + for (int i = 0; i < 4; i++) + { + if (!st.hasMoreTokens()) + { + throw new BadResumptionTokenException(); + } + + String s = st.nextToken(); + // If this value is a delimiter /, we have no value for this part + // of the resumption token. + if (s.equals("/")) + { + obj[i] = null; + } + else + { + obj[i] = s; + // Skip the delimiter + st.nextToken(); + } + log.debug("is: " + (String) obj[i]); + } + + if (!st.hasMoreTokens()) + { + throw new BadResumptionTokenException(); + } + + // Extract offset + try + { + obj[4] = new Integer(st.nextToken()); + } + catch (NumberFormatException nfe) + { + throw new BadResumptionTokenException(); + } + + return obj; + } } diff --git a/dspace/src/org/dspace/app/oai/DSpaceRecordFactory.java b/dspace/src/org/dspace/app/oai/DSpaceRecordFactory.java index 8f30cedbbe..e78a85d9d0 100644 --- a/dspace/src/org/dspace/app/oai/DSpaceRecordFactory.java +++ b/dspace/src/org/dspace/app/oai/DSpaceRecordFactory.java @@ -41,6 +41,7 @@ package org.dspace.app.oai; import java.sql.SQLException; +import java.util.Date; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -53,6 +54,7 @@ import ORG.oclc.oai.server.verb.CannotDisseminateFormatException; import org.dspace.content.Collection; import org.dspace.content.Community; +import org.dspace.content.DCDate; import org.dspace.content.DCValue; import org.dspace.content.Item; import org.dspace.core.LogManager; @@ -101,7 +103,9 @@ public class DSpaceRecordFactory extends RecordFactory protected String getDatestamp(Object nativeItem) { - return ((HarvestedItemInfo) nativeItem).datestamp; + Date d = ((HarvestedItemInfo) nativeItem).datestamp; + // Return as ISO8601 + return new DCDate(d).toString(); } diff --git a/dspace/src/org/dspace/app/webui/servlet/HandleServlet.java b/dspace/src/org/dspace/app/webui/servlet/HandleServlet.java index 3a72cfc767..6840267ea6 100644 --- a/dspace/src/org/dspace/app/webui/servlet/HandleServlet.java +++ b/dspace/src/org/dspace/app/webui/servlet/HandleServlet.java @@ -248,8 +248,7 @@ public class HandleServlet extends DSpaceServlet throws ServletException, IOException, SQLException, AuthorizeException { // Tombstone? - DCDate withdrawalDate = item.getWithdrawalDate(); - if (withdrawalDate != null) + if (item.isWithdrawn()) { JSPManager.showJSP(request, response, "tombstone.jsp"); return; diff --git a/dspace/src/org/dspace/content/DCDate.java b/dspace/src/org/dspace/content/DCDate.java index 9f10b92a13..3ce5ae030a 100644 --- a/dspace/src/org/dspace/content/DCDate.java +++ b/dspace/src/org/dspace/content/DCDate.java @@ -259,6 +259,21 @@ public class DCDate } + /** + * Get the date as a Java Date object + * + * @return a Date object + */ + public Date toDate() + { + GregorianCalendar utcGC = new GregorianCalendar( + TimeZone.getTimeZone("UTC")); + + utcGC.set(year, month - 1, day, hours, minutes, seconds); + return utcGC.getTime(); + } + + /** * Set the date. The date passed in is assumed to be in the current * time zone, and is adjusting to fit the current time zone. diff --git a/dspace/src/org/dspace/content/Item.java b/dspace/src/org/dspace/content/Item.java index 05be2e4526..e1ad207483 100644 --- a/dspace/src/org/dspace/content/Item.java +++ b/dspace/src/org/dspace/content/Item.java @@ -45,6 +45,7 @@ import java.io.InputStream; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.ListIterator; @@ -178,14 +179,13 @@ public class Item extends DSpaceObject TableRow resultRow = (TableRow) tri.next(); // Get the Dublin Core type - TableRow typeRow = DatabaseManager.find(ourContext, - "dctyperegistry", + String[] dcType = DCType.quickFind(context, resultRow.getIntColumn("dc_type_id")); // Make a DCValue object DCValue dcv = new DCValue(); - dcv.element = typeRow.getStringColumn("element"); - dcv.qualifier = typeRow.getStringColumn("qualifier"); + dcv.element = dcType[0]; + dcv.qualifier = dcType[1]; dcv.value = resultRow.getStringColumn("text_value"); dcv.language = resultRow.getStringColumn("text_lang"); @@ -319,7 +319,7 @@ public class Item extends DSpaceObject return new ItemIterator(context, rows); } - + /** * Get the internal ID of this item. In general, this shouldn't be * exposed to users @@ -349,6 +349,28 @@ public class Item extends DSpaceObject } + /** + * Find out if the item has been withdrawn + * + * @return true if the item has been withdrawn + */ + public boolean isWithdrawn() + { + return itemRow.getBooleanColumn("withdrawn"); + } + + + /** + * Get the date the item was last modified. + * + * @return the date the item was last modified. + */ + public Date getLastModified() + { + return itemRow.getDateColumn("last_modified"); + } + + /** * Set the "is_archived" flag. This is public and only * WorkflowItem.archive() should set this. @@ -991,6 +1013,9 @@ public class Item extends DSpaceObject "update_item", "item_id=" + getID())); + // Set the last modified date + itemRow.setColumn("last_modified", new Date()); + // Make sure that withdrawn and in_archive are non-null if (itemRow.isColumnNull("in_archive")) { @@ -1215,25 +1240,6 @@ public class Item extends DSpaceObject } - /** - * Item is withdrawn? If so, return the date it happened. - * null is returned if the item is not withdrawn. - * - * @return the date the item was withdrawn or - */ - public DCDate getWithdrawalDate() - { - if (itemRow.getBooleanColumn("withdrawn")) - { - return new DCDate(itemRow.getStringColumn("withdrawal_date")); - } - else - { - return null; - } - } - - /** * Delete (expunge) the item. Bundles and bitstreams are also deleted if * they are not also included in another item. The Dublin Core metadata is @@ -1329,7 +1335,7 @@ public class Item extends DSpaceObject return Constants.ITEM; } - + /** * remove all of the policies for item and replace them * with a new list of policies diff --git a/dspace/src/org/dspace/eperson/Subscribe.java b/dspace/src/org/dspace/eperson/Subscribe.java index b674ef4c00..422242e8a1 100644 --- a/dspace/src/org/dspace/eperson/Subscribe.java +++ b/dspace/src/org/dspace/eperson/Subscribe.java @@ -318,6 +318,8 @@ public class Subscribe c, startDate, endDate, + 0, // Limit and offset zero, get everything + 0, true, // Need item objects false, // But not containers false); // Or withdrawals diff --git a/dspace/src/org/dspace/search/Harvest.java b/dspace/src/org/dspace/search/Harvest.java index 8bf8a14d87..4b503418be 100644 --- a/dspace/src/org/dspace/search/Harvest.java +++ b/dspace/src/org/dspace/search/Harvest.java @@ -74,20 +74,26 @@ public class Harvest /** log4j logger */ private static Logger log = Logger.getLogger(Harvest.class); - /** ID of the date.available DC type */ - private static int dateAvailableDCTypeID = -1; - /** * Obtain information about items that have been created, modified or - * withdrawn within a given date range. - * FIXME: Assumes all in_archive items have date.available set, - * and have public metadata + * withdrawn within a given date range. You can also specify 'offset' + * and 'limit' so that a big harvest can be split up into smaller sections. + *

+ * Note that dates are passed in the standard ISO8601 format used by + * DSpace (and OAI-PMH).

+ * FIXME: Assumes all in_archive items have public metadata * * @param context DSpace context * @param scope a Community or Collection, or null * indicating the scope is all of DSpace * @param startDate start of date range, or null * @param endDate end of date range, or null + * @param offset for a partial harvest, the point in the overall list + * of matching items to start at. 0 means just start + * at the beginning. + * @param limit the number of matching items to return in a partial + * harvest. Specify 0 to return the whole list + * (or the rest of the list if an offset was specified.) * @param items if true the item field of * each HarvestedItemInfo object is * filled out @@ -102,13 +108,13 @@ public class Harvest DSpaceObject scope, String startDate, String endDate, + int offset, + int limit, boolean items, boolean containers, boolean withdrawn) throws SQLException { - int dcTypeID = getDateAvailableTypeID(context); - // SQL to add to the list of tables after the SELECT String scopeTableSQL = ""; @@ -135,23 +141,35 @@ public class Harvest // Theoretically, no other possibilities, won't bother to check } - // Put together our query + // Put together our query. Note there is no need for an + // "in_archive=true" condition, we are using the existence of + // Handles as our 'existence criterion'. String query = - "SELECT handle.handle, handle.resource_id, dcvalue.text_value FROM handle, dcvalue, item" + + "SELECT handle.handle, handle.resource_id, item.withdrawn, item.last_modified FROM handle, item" + scopeTableSQL + " WHERE handle.resource_type_id=" + Constants.ITEM + - " AND handle.resource_id=dcvalue.item_id AND dcvalue.item_id=item.item_id AND item.withdrawn=false AND dcvalue.dc_type_id=" + - dcTypeID + scopeWhereSQL; + " AND handle.resource_id=item.item_id" + scopeWhereSQL; if (startDate != null) { - query = query + " AND dcvalue.text_value >= '" + startDate + "'"; + query = query + " AND item.last_modified >= '" + startDate + "'"; } if (endDate != null) { - query = query + " AND dcvalue.text_value <= '" + endDate + "'"; + query = query + " AND item.last_modified <= '" + endDate + "'"; } + if (withdrawn = false) + { + // Exclude withdrawn items + query = query + " AND withdrawn=false"; + } + + // Order by item ID, so that for a given harvest the order will be + // consistent. This is so that big harvests can be broken up into + // several smaller operations (e.g. for OAI resumption tokens.) + query = query + " ORDER BY handle.resource_id"; + log.debug(LogManager.getHeader(context, "harvest SQL", query)); @@ -159,78 +177,43 @@ public class Harvest // Execute TableRowIterator tri = DatabaseManager.query(context, query); List infoObjects = new LinkedList(); + int index = 0; // Process results of query into HarvestedItemInfo objects while (tri.hasNext()) { TableRow row = tri.next(); - HarvestedItemInfo itemInfo = new HarvestedItemInfo(); - - itemInfo.handle = row.getStringColumn("handle"); - itemInfo.itemID = row.getIntColumn("resource_id"); - itemInfo.datestamp = row.getStringColumn("text_value"); - itemInfo.withdrawn = false; - if (containers) + /* + * This conditional ensures that we only process items within + * any constraints specified by 'offset' and 'limit' parameters. + */ + if (index >= offset && + (limit == 0 || index < offset + limit)) { - fillContainers(context, itemInfo); - } - - if (items) - { - // Get the item - itemInfo.item = Item.find(context, itemInfo.itemID); - } - - infoObjects.add(itemInfo); - } - - // Add information about deleted items if necessary - if (withdrawn) - { - // Put together our query - query = - "SELECT handle.handle, handle.resource_id, item.withdrawal_date FROM handle, item" + - scopeTableSQL + " WHERE handle.resource_type_id=" + Constants.ITEM + - " AND handle.resource_id=item.item_id AND item.withdrawn=true" - + scopeWhereSQL; - - if (startDate != null) - { - query = query + " AND item.withdrawal_date >= '" + startDate + "'"; - } - - if (endDate != null) - { - query = query + " AND item.withdrawal_date <= '" + endDate + "'"; - } - - log.debug(LogManager.getHeader(context, - "harvest SQL (withdrawals)", - query)); - - // Execute - tri = DatabaseManager.query(context, query); - - // Process results of query into HarvestedItemInfo objects - while (tri.hasNext()) - { - TableRow row = tri.next(); HarvestedItemInfo itemInfo = new HarvestedItemInfo(); itemInfo.handle = row.getStringColumn("handle"); itemInfo.itemID = row.getIntColumn("resource_id"); - itemInfo.datestamp = row.getStringColumn("withdrawal_date"); - itemInfo.withdrawn = true; - + // Put datestamp in ISO8601 + itemInfo.datestamp = row.getDateColumn("last_modified"); + itemInfo.withdrawn = row.getBooleanColumn("withdrawn"); + if (containers) { fillContainers(context, itemInfo); } - // Won't fill out item objects for withdrawn items + if (items) + { + // Get the item + itemInfo.item = Item.find(context, itemInfo.itemID); + } + infoObjects.add(itemInfo); } + + index++; } return infoObjects; @@ -268,23 +251,8 @@ public class Harvest itemInfo.item = i; itemInfo.handle = handle; - - DCDate withdrawalDate = i.getWithdrawalDate(); - - if (withdrawalDate == null) - { - // FIXME: Assume data.available is there - DCValue[] dateAvail = - i.getDC("date", "available", Item.ANY); - itemInfo.datestamp = dateAvail[0].value; - itemInfo.withdrawn = false; - } - else - { - itemInfo.datestamp = withdrawalDate.toString(); - itemInfo.withdrawn = true; - } - + itemInfo.withdrawn = i.isWithdrawn(); + itemInfo.datestamp = i.getLastModified(); itemInfo.itemID = i.getID(); // Get the sets @@ -322,24 +290,4 @@ public class Harvest itemInfo.containers[i][1] = r.getIntColumn("collection_id"); } } - - - /** - * Get the type ID of the "date.available" DC type. This gets the ID - * once then stores it to avoid repeat queries. - * - * @param context DSpace context - * @return the ID of the date.available DC type. - */ - private static int getDateAvailableTypeID(Context context) - throws SQLException - { - if (dateAvailableDCTypeID == -1) - { - DCType type = DCType.findByElement(context, "date", "available"); - dateAvailableDCTypeID = type.getID(); - } - - return dateAvailableDCTypeID; - } } diff --git a/dspace/src/org/dspace/search/HarvestedItemInfo.java b/dspace/src/org/dspace/search/HarvestedItemInfo.java index 58ebbd6275..e11bd6c904 100644 --- a/dspace/src/org/dspace/search/HarvestedItemInfo.java +++ b/dspace/src/org/dspace/search/HarvestedItemInfo.java @@ -40,6 +40,7 @@ package org.dspace.search; +import java.util.Date; import java.util.List; import org.dspace.content.Item; import org.dspace.core.Context; @@ -58,8 +59,8 @@ public class HarvestedItemInfo /** The Handle, with no prefix */ public String handle; - /** The datestamp (ISO8601) */ - public String datestamp; + /** The datestamp */ + public Date datestamp; /** The item. Only filled out if requested */ public Item item;