hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DS-732] Browse performance

Browse Source 
git-svn-id: http://scm.dspace.org/svn/repo/dspace/trunk@5699 9c30dcfa-912a-0410-8fc2-9e0234be79fd
This commit is contained in:
Graham Triggs
2010-10-31 09:04:54 +00:00
parent 294bd96916
commit a5658f7c19
5 changed files with 438 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dspace-api/src/main/java/org/dspace/browse/BrowseCreateDAO.java
View File

@@ -336,6 +336,8 @@ public interface BrowseCreateDAO
* @throws BrowseException
*/
public String createCommunityView(String table, String view, boolean execute) throws BrowseException;
public List<Integer> deleteMappingsByItemID(String mapTable, int itemID) throws BrowseException;
/**
* Create the table which will hold the distinct metadata values that appear in multiple
@@ -385,7 +387,7 @@ public interface BrowseCreateDAO
* @param withdrawn TODO
* @throws BrowseException
*/
public void pruneMapExcess(String map, boolean withdrawn) throws BrowseException;
public void pruneMapExcess(String map, boolean withdrawn, List<Integer> distinctIds) throws BrowseException;
/**
* So that there are no distinct values indexed which are no longer referenced from the
@@ -396,5 +398,5 @@ public interface BrowseCreateDAO
* @param map the name of the associated distinct mapping table.
* @throws BrowseException
*/
public void pruneDistinct(String table, String map) throws BrowseException;
public void pruneDistinct(String table, String map, List<Integer> distinctIds) throws BrowseException;
}

119
dspace-api/src/main/java/org/dspace/browse/BrowseCreateDAOOracle.java
View File

@@ -805,18 +805,72 @@ public class BrowseCreateDAOOracle implements BrowseCreateDAO
return true;
}
public List<Integer> deleteMappingsByItemID(String mapTable, int itemID) throws BrowseException
{
List<Integer> distinctIds = new ArrayList<Integer>();
TableRowIterator tri = null;
try
{
tri = DatabaseManager.queryTable(context, mapTable, "SELECT * FROM " + mapTable + " WHERE item_id=?", itemID);
if (tri != null)
{
while (tri.hasNext())
{
TableRow tr = tri.next();
distinctIds.add(tr.getIntColumn("distinct_id"));
DatabaseManager.delete(context, tr);
}
}
}
catch (SQLException e)
{
log.error("caught exception: ", e);
throw new BrowseException(e);
}
finally
{
if (tri != null)
{
tri.close();
}
}
return distinctIds;
}
/* (non-Javadoc)
* @see org.dspace.browse.BrowseCreateDAO#pruneDistinct(java.lang.String, java.lang.String)
*/
public void pruneDistinct(String table, String map) throws BrowseException
public void pruneDistinct(String table, String map, List<Integer> distinctIds) throws BrowseException
{
try
{
String query = "DELETE FROM " + table +
" WHERE NOT EXISTS (SELECT 1 FROM " + map +
" WHERE " + map + ".distinct_id = " + table +".id)";
StringBuilder query = new StringBuilder();
query.append("DELETE FROM ").append(table).append(" WHERE NOT EXISTS (SELECT 1 FROM ");
query.append(map).append(" WHERE ").append(map).append(".distinct_id = ").append(table).append(".id)");
DatabaseManager.updateQuery(context, query);
if (distinctIds != null && distinctIds.size() > 0)
{
query.append(" AND ").append(table).append(".id IN (");
for (int i = 0; i < distinctIds.size(); i++)
{
if (i > 0)
{
query.append(", ?");
}
else
{
query.append("?");
}
}
query.append(")");
DatabaseManager.updateQuery(context, query.toString(), distinctIds.toArray(new Integer[distinctIds.size()]));
}
else
{
DatabaseManager.updateQuery(context, query.toString());
}
}
catch (SQLException e)
{
@@ -832,10 +886,20 @@ public class BrowseCreateDAOOracle implements BrowseCreateDAO
{
try
{
String itemCriteria = withdrawn ? "item.withdrawn = 1" : "item.in_archive = 1 AND item.withdrawn = 0";
StringBuilder query = new StringBuilder();
String delete = "DELETE FROM " + table + " WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=" + table + ".item_id AND " + itemCriteria + ")";
DatabaseManager.updateQuery(context, delete);
query.append("DELETE FROM ").append(table).append(" WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=");
query.append(table).append(".item_id AND ");
if (withdrawn)
{
query.append("item.withdrawn = 1");
}
else
{
query.append("item.in_archive = 1 AND item.withdrawn = 0");
}
query.append(")");
DatabaseManager.updateQuery(context, query.toString());
}
catch (SQLException e)
{
@@ -847,14 +911,45 @@ public class BrowseCreateDAOOracle implements BrowseCreateDAO
/* (non-Javadoc)
* @see org.dspace.browse.BrowseCreateDAO#pruneMapExcess(java.lang.String, boolean)
*/
public void pruneMapExcess(String map, boolean withdrawn) throws BrowseException
public void pruneMapExcess(String map, boolean withdrawn, List<Integer> distinctIds) throws BrowseException
{
try
{
String itemCriteria = withdrawn ? "item.withdrawn = 1" : "item.in_archive = 1 AND item.withdrawn = 0";
StringBuilder query = new StringBuilder();
String deleteDistinct = "DELETE FROM " + map + " WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=" + map + ".item_id AND " + itemCriteria + ")";
DatabaseManager.updateQuery(context, deleteDistinct);
query.append("DELETE FROM ").append(map).append(" WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=");
query.append(map).append(".item_id AND ");
if (withdrawn)
{
query.append("item.withdrawn = 1");
}
else
{
query.append("item.in_archive = 1 AND item.withdrawn = 0");
}
query.append(")");
if (distinctIds != null && distinctIds.size() > 0)
{
query.append(" AND ").append(map).append(".distinct_id IN (");
for (int i = 0; i < distinctIds.size(); i++)
{
if (i > 0)
{
query.append(", ?");
}
else
{
query.append("?");
}
}
query.append(")");
DatabaseManager.updateQuery(context, query.toString(), distinctIds.toArray(new Integer[distinctIds.size()]));
}
else
{
DatabaseManager.updateQuery(context, query.toString());
}
}
catch (SQLException e)
{

126
dspace-api/src/main/java/org/dspace/browse/BrowseCreateDAOPostgres.java
View File

@@ -802,19 +802,73 @@ public class BrowseCreateDAOPostgres implements BrowseCreateDAO
return true;
}
public List<Integer> deleteMappingsByItemID(String mapTable, int itemID) throws BrowseException
{
List<Integer> distinctIds = new ArrayList<Integer>();
TableRowIterator tri = null;
try
{
tri = DatabaseManager.queryTable(context, mapTable, "SELECT * FROM " + mapTable + " WHERE item_id=?", itemID);
if (tri != null)
{
while (tri.hasNext())
{
TableRow tr = tri.next();
distinctIds.add(tr.getIntColumn("distinct_id"));
DatabaseManager.delete(context, tr);
}
}
}
catch (SQLException e)
{
log.error("caught exception: ", e);
throw new BrowseException(e);
}
finally
{
if (tri != null)
{
tri.close();
}
}
return distinctIds;
}
/* (non-Javadoc)
* @see org.dspace.browse.BrowseCreateDAO#pruneDistinct(java.lang.String, java.lang.String)
*/
public void pruneDistinct(String table, String map)
public void pruneDistinct(String table, String map, List<Integer> distinctIds)
throws BrowseException
{
try
{
String query = "DELETE FROM " + table +
" WHERE NOT EXISTS (SELECT 1 FROM " + map +
" WHERE " + map + ".distinct_id = " + table +".id)";
StringBuilder query = new StringBuilder();
query.append("DELETE FROM ").append(table).append(" WHERE NOT EXISTS (SELECT 1 FROM ");
query.append(map).append(" WHERE ").append(map).append(".distinct_id = ").append(table).append(".id)");
DatabaseManager.updateQuery(context, query);
if (distinctIds != null && distinctIds.size() > 0)
{
query.append(" AND ").append(table).append(".id IN (");
for (int i = 0; i < distinctIds.size(); i++)
{
if (i > 0)
{
query.append(", ?");
}
else
{
query.append("?");
}
}
query.append(")");
DatabaseManager.updateQuery(context, query.toString(), distinctIds.toArray(new Integer[distinctIds.size()]));
}
else
{
DatabaseManager.updateQuery(context, query.toString());
}
}
catch (SQLException e)
{
@@ -829,6 +883,29 @@ public class BrowseCreateDAOPostgres implements BrowseCreateDAO
public void pruneExcess(String table, boolean withdrawn)
throws BrowseException
{
try
{
StringBuilder query = new StringBuilder();
query.append("DELETE FROM ").append(table).append(" WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=");
query.append(table).append(".item_id AND ");
if (withdrawn)
{
query.append("item.withdrawn = true");
}
else
{
query.append("item.in_archive = true AND item.withdrawn = false");
}
query.append(")");
DatabaseManager.updateQuery(context, query.toString());
}
catch (SQLException e)
{
log.error("caught exception: ", e);
throw new BrowseException(e);
}
try
{
String itemCriteria = withdrawn ? "item.withdrawn = true" : "item.in_archive = true AND item.withdrawn = false";
@@ -846,15 +923,46 @@ public class BrowseCreateDAOPostgres implements BrowseCreateDAO
/* (non-Javadoc)
* @see org.dspace.browse.BrowseCreateDAO#pruneMapExcess(java.lang.String, boolean)
*/
public void pruneMapExcess(String map, boolean withdrawn)
public void pruneMapExcess(String map, boolean withdrawn, List<Integer> distinctIds)
throws BrowseException
{
try
{
String itemCriteria = withdrawn ? "item.withdrawn = true" : "item.in_archive = true AND item.withdrawn = false";
StringBuilder query = new StringBuilder();
String deleteDistinct = "DELETE FROM " + map + " WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=" + map + ".item_id AND " + itemCriteria + ")";
DatabaseManager.updateQuery(context, deleteDistinct);
query.append("DELETE FROM ").append(map).append(" WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=");
query.append(map).append(".item_id AND ");
if (withdrawn)
{
query.append("item.withdrawn = true");
}
else
{
query.append("item.in_archive = true AND item.withdrawn = false");
}
query.append(")");
if (distinctIds != null && distinctIds.size() > 0)
{
query.append(" AND ").append(map).append(".distinct_id IN (");
for (int i = 0; i < distinctIds.size(); i++)
{
if (i > 0)
{
query.append(", ?");
}
else
{
query.append("?");
}
}
query.append(")");
DatabaseManager.updateQuery(context, query.toString(), distinctIds.toArray(new Integer[distinctIds.size()]));
}
else
{
DatabaseManager.updateQuery(context, query.toString());
}
}
catch (SQLException e)
{

31
dspace-api/src/main/java/org/dspace/browse/IndexBrowse.java
View File

@@ -301,6 +301,16 @@ public class IndexBrowse
dao.deleteByItemID(table, itemID);
}
private void removeDistinctIndex(int itemID, String distinctTable, String mapTable)
throws BrowseException
{
List<Integer> distinctIds = dao.deleteMappingsByItemID(mapTable, itemID);
if (distinctIds != null && distinctIds.size() > 0)
{
dao.pruneDistinct(distinctTable, mapTable, distinctIds);
}
}
/**
* Prune indexes - called from the public interfaces or at the end of a batch indexing process
*/
@@ -312,8 +322,7 @@ public class IndexBrowse
if (bis[i].isMetadataIndex())
{
log.debug("Pruning metadata index: " + bis[i].getTableName());
dao.pruneMapExcess(bis[i].getMapTableName(), false);
dao.pruneDistinct(bis[i].getDistinctTableName(), bis[i].getMapTableName());
pruneDistinctIndex(bis[i], null);
}
}
@@ -321,6 +330,12 @@ public class IndexBrowse
dao.pruneExcess(BrowseIndex.getWithdrawnBrowseIndex().getTableName(), true);
}
private void pruneDistinctIndex(BrowseIndex bi, List<Integer> removedIds) throws BrowseException
{
dao.pruneMapExcess(bi.getMapTableName(), false, removedIds);
dao.pruneDistinct(bi.getDistinctTableName(), bi.getMapTableName(), removedIds);
}
/**
* Index the given item
*
@@ -344,7 +359,7 @@ public class IndexBrowse
indexItem(new ItemMetadataProxy(item));
// Ensure that we remove any invalid entries
pruneIndexes();
//pruneIndexes();
}
}
@@ -494,7 +509,7 @@ public class IndexBrowse
if (distIDSet.isEmpty())
{
// remove any old mappings
removeIndex(item.getID(), bis[i].getMapTableName());
removeDistinctIndex(item.getID(), bis[i].getDistinctTableName(), bis[i].getMapTableName());
}
else
{
@@ -506,6 +521,10 @@ public class IndexBrowse
distIDarr[didx++] = distID;
}
MappingResults results = dao.updateDistinctMappings(bis[i].getMapTableName(), item.getID(), distIDarr);
if (results.getRemovedDistinctIds() != null && results.getRemovedDistinctIds().size() > 0)
{
pruneDistinctIndex(bis[i], results.getRemovedDistinctIds());
}
}
}
}
@@ -645,7 +664,7 @@ public class IndexBrowse
dao.deleteCommunityMappings(itemID);
// Ensure that we remove any invalid entries
pruneIndexes();
//pruneIndexes();
return true;
}
@@ -1158,7 +1177,7 @@ public class IndexBrowse
// penultimately we have to delete any items that couldn't be located in the
// index list
pruneIndexes();
//pruneIndexes();
// Make sure the deletes are written back
context.commit();

212
dspace-api/src/test/java/org/dspace/browse/MockBrowseCreateDAOOracle.java
View File

@@ -36,6 +36,7 @@ package org.dspace.browse;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -286,12 +287,10 @@ public class MockBrowseCreateDAOOracle
}
}
/* (non-Javadoc)
* @see org.dspace.browse.BrowseCreateDAO#updateDistinctMapping(java.lang.String, int, int)
*/
@Mock
public boolean updateDistinctMappings(String table, int itemID, int[] distinctIDs) throws BrowseException
public MappingResults updateDistinctMappings(String table, int itemID, int[] distinctIDs) throws BrowseException
{
BrowseMappingResults results = new BrowseMappingResults();
try
{
checkContext();
@@ -299,7 +298,9 @@ public class MockBrowseCreateDAOOracle
for (int i = 0; i < distinctIDs.length; i++)
{
if (!isFirstOccurrence(distinctIDs, i))
{
distinctIDs[i] = -1;
}
}
// Find all existing mappings for this item
@@ -320,6 +321,7 @@ public class MockBrowseCreateDAOOracle
// Found this mapping
if (distinctIDs[i] == trDistinctID)
{
results.addRetainedDistinctId(trDistinctID);
// Flag it, and remove (-1) from the item mappings
itemIsMapped = true;
distinctIDs[i] = -1;
@@ -328,7 +330,10 @@ public class MockBrowseCreateDAOOracle
// The item is no longer mapped to this community, so remove the database record
if (!itemIsMapped)
{
results.addRemovedDistinctId(trDistinctID);
DatabaseManager.delete(internalContext, tr);
}
}
}
finally
@@ -338,14 +343,15 @@ public class MockBrowseCreateDAOOracle
}
// Any remaining mappings need to be added to the database
for (int i = 0; i < distinctIDs.length; i++)
for (int distinctID : distinctIDs)
{
if (distinctIDs[i] > -1)
if (distinctID > -1)
{
TableRow row = DatabaseManager.create(internalContext, table);
row.setColumn("item_id", itemID);
row.setColumn("distinct_id", distinctIDs[i]);
row.setColumn("distinct_id", distinctID);
DatabaseManager.update(internalContext, row);
results.addAddedDistinctId(distinctID);
}
}
}
@@ -360,7 +366,7 @@ public class MockBrowseCreateDAOOracle
cleanContext();
}
return false;
return results;
}
/* (non-Javadoc)
@@ -904,20 +910,84 @@ public class MockBrowseCreateDAOOracle
return true;
}
@Mock
public List<Integer> deleteMappingsByItemID(String mapTable, int itemID) throws BrowseException
{
System.out.println("Map: " + mapTable + ", item: " + itemID);
List<Integer> distinctIds = new ArrayList<Integer>();
TableRowIterator tri = null;
try
{
checkContext();
try
{
tri = DatabaseManager.queryTable(internalContext, mapTable, "SELECT * FROM " + mapTable + " WHERE item_id=?", itemID);
if (tri != null)
{
while (tri.hasNext())
{
TableRow tr = tri.next();
distinctIds.add(tr.getIntColumn("distinct_id"));
DatabaseManager.delete(internalContext, tr);
}
}
}
catch (SQLException e)
{
log.error("caught exception: ", e);
throw new BrowseException(e);
}
finally
{
if (tri != null)
{
tri.close();
}
}
}
finally
{
cleanContext();
}
return distinctIds;
}
/* (non-Javadoc)
* @see org.dspace.browse.BrowseCreateDAO#pruneDistinct(java.lang.String, java.lang.String)
*/
@Mock
public void pruneDistinct(String table, String map) throws BrowseException
public void pruneDistinct(String table, String map, List<Integer> distinctIds) throws BrowseException
{
try
{
checkContext();
String query = "DELETE FROM " + table +
" WHERE id IN (SELECT id FROM " + table +
" MINUS SELECT distinct_id AS id FROM " + map + ")";
StringBuilder query = new StringBuilder();
query.append("DELETE FROM ").append(table).append(" WHERE NOT EXISTS (SELECT 1 FROM ");
query.append(map).append(" WHERE ").append(map).append(".distinct_id = ").append(table).append(".id)");
DatabaseManager.updateQuery(internalContext, query);
if (distinctIds != null && distinctIds.size() > 0)
{
query.append(" AND ").append(table).append(".id IN (");
for (int i = 0; i < distinctIds.size(); i++)
{
if (i > 0)
{
query.append(", ?");
}
else
{
query.append("?");
}
}
query.append(")");
DatabaseManager.updateQuery(internalContext, query.toString(), distinctIds.toArray(new Integer[distinctIds.size()]));
}
else
{
DatabaseManager.updateQuery(internalContext, query.toString());
}
}
catch (SQLException e)
{
@@ -934,26 +1004,77 @@ public class MockBrowseCreateDAOOracle
* @see org.dspace.browse.BrowseCreateDAO#pruneExcess(java.lang.String, java.lang.String)
*/
@Mock
public void pruneExcess(String table, String map, boolean withdrawn) throws BrowseException
public void pruneExcess(String table, boolean withdrawn) throws BrowseException
{
try
{
checkContext();
String itemQuery = "SELECT item_id FROM item WHERE ";
StringBuilder query = new StringBuilder();
query.append("DELETE FROM ").append(table).append(" WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=");
query.append(table).append(".item_id AND ");
if (withdrawn)
itemQuery += "withdrawn = 1";
else
itemQuery += "in_archive = 1 AND withdrawn = 0";
String delete = "DELETE FROM " + table + " WHERE item_id IN ( SELECT item_id FROM " + table + " MINUS " + itemQuery + ")";
DatabaseManager.updateQuery(internalContext, delete);
if (map != null)
{
String deleteDistinct = "DELETE FROM " + map + " WHERE item_id IN ( SELECT item_id FROM " + map + " MINUS " + itemQuery + ")";
DatabaseManager.updateQuery(internalContext, deleteDistinct);
internalContext.commit();
query.append("item.withdrawn = 1");
}
else
{
query.append("item.in_archive = 1 AND item.withdrawn = 0");
}
query.append(")");
DatabaseManager.updateQuery(internalContext, query.toString());
}
catch (SQLException e)
{
log.error("caught exception: ", e);
throw new BrowseException(e);
}
finally
{
cleanContext();
}
}
@Mock
public void pruneMapExcess(String map, boolean withdrawn, List<Integer> distinctIds) throws BrowseException
{
try
{
checkContext();
StringBuilder query = new StringBuilder();
query.append("DELETE FROM ").append(map).append(" WHERE NOT EXISTS (SELECT 1 FROM item WHERE item.item_id=");
query.append(map).append(".item_id AND ");
if (withdrawn)
{
query.append("item.withdrawn = 1");
}
else
{
query.append("item.in_archive = 1 AND item.withdrawn = 0");
}
query.append(")");
if (distinctIds != null && distinctIds.size() > 0)
{
query.append(" AND ").append(map).append(".distinct_id IN (");
for (int i = 0; i < distinctIds.size(); i++)
{
if (i > 0)
{
query.append(", ?");
}
else
{
query.append("?");
}
}
query.append(")");
DatabaseManager.updateQuery(internalContext, query.toString(), distinctIds.toArray(new Integer[distinctIds.size()]));
}
else
{
DatabaseManager.updateQuery(internalContext, query.toString());
}
}
catch (SQLException e)
@@ -1203,4 +1324,41 @@ public class MockBrowseCreateDAOOracle
return true;
}
private static class BrowseMappingResults implements MappingResults
{
private List<Integer> addedDistinctIds = new ArrayList<Integer>();
private List<Integer> retainedDistinctIds = new ArrayList<Integer>();
private List<Integer> removedDistinctIds = new ArrayList<Integer>();
private void addAddedDistinctId(int id)
{
addedDistinctIds.add(id);
}
private void addRetainedDistinctId(int id)
{
retainedDistinctIds.add(id);
}
private void addRemovedDistinctId(int id)
{
removedDistinctIds.add(id);
}
public List<Integer> getAddedDistinctIds()
{
return addedDistinctIds;
}
public List<Integer> getRetainedDistinctIds()
{
return retainedDistinctIds;
}
public List<Integer> getRemovedDistinctIds()
{
return Collections.unmodifiableList(removedDistinctIds);
}
}
}