hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-13 13:03:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

DS-4440 GDPR - Anonymize statistics feature - remove shard support

Browse Source
This commit is contained in:
Samuel
2020-11-06 10:55:48 +01:00
parent c287062090
commit feb6ba6d18
4 changed files with 10 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
dspace-api/src/main/java/org/dspace/statistics/AnonymizeStatistics.java
View File

@@ -14,7 +14,6 @@ import static java.util.Arrays.asList;
import static java.util.Calendar.DAY_OF_YEAR;
import static java.util.Collections.singletonList;
import static org.apache.commons.cli.Option.builder;
import static org.apache.commons.lang.StringUtils.isNotBlank;
import static org.apache.commons.lang.time.DateFormatUtils.format;
import static org.apache.log4j.Logger.getLogger;
import static org.dspace.core.LogManager.getHeader;
@@ -25,8 +24,6 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -197,31 +194,18 @@ public class AnonymizeStatistics {
// list of the processing callables to execute
Collection<DoProcessing> callables = new ArrayList<>();
// list of the shards to commit
Set<String> shards = new HashSet<>();
for (SolrDocument document : documents.getResults()) {
updated++;
callables.add(new DoProcessing(document, updated));
String shard = (String) document.getFieldValue("[shard]");
if (isNotBlank(shard)) {
shards.add(shard);
}
}
// execute the processing callables
executorService.invokeAll(callables);
// Commit the main core
// Commit the solr core
solrLoggerService.commit();
// Commit all relevant solr shards
for (String shard : shards) {
solrLoggerService.commitShard(shard);
}
System.out.println("processed " + updated + " records");
} while (documents.getResults().getNumFound() > 0);
@@ -231,7 +215,6 @@ public class AnonymizeStatistics {
} else {
printWarning("not all relevant documents were updated, check the DSpace logs for more details");
}
} catch (Exception e) {
printError(e);
}
@@ -259,7 +242,7 @@ public class AnonymizeStatistics {
"ip:*",
"time:[* TO " + TIME_LIMIT + "] AND -dns:" + DNS_MASK,
null, batchSize, -1, null, null, null, null,
null, false, -1, false, true
null, false, -1, false
);
}