diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java index 139b72b4fe51..66a9d30450ae 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java @@ -58,6 +58,7 @@ import org.springframework.jmx.export.annotation.ManagedResource; import javax.management.ObjectName; +import java.util.Collection; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -80,7 +81,7 @@ public class BatchOpsMetadataReindexer extends MetadataIndexerProcessor implemen .removalListener(removalListener) .build(); - private final Set metadata; + private final Collection metadata; private ExecutorService executor = null; private ObjectName probeName; private final int toProcessCount; @@ -90,7 +91,7 @@ public class BatchOpsMetadataReindexer extends MetadataIndexerProcessor implemen private final MBeanExporter exporter; private final EsSearchManager esSearchManager; - public BatchOpsMetadataReindexer(DataManager dm, Set metadata) { + public BatchOpsMetadataReindexer(DataManager dm, Collection metadata) { super(dm); this.metadata = metadata; this.toProcessCount = metadata.size(); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManagerImpl.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManagerImpl.java index 6f8bf6680d94..37daf5e1a36b 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManagerImpl.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManagerImpl.java @@ -30,10 +30,7 @@ import org.fao.geonet.api.tools.i18n.TranslationPackBuilder; import org.fao.geonet.constants.Edit; import org.fao.geonet.constants.Geonet; -import org.fao.geonet.domain.HarvestHistory; -import org.fao.geonet.domain.ISODate; -import org.fao.geonet.domain.Metadata; -import org.fao.geonet.domain.Profile; +import org.fao.geonet.domain.*; import org.fao.geonet.exceptions.BadInputEx; import org.fao.geonet.exceptions.JeevesException; import org.fao.geonet.exceptions.MissingParameterEx; @@ -41,10 +38,12 @@ import org.fao.geonet.kernel.AccessManager; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.HarvestInfoProvider; +import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.harvest.Common.OperResult; import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester; import org.fao.geonet.kernel.harvest.harvester.AbstractParams; import org.fao.geonet.kernel.harvest.harvester.HarversterJobListener; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.HarvesterSettingsManager; import org.fao.geonet.repository.HarvestHistoryRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -79,6 +78,7 @@ public class HarvestManagerImpl implements HarvestInfoProvider, HarvestManager { "ownerGroup", "ownerUser", "apiKey", "apiKeyHeader"); private HarvesterSettingsManager settingMan; private DataManager dataMan; + private IMetadataUtils metadataUtils; private Path xslPath; private ServiceContext context; private boolean readOnly; @@ -108,6 +108,7 @@ public ConfigurableApplicationContext getApplicationContext() { public void init(ServiceContext context, boolean isReadOnly) throws Exception { this.context = context; this.dataMan = context.getBean(DataManager.class); + this.metadataUtils = context.getBean(IMetadataUtils.class); this.settingMan = context.getBean(HarvesterSettingsManager.class); this.translationPackBuilder = context.getBean(TranslationPackBuilder.class); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java index cf9347750a02..0c7a813a9802 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java @@ -29,15 +29,7 @@ import org.fao.geonet.Logger; import org.fao.geonet.constants.Geonet; import org.fao.geonet.csw.common.exceptions.InvalidParameterValueEx; -import org.fao.geonet.domain.AbstractMetadata; -import org.fao.geonet.domain.Group; -import org.fao.geonet.domain.HarvestHistory; -import org.fao.geonet.domain.HarvestHistory_; -import org.fao.geonet.domain.ISODate; -import org.fao.geonet.domain.Profile; -import org.fao.geonet.domain.Source; -import org.fao.geonet.domain.SourceType; -import org.fao.geonet.domain.User; +import org.fao.geonet.domain.*; import org.fao.geonet.exceptions.BadInputEx; import org.fao.geonet.exceptions.BadParameterEx; import org.fao.geonet.exceptions.JeevesException; @@ -51,6 +43,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.harvest.Common.OperResult; import org.fao.geonet.kernel.harvest.Common.Status; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.security.SecurityProviderConfiguration; import org.fao.geonet.kernel.security.SecurityProviderUtil; import org.fao.geonet.kernel.setting.HarvesterSettingsManager; @@ -128,6 +121,10 @@ public abstract class AbstractHarvester errors = Collections.synchronizedList(new LinkedList<>()); + /** + * true if this harvester is currenly reindexing, false otherwise + */ + private final AtomicBoolean reindexing = new AtomicBoolean(false); protected ServiceContext context; @@ -507,6 +504,30 @@ public void update(Element node) throws BadInputEx, SQLException, SchedulerExcep } } + /** + * Reindexes all records of this harvester + * @return False is another harvester was running, True if the reindexing was successful + * @throws Exception If the indexing failed + */ + public boolean reindex() throws Exception { + String harvesterUUID = getParams().getUuid(); + final Specification specification = (Specification) MetadataSpecs.hasHarvesterUuid(harvesterUUID); + + var wasRunning = reindexing.compareAndExchange(false, true); + if (wasRunning) { + return false; + } + + try { + List listToReindex = metadataUtils.findAllIdsBy(specification); + BatchOpsMetadataReindexer reindexer = new BatchOpsMetadataReindexer(dataMan, listToReindex); + reindexer.process(settingManager.getSiteId()); + } finally { + reindexing.compareAndExchange(true, false); + } + return true; + } + public String getID() { return id; } @@ -518,6 +539,7 @@ public void addInfo(Element node) { Element info = node.getChild("info"); info.addContent(new Element("running").setText(running + "")); + info.addContent(new Element("reindexing").setText(reindexing + "")); //--- harvester specific info doAddInfo(node); diff --git a/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java b/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java index 1e6973dde27c..58d434445f9f 100644 --- a/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java +++ b/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java @@ -41,6 +41,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.harvest.Common; import org.fao.geonet.kernel.harvest.HarvestManager; import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester; import org.fao.geonet.repository.HarvestHistoryRepository; @@ -173,6 +174,42 @@ public HttpEntity assignHarvestedRecordToSource( } + @io.swagger.v3.oas.annotations.Operation( + summary = "Reindexes all records of an harvester", + description = "" + ) + @RequestMapping( + value = "/{harvesterUuid}/reindex", + method = RequestMethod.POST + ) + @ResponseStatus(value = HttpStatus.OK) + @PreAuthorize("hasAuthority('UserAdmin')") + @ApiResponses(value = { + @ApiResponse(responseCode = "204", description = "Reindexing was successful"), + @ApiResponse(responseCode = "409", description = "Reindexing already running"), + @ApiResponse(responseCode = "404", description = ApiParams.API_RESPONSE_RESOURCE_NOT_FOUND), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + public ResponseEntity reindexHarvester( + @Parameter( + description = "The harvester UUID" + ) + @PathVariable + String harvesterUuid) throws Exception { + final AbstractHarvester harvester = harvestManager.getHarvester(harvesterUuid); + if (harvester == null) { + throw new ResourceNotFoundException(String.format( + "Harvester with UUID '%s' not found. Cannot reindex harvester.", + harvesterUuid)); + } + var successful = harvester.reindex(); + if (successful) { + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } else { + return new ResponseEntity<>(HttpStatus.CONFLICT); + } + } + @io.swagger.v3.oas.annotations.Operation( summary = "Check if a harvester name or host already exist", description = "" diff --git a/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js b/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js index 1116e7aa9f0c..5682d8933537 100644 --- a/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js +++ b/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js @@ -528,6 +528,27 @@ } ); }; + $scope.reindexHarvesterRecord = function () { + return $http + .post( + "../api/harvesters/" + + $scope.harvesterSelected.site.uuid + + "/reindex?source=" + + gnConfig["system.site.siteId"] + ) + .then( + function (response) { + $scope.harvesterSelected = {}; + $scope.harvesterUpdated = false; + $scope.harvesterNew = false; + $scope.$parent.loadHarvesters(); + }, + function (response) { + console.log(response.data); + } + ); + }; + $scope.assignHarvestedRecordToLocalNode = function () { $http .post( diff --git a/web-ui/src/main/resources/catalog/locales/de-admin.json b/web-ui/src/main/resources/catalog/locales/de-admin.json index 4612db5244de..4c46e15b6ead 100644 --- a/web-ui/src/main/resources/catalog/locales/de-admin.json +++ b/web-ui/src/main/resources/catalog/locales/de-admin.json @@ -570,6 +570,8 @@ "recordsStatistics": "Metadatenstatistiken", "recordsType": "Art der Metadatensätze", "refreshHarvester": "Harvester aktualisieren", + "reindexHarvesterRecords": "Reindexieren von {{records}} Datensätzen", + "reindexHarvesterRecordsHelp": "Reindexiert alle Datensätze eines Harvesters.", "reindexRecords": "Reindexieren {{records}} verknüpfter Datensätze", "reindexRecords-help": "Nach dem Aktualisieren von Schlagwörtern und wenn Datensätze XLinks verwenden, sollten Datensätze, die diesen Thesaurus verwenden, neu indiziert werden (aufgrund des XLinks-Cache-Systems).", "related": "Verknüpft", diff --git a/web-ui/src/main/resources/catalog/locales/en-admin.json b/web-ui/src/main/resources/catalog/locales/en-admin.json index 93a09923e44d..56677b4597ef 100644 --- a/web-ui/src/main/resources/catalog/locales/en-admin.json +++ b/web-ui/src/main/resources/catalog/locales/en-admin.json @@ -581,6 +581,8 @@ "recordsStatistics": "Metadata statistics", "recordsType": "Type of metadata records", "refreshHarvester": "Refresh harvester", + "reindexHarvesterRecords": "Reindex {{records}} records", + "reindexHarvesterRecordsHelp": "Reindex all records of the harvester.", "reindexRecords": "Reindex {{records}} related record(s)", "reindexRecords-help": "After updating keywords and when records are using XLinks, records using this thesaurus should be reindexed (due to the XLinks cache system).", "related": "Related", diff --git a/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html b/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html index 284395edc850..33e59c43ce6d 100644 --- a/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html +++ b/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html @@ -525,6 +525,23 @@

deleteHarvesterRecords + +