Add support for reindexing all metadata of a harvester

core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java

@@ -58,6 +58,7 @@
 import org.springframework.jmx.export.annotation.ManagedResource;
 
 import javax.management.ObjectName;
+import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -80,7 +81,7 @@ public class BatchOpsMetadataReindexer extends MetadataIndexerProcessor implemen
         .removalListener(removalListener)
         .build();
 
-    private final Set<Integer> metadata;
+    private final Collection<Integer> metadata;
     private ExecutorService executor = null;
     private ObjectName probeName;
     private final int toProcessCount;
@@ -90,7 +91,7 @@ public class BatchOpsMetadataReindexer extends MetadataIndexerProcessor implemen
     private final MBeanExporter exporter;
     private final EsSearchManager esSearchManager;
 
-    public BatchOpsMetadataReindexer(DataManager dm, Set<Integer> metadata) {
+    public BatchOpsMetadataReindexer(DataManager dm, Collection<Integer> metadata) {
         super(dm);
         this.metadata = metadata;
         this.toProcessCount = metadata.size();

harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManagerImpl.java

@@ -30,21 +30,20 @@
 import org.fao.geonet.api.tools.i18n.TranslationPackBuilder;
 import org.fao.geonet.constants.Edit;
 import org.fao.geonet.constants.Geonet;
-import org.fao.geonet.domain.HarvestHistory;
-import org.fao.geonet.domain.ISODate;
-import org.fao.geonet.domain.Metadata;
-import org.fao.geonet.domain.Profile;
+import org.fao.geonet.domain.*;
 import org.fao.geonet.exceptions.BadInputEx;
 import org.fao.geonet.exceptions.JeevesException;
 import org.fao.geonet.exceptions.MissingParameterEx;
 import org.fao.geonet.exceptions.OperationAbortedEx;
 import org.fao.geonet.kernel.AccessManager;
 import org.fao.geonet.kernel.DataManager;
 import org.fao.geonet.kernel.HarvestInfoProvider;
+import org.fao.geonet.kernel.datamanager.IMetadataUtils;
 import org.fao.geonet.kernel.harvest.Common.OperResult;
 import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester;
 import org.fao.geonet.kernel.harvest.harvester.AbstractParams;
 import org.fao.geonet.kernel.harvest.harvester.HarversterJobListener;
+import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer;
 import org.fao.geonet.kernel.setting.HarvesterSettingsManager;
 import org.fao.geonet.repository.HarvestHistoryRepository;
 import org.fao.geonet.repository.specification.MetadataSpecs;
@@ -79,6 +78,7 @@ public class HarvestManagerImpl implements HarvestInfoProvider, HarvestManager {
             "ownerGroup", "ownerUser", "apiKey", "apiKeyHeader");
     private HarvesterSettingsManager settingMan;
     private DataManager dataMan;
+    private IMetadataUtils metadataUtils;
     private Path xslPath;
     private ServiceContext context;
     private boolean readOnly;
@@ -108,6 +108,7 @@ public ConfigurableApplicationContext getApplicationContext() {
     public void init(ServiceContext context, boolean isReadOnly) throws Exception {
         this.context = context;
         this.dataMan = context.getBean(DataManager.class);
+        this.metadataUtils = context.getBean(IMetadataUtils.class);
         this.settingMan = context.getBean(HarvesterSettingsManager.class);
         this.translationPackBuilder = context.getBean(TranslationPackBuilder.class);
 

harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java

@@ -29,15 +29,7 @@
 import org.fao.geonet.Logger;
 import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.csw.common.exceptions.InvalidParameterValueEx;
-import org.fao.geonet.domain.AbstractMetadata;
-import org.fao.geonet.domain.Group;
-import org.fao.geonet.domain.HarvestHistory;
-import org.fao.geonet.domain.HarvestHistory_;
-import org.fao.geonet.domain.ISODate;
-import org.fao.geonet.domain.Profile;
-import org.fao.geonet.domain.Source;
-import org.fao.geonet.domain.SourceType;
-import org.fao.geonet.domain.User;
+import org.fao.geonet.domain.*;
 import org.fao.geonet.exceptions.BadInputEx;
 import org.fao.geonet.exceptions.BadParameterEx;
 import org.fao.geonet.exceptions.JeevesException;
@@ -51,6 +43,7 @@
 import org.fao.geonet.kernel.datamanager.IMetadataUtils;
 import org.fao.geonet.kernel.harvest.Common.OperResult;
 import org.fao.geonet.kernel.harvest.Common.Status;
+import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer;
 import org.fao.geonet.kernel.security.SecurityProviderConfiguration;
 import org.fao.geonet.kernel.security.SecurityProviderUtil;
 import org.fao.geonet.kernel.setting.HarvesterSettingsManager;
@@ -128,6 +121,10 @@ public abstract class AbstractHarvester<T extends HarvestResult, P extends Abstr
      * Contains all the errors that were thrown during harvesting and that may have caused the harvesting to abort
      */
     protected final List<HarvestError> errors = Collections.synchronizedList(new LinkedList<>());
+    /**
+     * true if this harvester is currenly reindexing, false otherwise
+     */
+    private final AtomicBoolean reindexing = new AtomicBoolean(false);
 
     protected ServiceContext context;
 
@@ -507,6 +504,30 @@ public void update(Element node) throws BadInputEx, SQLException, SchedulerExcep
         }
     }
 
+    /**
+     * Reindexes all records of this harvester
+     * @return False is another harvester was running, True if the reindexing was successful
+     * @throws Exception If the indexing failed
+     */
+    public boolean reindex() throws Exception {
+        String harvesterUUID = getParams().getUuid();
+        final Specification<Metadata> specification = (Specification<Metadata>) MetadataSpecs.hasHarvesterUuid(harvesterUUID);
+
+        var wasRunning = reindexing.compareAndExchange(false, true);
+        if (wasRunning) {
+            return false;
+        }
+
+        try {
+            List<Integer> listToReindex = metadataUtils.findAllIdsBy(specification);
+            BatchOpsMetadataReindexer reindexer = new BatchOpsMetadataReindexer(dataMan, listToReindex);
+            reindexer.process(settingManager.getSiteId());
+        } finally {
+            reindexing.compareAndExchange(true, false);
+        }
+        return true;
+    }
+
     public String getID() {
         return id;
     }
@@ -518,6 +539,7 @@ public void addInfo(Element node) {
         Element info = node.getChild("info");
 
         info.addContent(new Element("running").setText(running + ""));
+        info.addContent(new Element("reindexing").setText(reindexing + ""));
 
         //--- harvester specific info
         doAddInfo(node);

services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java

@@ -41,6 +41,7 @@
 import org.fao.geonet.kernel.DataManager;
 import org.fao.geonet.kernel.datamanager.IMetadataManager;
 import org.fao.geonet.kernel.datamanager.IMetadataUtils;
+import org.fao.geonet.kernel.harvest.Common;
 import org.fao.geonet.kernel.harvest.HarvestManager;
 import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester;
 import org.fao.geonet.repository.HarvestHistoryRepository;
@@ -173,6 +174,42 @@ public HttpEntity<HttpStatus> assignHarvestedRecordToSource(
     }
 
 
+    @io.swagger.v3.oas.annotations.Operation(
+            summary = "Reindexes all records of an harvester",
+            description = ""
+    )
+    @RequestMapping(
+            value = "/{harvesterUuid}/reindex",
+            method = RequestMethod.POST
+    )
+    @ResponseStatus(value = HttpStatus.OK)
+    @PreAuthorize("hasAuthority('UserAdmin')")
+    @ApiResponses(value = {
+            @ApiResponse(responseCode = "204", description = "Reindexing was successful"),
+            @ApiResponse(responseCode = "409", description = "Reindexing already running"),
+            @ApiResponse(responseCode = "404", description = ApiParams.API_RESPONSE_RESOURCE_NOT_FOUND),
+            @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN)
+    })
+    public ResponseEntity<HttpStatus> reindexHarvester(
+            @Parameter(
+                    description = "The harvester UUID"
+            )
+            @PathVariable
+            String harvesterUuid) throws Exception {
+        final AbstractHarvester<?, ?> harvester = harvestManager.getHarvester(harvesterUuid);
+        if (harvester == null) {
+            throw new ResourceNotFoundException(String.format(
+                    "Harvester with UUID '%s' not found. Cannot reindex harvester.",
+                    harvesterUuid));
+        }
+        var successful = harvester.reindex();
+        if (successful) {
+            return new ResponseEntity<>(HttpStatus.NO_CONTENT);
+        } else {
+            return new ResponseEntity<>(HttpStatus.CONFLICT);
+        }
+    }
+
     @io.swagger.v3.oas.annotations.Operation(
         summary = "Check if a harvester name or host already exist",
         description = ""

web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js

@@ -528,6 +528,27 @@
             }
           );
       };
+      $scope.reindexHarvesterRecord = function () {
+        return $http
+          .post(
+            "../api/harvesters/" +
+              $scope.harvesterSelected.site.uuid +
+              "/reindex?source=" +
+              gnConfig["system.site.siteId"]
+          )
+          .then(
+            function (response) {
+              $scope.harvesterSelected = {};
+              $scope.harvesterUpdated = false;
+              $scope.harvesterNew = false;
+              $scope.$parent.loadHarvesters();
+            },
+            function (response) {
+              console.log(response.data);
+            }
+          );
+      };
+
       $scope.assignHarvestedRecordToLocalNode = function () {
         $http
           .post(

web-ui/src/main/resources/catalog/locales/de-admin.json

@@ -570,6 +570,8 @@
     "recordsStatistics": "Metadatenstatistiken",
     "recordsType": "Art der Metadatensätze",
     "refreshHarvester": "Harvester aktualisieren",
+    "reindexHarvesterRecords": "Reindexieren von <strong>{{records}}</strong> Datensätzen",
+    "reindexHarvesterRecordsHelp": "Reindexiert alle Datensätze eines Harvesters.",
     "reindexRecords": "Reindexieren {{records}} verknüpfter Datensätze",
     "reindexRecords-help": "Nach dem Aktualisieren von Schlagwörtern und wenn Datensätze XLinks verwenden, sollten Datensätze, die diesen Thesaurus verwenden, neu indiziert werden (aufgrund des XLinks-Cache-Systems).",
     "related": "Verknüpft",

web-ui/src/main/resources/catalog/locales/en-admin.json

@@ -581,6 +581,8 @@
     "recordsStatistics": "Metadata statistics",
     "recordsType": "Type of metadata records",
     "refreshHarvester": "Refresh harvester",
+    "reindexHarvesterRecords": "Reindex <strong>{{records}}</strong> records",
+    "reindexHarvesterRecordsHelp": "Reindex all records of the harvester.",
     "reindexRecords": "Reindex {{records}} related record(s)",
     "reindexRecords-help": "After updating keywords and when records are using XLinks, records using this thesaurus should be reindexed (due to the XLinks cache system).",
     "related": "Related",

web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html

@@ -525,6 +525,23 @@ <h4 data-ng-show="h.deleted == 'false'" class="timeline-title">
                     deleteHarvesterRecords</span
                   >
                 </button>
+
+                <button
+                  id="gn-harvest-records-buttons-reindex"
+                  type="button"
+                  class="btn btn-warning"
+                  data-gn-click-and-spin="reindexHarvesterRecord(harvesterSelected['@id'])"
+                  data-ng-disabled="harvesterSelected.info.running === true || harvesterSelected.info.reindexing === true || deleting.indexOf(harvesterSelected['@id']) > -1"
+                  title="{{'reindexHarvesterRecordsHelp' | translate}}"
+                >
+                  <span class="fa fa-refresh"></span>&nbsp;
+                  <span
+                    data-translate=""
+                    data-translate-values="{records: '{{searchResults.count}}'}"
+                  >
+                    reindexHarvesterRecords</span
+                  >
+                </button>
               </div>
 
               <div