Added more geo-options (srs and spatial index file) and added support to resolve globs against the classpath.

Author: Aklakan

pom.xml

@@ -334,6 +334,12 @@
 				<version>0.0.1-3-SNAPSHOT</version>
 			</dependency>
 
+			<dependency>
+				<groupId>io.github.classgraph</groupId>
+				<artifactId>classgraph</artifactId>
+				<version>4.8.129</version>
+			</dependency>
+
 			<dependency>
 				<groupId>org.apache.commons</groupId>
 				<artifactId>commons-vfs2</artifactId>

rdf-processing-toolkit-cli/pom.xml

@@ -65,12 +65,12 @@
 			<groupId>org.aksw.jenax</groupId>
 			<artifactId>jenax-arq-connection-qlever</artifactId>
 		</dependency>
-		
+
 		<dependency>
 			<groupId>org.aksw.jenax</groupId>
 			<artifactId>jenax-graphql-v2-schemagen</artifactId>
 		</dependency>
-		
+
 		<dependency>
 			<groupId>org.aksw.jenax</groupId>
 			<artifactId>jenax-io-core</artifactId>
@@ -89,6 +89,12 @@
 			<groupId>org.aksw.rmltk</groupId>
 			<artifactId>rml-jena-arq</artifactId>
 		</dependency>
+
+		<dependency>
+			<groupId>io.github.classgraph</groupId>
+			<artifactId>classgraph</artifactId>
+		</dependency>
+
 		<dependency>
 			<groupId>net.sansa-stack</groupId>
 			<artifactId>sansa-cmds-picocli_2.13</artifactId>

rdf-processing-toolkit-cli/src/main/java/org/aksw/sparql_integrate/cli/cmd/CmdSparqlIntegrateMain.java

@@ -60,6 +60,8 @@ public class CmdSparqlIntegrateMain
     @Option(names = { "--read-only" },  description="Disable SPARQL update on the server")
     public boolean readOnlyMode = false;
 
+    @Option(names = { "--db-loader-set" }, description="Set an engine loader option")
+    public Map<String, String> dbLoaderOptions = new LinkedHashMap<>();
 
     @Option(names = { "--env" },  description="Set property that can be accessed using the SPARQL function sys:getenv(key).")
     public Map<String, String> env;

rdf-processing-toolkit-cli/src/main/java/org/aksw/sparql_integrate/cli/main/ClassPathResourceResolver.java

@@ -0,0 +1,64 @@
+package org.aksw.sparql_integrate.cli.main;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import io.github.classgraph.ClassGraph;
+import io.github.classgraph.Resource;
+import io.github.classgraph.ResourceList;
+import io.github.classgraph.ScanResult;
+
+public class ClassPathResourceResolver {
+
+    /** Try to resolve the glob to a list of resources on the class path. */
+    public static List<String> resolve(String glob) {
+        // Replace all unescaped backslashes
+        // TODO Fix pattern
+        String normalized = glob.replace('\\', '/');
+
+        String prefix = longestLiteralPrefix(normalized);
+        // Remove leading slashes from the prefix because ClassGraph resources don't start with a slash.
+        normalized = normalized.replaceAll("^//+", "");
+
+        List<String> result = new ArrayList<>();
+        try (ScanResult scanResult = new ClassGraph().acceptPaths(prefix).scan()) {
+            ResourceList resourceList;
+            try {
+                resourceList = scanResult.getResourcesMatchingWildcard(normalized);
+                //ResourceList resourceList = scanResult.getAllResources();
+                for (Resource r : resourceList) {
+                    result.add(r.getPath());
+                }
+            } catch (PatternSyntaxException e) {
+                // Invalid glob pattern - don't match anything.
+            }
+        }
+        Collections.sort(result);
+        return result;
+    }
+
+    /** Longest literal (meta-free) prefix of a Unix-style glob. */
+    public static String longestLiteralPrefix(String glob) {
+        if (glob == null) {
+            throw new NullPointerException("glob");
+        }
+
+        int cut = StringUtils.indexOfAny(glob, '*', '?', '[');
+        if (cut == -1) {
+            cut = glob.length();
+        }
+
+        // Back up over any trailing slashes.
+        int end = cut;
+        while (end > 0 && glob.charAt(end - 1) == '/') {
+            end--;
+        }
+        String prefix = glob.substring(0, end);
+        return prefix;
+    }
+
+}

rdf-processing-toolkit-cli/src/main/java/org/aksw/sparql_integrate/cli/main/SparqlIntegrateCmdImpls.java

@@ -55,7 +55,7 @@
 import org.aksw.jenax.dataaccess.sparql.connection.common.RDFConnectionUtils;
 import org.aksw.jenax.dataaccess.sparql.creator.RDFDatabase;
 import org.aksw.jenax.dataaccess.sparql.creator.RDFDatabaseFactory;
-import org.aksw.jenax.dataaccess.sparql.creator.RdfDatabaseBuilder;
+import org.aksw.jenax.dataaccess.sparql.creator.RDFDatabaseBuilder;
 import org.aksw.jenax.dataaccess.sparql.datasource.RDFDataSource;
 import org.aksw.jenax.dataaccess.sparql.engine.RDFEngine;
 import org.aksw.jenax.dataaccess.sparql.engine.RDFEngines;
@@ -269,7 +269,19 @@ public static int sparqlIntegrate(CmdSparqlIntegrateMain cmd) throws Exception {
             });
         }
 
-        List<String> args = cmd.nonOptionArgs;
+        List<String> rawArgs = cmd.nonOptionArgs;
+
+        List<String> args = new ArrayList<>();
+
+        for (String rawArg : rawArgs) {
+            List<String> resolved = ClassPathResourceResolver.resolve(rawArg);
+            if (resolved.isEmpty()) {
+                args.add(rawArg);
+            } else {
+                logger.info("Resolved " + rawArg + " to " + resolved);
+                args.addAll(resolved);
+            }
+        }
 
         // If an in/out file is given then prepend it to the arguments
         Path outFile = null;
@@ -437,7 +449,10 @@ public static int sparqlIntegrate(CmdSparqlIntegrateMain cmd) throws Exception {
         if (databaseFactory != null) {
             closeablePath = setupDbFolder(cmd);
 
-            RdfDatabaseBuilder<?> databaseBuilder = databaseFactory.newBuilder();
+            RDFDatabaseBuilder<?> databaseBuilder = databaseFactory.newBuilder();
+
+            databaseBuilder.setProperties(cmd.dbLoaderOptions);
+
             Path outputFolder = closeablePath.path();
             databaseBuilder.setOutputFolder(outputFolder);
 
@@ -681,7 +696,7 @@ public static int sparqlIntegrate(CmdSparqlIntegrateMain cmd) throws Exception {
             Document metaDoc = parser.parseDocument(metaSchemaRawStr);
 
             String graphQlSchemaRawStr = toStringUtf8(streamMgr, cmd.graphQlSchema);
-            Document schemaDoc = parser.parseDocument(graphQlSchemaRawStr);
+            Document schemaDoc = GraphQlUtils.parseUnrestricted(graphQlSchemaRawStr);
 
             List<Definition> mergedDefinitions = new ArrayList<>();
             mergedDefinitions.addAll(metaDoc.getDefinitions());
@@ -795,7 +810,7 @@ public void beforeExec() {
                             public void beforeExec() {
                                 Context finalDatasetCxt = finalDataset.getContext();
                                 if (finalDatasetCxt != null && finalDatasetCxt.isFalseOrUndef(SPATIAL_INDEX_IS_CLEAN)) {
-                                    updateSpatialIndex(finalDataset);
+                                    updateSpatialIndex(finalDataset, cmd.arqConfig.geoindexSrs, cmd.arqConfig.geoindexFile);
 
                                     // The the spatial index symbol is in the dataset's context
                                     // copy it into the query exec's context.
@@ -851,7 +866,7 @@ public void beforeExec() {
                                 if (finalDatasetCxt != null && finalDatasetCxt.isFalseOrUndef(SPATIAL_INDEX_IS_CLEAN)) {
                                     spatialUpdateNeeded = isSpatialIndexUpdateImmediatelyRequired(ur);
                                     if (spatialUpdateNeeded) {
-                                        updateSpatialIndex(finalDataset);
+                                        updateSpatialIndex(finalDataset, cmd.arqConfig.geoindexSrs, cmd.arqConfig.geoindexFile);
 
                                         // The the spatial index symbol is in the dataset's context
                                         // copy it into the query exec's context.
@@ -875,7 +890,7 @@ public void afterExec() {
                                 // Defer update
                                 if (false && spatialUpdateNeeded) {
                                     // Note: The update runs in the same transaction as the update!
-                                    updateSpatialIndex(finalDataset);
+                                    updateSpatialIndex(finalDataset, cmd.arqConfig.geoindexSrs, cmd.arqConfig.geoindexFile);
                                 }
                             }
                         };
@@ -1112,11 +1127,11 @@ private static void loadMacros(Set<String> macroProfiles, Map<String, UserDefine
     }
 
     /** Be careful not to call within a read transaction! */
-    public static void updateSpatialIndex(Dataset dataset) {
+    public static void updateSpatialIndex(Dataset dataset, String srs, Path file) {
         Context cxt = dataset.getContext();
-        logger.info("(Re-)computing geo index");
+        // logger.info("(Re-)computing geo index");
         try {
-            GeoSPARQLConfig.setupSpatialIndex(dataset);
+            GeoSPARQLConfig.setupSpatialIndex(dataset, srs, file);
             cxt.setTrue(SPATIAL_INDEX_IS_CLEAN);
         } catch (Exception e) {
             if (e.getMessage().toLowerCase().contains("no srs found")) {