Merge pull request #50 from ncsa/release/4.8.0

Release 4.8.0

Author: navarroc

CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [4.8.0] - 2025-07-23
+
+### Added
+- Will now use nodes with datawolf/node-purpose={core,worker} to launch engine and jobs.[#49](https://github.com/ncsa/datawolf/issues/49)
+
+### Changed
+- WorkflowToolData to allow optional datasets [#39](https://github.com/ncsa/datawolf/issues/39)
+- IncoreFileStorage to use the filename as the dataset name [#47](https://github.com/ncsa/datawolf/issues/47)
+
+### Fixed
+- Build failure from update to maven assembly plugin configuration element [#44](https://github.com/ncsa/datawolf/issues/44)
+- Workflow steps didn't fail when tools used wildcards to find outputs and no outputs were found [#41](https://github.com/ncsa/datawolf/issues/41)
+
+
 ## [4.7.0] - 2024-10-29
 
 ### Added

Dockerfile

@@ -23,6 +23,7 @@ ENV [email protected] \
     KUBERNETES_DATA="/data" \
     KUBERNETES_CPU=2 \
     KUBERNETES_MEMORY=4 \
+    KUBERNETES_WORKER_NODE_AFFINITY_REQUIRED=false \
     DATASET_PERMISSIONS=private
 
 EXPOSE 8888

charts/datawolf/Chart.yaml

@@ -13,7 +13,7 @@ version: 1.1.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 4.7.0
+appVersion: 4.8.0
 
 # List of people that maintain this helm chart.
 maintainers:

charts/datawolf/templates/deployment.yaml

@@ -94,6 +94,8 @@ spec:
               value: {{ .Values.jobs.cpu | quote }}
             - name: KUBERNETES_MEMORY
               value: {{ .Values.jobs.memory | quote }}
+            - name: KUBERNETES_WORKER_NODE_AFFINITY_REQUIRED
+              value: {{ .Values.jobs.nodeAffinityRequired | quote }}
             {{- if .Values.extraEnvVars }}
             {{ .Values.extraEnvVars | toYaml | nindent 12 }}
             {{- end }}
@@ -131,8 +133,27 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
       affinity:
+        nodeAffinity:
+          {{- if .Values.engine.nodeAffinityRequired }}
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: datawolf/node-purpose
+                operator: In
+                values:
+                - core
+          {{- else }}
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            preference:
+              matchExpressions:
+              - key: datawolf/node-purpose
+                operator: In
+                values:
+                - core
+          {{- end }}
+      {{- with .Values.affinity }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
       {{- with .Values.tolerations }}

charts/datawolf/values.yaml

@@ -26,12 +26,18 @@ engine:
   extralocalexecutor: 1
   localexecutorthreads: 8
   pagesize: 250
+  # default node selector for engine, this will use the nodes labeled with datawolf/node-purpose=core
+  # if set to true the label must be present on the node, otherwise it will prefer the node.
+  nodeAffinityRequired: false
 
 jobs:
   # default number of cpus per job
   cpu: 2.0
   # default memory in GB per job
   memory: 4.0
+  # default node selector for jobs, this will use the nodes labeled with datawolf/node-purpose=core
+  # if set to true the label must be present on the node, otherwise it will prefer the node.
+  nodeAffinityRequired: false
 
 extraEnvVars: {}
 

datawolf-core/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>edu.illinois.ncsa</groupId>
 		<artifactId>datawolf</artifactId>
-		<version>4.7.0</version>
+		<version>4.8.0</version>
 	</parent>
 	<artifactId>datawolf-core</artifactId>
 

datawolf-core/src/main/java/edu/illinois/ncsa/datawolf/Engine.java

@@ -45,6 +45,7 @@
 import javax.inject.Named;
 import javax.inject.Singleton;
 
+import edu.illinois.ncsa.datawolf.domain.WorkflowToolData;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -566,17 +567,22 @@ public void run() {
                                 if (local >= (LocalExecutor.getWorkers() + getExtraLocalExecutor())) {
                                     canrun = 1;
                                 } else {
-                                    // Transaction transaction = null;
+                                    // Map of step inputs and whether they are required
+                                    Map<String, Boolean> requiredInputs = new HashMap<>();
                                     try {
                                         unitOfWork.begin();
-                                        // transaction =
-// SpringData.getTransaction();
-                                        // transaction.start();
 
                                         execution = executionDao.findOne(exec.getExecutionId());
                                         execution.getDatasets().values();
                                         step = workflowStepDao.findOne(exec.getStepId());
-                                        step.getInputs().values();
+
+                                        // Populate map of step inputs and whether they are required
+                                        for(String inputKey : step.getInputs().values()) {
+                                            WorkflowToolData inputData = step.getInput(inputKey);
+                                            requiredInputs.put(inputKey, inputData.isAllowNull());
+                                        }
+
+//                                        step.getInputs().values();
                                     } catch (Exception e) {
                                         logger.error("Error getting job information.", e);
                                     } finally {
@@ -594,7 +600,8 @@ public void run() {
                                     if (step != null) {
                                         for (String id : step.getInputs().values()) {
                                             if (execution != null) {
-                                                if (!execution.hasDataset(id)) {
+                                                boolean allowNull = requiredInputs.get(id);
+                                                if (!execution.hasDataset(id) && !allowNull) {
                                                     canrun = 1;
                                                 } else if (execution.getDataset(id) == null) {
                                                     canrun = 2;

datawolf-doc/doc/manual/index.html

@@ -247,18 +247,18 @@
 
 https://opensource.ncsa.illinois.edu/projects/artifacts.php?key=WOLF
 
-By default, the latest release is selected in the page (currently 4.7.0). To get early access to development releases, check the box **Show also prereleases.**
+By default, the latest release is selected in the page (currently 4.8.0). To get early access to development releases, check the box **Show also prereleases.**
 
 * Click on **Version** 
-    * Select **4.7.0**
-    * Under **Files** select **datawolf-webapp-all-4.7.0-bin.zip**
+    * Select **4.8.0**
+    * Under **Files** select **datawolf-webapp-all-4.8.0-bin.zip**
 * Click **I Accept** to accept the License.
 
 This will give you the latest stable build that includes both the Data Wolf Server and the Web Editor. You can also find links to the javacode there as well as the manual. The link to the source code can be found at the end of this document.
 
 ### Installation and Setup
 
-To install the files necessary for the Server and Editor, find where you downloaded Data Wolf and unzip it somewhere. This will create a folder called **datawolf-webapp-all-4.7.0**. In the next few sections, we'll discuss some of the important files that come with the installation you just unzipped so you can tailor your setup to meet your needs. If you wish to skip this, you can go directly to the section **Running Data Wolf Server and Editor**.
+To install the files necessary for the Server and Editor, find where you downloaded Data Wolf and unzip it somewhere. This will create a folder called **datawolf-webapp-all-4.8.0**. In the next few sections, we'll discuss some of the important files that come with the installation you just unzipped so you can tailor your setup to meet your needs. If you wish to skip this, you can go directly to the section **Running Data Wolf Server and Editor**.
 
 #### Data Wolf properties
 
@@ -443,7 +443,7 @@
 
 #### Launch Scripts
 
-If you go back to the folder **Data Wolf-webapp-all-4.7.0** you will see a sub-folder called **bin**, open this. Inside you will find two scripts, **datawolf-service** and **datawolf-service.bat**. The latter is intended for running Data Wolf on a Windows machine and the former is for running on Mac & Linux. As with the previous section, knowledge of this file is not required unless you are interested in configuring the Data Wolf Server and Editor beyond the default settings. We will show snippets of the file **datawolf-service** and discuss what each section is configuring.
+If you go back to the folder **Data Wolf-webapp-all-4.8.0** you will see a sub-folder called **bin**, open this. Inside you will find two scripts, **datawolf-service** and **datawolf-service.bat**. The latter is intended for running Data Wolf on a Windows machine and the former is for running on Mac & Linux. As with the previous section, knowledge of this file is not required unless you are interested in configuring the Data Wolf Server and Editor beyond the default settings. We will show snippets of the file **datawolf-service** and discuss what each section is configuring.
 
 ```
 # port for the jetty server

datawolf-doc/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <groupId>edu.illinois.ncsa</groupId>
     <artifactId>datawolf</artifactId>
-    <version>4.7.0</version>
+    <version>4.8.0</version>
   </parent>
   <artifactId>datawolf-doc</artifactId>
 </project>

datawolf-domain/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>edu.illinois.ncsa</groupId>
 		<artifactId>datawolf</artifactId>
-		<version>4.7.0</version>
+		<version>4.8.0</version>
 	</parent>
 	<artifactId>datawolf-domain</artifactId>