Add support for infoset dataType member

daffodil-core/src/main/java/org/apache/daffodil/api/infoset/InfosetOutputter.java

@@ -47,19 +47,28 @@ public abstract class InfosetOutputter {
    * blob directory path which defaults to java temp dir
    */
   private Path blobDirectory = Paths.get(System.getProperty("java.io.tmpdir"));
+
   /**
    * blob prefix which defaults to daffodil-
    */
   private String blobPrefix = "daffodil-";
+
   /**
    * blob suffix which defaults to .blob
    */
   private String blobSuffix = ".blob";
+
   /**
    * list of blob paths output in the infoset
    */
   private List<Path> blobPaths;
 
+  /**
+   * whether or not to enable the dataType infoset member. It is up to
+   * InfosetOutputter implementations if and how to include member.
+   */
+  private boolean includeDataType = false;
+
   /**
    * Reset the internal state of this InfosetOutputter. This should be called
    * in between calls to the parse method.
@@ -205,4 +214,22 @@ final public List<Path> getBlobPaths() {
   final public void setBlobPaths(List<Path> blobPaths) {
     this.blobPaths = blobPaths;
   }
+
+  /**
+   * Set whether the InfosetOutputter should include the dataType member when
+   * it outputs infoset elements. It is up to InfosetOutputter implementations
+   * if and how to output the dataType
+   */
+  final public void setIncludeDataType(boolean includeDataType) {
+    this.includeDataType = includeDataType;
+  }
+
+  /**
+   * Get whether the InfosetOutputter should include the dataType member when
+   * it outputs infoset elements. It is up to InfosetOutputter implementations
+   * if and how to output this dataType
+   */
+  final public boolean getIncludeDataType() {
+    return includeDataType;
+  }
 }

daffodil-core/src/main/scala/org/apache/daffodil/lib/xml/XMLUtils.scala

@@ -912,6 +912,18 @@ Differences were (path, expected, actual):
     res
   }
 
+  // Normalize xmlns mappings to improve comparisons. Avoids differences in whitespace,
+  // namespace ordering, and bindings like xmlns:xsi which is optionally added by some infoset
+  // outputters that have support for xsi:type
+  private def normalizeMappings(mappings: String): String = {
+    mappings
+      .trim()
+      .split("\\s+")
+      .filterNot(_.startsWith("xmlns:xsi="))
+      .sorted
+      .mkString(" ")
+  }
+
   def computeDiffOne(
     an: Node,
     bn: Node,
@@ -939,13 +951,21 @@ Differences were (path, expected, actual):
             (prefixB, labelB, attribsB, nsbB, childrenB)
           case x => Assert.invariantFailed(s"Expected elem, found $x")
         }
-        val typeA: Option[String] = getXSIType(a)
-        val typeB: Option[String] = getXSIType(b)
+        // some TDML files use xsd prefixes for xsi:type values. Instead of trying to resolve
+        // the prefix, just replace "xsd:" with "xs:"--the rest of our code assumes xs prefixes
+        val typeA: Option[String] = getXSIType(a).map { t =>
+          if (t.startsWith("xsd:")) "xs:" + t.substring(4) else t
+        }
+        val typeB: Option[String] = getXSIType(b).map { t =>
+          if (t.startsWith("xsd:")) "xs:" + t.substring(4) else t
+        }
         val maybeType: Option[String] = Option(typeA.getOrElse(typeB.getOrElse(null)))
         val nilledA = a.attribute(XSI_NAMESPACE.toString, "nil")
         val nilledB = b.attribute(XSI_NAMESPACE.toString, "nil")
-        val mappingsA = if (checkNamespaces) nsbA.buildString(aParentScope).trim else ""
-        val mappingsB = if (checkNamespaces) nsbB.buildString(bParentScope).trim else ""
+        val mappingsA =
+          if (checkNamespaces) normalizeMappings(nsbA.buildString(aParentScope)) else ""
+        val mappingsB =
+          if (checkNamespaces) normalizeMappings(nsbB.buildString(bParentScope)) else ""
 
         if (labelA != labelB) {
           // different label
@@ -965,15 +985,9 @@ Differences were (path, expected, actual):
               nilledB.map(_.toString).getOrElse("")
             )
           )
-        } else if (typeA != typeB && typeA.isDefined && typeB.isDefined) {
+        } else if (typeA.isDefined && typeB.isDefined && typeA.get != typeB.get) {
           // different xsi:type (if both suppplied)
-          List(
-            (
-              zPath + "/" + labelA + "@xsi:type",
-              typeA.map(_.toString).getOrElse(""),
-              typeA.map(_.toString).getOrElse("")
-            )
-          )
+          List((zPath + "/" + labelA + "@xsi:type", typeA.get, typeB.get))
         } else {
           val pathLabel = labelA + maybeIndex.map("[" + _ + "]").getOrElse("")
           val thisPathStep = pathLabel +: parentPathSteps

daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/InfosetImpl.scala

@@ -1543,6 +1543,10 @@ sealed class DISimple(override val erd: ElementRuntimeData)
           else if (f == Float.NegativeInfinity) XMLUtils.NegativeInfinityString
           else f.toString
         }
+        case d: java.math.BigDecimal => {
+          // scientific notation is not allowed by XSD for xs:decimal
+          d.toPlainString
+        }
         case x => x.toString
       }
     }

daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/ScalaXMLInfosetOutputter.scala

@@ -19,7 +19,9 @@ package org.apache.daffodil.runtime1.infoset
 
 import scala.collection.mutable.ListBuffer
 import scala.xml.MetaData
+import scala.xml.NamespaceBinding
 import scala.xml.Null
+import scala.xml.PrefixedAttribute
 import scala.xml.UnprefixedAttribute
 
 import org.apache.daffodil.api.DFDLPrimType
@@ -54,6 +56,16 @@ class ScalaXMLInfosetOutputter(showFreedInfo: Boolean = false)
     resultNode = Maybe(root(0))
   }
 
+  private def getScope(diElem: DIElement): NamespaceBinding = {
+    val minScope = diElem.metadata.minimizedScope
+    // if including xsi:type is enabled, ensure the xsi namespace is defined on the root element
+    if (getIncludeDataType() && stack.length == 1 && minScope.getURI("xsi") == null) {
+      NamespaceBinding("xsi", XMLUtils.XSI_NAMESPACE, minScope)
+    } else {
+      minScope
+    }
+  }
+
   private def getAttributes(diElem: DIElement): MetaData = {
     val nilAttr = if (diElem.isNilled) XMLUtils.xmlNilAttribute else Null
     val freedAttr =
@@ -80,7 +92,14 @@ class ScalaXMLInfosetOutputter(showFreedInfo: Boolean = false)
       } else {
         nilAttr
       }
-    freedAttr
+    val typedAttr =
+      if (getIncludeDataType() && diElem.isSimple) {
+        val primName = diElem.erd.optPrimType.get.name
+        new PrefixedAttribute("xsi", "type", "xs:" + primName, freedAttr)
+      } else {
+        freedAttr
+      }
+    typedAttr
   }
 
   override def startSimple(se: InfosetSimpleElement): Unit = {
@@ -105,7 +124,7 @@ class ScalaXMLInfosetOutputter(showFreedInfo: Boolean = false)
         diSimple.metadata.prefix,
         diSimple.metadata.name,
         attributes,
-        diSimple.metadata.minimizedScope,
+        getScope(diSimple),
         minimizeEmpty = true,
         children*
       )
@@ -130,7 +149,7 @@ class ScalaXMLInfosetOutputter(showFreedInfo: Boolean = false)
         diComplex.metadata.prefix,
         diComplex.metadata.name,
         attributes,
-        diComplex.metadata.minimizedScope,
+        getScope(diComplex),
         minimizeEmpty = true,
         children*
       )

daffodil-core/src/main/scala/org/apache/daffodil/runtime1/infoset/XMLTextInfosetOutputter.scala

@@ -82,9 +82,12 @@ class XMLTextInfosetOutputter private (
    */
   private var inScopeComplexElementHasChildren = false
 
+  private var hasStartedRoot = false
+
   override def reset(): Unit = {
     resetIndentation()
     inScopeComplexElementHasChildren = false
+    hasStartedRoot = false
   }
 
   private def outputTagName(elem: DIElement): Unit = {
@@ -109,6 +112,16 @@ class XMLTextInfosetOutputter private (
         nsbStart.buildString(sb, nsbEnd)
         writer.write(sb.toString)
       }
+      // if including xsi:type is enabled, ensure the xsi namespace is defined on the root
+      // element
+      if (getIncludeDataType() && !hasStartedRoot && nsbStart.getURI("xsi") == null) {
+        writer.write(" xmlns:xsi=\"" + XMLUtils.XSI_NAMESPACE + "\"")
+      }
+    }
+
+    if (getIncludeDataType() && elem.isSimple) {
+      val primName = elem.erd.optPrimType.get.name
+      writer.write(" xsi:type=\"xs:" + primName + "\"")
     }
 
     if (elem.isNilled) {
@@ -223,6 +236,7 @@ class XMLTextInfosetOutputter private (
 
     outputEndTag(simple)
     inScopeComplexElementHasChildren = true
+    hasStartedRoot = true
   }
 
   override def endSimple(simple: InfosetSimpleElement): Unit = {
@@ -238,6 +252,7 @@ class XMLTextInfosetOutputter private (
     outputStartTag(complex)
     incrementIndentation()
     inScopeComplexElementHasChildren = false
+    hasStartedRoot = true
   }
 
   override def endComplex(ce: InfosetComplexElement): Unit = {

daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala

@@ -287,6 +287,11 @@ class DataProcessor(
     output: api.infoset.InfosetOutputter
   ): DFDL.ParseResult = {
     checkNotError()
+
+    if (tunables.infosetIncludeDataType) {
+      output.setIncludeDataType(true)
+    }
+
     // If full validation is enabled, tee all the infoset events to a second
     // infoset outputter that writes the infoset to a byte array, and then
     // we'll validate that byte array upon a successful parse.

daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd

@@ -273,6 +273,14 @@
             </xs:restriction>
           </xs:simpleType>
         </xs:element>
+        <xs:element name="infosetIncludeDataType" type="xs:boolean" default="false" minOccurs="0">
+          <xs:annotation>
+            <xs:documentation>
+              Whether or not to include the dataType infoset member when outputting an infoset. If
+              and how this is represented in the infoset is up to InfosetOutputter implementations.
+            </xs:documentation>
+          </xs:annotation>
+        </xs:element>
         <xs:element name="inputFileMemoryMapLowThreshold" type="xs:int" default="33554432" minOccurs="0">
           <xs:annotation>
             <xs:documentation>

daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/TDMLInfosetOutputter.scala

@@ -57,6 +57,7 @@ class TDMLInfosetOutputterScala(scalaOut: ScalaXMLInfosetOutputter)
 object TDMLInfosetOutputterScala {
   def apply(): TDMLInfosetOutputterScala = {
     val scalaOut = new ScalaXMLInfosetOutputter()
+    scalaOut.setIncludeDataType(true)
     new TDMLInfosetOutputterScala(scalaOut)
   }
 }
@@ -99,6 +100,10 @@ object TDMLInfosetOutputterAll {
     val jsonOut = new JsonInfosetOutputter(jsonStream, false)
     val xmlOut = new XMLTextInfosetOutputter(xmlStream, false)
 
+    Seq(scalaOut, jdomOut, w3cdomOut, jsonOut, xmlOut).foreach { out =>
+      out.setIncludeDataType(true)
+    }
+
     new TDMLInfosetOutputterAll(
       jsonStream,
       xmlStream,

daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestTDMLRunner.scala

@@ -690,7 +690,7 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
     runner.runOneTest("testTDMLHexBinaryTypeAwareSuccess")
   }
 
-  @Test def testTDMLHexBinaryTypeAwareFailure(): Unit = {
+  @Test def testTDMLHexBinaryTypeAwareSuccess_03(): Unit = {
     val testSuite = <ts:testSuite xmlns:ts={tdml} suiteName="theSuiteName" xmlns:xs={
       xsd
     } xmlns:dfdl={dfdl} xmlns:tns={example}>
@@ -699,7 +699,7 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
                         <dfdl:format ref="tns:GeneralFormat"/>
                         <xs:element name="data" type="xs:hexBinary" dfdl:lengthKind="explicit" dfdl:length="4"/>
                       </ts:defineSchema>
-                      <ts:parserTestCase ID="some identifier" name="testTDMLHexBinaryTypeAwareFailure"
+                      <ts:parserTestCase ID="some identifier" name="testTDMLHexBinaryTypeAwareSuccess"
                         root="data" model="mySchema">
                         <ts:document>
                           <ts:documentPart type="byte">A1B2C3D4</ts:documentPart>
@@ -712,13 +712,7 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
                       </ts:parserTestCase>
                     </ts:testSuite>
     val runner = new Runner(testSuite)
-    val e = intercept[Exception] {
-      runner.runOneTest("testTDMLHexBinaryTypeAwareFailure")
-    }
-    val msg = e.getMessage()
-    assertTrue(msg.contains("Comparison failed"))
-    assertTrue(msg.contains("a1b2c3d4"))
-    assertTrue(msg.contains("A1B2C3D4"))
+    runner.runOneTest("testTDMLHexBinaryTypeAwareSuccess")
   }
 
   @Test def testTDMLDateTimeTypeAwareSuccess_01(): Unit = {
@@ -825,7 +819,7 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
     runner.runOneTest("testTDMLDateTimeTypeAwareSuccess")
   }
 
-  @Test def testTDMLDateTimeTypeAwareFailure(): Unit = {
+  @Test def testTDMLDateTimeTypeAwareSuccess_05(): Unit = {
     val testSuite = <ts:testSuite xmlns:ts={tdml} suiteName="theSuiteName" xmlns:xs={
       xsd
     } xmlns:dfdl={dfdl} xmlns:tns={example}>
@@ -836,7 +830,7 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
                           dfdl:calendarPatternKind="explicit"
                           dfdl:calendarPattern="uuuu-MM-dd'T'HH:mm:ss.SSSSSSxxxxx" />
                       </ts:defineSchema>
-                      <ts:parserTestCase ID="some identifier" name="testTDMLDateTimeTypeAwareFailure"
+                      <ts:parserTestCase ID="some identifier" name="testTDMLDateTimeTypeAwareSuccess"
                         root="data" model="mySchema">
                         <ts:document>1995-03-24T01:30:00.000000+00:00</ts:document>
                         <ts:infoset>
@@ -847,13 +841,7 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
                       </ts:parserTestCase>
                     </ts:testSuite>
     val runner = new Runner(testSuite)
-    val e = intercept[Exception] {
-      runner.runOneTest("testTDMLDateTimeTypeAwareFailure")
-    }
-    val msg = e.getMessage()
-    assertTrue(msg.contains("Comparison failed"))
-    assertTrue(msg.contains("1995-03-24T01:30:00Z"))
-    assertTrue(msg.contains("1995-03-24T01:30:00+00:00"))
+    runner.runOneTest("testTDMLDateTimeTypeAwareSuccess")
   }
 
   /**

daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml

@@ -3345,7 +3345,7 @@
     </tdml:document>
     <tdml:infoset>
       <tdml:dfdlInfoset>
-        <dec_03 xsi:type="xs:double">1.23456789E+13</dec_03>
+        <dec_03 xsi:type="xs:decimal">12345678900000</dec_03>
       </tdml:dfdlInfoset>
     </tdml:infoset>
   </tdml:parserTestCase>
@@ -3357,7 +3357,7 @@
     </tdml:document>
     <tdml:infoset>
       <tdml:dfdlInfoset>
-        <dec_04 xsi:type="xs:double">1E-200</dec_04>
+        <dec_04 xsi:type="xs:decimal">0.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001</dec_04>
       </tdml:dfdlInfoset>
     </tdml:infoset>
   </tdml:parserTestCase>
@@ -3369,7 +3369,7 @@
     </tdml:document>
     <tdml:infoset>
       <tdml:dfdlInfoset>
-        <dec_05 xsi:type="xs:double">1E+200</dec_05>
+        <dec_05 xsi:type="xs:decimal">100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000</dec_05>
       </tdml:dfdlInfoset>
     </tdml:infoset>
   </tdml:parserTestCase>