fix: reject vacuous equivalence and deserialization error false matches in Java comparator

.github/workflows/unit-tests.yaml

@@ -40,6 +40,19 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: maven
+
+      - name: Build codeflash-runtime JAR
+        run: |
+          cd codeflash-java-runtime
+          mvn clean package -q -DskipTests
+          mvn install -q -DskipTests
+
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:

codeflash-java-runtime/src/main/java/com/codeflash/Comparator.java

@@ -50,9 +50,6 @@ public static void main(String[] args) {
             return;
         }
 
-        String originalDbPath = args[0];
-        String candidateDbPath = args[1];
-
         try {
             Class.forName("org.sqlite.JDBC");
         } catch (ClassNotFoundException e) {
@@ -61,80 +58,100 @@ public static void main(String[] args) {
             return;
         }
 
-        Map<String, byte[]> originalResults;
-        Map<String, byte[]> candidateResults;
-
+        String result;
         try {
-            originalResults = readTestResults(originalDbPath);
+            result = compareDatabases(args[0], args[1]);
         } catch (Exception e) {
-            printError("Failed to read original database: " + e.getMessage());
+            printError(e.getMessage());
             System.exit(2);
             return;
         }
 
-        try {
-            candidateResults = readTestResults(candidateDbPath);
-        } catch (Exception e) {
-            printError("Failed to read candidate database: " + e.getMessage());
-            System.exit(2);
-            return;
-        }
+        System.out.println(result);
+        boolean equivalent = result.startsWith("{\"equivalent\":true");
+        System.exit(equivalent ? 0 : 1);
+    }
+
+    static String compareDatabases(String originalDbPath, String candidateDbPath) throws Exception {
+        Map<String, byte[]> originalResults = readTestResults(originalDbPath);
+        Map<String, byte[]> candidateResults = readTestResults(candidateDbPath);
 
         Set<String> allKeys = new LinkedHashSet<>();
         allKeys.addAll(originalResults.keySet());
         allKeys.addAll(candidateResults.keySet());
 
         List<String> diffs = new ArrayList<>();
         int totalInvocations = allKeys.size();
+        int actualComparisons = 0;
+        int skippedPlaceholders = 0;
+        int skippedDeserializationErrors = 0;
 
         for (String key : allKeys) {
             byte[] origBytes = originalResults.get(key);
             byte[] candBytes = candidateResults.get(key);
 
             if (origBytes == null && candBytes == null) {
-                // Both null (void methods) — equivalent
+                // Both null (void methods) — a real comparison (void-to-void match)
+                actualComparisons++;
                 continue;
             }
 
             if (origBytes == null) {
                 Object candObj = safeDeserialize(candBytes);
                 diffs.add(formatDiff("missing", key, 0, null, safeToString(candObj)));
+                actualComparisons++;
                 continue;
             }
 
             if (candBytes == null) {
                 Object origObj = safeDeserialize(origBytes);
                 diffs.add(formatDiff("missing", key, 0, safeToString(origObj), null));
+                actualComparisons++;
                 continue;
             }
 
             Object origObj = safeDeserialize(origBytes);
             Object candObj = safeDeserialize(candBytes);
 
+            if (isDeserializationError(origObj) || isDeserializationError(candObj)) {
+                skippedDeserializationErrors++;
+                continue;
+            }
+
             try {
                 if (!compare(origObj, candObj)) {
                     diffs.add(formatDiff("return_value", key, 0, safeToString(origObj), safeToString(candObj)));
                 }
+                actualComparisons++;
             } catch (KryoPlaceholderAccessException e) {
-                // Placeholder detected — skip comparison for this invocation
+                skippedPlaceholders++;
                 continue;
             }
         }
 
-        boolean equivalent = diffs.isEmpty();
+        boolean equivalent = diffs.isEmpty() && actualComparisons > 0;
+
+        System.err.println("[codeflash-comparator] total=" + totalInvocations
+            + " compared=" + actualComparisons
+            + " skipped_placeholders=" + skippedPlaceholders
+            + " skipped_deser_errors=" + skippedDeserializationErrors
+            + " diffs=" + diffs.size()
+            + " equivalent=" + equivalent);
 
         StringBuilder json = new StringBuilder();
         json.append("{\"equivalent\":").append(equivalent);
         json.append(",\"totalInvocations\":").append(totalInvocations);
+        json.append(",\"actualComparisons\":").append(actualComparisons);
+        json.append(",\"skippedPlaceholders\":").append(skippedPlaceholders);
+        json.append(",\"skippedDeserializationErrors\":").append(skippedDeserializationErrors);
         json.append(",\"diffs\":[");
         for (int i = 0; i < diffs.size(); i++) {
             if (i > 0) json.append(",");
             json.append(diffs.get(i));
         }
         json.append("]}");
 
-        System.out.println(json.toString());
-        System.exit(equivalent ? 0 : 1);
+        return json.toString();
     }
 
     private static Map<String, byte[]> readTestResults(String dbPath) throws Exception {
@@ -178,6 +195,11 @@ private static Object safeDeserialize(byte[] data) {
         }
     }
 
+    static boolean isDeserializationError(Object obj) {
+        if (!(obj instanceof Map)) return false;
+        return "DeserializationError".equals(((Map<?, ?>) obj).get("__type"));
+    }
+
     private static String safeToString(Object obj) {
         if (obj == null) {
             return "null";