Feature/fdb 327 fdb compare

src/fdb5/CMakeLists.txt

@@ -520,6 +520,27 @@ ecbuild_add_executable(
             remote/fdb-server.cc
         LIBS fdb5 )
 
+ecbuild_add_executable( TARGET    fdb-compare
+                        CONDITION HAVE_FDB_BUILD_TOOLS
+                        SOURCES   tools/compare/fdb-compare.cc
+                                  tools/compare/common/ComparisonMap.cc
+                                  tools/compare/common/DataMap.cc
+                                  tools/compare/common/Scope.cc
+                                  tools/compare/common/Util.cc
+                                  tools/compare/mars/Compare.cc
+                                  tools/compare/grib/Compare.cc
+                                  tools/compare/grib/CompareKeys.cc
+                                  tools/compare/grib/CompareHash.cc
+                                  tools/compare/grib/CompareDatasection.cc
+                                  tools/compare/grib/CompareBitwise.cc
+                                  tools/compare/grib/Utils.cc
+                        INCLUDES  ${ECCODES_INCLUDE_DIRS}  # Please don't remove me, I am needed
+                                  tools/compare/grib/
+                                  tools/compare/mars/
+                                  tools/compare/common/
+                                  tools/compare/
+                        LIBS      fdb5 )
+
 if ( HAVE_FDB_BUILD_TOOLS )
     target_sources( fdb-lock PRIVATE tools/FDBLock.cc tools/FDBLock.h )
     target_sources( fdb-unlock PRIVATE tools/FDBLock.cc tools/FDBLock.h )

src/fdb5/tools/compare/common/ComparisonMap.cc

@@ -0,0 +1,47 @@
+/*
+ * (C) Copyright 2025- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+#include "common/ComparisonMap.h"
+
+
+namespace compare {
+
+bool isSubset(const KeyValueMap& a, const KeyValueMap& b) {
+    for (const auto& kv : a) {
+        auto it = b.find(kv.first);
+        if (it == b.end() || it->second != kv.second)
+            return false;
+    }
+    return true;
+}
+
+
+DataIndex assembleCompareMap(fdb5::FDB& fdb, const fdb5::FDBToolRequest& req, const KeyValueMap& ignore) {
+    DataIndex out;
+
+    auto list = fdb.list(req);
+    fdb5::ListElement elem;
+
+    std::cout << req << std::endl;
+    std::cout << ignore << std::endl;
+    while (list.next(elem)) {
+        KeyValueMap km;
+        for (const auto& bit : elem.keys()) {
+            auto d = bit.keyDict();
+            km.insert(d.begin(), d.end());
+        }
+        if (ignore.empty() || !isSubset(ignore, km)) {
+            out.emplace(km, DataLocation{elem.location().uri().path(), static_cast<long long>(elem.location().offset()),
+                                         static_cast<long long>(elem.location().length())});
+        }
+    }
+    std::cout << "[LOG] FDB request: " << req << " resulted in " << out.size() << " entries.\n";
+    return out;
+}
+}  // namespace compare

src/fdb5/tools/compare/common/ComparisonMap.h

@@ -0,0 +1,25 @@
+/*
+ * (C) Copyright 2025- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+#pragma once
+#include <fdb5/api/FDB.h>
+#include <fdb5/api/helpers/FDBToolRequest.h>
+#include "DataMap.h"
+
+namespace compare {
+
+/// Runs a request on a fdb to create a list of filtered entries to compared on.
+///
+/// @param fdb FDB on which to operate on
+/// @param req Request which performs a selection on the fdb
+/// @param ignore Map of key-value pairs to ignore
+/// @return filtered DataIndex
+DataIndex assembleCompareMap(fdb5::FDB& fdb, const fdb5::FDBToolRequest& req, const KeyValueMap& ignore);
+
+}  // namespace compare

src/fdb5/tools/compare/common/DataMap.cc

@@ -0,0 +1,137 @@
+/*
+ * (C) Copyright 2025- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+#include "DataMap.h"
+
+#include <sstream>
+
+namespace compare {
+
+std::ostream& operator<<(std::ostream& os, const KeyValueMap& km) {
+    os << "{";
+    for (const auto& [k, v] : km) {
+        os << k << "=" << v << ", ";
+    }
+    os << "}";
+    return os;
+}
+
+std::ostream& operator<<(std::ostream& os, const KeyDiffMap& km) {
+    os << "{";
+    for (const auto& [k, v] : km) {
+        os << k << "=" << "(" << (v.first ? *v.first : "<MISSING>") << ", " << (v.second ? *v.second : "<MISSING>")
+           << ")" << ", ";
+    }
+    os << "}";
+    return os;
+}
+
+std::ostream& operator<<(std::ostream& os, const KeySet& km) {
+    os << "{";
+    for (const auto& k : km) {
+        os << k << ", ";
+    }
+    os << "}";
+    return os;
+}
+
+std::ostream& operator<<(std::ostream& os, const DataLocation& loc) {
+    return os << "(" << loc.path << ", " << loc.offset << ", " << loc.length << ")";
+}
+
+std::ostream& operator<<(std::ostream& os, const DataIndex& idx) {
+    for (const auto& [km, loc] : idx) {
+        os << "Key: " << km << " -> Value: " << loc << "\n";
+    }
+    return os;
+}
+
+
+void parseKeyValues(KeyValueMap& container, const std::string& keyValueStr) {
+    std::istringstream stream(keyValueStr);
+    std::string entry;
+
+    while (std::getline(stream, entry, ',')) {
+        std::istringstream pairStream(entry);
+        std::string key, value;
+        if (std::getline(pairStream, key, '=') && std::getline(pairStream, value)) {
+            container[key] = value;  // Update or insert key-value pair
+        }
+        else {
+            std::cerr << "Invalid key-value pair: " << entry << std::endl;
+        }
+    }
+}
+
+KeyValueMap parseKeyValues(const std::string& keyValueStr) {
+    KeyValueMap container;
+    parseKeyValues(container, keyValueStr);
+    return container;
+}
+
+void parseKeySet(KeySet& container, const std::string& keyStr) {
+    std::istringstream stream(keyStr);
+    std::string entry;
+
+    while (std::getline(stream, entry, ',')) {
+        container.insert(entry);  // Insert the whole entry as a single string
+    }
+}
+
+KeySet parseKeySet(const std::string& keyStr) {
+    KeySet container;
+    parseKeySet(container, keyStr);
+    return container;
+}
+
+compare::KeyDiffMap requestDiff(const KeyValueMap& l, const KeyValueMap& r) {
+    compare::KeyDiffMap res;
+    for (const auto& [lk, lv] : l) {
+        auto search = r.find(lk);
+        if (search != r.end()) {
+            if (search->second != lv) {
+                res.insert({lk, {lv, search->second}});
+            }
+        }
+        else {
+            res.insert({lk, {lv, {}}});
+        }
+    }
+
+    for (const auto& [rk, rv] : r) {
+        auto search = l.find(rk);
+        if (search == r.end()) {
+            res.insert({rk, {{}, rv}});
+        }
+    }
+    return res;
+}
+
+
+KeyValueMap applyKeyDiff(KeyValueMap k, const KeyDiffMap& diff, bool swapDiff) {
+    for (const auto& [field, val_pair] : diff) {
+        const auto& val = swapDiff ? val_pair.first : val_pair.second;
+        if (val) {
+            auto it = k.find(field);
+            if (it != k.end()) {
+                it->second = *val;  // replace value
+            }
+            else {
+                k.insert({field, *val});
+            }
+        }
+        else {
+            // Delete
+            k.erase(field);
+        }
+    }
+    return k;  // return modified copy
+};
+
+}  // namespace compare

src/fdb5/tools/compare/common/DataMap.h

@@ -0,0 +1,81 @@
+/*
+ * (C) Copyright 2025- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+#pragma once
+#include <functional>
+#include <iostream>
+#include <map>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+namespace compare {
+
+struct DataLocation {
+    std::string path;
+    long long offset = 0;
+    long long length = 0;
+};
+
+using KeyValueMap = std::map<std::string, std::string>;
+using ValueDiff   = std::pair<std::optional<std::string>, std::optional<std::string>>;
+using KeyDiffMap  = std::map<std::string, ValueDiff>;
+
+using KeySet = std::unordered_set<std::string>;
+
+struct MapHash {
+    size_t operator()(const KeyValueMap& m) const noexcept {
+        size_t h = 0;
+        std::hash<std::string> hs;
+        for (const auto& kv : m) {
+            h ^= (hs(kv.first) * 1315423911u) ^ hs(kv.second);
+        }
+        return h;
+    }
+};
+
+struct MapEqual {
+    bool operator()(const KeyValueMap& a, const KeyValueMap& b) const noexcept { return a == b; }
+};
+
+using DataIndex = std::unordered_map<KeyValueMap, DataLocation, MapHash, MapEqual>;
+
+/// Print a KeyValueMap
+std::ostream& operator<<(std::ostream& os, const KeyValueMap& km);
+
+/// Print a KeyDiffMap
+std::ostream& operator<<(std::ostream& os, const KeyDiffMap& km);
+
+/// Print a KeySet
+std::ostream& operator<<(std::ostream& os, const KeySet& km);
+
+/// Print a DataLocation
+std::ostream& operator<<(std::ostream& os, const DataLocation& loc);
+
+/// Print a DataIndex
+std::ostream& operator<<(std::ostream& os, const DataIndex& idx);
+
+
+void parseKeyValues(KeyValueMap& container, const std::string& keyValueStr);
+KeyValueMap parseKeyValues(const std::string& keyValueStr);
+
+void parseKeySet(KeySet& container, const std::string& keyValueStr);
+KeySet parseKeySet(const std::string& keyValueStr);
+
+compare::KeyDiffMap requestDiff(const KeyValueMap& l, const KeyValueMap& r);
+
+/// @param km Map of keys with values
+/// @param diff Map of keys with differences
+/// @param swapPair Optional - If false the second pair of the difference is put in the resulting request.
+//                  If true the first one is used.
+/// @return New KeyValueMap with differences applied
+KeyValueMap applyKeyDiff(KeyValueMap km, const KeyDiffMap& diff, bool swapPair = false);
+
+}  // namespace compare