Begin prep for final API

Author: BrianMichell

mdio/intersection.h mdio/coordinate_selector.hmdio/intersection.h renamed to mdio/coordinate_selector.h

@@ -14,44 +14,44 @@
 #include "tensorstore/index_space/index_transform_builder.h"
 #include "tensorstore/box.h"
 
+#define MDIO_INTERNAL_PROFILING 0  // TODO(BrianMichell): Remove simple profiling code once we approach a more mature API access.
+
 namespace mdio {
 
+#ifdef MDIO_INTERNAL_PROFILING
+void timer(std::chrono::high_resolution_clock::time_point start) {
+  auto end = std::chrono::high_resolution_clock::now();
+  auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+  std::cout << "Time taken: " << duration.count() << " microseconds" << std::endl;
+}
+#endif
 
 /// \brief Collects valid index selections per dimension for a Dataset without
 /// performing slicing immediately.
 ///
-/// Only dimensions explicitly filtered via add_selection appear in the map;
+/// Only dimensions explicitly filtered via filterByCoordinate appear in the map;
 /// any dimension not present should be treated as having its full index range.
-class IndexSelection {
+class CoordinateSelector {
 public:
   /// Construct from an existing Dataset (captures its full domain).
-  explicit IndexSelection(const Dataset& dataset)
+  explicit CoordinateSelector(const Dataset& dataset)
       : dataset_(dataset), base_domain_(dataset.domain) {}
 
 
   template <typename T>
-  mdio::Future<void> add_selection(const ValueDescriptor<T>& descriptor) {
+  mdio::Future<void> filterByCoordinate(const ValueDescriptor<T>& descriptor) {
     if (kept_runs_.empty()) {
       return _init_runs(descriptor);
     } else {
       return _add_new_run(descriptor);
     }
   }
 
-  /// \brief Emit a RangeDescriptor per surviving tuple coordinate, without coalescing.
-  std::vector<RangeDescriptor<Index>> range_descriptors() const {
-    std::vector<mdio::RangeDescriptor<mdio::Index>> ret;
-    ret.reserve(kept_runs_.size() * kept_runs_[0].size());
-    for (auto const& run : kept_runs_) {
-      for (auto const& interval : run) {  // TODO: This is not an interval!
-        ret.emplace_back(RangeDescriptor<mdio::Index>{interval.label, interval.start, interval.stop, 1});
-      }
-    }
-    return ret;
-  }
-
   template <typename T>
-  Future<void> sort_runs(const std::string& sort_key) {
+  Future<void> sortSelectionByKey(const std::string& sort_key) {
+    #ifdef MDIO_INTERNAL_PROFILING
+    auto start = std::chrono::high_resolution_clock::now();
+    #endif
     auto non_const_ds = dataset_;
     const size_t n = kept_runs_.size();
 
@@ -66,6 +66,11 @@ class IndexSelection {
 
     std::vector<T> keys;
     keys.reserve(n);
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Set up sorting of  " << sort_key << " ... ";
+    timer(start);
+    start = std::chrono::high_resolution_clock::now();
+    #endif
     for (auto &f : reads) {
       // if (!f.status().ok()) return f.status();
       // auto data = f.value();
@@ -77,6 +82,11 @@ class IndexSelection {
       // auto n = std::get<3>(resolution);  // Not required
       keys.push_back(data_ptr[offset]);
     }
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Waiting for reads to complete for " << sort_key << " ... ";
+    timer(start);
+    start = std::chrono::high_resolution_clock::now();
+    #endif
 
     // 2) Build and stable-sort an index array [0…n-1] by key
     std::vector<size_t> idx(n);
@@ -85,6 +95,11 @@ class IndexSelection {
       idx.begin(), idx.end(),
       [&](size_t a, size_t b) { return keys[a] < keys[b]; }
     );
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Sorting time for " << sort_key << " ... ";
+    timer(start);
+    start = std::chrono::high_resolution_clock::now();
+    #endif
 
     // 3) One linear, move-only pass into a temp buffer
     using Desc = std::decay_t<decltype(kept_runs_)>::value_type;
@@ -96,11 +111,18 @@ class IndexSelection {
 
     // 4) Steal the buffer back
     kept_runs_ = std::move(tmp);
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Stealing buffer back time for " << sort_key << " ... ";
+    timer(start);
+    #endif
     return absl::OkStatus();
   }
 
   template <typename T>
-  Future<std::vector<T>> run_values(const std::string& output_variable) {
+  Future<std::vector<T>> readSelection(const std::string& output_variable) {
+    #ifdef MDIO_INTERNAL_PROFILING
+    auto start = std::chrono::high_resolution_clock::now();
+    #endif
     auto non_const_ds = dataset_;
     std::vector<Future<VariableData<T>>> reads;
     reads.reserve(kept_runs_.size());
@@ -116,6 +138,11 @@ class IndexSelection {
       }
     }
 
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Set up reading of " << output_variable << " ... ";
+    timer(start);
+    start = std::chrono::high_resolution_clock::now();
+    #endif
 
     for (auto& f : reads) {
       MDIO_ASSIGN_OR_RETURN(auto resolution, _resolve_future<T>(f));
@@ -127,6 +154,11 @@ class IndexSelection {
       std::memcpy(buffer.data(), data_ptr + offset, n * sizeof(T));
       ret.insert(ret.end(), buffer.begin(), buffer.end());
     }
+
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Reading time for " << output_variable << " ... ";
+    timer(start);
+    #endif
     return ret;
   }
 
@@ -135,9 +167,23 @@ class IndexSelection {
   tensorstore::IndexDomain<> base_domain_;
   std::vector<std::vector<mdio::RangeDescriptor<mdio::Index>>> kept_runs_;
 
+  /*
+  TODO: The built RangeDescriptors aren't behaving as I hoped.
+  They are building the longest runs possible properly, however
+  as it becomes disjointed we start to lose some info.
+
+  e.g. We can have [0,1], [0, 25], [0, 120] but
+  the last dimension is actually [0, 1000]. 
+
+  What we should get instead is [0, 1], [0, 24], [0, 1000] and [0, 1], [24, 25], [0, 120]
+  */
+
   template <typename T>
   Future<void> _init_runs(const ValueDescriptor<T>& descriptor) {
     using Interval = typename Variable<T>::Interval;
+    #ifdef MDIO_INTERNAL_PROFILING
+    auto start = std::chrono::high_resolution_clock::now();
+    #endif
     MDIO_ASSIGN_OR_RETURN(auto var, dataset_.variables.get<T>(std::string(descriptor.label.label())));
     auto fut = var.Read();
     MDIO_ASSIGN_OR_RETURN(auto intervals, var.get_intervals());
@@ -153,6 +199,12 @@ class IndexSelection {
 
     std::size_t run_idx = offset;
 
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Initialize and read time... ";
+    timer(start);
+    start = std::chrono::high_resolution_clock::now();
+    #endif
+
     for (mdio::Index idx = offset; idx < offset + n_samples; ++idx) {
       bool is_match = data_ptr[idx] == descriptor.value;
 
@@ -189,19 +241,97 @@ class IndexSelection {
       }
     }
 
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Build runs time... ";
+    timer(start);
+    start = std::chrono::high_resolution_clock::now();
+    #endif
+
     if (local_runs.empty()) {
       std::stringstream ss;
       ss << "No matches for coordinate '" << descriptor.label.label() << "'";
       return absl::NotFoundError(ss.str());
     }
 
     kept_runs_ = _from_intervals<T>(local_runs);
+    #ifdef MDIO_INTERNAL_PROFILING
+    std::cout << "Finalize time... ";
+    timer(start); 
+    #endif
     return absl::OkStatus();
   }
 
+  /**
+   * @brief Using the existing runs, further filter the Dataset by the new coordiante.
+   */
   template <typename T>
   Future<void> _add_new_run(const ValueDescriptor<T>& descriptor) {
-    return absl::UnimplementedError("Adding selection to an existing IndexSelection is not yet implemented");
+    using Interval = typename Variable<T>::Interval;
+    std::vector<std::vector<Interval>> new_runs;
+
+    std::vector<std::vector<Interval>> stored_intervals; // Use this to ensure everything remains in memory until the Intervals are no longer needed.
+    stored_intervals.reserve(kept_runs_.size());
+
+    auto non_const_ds = dataset_;
+
+    for (const auto& desc : kept_runs_) {
+      MDIO_ASSIGN_OR_RETURN(auto ds, non_const_ds.isel(desc));
+      MDIO_ASSIGN_OR_RETURN(auto var, ds.variables.get<T>(std::string(descriptor.label.label())));
+      auto fut = var.Read();
+      MDIO_ASSIGN_OR_RETURN(auto intervals, var.get_intervals());
+      stored_intervals.push_back(std::move(intervals));  // Just to ensure nothing gets freed prematurely.
+      MDIO_ASSIGN_OR_RETURN(auto resolution, _resolve_future<T>(fut));
+      auto data = std::get<0>(resolution);
+      auto data_ptr = std::get<1>(resolution);
+      auto offset = std::get<2>(resolution);
+      auto n = std::get<3>(resolution);
+
+      auto current_pos = intervals;
+      bool isInRun = false;
+
+      std::size_t run_idx = offset;
+
+      for (Index idx = offset; idx < offset + n; ++idx) {
+        bool is_match = data_ptr[idx] == descriptor.value;
+        if (is_match && !isInRun) {
+          isInRun = true;
+          for (auto i=run_idx; i<idx; ++i) {
+            _current_position_increment<T>(current_pos, intervals);
+          }
+          run_idx = idx;
+          std::vector<Interval> run = current_pos;
+          new_runs.push_back(std::move(run));
+        } else if (is_match && isInRun) {
+          // Somewhere in the middle of a run
+          // do nothing TODO: Remove me
+        } else if (!is_match && isInRun) {
+          // The end of a run
+          isInRun = false;
+          for (auto i=run_idx; i<idx; ++i) {
+            _current_position_increment<T>(current_pos, intervals);
+          }
+          run_idx = idx;
+          auto& last_run = new_runs.back();
+          for (auto i=0; i<current_pos.size(); ++i) {
+            last_run[i].exclusive_max = current_pos[i].inclusive_min + 1;
+          }
+        } else if (!is_match && !isInRun) {
+          // No run at all
+          // do nothing TODO: Remove me
+        } else {
+          // base case TODO: Remove me
+        }
+      }
+    }
+
+    if (new_runs.empty()) {
+      std::stringstream ss;
+      ss << "No matches for coordinate '" << descriptor.label.label() << "'";
+      return absl::NotFoundError(ss.str());
+    }
+
+    kept_runs_ = _from_intervals<T>(new_runs);  // TODO: We need to ensure we don't accidentally drop any pre-sliced dimensions...
+    return absl::OkStatus();
   }
 
   template <typename T>