[feature](filecache)A 2Q-LRU mechanism for protecting hotspot data in the normal queue

be/src/common/config.cpp

@@ -1232,6 +1232,12 @@ DEFINE_mInt64(file_cache_background_lru_dump_update_cnt_threshold, "1000");
 DEFINE_mInt64(file_cache_background_lru_dump_tail_record_num, "5000000");
 DEFINE_mInt64(file_cache_background_lru_log_replay_interval_ms, "1000");
 DEFINE_mBool(enable_evaluate_shadow_queue_diff, "false");
+DEFINE_mBool(enable_file_cache_normal_queue_2qlru, "false");
+DEFINE_Int32(file_cache_2qlru_cold_blocks_percent, "20");
+// Inspired by MySQL's innodb_old_blocks_time, defines the time window (in milliseconds)
+// during which accesses to a cold block do not promote it to the hot queue.
+// Reference: https://dev.mysql.com/doc/refman/8.4/en/innodb-parameters.html#sysvar_innodb_old_blocks_time
+DEFINE_mInt32(file_cache_2qlru_cold_blocks_promotion_ms, "1000");
 
 DEFINE_mBool(file_cache_enable_only_warm_up_idx, "false");
 

be/src/common/config.h

@@ -1275,6 +1275,9 @@ DECLARE_mInt64(file_cache_background_lru_dump_update_cnt_threshold);
 DECLARE_mInt64(file_cache_background_lru_dump_tail_record_num);
 DECLARE_mInt64(file_cache_background_lru_log_replay_interval_ms);
 DECLARE_mBool(enable_evaluate_shadow_queue_diff);
+DECLARE_mBool(enable_file_cache_normal_queue_2qlru);
+DECLARE_Int32(file_cache_2qlru_cold_blocks_percent);
+DECLARE_mInt32(file_cache_2qlru_cold_blocks_promotion_ms);
 
 DECLARE_mBool(file_cache_enable_only_warm_up_idx);
 

be/src/io/cache/block_file_cache.h

@@ -130,8 +130,9 @@ struct FileBlockCell {
     std::optional<LRUQueue::Iterator> queue_iterator;
 
     mutable int64_t atime {0};
+
     void update_atime() const {
-        atime = std::chrono::duration_cast<std::chrono::seconds>(
+        atime = std::chrono::duration_cast<std::chrono::milliseconds>(
                         std::chrono::steady_clock::now().time_since_epoch())
                         .count();
     }
@@ -408,7 +409,7 @@ class BlockFileCache {
 
     void update_block_lru(FileBlockSPtr block, std::lock_guard<std::mutex>& cache_lock);
 
-    void use_cell(const FileBlockCell& cell, FileBlocks* result, bool not_need_move,
+    void use_cell(FileBlockCell& cell, FileBlocks* result, bool not_need_move,
                   std::lock_guard<std::mutex>& cache_lock);
 
     bool try_reserve_for_lru(const UInt128Wrapper& hash, QueryFileCacheContextPtr query_context,
@@ -539,6 +540,7 @@ class BlockFileCache {
     LRUQueue _normal_queue;
     LRUQueue _disposable_queue;
     LRUQueue _ttl_queue;
+    LRUQueue _cold_normal_queue;
 
     // keys for async remove
     RecycleFileCacheKeys _recycle_keys;
@@ -555,19 +557,21 @@ class BlockFileCache {
     std::shared_ptr<bvar::Status<size_t>> _cur_ttl_cache_lru_queue_element_count_metrics;
     std::shared_ptr<bvar::Status<size_t>> _cur_normal_queue_element_count_metrics;
     std::shared_ptr<bvar::Status<size_t>> _cur_normal_queue_cache_size_metrics;
+    std::shared_ptr<bvar::Status<size_t>> _cur_cold_normal_queue_element_count_metrics;
+    std::shared_ptr<bvar::Status<size_t>> _cur_cold_normal_queue_cache_size_metrics;
     std::shared_ptr<bvar::Status<size_t>> _cur_index_queue_element_count_metrics;
     std::shared_ptr<bvar::Status<size_t>> _cur_index_queue_cache_size_metrics;
     std::shared_ptr<bvar::Status<size_t>> _cur_disposable_queue_element_count_metrics;
     std::shared_ptr<bvar::Status<size_t>> _cur_disposable_queue_cache_size_metrics;
-    std::array<std::shared_ptr<bvar::Adder<size_t>>, 4> _queue_evict_size_metrics;
+    std::array<std::shared_ptr<bvar::Adder<size_t>>, 5> _queue_evict_size_metrics;
     std::shared_ptr<bvar::Adder<size_t>> _total_read_size_metrics;
     std::shared_ptr<bvar::Adder<size_t>> _total_hit_size_metrics;
     std::shared_ptr<bvar::Adder<size_t>> _total_evict_size_metrics;
     std::shared_ptr<bvar::Adder<size_t>> _gc_evict_bytes_metrics;
     std::shared_ptr<bvar::Adder<size_t>> _gc_evict_count_metrics;
-    std::shared_ptr<bvar::Adder<size_t>> _evict_by_time_metrics_matrix[4][4];
-    std::shared_ptr<bvar::Adder<size_t>> _evict_by_size_metrics_matrix[4][4];
-    std::shared_ptr<bvar::Adder<size_t>> _evict_by_self_lru_metrics_matrix[4];
+    std::shared_ptr<bvar::Adder<size_t>> _evict_by_time_metrics_matrix[5][5];
+    std::shared_ptr<bvar::Adder<size_t>> _evict_by_size_metrics_matrix[5][5];
+    std::shared_ptr<bvar::Adder<size_t>> _evict_by_self_lru_metrics_matrix[5];
     std::shared_ptr<bvar::Adder<size_t>> _evict_by_try_release;
 
     std::shared_ptr<bvar::Window<bvar::Adder<size_t>>> _num_hit_blocks_5m;

be/src/io/cache/cache_lru_dumper.cpp

@@ -482,6 +482,10 @@ void CacheLRUDumper::restore_queue(LRUQueue& queue, const std::string& queue_nam
                 ctx.cache_type = FileCacheType::NORMAL;
             } else if (queue_name == "disposable") {
                 ctx.cache_type = FileCacheType::DISPOSABLE;
+            } else if (queue_name == "cold_normal") {
+                ctx.cache_type = config::enable_file_cache_normal_queue_2qlru
+                                         ? FileCacheType::COLD_NORMAL
+                                         : FileCacheType::NORMAL;
             } else {
                 LOG_WARNING("unknown queue type for lru restore, skip");
                 DCHECK(false);
@@ -498,7 +502,7 @@ void CacheLRUDumper::restore_queue(LRUQueue& queue, const std::string& queue_nam
 };
 
 void CacheLRUDumper::remove_lru_dump_files() {
-    std::vector<std::string> queue_names = {"disposable", "index", "normal", "ttl"};
+    std::vector<std::string> queue_names = {"disposable", "index", "normal", "ttl", "cold_normal"};
     for (const auto& queue_name : queue_names) {
         std::string filename =
                 fmt::format("{}/lru_dump_{}.tail", _mgr->_cache_base_path, queue_name);

be/src/io/cache/file_block.cpp

@@ -75,6 +75,11 @@ uint64_t FileBlock::get_caller_id() {
     return id;
 }
 
+void FileBlock::set_cache_type(FileCacheType new_type) {
+    std::lock_guard block_lock(_mutex);
+    _key.meta.type = new_type;
+}
+
 uint64_t FileBlock::get_or_set_downloader() {
     std::lock_guard block_lock(_mutex);
 

be/src/io/cache/file_block.h

@@ -115,6 +115,8 @@ class FileBlock {
 
     FileCacheType cache_type() const { return _key.meta.type; }
 
+    void set_cache_type(FileCacheType new_type);
+
     int64_t tablet_id() const { return _key.meta.tablet_id; }
 
     void set_tablet_id(int64_t id) { _key.meta.tablet_id = id; }

be/src/io/cache/file_cache_common.cpp

@@ -36,6 +36,8 @@ std::string cache_type_to_surfix(FileCacheType type) {
         return "";
     case FileCacheType::TTL:
         return "_ttl";
+    case FileCacheType::COLD_NORMAL:
+        return "_cold_normal";
     }
     return "";
 }
@@ -47,6 +49,8 @@ FileCacheType surfix_to_cache_type(const std::string& str) {
         return FileCacheType::DISPOSABLE;
     } else if (str == "ttl") {
         return FileCacheType::TTL;
+    } else if (str == "cold_normal") {
+        return FileCacheType::COLD_NORMAL;
     }
     DCHECK(false) << "The string is " << str;
     return FileCacheType::DISPOSABLE;
@@ -61,6 +65,8 @@ FileCacheType string_to_cache_type(const std::string& str) {
         return FileCacheType::DISPOSABLE;
     } else if (str == "ttl") {
         return FileCacheType::TTL;
+    } else if (str == "cold_normal") {
+        return FileCacheType::COLD_NORMAL;
     }
     DCHECK(false) << "The string is " << str;
     return FileCacheType::NORMAL;
@@ -75,6 +81,8 @@ std::string cache_type_to_string(FileCacheType type) {
         return "normal";
     case FileCacheType::TTL:
         return "ttl";
+    case FileCacheType::COLD_NORMAL:
+        return "cold_normal";
     }
     DCHECK(false) << "unknown type: " << type;
     return "normal";
@@ -90,7 +98,9 @@ std::string FileCacheSettings::to_string() const {
        << ", index_queue_elements: " << index_queue_elements
        << ", ttl_queue_size: " << ttl_queue_size << ", ttl_queue_elements: " << ttl_queue_elements
        << ", query_queue_size: " << query_queue_size
-       << ", query_queue_elements: " << query_queue_elements << ", storage: " << storage;
+       << ", query_queue_elements: " << query_queue_elements
+       << ", cold_query_queue_size: " << cold_query_queue_size
+       << ", cold_query_queue_elements: " << cold_query_queue_elements << ", storage: " << storage;
     return ss.str();
 }
 
@@ -125,6 +135,20 @@ FileCacheSettings get_file_cache_settings(size_t capacity, size_t max_query_cach
     settings.query_queue_elements =
             std::max(settings.query_queue_size / settings.max_file_block_size,
                      REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
+
+    if (config::enable_file_cache_normal_queue_2qlru) {
+        size_t normal_queue_per_size = settings.query_queue_size / 100;
+        size_t normal_queue_per_elements = settings.query_queue_elements / 100;
+
+        settings.cold_query_queue_size =
+                normal_queue_per_size * config::file_cache_2qlru_cold_blocks_percent;
+        settings.cold_query_queue_elements =
+                normal_queue_per_elements * config::file_cache_2qlru_cold_blocks_percent;
+
+        settings.query_queue_size -= settings.cold_query_queue_size;
+        settings.query_queue_elements -= settings.cold_query_queue_elements;
+    }
+
     settings.storage = storage;
     return settings;
 }
@@ -136,7 +160,10 @@ std::string UInt128Wrapper::to_string() const {
 FileBlocksHolderPtr FileCacheAllocatorBuilder::allocate_cache_holder(size_t offset, size_t size,
                                                                      int64_t tablet_id) const {
     CacheContext ctx;
-    ctx.cache_type = _expiration_time == 0 ? FileCacheType::NORMAL : FileCacheType::TTL;
+    ctx.cache_type = _expiration_time == 0 ? (config::enable_file_cache_normal_queue_2qlru
+                                                      ? FileCacheType::COLD_NORMAL
+                                                      : FileCacheType::NORMAL)
+                                           : FileCacheType::TTL;
     ctx.expiration_time = _expiration_time;
     ctx.is_cold_data = _is_cold_data;
     ctx.tablet_id = tablet_id;

be/src/io/cache/file_cache_common.h

@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <vector>
 
+#include "common/config.h"
 #include "io/io_common.h"
 #include "vec/common/uint128.h"
 
@@ -41,6 +42,7 @@ enum FileCacheType {
     NORMAL = 1,
     DISPOSABLE = 0,
     TTL = 3,
+    COLD_NORMAL = 4,
 };
 std::string cache_type_to_surfix(FileCacheType type);
 FileCacheType surfix_to_cache_type(const std::string& str);
@@ -129,6 +131,8 @@ struct FileCacheSettings {
     size_t index_queue_elements {0};
     size_t query_queue_size {0};
     size_t query_queue_elements {0};
+    size_t cold_query_queue_size {0};
+    size_t cold_query_queue_elements {0};
     size_t ttl_queue_size {0};
     size_t ttl_queue_elements {0};
     size_t max_file_block_size {0};
@@ -156,7 +160,9 @@ struct CacheContext {
         } else if (io_context->is_disposable) {
             cache_type = FileCacheType::DISPOSABLE;
         } else {
-            cache_type = FileCacheType::NORMAL;
+            cache_type = doris::config::enable_file_cache_normal_queue_2qlru
+                                 ? FileCacheType::COLD_NORMAL
+                                 : FileCacheType::NORMAL;
         }
         query_id = io_context->query_id ? *io_context->query_id : TUniqueId();
         is_warmup = io_context->is_warmup;

be/src/io/cache/fs_file_cache_storage.cpp

@@ -591,7 +591,8 @@ Status FSFileCacheStorage::parse_filename_suffix_to_cache_type(
     if (expiration_time > 0) {
         *cache_type = FileCacheType::TTL;
     } else if (*cache_type == FileCacheType::TTL && expiration_time == 0) {
-        *cache_type = FileCacheType::NORMAL;
+        *cache_type = config::enable_file_cache_normal_queue_2qlru ? FileCacheType::COLD_NORMAL
+                                                                   : FileCacheType::NORMAL;
     }
 
     if (!parsed) {

be/src/io/cache/lru_queue_recorder.cpp

@@ -105,6 +105,8 @@ LRUQueue& LRUQueueRecorder::get_shadow_queue(FileCacheType type) {
         return _shadow_normal_queue;
     case FileCacheType::TTL:
         return _shadow_ttl_queue;
+    case FileCacheType::COLD_NORMAL:
+        return _shadow_cold_normal_queue;
     default:
         LOG(WARNING) << "invalid shadow queue type";
         DCHECK(false);
@@ -122,6 +124,8 @@ CacheLRULogQueue& LRUQueueRecorder::get_lru_log_queue(FileCacheType type) {
         return _normal_lru_log_queue;
     case FileCacheType::TTL:
         return _ttl_lru_log_queue;
+    case FileCacheType::COLD_NORMAL:
+        return _cold_normal_lru_log_queue;
     default:
         LOG(WARNING) << "invalid lru log queue type";
         DCHECK(false);

be/src/io/cache/lru_queue_recorder.h

@@ -54,6 +54,7 @@ class LRUQueueRecorder {
         _lru_queue_update_cnt_from_last_dump[FileCacheType::NORMAL] = 0;
         _lru_queue_update_cnt_from_last_dump[FileCacheType::INDEX] = 0;
         _lru_queue_update_cnt_from_last_dump[FileCacheType::TTL] = 0;
+        _lru_queue_update_cnt_from_last_dump[FileCacheType::COLD_NORMAL] = 0;
     }
     void record_queue_event(FileCacheType type, CacheLRULogType log_type, const UInt128Wrapper hash,
                             const size_t offset, const size_t size);
@@ -74,11 +75,13 @@ class LRUQueueRecorder {
     LRUQueue _shadow_normal_queue;
     LRUQueue _shadow_disposable_queue;
     LRUQueue _shadow_ttl_queue;
+    LRUQueue _shadow_cold_normal_queue;
 
     CacheLRULogQueue _ttl_lru_log_queue;
     CacheLRULogQueue _index_lru_log_queue;
     CacheLRULogQueue _normal_lru_log_queue;
     CacheLRULogQueue _disposable_lru_log_queue;
+    CacheLRULogQueue _cold_normal_lru_log_queue;
 
     std::unordered_map<FileCacheType, size_t> _lru_queue_update_cnt_from_last_dump;
 

be/src/olap/options.cpp

@@ -258,7 +258,15 @@ Status parse_conf_cache_paths(const std::string& config_path, std::vector<CacheP
         if (total_size < 0 || (config::enable_file_cache_query_limit && query_limit_bytes < 0)) {
             return Status::InvalidArgument("total_size or query_limit should not less than zero");
         }
-
+        if (config::enable_file_cache_normal_queue_2qlru) {
+            // The configurable range is inspired by MySQL's innodb_old_blocks_pct
+            // Reference: https://dev.mysql.com/doc/refman/8.4/en/innodb-parameters.html#sysvar_innodb_old_blocks_pct
+            if (config::file_cache_2qlru_cold_blocks_percent < 5 ||
+                config::file_cache_2qlru_cold_blocks_percent > 95) {
+            }
+            return Status::InvalidArgument(
+                    "file_cache_2qlru_cold_blocks_percent must be between 5 and 95.");
+        }
         // percent
         auto get_percent_value = [&](const std::string& key, size_t& percent) {
             auto& value = map.FindMember(key.c_str())->value;