diff --git a/cpp/README-zh.md b/cpp/README-zh.md index bb20dcd92..a8af952b1 100644 --- a/cpp/README-zh.md +++ b/cpp/README-zh.md @@ -99,6 +99,12 @@ sudo apt-get install -y cmake make g++ clang-format libuuid-dev bash build.sh ``` +`build.sh` 默认只编译,不执行安装。如果需要安装到 CMake 的安装前缀目录,显式传入 `install` 参数: + +```bash +bash build.sh install +``` + 如果你安装了 Maven 工具,也可以运行: ```bash diff --git a/cpp/build.sh b/cpp/build.sh index d2950595b..f51c4eded 100644 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -21,6 +21,7 @@ build_type=Release build_test=0 build_bench=0 +do_install=0 use_cpp11=1 enable_cov=0 debug_se=0 @@ -39,10 +40,37 @@ get_key_value() { echo "${1#*=}" } +usage() +{ + cat <, -t Build type: Debug, Release, RelWithDebInfo, MinSizeRel. + -a= Enable or disable AddressSanitizer. + -c= Enable or disable code coverage. + --enable-antlr4= + --disable-antlr4 + --enable-snappy= + --disable-snappy + --enable-lz4= + --disable-lz4 + --enable-lzokay= + --disable-lzokay + --enable-zlib= + --disable-zlib + -h, --help Show this help message. +EOF +} + function print_config() { echo "build_type=$build_type" echo "build_test=$build_test" + echo "do_install=$do_install" echo "use_cpp11=$use_cpp11" echo "enable_cov=$enable_cov" echo "enable_asan=$enable_asan" @@ -68,6 +96,8 @@ parse_options() do_clean=1;; run_cov) run_cov_only=1;; + install | --install) + do_install=1;; -t=*) build_type=$(get_key_value "$1");; -t) @@ -103,18 +133,19 @@ parse_options() enable_lzokay=OFF;; --disable-zlib) enable_zlib=OFF;; - #-h | --help) - # usage - # exit 0;; - #*) - # echo "Unknown option '$1'" - # exit 1;; + -h | --help) + usage + exit 0;; + *) + echo "Unknown option '$1'" + usage + exit 1;; esac shift done } -parse_options $* +parse_options "$@" print_config if [[ ${run_cov_only} -eq 1 ]] @@ -171,4 +202,9 @@ cmake ../../ \ -DENABLE_ZLIB=$enable_zlib VERBOSE=1 make -VERBOSE=1 make install \ No newline at end of file +if [ ${do_install} -eq 1 ] +then + VERBOSE=1 make install +else + echo "Skip install. Pass 'install' to run 'make install'." +fi diff --git a/cpp/test/tools/cli_args_test.cc b/cpp/test/tools/cli_args_test.cc index 614329463..42b7eb650 100644 --- a/cpp/test/tools/cli_args_test.cc +++ b/cpp/test/tools/cli_args_test.cc @@ -92,6 +92,34 @@ TEST(ParseArgsTest, LimitOffsetAndTimeRange) { EXPECT_EQ(p.end, 200); } +TEST(ParseArgsTest, TagFilterParsed) { + auto p = tsfile_cli::parse_args( + {"cat", "--tag-filter", "id1", "eq", "dev_a", "data.tsfile"}); + EXPECT_TRUE(p.error.empty()); + EXPECT_TRUE(p.has_tag_filter); + EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kEq); + EXPECT_EQ(p.tag_filter_column, "id1"); + EXPECT_EQ(p.tag_filter_value, "dev_a"); +} + +TEST(ParseArgsTest, TagBetweenParsed) { + auto p = tsfile_cli::parse_args( + {"cat", "--tag-between", "id1", "dev_a", "dev_c", "data.tsfile"}); + EXPECT_TRUE(p.error.empty()); + EXPECT_TRUE(p.has_tag_filter); + EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kBetween); + EXPECT_EQ(p.tag_filter_column, "id1"); + EXPECT_EQ(p.tag_filter_value, "dev_a"); + EXPECT_EQ(p.tag_filter_value2, "dev_c"); +} + +TEST(ParseArgsTest, DuplicateTagFilterIsError) { + auto p = tsfile_cli::parse_args({"cat", "--tag-filter", "id1", "eq", + "dev_a", "--tag-between", "id1", "a", "z", + "data.tsfile"}); + EXPECT_FALSE(p.error.empty()); +} + TEST(ParseArgsTest, UnknownFlagIsError) { auto p = tsfile_cli::parse_args({"ls", "--bogus", "data.tsfile"}); EXPECT_FALSE(p.error.empty()); diff --git a/cpp/test/tools/cli_test_util.h b/cpp/test/tools/cli_test_util.h index 0d1dccb56..5b4e532d9 100644 --- a/cpp/test/tools/cli_test_util.h +++ b/cpp/test/tools/cli_test_util.h @@ -99,6 +99,49 @@ inline std::string write_table_fixture() { return out_path; } +inline std::string write_tag_filter_fixture() { + storage::libtsfile_init(); + std::string out_path = + unique_temp_path("tsfile_cli_tag_filter_fixture", ".tsfile"); + std::string table_name = "t1"; + + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(out_path, flags, 0666); + + auto* schema = new storage::TableSchema( + table_name, + { + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("s1", common::INT64, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD), + }); + + auto* writer = new storage::TsFileTableWriter(&file, schema); + storage::Tablet tablet( + table_name, {"id1", "s1"}, {common::STRING, common::INT64}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 10); + + const char* tags[] = {"dev_a", "dev_b", "dev_b", "dev_c"}; + for (int row = 0; row < 4; ++row) { + tablet.add_timestamp(row, static_cast(row)); + tablet.add_value(row, "id1", tags[row]); + tablet.add_value(row, "s1", static_cast((row + 1) * 10)); + } + + writer->write_table(tablet); + writer->flush(); + writer->close(); + + delete writer; + delete schema; + return out_path; +} + } // namespace tsfile_cli_test #endif // TSFILE_CLI_TEST_UTIL_H diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index ccd3eba9b..de03cf782 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -34,6 +34,11 @@ struct Fixture { ~Fixture() { std::remove(path.c_str()); } }; +struct TagFilterFixture { + std::string path = tsfile_cli_test::write_tag_filter_fixture(); + ~TagFilterFixture() { std::remove(path.c_str()); } +}; + size_t count_lines(const std::string& s) { size_t n = 0; for (char c : s) { @@ -134,6 +139,28 @@ TEST(CliE2E, CatReturnsAllRows) { EXPECT_NE(out.str().find("time\ts1\n"), std::string::npos); } +TEST(CliE2E, CatPushesDownOffsetAndLimit) { + Fixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"cat", "-m", "s1", "--offset", "2", "-n", "2", "-f", "tsv", f.path}, + out, err); + EXPECT_EQ(code, 0); + EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); +} + +TEST(CliE2E, HeadPushesDownOffsetAndLimit) { + Fixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"head", "-m", "s1", "--offset", "1", "-n", "3", "-f", "tsv", f.path}, + out, err); + EXPECT_EQ(code, 0); + EXPECT_EQ(out.str(), "time\ts1\n1\t10\n2\t20\n3\t30\n"); +} + TEST(CliE2E, CatWithTimeRange) { Fixture f; std::ostringstream out; @@ -145,6 +172,65 @@ TEST(CliE2E, CatWithTimeRange) { EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); } +TEST(CliE2E, CatAppliesOffsetAfterTimeRange) { + Fixture f; + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "1", "--end", "4", + "--offset", "1", "-n", "2", "-f", "tsv", f.path}, + out, err); + EXPECT_EQ(code, 0); + EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); +} + +TEST(CliE2E, CatFiltersRowsByTagEq) { + TagFilterFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "eq", "dev_b", "-f", "tsv", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); +} + +TEST(CliE2E, HeadFiltersRowsByTagBetween) { + TagFilterFixture f; + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"head", "-m", "s1", "--tag-between", "id1", + "dev_b", "dev_c", "-n", "10", "-f", "tsv", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n3\t40\n"); +} + +TEST(CliE2E, SampleFiltersRowsByTagEq) { + TagFilterFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"sample", "-m", "s1", "--tag-filter", "id1", "eq", "dev_b", "-n", "10", + "--seed", "1", "-f", "tsv", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); +} + +TEST(CliE2E, TagFilterRejectsFieldColumn) { + TagFilterFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "s1", + "eq", "20", "-f", "tsv", f.path}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_NE(err.str().find("invalid tag filter column"), std::string::npos) + << err.str(); +} + TEST(CliE2E, CatJsonIsNdjson) { Fixture f; std::ostringstream out; @@ -181,6 +267,39 @@ TEST(CliE2E, CountReportsSeriesCountsAndTotal) { EXPECT_NE(out.str().find("total\t\t"), std::string::npos); } +TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { + Fixture f; + + std::ostringstream schema_out; + std::ostringstream schema_err; + EXPECT_EQ( + tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "tsv", f.path}, + schema_out, schema_err), + 0); + EXPECT_NE(schema_out.str().find("table1\ts1\tINT64"), std::string::npos) + << schema_out.str(); + + std::ostringstream count_out; + std::ostringstream count_err; + EXPECT_EQ( + tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "tsv", f.path}, + count_out, count_err), + 0); + EXPECT_NE(count_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + std::string::npos) + << count_out.str(); + + std::ostringstream stats_out; + std::ostringstream stats_err; + EXPECT_EQ( + tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "tsv", f.path}, + stats_out, stats_err), + 0); + EXPECT_NE(stats_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + std::string::npos) + << stats_out.str(); +} + TEST(CliE2E, SampleIsReproducibleWithSeed) { Fixture f; std::ostringstream out1; diff --git a/cpp/tools/README.md b/cpp/tools/README.md index bc8e5617f..42269be9a 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -44,6 +44,7 @@ Choose any one of the following. ```bash bash build.sh -t=Debug # -> cpp/build/Debug/bin/tsfile-cli bash build.sh # Release (default) -> cpp/build/Release/bin/tsfile-cli +bash build.sh install # Release build, then run make install ``` **2. Maven (builds the whole C++ module).** From the repository root: @@ -77,9 +78,10 @@ Verify the binary: ``` The executable links the `tsfile` shared library built alongside it. To run it from -anywhere, either run it in place by its full path, or use CMake's install step -(`cmake --install .` / `make install`), which installs the binary to `/bin` and -`libtsfile` to `/lib`. +anywhere, either run it in place by its full path, or explicitly install it with +`bash build.sh install`, `cmake --install .`, or `make install`. The install step places +the binary under `/bin` and `libtsfile` under `/lib`. The build script +does not install by default. ## Usage @@ -117,6 +119,7 @@ Shared options: | `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`; `--offset` not valid for `sample`) | | `--start ` / `--end ` | Inclusive epoch-millisecond time range (`head`, `cat`, `sample`) | | `--seed N` | Reproducible sampling seed (`sample` only) | +| `--tag-filter C OP V` / `--tag-between C L U` / `--tag-not-between C L U` | Table TAG predicate for `head`, `cat`, `sample`; `OP` is `eq`, `neq`, `lt`, `lteq`, `gt`, `gteq`, `regexp`, or `not-regexp` | | `--no-header` | Omit the header row | | `--model tree\|table` | Force the model (otherwise auto-detected) | @@ -130,6 +133,7 @@ BIN=cpp/build/Debug/bin/tsfile-cli $BIN ls -f tsv data.tsfile # list tables / devices $BIN meta data.tsfile # quick file overview $BIN count -t table1 -f tsv data.tsfile # row counts, no page scan +$BIN cat -t table1 --tag-filter device eq dev_1 -m temp -f tsv data.tsfile $BIN cat -m temp,humidity --start 1700000000000 -f csv data.tsfile | head $BIN sample -m temp -n 20 --seed 42 -f json data.tsfile | jq . ``` diff --git a/cpp/tools/cli/cli_args.cc b/cpp/tools/cli/cli_args.cc index c34145da9..731db6ff6 100644 --- a/cpp/tools/cli/cli_args.cc +++ b/cpp/tools/cli/cli_args.cc @@ -67,6 +67,29 @@ bool parse_format(const std::string& s, ParsedArgs::Format& out) { return true; } +bool parse_tag_filter_op(const std::string& s, ParsedArgs::TagFilterOp& out) { + if (s == "eq" || s == "=" || s == "==") { + out = ParsedArgs::TagFilterOp::kEq; + } else if (s == "neq" || s == "ne" || s == "!=") { + out = ParsedArgs::TagFilterOp::kNeq; + } else if (s == "lt" || s == "<") { + out = ParsedArgs::TagFilterOp::kLt; + } else if (s == "lteq" || s == "lte" || s == "le" || s == "<=") { + out = ParsedArgs::TagFilterOp::kLteq; + } else if (s == "gt" || s == ">") { + out = ParsedArgs::TagFilterOp::kGt; + } else if (s == "gteq" || s == "gte" || s == "ge" || s == ">=") { + out = ParsedArgs::TagFilterOp::kGteq; + } else if (s == "regexp" || s == "regex" || s == "=~") { + out = ParsedArgs::TagFilterOp::kRegexp; + } else if (s == "not-regexp" || s == "not-regex" || s == "!~") { + out = ParsedArgs::TagFilterOp::kNotRegexp; + } else { + return false; + } + return true; +} + } // namespace ParsedArgs parse_args(const std::vector& args) { @@ -99,6 +122,17 @@ ParsedArgs parse_args(const std::vector& args) { dst = args[++i]; return true; }; + auto need_tag_filter_slot = [&](const std::string& flag) -> bool { + if (p.has_tag_filter) { + p.error = "Only one tag filter predicate is supported"; + return false; + } + if (i + 3 >= args.size()) { + p.error = "Missing value for " + flag; + return false; + } + return true; + }; for (; i < args.size(); ++i) { const std::string& a = args[i]; @@ -180,6 +214,30 @@ ParsedArgs parse_args(const std::vector& args) { p.verbose = true; } else if (a == "--header-match") { p.header_match = true; + } else if (a == "--tag-filter") { + if (!need_tag_filter_slot(a)) { + return p; + } + p.has_tag_filter = true; + p.tag_filter_column = args[++i]; + std::string op = args[++i]; + if (!parse_tag_filter_op(op, p.tag_filter_op)) { + p.error = "Invalid --tag-filter operator: " + op + + " (use eq|neq|lt|lteq|gt|gteq|regexp|not-regexp)"; + return p; + } + p.tag_filter_value = args[++i]; + } else if (a == "--tag-between" || a == "--tag-not-between") { + if (!need_tag_filter_slot(a)) { + return p; + } + p.has_tag_filter = true; + p.tag_filter_op = (a == "--tag-between") + ? ParsedArgs::TagFilterOp::kBetween + : ParsedArgs::TagFilterOp::kNotBetween; + p.tag_filter_column = args[++i]; + p.tag_filter_value = args[++i]; + p.tag_filter_value2 = args[++i]; } else if (a == "--model") { if (!need_value(a, val)) { return p; diff --git a/cpp/tools/cli/cli_args.h b/cpp/tools/cli/cli_args.h index 90fa95e79..56199997a 100644 --- a/cpp/tools/cli/cli_args.h +++ b/cpp/tools/cli/cli_args.h @@ -28,6 +28,19 @@ namespace tsfile_cli { struct ParsedArgs { enum class Format { kAuto, kCsv, kTsv, kJson, kTable }; + enum class TagFilterOp { + kNone, + kEq, + kNeq, + kLt, + kLteq, + kGt, + kGteq, + kRegexp, + kNotRegexp, + kBetween, + kNotBetween, + }; std::string command; // subcommand, e.g. "ls"/"write" (args[0]) std::string file; // positional ; for write, the CSV/TSV @@ -50,6 +63,11 @@ struct ParsedArgs { std::string columns; // --columns spec for write (name:TYPE:cat,..) bool verbose = false; // -v/--verbose; write progress to stderr bool header_match = false; // --header-match; validate write header row + bool has_tag_filter = false; // --tag-filter/--tag-between was supplied + TagFilterOp tag_filter_op = TagFilterOp::kNone; + std::string tag_filter_column; // TAG column name for table row queries + std::string tag_filter_value; // comparison value or BETWEEN lower bound + std::string tag_filter_value2; // BETWEEN upper bound bool help = false; // -h/--help requested bool version = false; // --version requested std::string error; // non-empty if parsing failed (the message) diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index d7ca285a1..27ca751f0 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -70,6 +70,10 @@ void print_usage(std::ostream& os) { " --start inclusive lower time bound\n" " --end inclusive upper time bound\n" " --seed N RNG seed for sample\n" + " --tag-filter C OP V table TAG predicate; OP is " + "eq|neq|lt|lteq|gt|gteq|regexp|not-regexp\n" + " --tag-between C L U table TAG predicate: L <= C <= U\n" + " --tag-not-between C L U table TAG predicate outside [L,U]\n" " --no-header omit the header row\n" " --model tree|table force the data model (else auto)\n" " -h, --help print this help\n" @@ -141,6 +145,10 @@ bool validate_write_flags(const ParsedArgs& p, std::ostream& err) { err << "Error: --header-match cannot be combined with --no-header\n"; return false; } + if (p.has_tag_filter) { + err << "Error: tag filter flags are not valid for write\n"; + return false; + } // Name the offending flag so the user does not have to guess which of // the read-only options triggered the rejection. if (!p.measurements.empty()) { @@ -211,6 +219,18 @@ bool validate_read_flag_applicability(const ParsedArgs& p, std::ostream& err) { err << "Error: --start/--end are only valid for head/cat/sample\n"; return false; } + if (p.has_tag_filter && !is_row) { + err << "Error: tag filter flags are only valid for head/cat/sample\n"; + return false; + } + if (p.has_tag_filter && p.model == "tree") { + err << "Error: tag filter flags are only valid for table model\n"; + return false; + } + if (p.has_tag_filter && !p.device.empty()) { + err << "Error: tag filter flags cannot be combined with -d/--device\n"; + return false; + } if (!scoped && !p.device.empty()) { err << "Error: -d/--device is not valid for " << c << "\n"; return false; diff --git a/cpp/tools/commands/cmd_sample.cc b/cpp/tools/commands/cmd_sample.cc index 1123df26a..0a9c06360 100644 --- a/cpp/tools/commands/cmd_sample.cc +++ b/cpp/tools/commands/cmd_sample.cc @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -25,6 +26,7 @@ #include "commands/commands.h" #include "common/schema.h" #include "format/result_set_format.h" +#include "reader/filter/filter.h" #include "reader/tsfile_reader.h" namespace tsfile_cli { @@ -37,6 +39,7 @@ int cmd_sample(const ParsedArgs& args, storage::TsFileReader& reader, : std::numeric_limits::max(); storage::ResultSet* rs = nullptr; int qret = 0; + std::unique_ptr tag_filter; if (is_table_model(args, reader)) { std::string table_name = args.table; @@ -55,8 +58,16 @@ int cmd_sample(const ParsedArgs& args, storage::TsFileReader& reader, cols = ts->get_measurement_names(); } } - qret = reader.query(table_name, cols, start, end, rs); + tag_filter = build_table_tag_filter(args, reader, table_name, err); + if (args.has_tag_filter && tag_filter == nullptr) { + return kExitUsage; + } + qret = reader.query(table_name, cols, start, end, rs, tag_filter.get()); } else { + if (args.has_tag_filter) { + err << "Error: tag filter flags are only valid for table model\n"; + return kExitUsage; + } std::vector paths = collect_tree_query_paths(args, reader); if (paths.empty()) { err << "Error: no time series found\n"; diff --git a/cpp/tools/commands/cmd_schema.cc b/cpp/tools/commands/cmd_schema.cc index 3e03f4f99..9b8442d8c 100644 --- a/cpp/tools/commands/cmd_schema.cc +++ b/cpp/tools/commands/cmd_schema.cc @@ -27,18 +27,21 @@ #include "commands/commands.h" #include "common/schema.h" #include "reader/tsfile_reader.h" +#include "utils/storage_utils.h" namespace tsfile_cli { namespace { void write_table_schema_rows(const ParsedArgs& args, storage::TsFileReader& reader, RowWriter& w) { + const std::string target_table_name = storage::to_lower(args.table); auto schemas = reader.get_all_table_schemas(); for (auto& schema : schemas) { if (!schema) { continue; } - if (!args.table.empty() && schema->get_table_name() != args.table) { + if (!target_table_name.empty() && + schema->get_table_name() != target_table_name) { continue; } for (const auto& ms : schema->get_measurement_schemas()) { diff --git a/cpp/tools/commands/commands.h b/cpp/tools/commands/commands.h index 5e26fd64b..ea7038ff5 100644 --- a/cpp/tools/commands/commands.h +++ b/cpp/tools/commands/commands.h @@ -20,6 +20,7 @@ #ifndef TSFILE_CLI_COMMANDS_H #define TSFILE_CLI_COMMANDS_H +#include #include #include #include @@ -28,6 +29,7 @@ #include "format/output_format.h" namespace storage { +class Filter; class TsFileReader; } // namespace storage @@ -43,6 +45,10 @@ bool is_table_model(const ParsedArgs& args, storage::TsFileReader& reader); std::vector collect_tree_query_paths( const ParsedArgs& args, storage::TsFileReader& reader); +std::unique_ptr build_table_tag_filter( + const ParsedArgs& args, storage::TsFileReader& reader, + const std::string& table_name, std::ostream& err); + int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& err, long long offset, long long limit); diff --git a/cpp/tools/commands/row_query.cc b/cpp/tools/commands/row_query.cc index 702deefd6..5acb63a4b 100644 --- a/cpp/tools/commands/row_query.cc +++ b/cpp/tools/commands/row_query.cc @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -28,9 +29,87 @@ #include "common/device_id.h" #include "common/schema.h" #include "format/result_set_format.h" +#include "reader/filter/tag_filter.h" #include "reader/tsfile_reader.h" namespace tsfile_cli { +namespace { + +bool can_push_down_row_window(const ParsedArgs& args, long long offset, + long long limit) { + return !args.has_start && !args.has_end && offset <= INT_MAX && + (limit < 0 || limit <= INT_MAX); +} + +int to_reader_row_bound(long long value) { + return value < 0 ? -1 : static_cast(value); +} + +} // namespace + +std::unique_ptr build_table_tag_filter( + const ParsedArgs& args, storage::TsFileReader& reader, + const std::string& table_name, std::ostream& err) { + if (!args.has_tag_filter) { + return std::unique_ptr(); + } + auto schema = reader.get_table_schema(table_name); + if (!schema) { + err << "Error: no schema found for table " << table_name << "\n"; + return std::unique_ptr(); + } + + storage::TagFilterBuilder builder(schema.get()); + storage::Filter* filter = nullptr; + switch (args.tag_filter_op) { + case ParsedArgs::TagFilterOp::kEq: + filter = builder.eq(args.tag_filter_column, args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kNeq: + filter = builder.neq(args.tag_filter_column, args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kLt: + filter = builder.lt(args.tag_filter_column, args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kLteq: + filter = + builder.lteq(args.tag_filter_column, args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kGt: + filter = builder.gt(args.tag_filter_column, args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kGteq: + filter = + builder.gteq(args.tag_filter_column, args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kRegexp: + filter = + builder.reg_exp(args.tag_filter_column, args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kNotRegexp: + filter = builder.not_reg_exp(args.tag_filter_column, + args.tag_filter_value); + break; + case ParsedArgs::TagFilterOp::kBetween: + filter = builder.between_and(args.tag_filter_column, + args.tag_filter_value, + args.tag_filter_value2); + break; + case ParsedArgs::TagFilterOp::kNotBetween: + filter = builder.not_between_and(args.tag_filter_column, + args.tag_filter_value, + args.tag_filter_value2); + break; + case ParsedArgs::TagFilterOp::kNone: + break; + } + if (filter == nullptr) { + err << "Error: invalid tag filter column '" << args.tag_filter_column + << "' for table " << table_name << "\n"; + return std::unique_ptr(); + } + return std::unique_ptr(filter); +} std::vector collect_tree_query_paths( const ParsedArgs& args, storage::TsFileReader& reader) { @@ -91,6 +170,8 @@ int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, storage::ResultSet* rs = nullptr; int qret = 0; + const bool push_down = can_push_down_row_window(args, offset, limit); + std::unique_ptr tag_filter; if (is_table_model(args, reader)) { std::string table_name = args.table; @@ -109,14 +190,35 @@ int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, cols = ts->get_measurement_names(); } } - qret = reader.query(table_name, cols, start, end, rs); + tag_filter = build_table_tag_filter(args, reader, table_name, err); + if (args.has_tag_filter && tag_filter == nullptr) { + return kExitUsage; + } + if (push_down) { + qret = reader.queryByRow(table_name, cols, + to_reader_row_bound(offset), + to_reader_row_bound(limit), rs, + tag_filter.get()); + } else { + qret = reader.query(table_name, cols, start, end, rs, + tag_filter.get()); + } } else { + if (args.has_tag_filter) { + err << "Error: tag filter flags are only valid for table model\n"; + return kExitUsage; + } std::vector paths = collect_tree_query_paths(args, reader); if (paths.empty()) { err << "Error: no time series found\n"; return kExitRuntime; } - qret = reader.query(paths, start, end, rs); + if (push_down) { + qret = reader.queryByRow(paths, to_reader_row_bound(offset), + to_reader_row_bound(limit), rs); + } else { + qret = reader.query(paths, start, end, rs); + } } if (qret != 0 || rs == nullptr) { @@ -127,7 +229,10 @@ int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, return kExitRuntime; } - int wret = emit_result_set(rs, fmt, args.no_header, out, offset, limit); + int wret = push_down + ? emit_result_set(rs, fmt, args.no_header, out) + : emit_result_set(rs, fmt, args.no_header, out, offset, + limit); reader.destroy_query_data_set(rs); if (wret != 0) { err << "Error: failed to read rows: " << error_code_message(wret) diff --git a/cpp/tools/commands/statistics.cc b/cpp/tools/commands/statistics.cc index f1c21b994..acb7e01b9 100644 --- a/cpp/tools/commands/statistics.cc +++ b/cpp/tools/commands/statistics.cc @@ -27,6 +27,7 @@ #include "commands/commands.h" #include "common/statistic.h" #include "reader/tsfile_reader.h" +#include "utils/storage_utils.h" namespace tsfile_cli { namespace { @@ -138,6 +139,7 @@ StatisticCells statistic_value_cells(storage::Statistic* st) { std::vector collect_series_stats(const ParsedArgs& args, storage::TsFileReader& reader) { std::vector rows; + const std::string target_table_name = storage::to_lower(args.table); storage::DeviceTimeseriesMetadataMap meta = reader.get_timeseries_metadata(); for (auto& kv : meta) { @@ -145,8 +147,8 @@ std::vector collect_series_stats(const ParsedArgs& args, if (!args.device.empty() && target != args.device) { continue; } - if (!args.table.empty() && kv.first && - kv.first->get_table_name() != args.table) { + if (!target_table_name.empty() && kv.first && + kv.first->get_table_name() != target_table_name) { continue; } for (auto& ts : kv.second) { diff --git a/cpp/tools/skills/tsfile-cli/SKILL.md b/cpp/tools/skills/tsfile-cli/SKILL.md index 4405050cd..9ff82db64 100644 --- a/cpp/tools/skills/tsfile-cli/SKILL.md +++ b/cpp/tools/skills/tsfile-cli/SKILL.md @@ -58,9 +58,11 @@ Table model + row verbs (`head/cat/sample`): without `-t`, only the **first** ta opts: -f csv|tsv|json|table (default TTY→table, pipe→tsv) -d | -t (mutually exclusive) -m a,b,c (projection) · -n N · --offset N · --start · --end (inclusive) + --tag-filter C OP V · --tag-between C L U · --tag-not-between C L U (table TAG predicates) --seed N · --no-header · --model tree|table (else auto) applies: -m → schema/stats/count/head/cat/sample · -d/-t → row cmds/schema/stats/count - (-d needs tree model, -t needs table model in head/cat/sample/schema) · --offset ∉ sample + (-d needs tree model, -t needs table model in head/cat/sample/schema) · --offset ∉ sample + tag filters → head/cat/sample table model; OP=eq|neq|lt|lteq|gt|gteq|regexp|not-regexp json=NDJSON (num/bool bare, else quoted, null→null, NaN/Inf→null) · csv=RFC4180 · ts=raw epoch ms exit: 0 ok · 1 usage · 2 file open/corrupt · 3 query/runtime ``` @@ -68,6 +70,7 @@ exit: 0 ok · 1 usage · 2 file open/corrupt · 3 query/runtime ```sh B=cpp/build/Debug/bin/tsfile-cli $B meta data.tsfile; $B count -t table1 -f tsv data.tsfile +$B cat -t table1 --tag-filter device eq dev_1 -m temp -f tsv data.tsfile $B cat -m temp --start 1700000000000 -f csv data.tsfile 2>/dev/null | head ```