Skip to content

Commit 1604506

Browse files
linrrzqqqlinrrzqqq
committed
[test]
1 parent d330769 commit 1604506

13 files changed

Lines changed: 701 additions & 239 deletions

File tree

‎be/src/core/value/bitmap_value.h‎

Lines changed: 347 additions & 160 deletions
Large diffs are not rendered by default.

‎be/src/exprs/function/function_bitmap.cpp‎

Lines changed: 9 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -637,32 +637,20 @@ struct BitmapAndNotCount {
637637

638638
static void vector_vector(const TData& lvec, const TData& rvec, ResTData* res) {
639639
size_t size = lvec.size();
640-
BitmapValue mid_data;
641640
for (size_t i = 0; i < size; ++i) {
642-
mid_data = lvec[i];
643-
mid_data &= rvec[i];
644-
res[i] = lvec[i].andnot_cardinality(mid_data);
645-
mid_data.reset();
641+
res[i] = lvec[i].andnot_cardinality(rvec[i]);
646642
}
647643
}
648644
static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData* res) {
649645
size_t size = rvec.size();
650-
BitmapValue mid_data;
651646
for (size_t i = 0; i < size; ++i) {
652-
mid_data = lval;
653-
mid_data &= rvec[i];
654-
res[i] = lval.andnot_cardinality(mid_data);
655-
mid_data.reset();
647+
res[i] = lval.andnot_cardinality(rvec[i]);
656648
}
657649
}
658650
static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData* res) {
659651
size_t size = lvec.size();
660-
BitmapValue mid_data;
661652
for (size_t i = 0; i < size; ++i) {
662-
mid_data = lvec[i];
663-
mid_data &= rval;
664-
res[i] = lvec[i].andnot_cardinality(mid_data);
665-
mid_data.reset();
653+
res[i] = lvec[i].andnot_cardinality(rval);
666654
}
667655
}
668656
};
@@ -904,25 +892,19 @@ struct BitmapHasAny {
904892
static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) {
905893
size_t size = lvec.size();
906894
for (size_t i = 0; i < size; ++i) {
907-
auto bitmap = lvec[i];
908-
bitmap &= rvec[i];
909-
res[i] = bitmap.cardinality() != 0;
895+
res[i] = lvec[i].intersects(rvec[i]);
910896
}
911897
}
912898
static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) {
913899
size_t size = lvec.size();
914900
for (size_t i = 0; i < size; ++i) {
915-
auto bitmap = lvec[i];
916-
bitmap &= rval;
917-
res[i] = bitmap.cardinality() != 0;
901+
res[i] = lvec[i].intersects(rval);
918902
}
919903
}
920904
static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) {
921905
size_t size = rvec.size();
922906
for (size_t i = 0; i < size; ++i) {
923-
auto bitmap = lval;
924-
bitmap &= rvec[i];
925-
res[i] = bitmap.cardinality() != 0;
907+
res[i] = lval.intersects(rvec[i]);
926908
}
927909
}
928910
};
@@ -942,28 +924,19 @@ struct BitmapHasAll {
942924
static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) {
943925
size_t size = lvec.size();
944926
for (size_t i = 0; i < size; ++i) {
945-
uint64_t lhs_cardinality = lvec[i].cardinality();
946-
auto bitmap = lvec[i];
947-
bitmap |= rvec[i];
948-
res[i] = bitmap.cardinality() == lhs_cardinality;
927+
res[i] = lvec[i].contains_all(rvec[i]);
949928
}
950929
}
951930
static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) {
952931
size_t size = lvec.size();
953932
for (size_t i = 0; i < size; ++i) {
954-
uint64_t lhs_cardinality = lvec[i].cardinality();
955-
auto bitmap = lvec[i];
956-
bitmap |= rval;
957-
res[i] = bitmap.cardinality() == lhs_cardinality;
933+
res[i] = lvec[i].contains_all(rval);
958934
}
959935
}
960936
static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) {
961937
size_t size = rvec.size();
962-
uint64_t lhs_cardinality = lval.cardinality();
963938
for (size_t i = 0; i < size; ++i) {
964-
auto bitmap = lval;
965-
bitmap |= rvec[i];
966-
res[i] = bitmap.cardinality() == lhs_cardinality;
939+
res[i] = lval.contains_all(rvec[i]);
967940
}
968941
}
969942
};

‎be/src/exprs/function/function_bitmap_min_or_max.h‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,20 +69,24 @@ class FunctionBitmapSingle : public IFunction {
6969
private:
7070
void execute_straight(const ColumnBitmap* date_column, ColumnInt64* result_column,
7171
NullMap& result_null_map, size_t input_rows_count) const {
72+
const auto& data = date_column->get_data();
73+
auto& result_data = result_column->get_data();
74+
result_data.resize(input_rows_count);
75+
7276
for (size_t i = 0; i < input_rows_count; i++) {
7377
if (result_null_map[i]) {
74-
result_column->insert_default();
78+
result_data[i] = 0;
7579
continue;
7680
}
7781

78-
BitmapValue value = date_column->get_element(i);
79-
if (!value.cardinality()) {
82+
const BitmapValue& value = data[i];
83+
if (value.empty()) {
8084
result_null_map[i] = true;
81-
result_column->insert_default();
85+
result_data[i] = 0;
8286
continue;
8387
}
8488

85-
result_column->insert(Field::create_field<TYPE_BIGINT>(Impl::calculate(value)));
89+
result_data[i] = Impl::calculate(value);
8690
}
8791
}
8892
};

‎be/src/exprs/function/function_bitmap_variadic.cpp‎

Lines changed: 108 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include "core/block/column_with_type_and_name.h"
3232
#include "core/column/column.h"
3333
#include "core/column/column_complex.h"
34+
#include "core/column/column_const.h"
3435
#include "core/column/column_nullable.h"
3536
#include "core/column/column_vector.h"
3637
#include "core/data_type/data_type.h"
@@ -156,6 +157,87 @@ BITMAP_FUNCTION_COUNT_VARIADIC(BitmapOrCount, bitmap_or_count, |=);
156157
BITMAP_FUNCTION_COUNT_VARIADIC(BitmapAndCount, bitmap_and_count, &=);
157158
BITMAP_FUNCTION_COUNT_VARIADIC(BitmapXorCount, bitmap_xor_count, ^=);
158159

160+
template <typename Impl>
161+
Status execute_binary_bitmap_count(const ColumnPtr& lhs_column, const ColumnPtr& rhs_column,
162+
size_t input_rows_count, ColumnInt64::Container& res) {
163+
struct BitmapColumnAccessor {
164+
const std::vector<BitmapValue>* values = nullptr;
165+
const BitmapValue* const_value = nullptr;
166+
const ColumnUInt8::value_type* null_map_data = nullptr;
167+
bool is_const = false;
168+
bool is_const_null = false;
169+
170+
bool is_null_at(size_t row) const {
171+
return is_const ? is_const_null : (null_map_data && null_map_data[row]);
172+
}
173+
174+
const BitmapValue& get_value(size_t row) const {
175+
return is_const ? *const_value : (*values)[row];
176+
}
177+
};
178+
179+
auto make_accessor = [](const ColumnPtr& column) {
180+
BitmapColumnAccessor accessor;
181+
const IColumn* data_column = column.get();
182+
183+
if (const auto* const_column = check_and_get_column<ColumnConst>(*data_column)) {
184+
accessor.is_const = true;
185+
data_column = &const_column->get_data_column();
186+
}
187+
188+
if (const auto* nullable = check_and_get_column<ColumnNullable>(*data_column)) {
189+
if (accessor.is_const) {
190+
accessor.is_const_null = nullable->is_null_at(0);
191+
} else {
192+
accessor.null_map_data = nullable->get_null_map_data().data();
193+
}
194+
data_column = nullable->get_nested_column_ptr().get();
195+
}
196+
197+
const auto* bitmap_column = assert_cast<const ColumnBitmap*>(data_column);
198+
if (accessor.is_const) {
199+
accessor.const_value = &bitmap_column->get_data()[0];
200+
} else {
201+
accessor.values = &bitmap_column->get_data();
202+
}
203+
204+
return accessor;
205+
};
206+
207+
const auto lhs = make_accessor(lhs_column);
208+
const auto rhs = make_accessor(rhs_column);
209+
210+
for (size_t row = 0; row < input_rows_count; ++row) {
211+
const bool lhs_is_null = lhs.is_null_at(row);
212+
const bool rhs_is_null = rhs.is_null_at(row);
213+
214+
if constexpr (std::is_same_v<Impl, BitmapOrCount>) {
215+
if (lhs_is_null) {
216+
res[row] = rhs_is_null ? 0 : rhs.get_value(row).cardinality();
217+
} else if (rhs_is_null) {
218+
res[row] = lhs.get_value(row).cardinality();
219+
} else {
220+
res[row] = lhs.get_value(row).or_cardinality(rhs.get_value(row));
221+
}
222+
} else if constexpr (std::is_same_v<Impl, BitmapAndCount>) {
223+
res[row] = (lhs_is_null || rhs_is_null)
224+
? 0
225+
: lhs.get_value(row).and_cardinality(rhs.get_value(row));
226+
} else {
227+
if (lhs_is_null || rhs_is_null) {
228+
res[row] = 0;
229+
} else {
230+
const auto& lhs_value = lhs.get_value(row);
231+
const auto& rhs_value = rhs.get_value(row);
232+
uint64_t inter = lhs_value.and_cardinality(rhs_value);
233+
res[row] = lhs_value.cardinality() + rhs_value.cardinality() - 2 * inter;
234+
}
235+
}
236+
}
237+
238+
return Status::OK();
239+
}
240+
159241
Status execute_bitmap_op_count_null_to_zero(
160242
FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result,
161243
size_t input_rows_count,
@@ -219,12 +301,6 @@ class FunctionBitMapVariadic : public IFunction {
219301
const ColumnNumbers& arguments, uint32_t result,
220302
size_t input_rows_count) const {
221303
size_t argument_size = arguments.size();
222-
std::vector<ColumnPtr> argument_columns(argument_size);
223-
224-
for (size_t i = 0; i < argument_size; ++i) {
225-
argument_columns[i] =
226-
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
227-
}
228304

229305
using ResultDataType = typename Impl::ResultDataType; //DataTypeBitMap or DataTypeInt64
230306
using ColVecResult = std::conditional_t<is_complex_v<ResultDataType::PType>,
@@ -244,8 +320,31 @@ class FunctionBitMapVariadic : public IFunction {
244320
auto& vec_res = col_res->get_data();
245321
vec_res.resize(input_rows_count);
246322

247-
RETURN_IF_ERROR(Impl::vector_vector(argument_columns.data(), argument_size,
248-
input_rows_count, vec_res, col_res_nulls.get()));
323+
if constexpr (std::is_same_v<Impl, BitmapOrCount> || std::is_same_v<Impl, BitmapAndCount> ||
324+
std::is_same_v<Impl, BitmapXorCount>) {
325+
if (argument_size == 2) {
326+
RETURN_IF_ERROR(execute_binary_bitmap_count<Impl>(
327+
block.get_by_position(arguments[0]).column,
328+
block.get_by_position(arguments[1]).column, input_rows_count, vec_res));
329+
} else {
330+
std::vector<ColumnPtr> argument_columns(argument_size);
331+
for (size_t i = 0; i < argument_size; ++i) {
332+
argument_columns[i] = block.get_by_position(arguments[i])
333+
.column->convert_to_full_column_if_const();
334+
}
335+
RETURN_IF_ERROR(Impl::vector_vector(argument_columns.data(), argument_size,
336+
input_rows_count, vec_res,
337+
col_res_nulls.get()));
338+
}
339+
} else {
340+
std::vector<ColumnPtr> argument_columns(argument_size);
341+
for (size_t i = 0; i < argument_size; ++i) {
342+
argument_columns[i] = block.get_by_position(arguments[i])
343+
.column->convert_to_full_column_if_const();
344+
}
345+
RETURN_IF_ERROR(Impl::vector_vector(argument_columns.data(), argument_size,
346+
input_rows_count, vec_res, col_res_nulls.get()));
347+
}
249348
if (!use_default_implementation_for_nulls() && result_info.type->is_nullable()) {
250349
block.replace_by_position(
251350
result, ColumnNullable::create(std::move(col_res), std::move(col_res_nulls)));
@@ -277,4 +376,4 @@ void register_function_bitmap_variadic(SimpleFunctionFactory& factory) {
277376
factory.register_function<FunctionBitmapAndCount>();
278377
factory.register_function<FunctionBitmapXorCount>();
279378
}
280-
} // namespace doris
379+
} // namespace doris

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.nereids.rules.expression.rules.AddMinMax;
2121
import org.apache.doris.nereids.rules.expression.rules.ArrayContainToArrayOverlap;
2222
import org.apache.doris.nereids.rules.expression.rules.BetweenToEqual;
23+
import org.apache.doris.nereids.rules.expression.rules.BitmapCountToBitmapOpCount;
2324
import org.apache.doris.nereids.rules.expression.rules.CaseWhenToCompoundPredicate;
2425
import org.apache.doris.nereids.rules.expression.rules.CaseWhenToIf;
2526
import org.apache.doris.nereids.rules.expression.rules.CondReplaceNullWithFalse;
@@ -62,6 +63,7 @@ public class ExpressionOptimization extends ExpressionRewrite {
6263
DistinctPredicatesRule.INSTANCE,
6364
ExtractCommonFactorRule.INSTANCE,
6465

66+
BitmapCountToBitmapOpCount.INSTANCE,
6567
DateFunctionRewrite.INSTANCE,
6668
ArrayContainToArrayOverlap.INSTANCE,
6769
CondReplaceNullWithFalse.INSTANCE,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
public enum ExpressionRuleType {
2424
ADD_MIN_MAX,
2525
ARRAY_CONTAIN_TO_ARRAY_OVERLAP,
26+
BITMAP_COUNT_TO_BITMAP_OP_COUNT,
2627
BETWEEN_TO_EQUAL,
2728
CASE_WHEN_TO_COMPOUND_PREDICATE,
2829
CASE_WHEN_TO_IF,

0 commit comments

Comments
 (0)