diff --git a/dbms/src/Columns/ColumnFunction.cpp b/dbms/src/Columns/ColumnFunction.cpp index ce356a5c09b..e78098cd944 100644 --- a/dbms/src/Columns/ColumnFunction.cpp +++ b/dbms/src/Columns/ColumnFunction.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,14 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -ColumnFunction::ColumnFunction(size_t size, FunctionBasePtr function, const ColumnsWithTypeAndName & columns_to_capture) +ColumnFunction::ColumnFunction( + size_t size, + FunctionBasePtr function, + const ColumnsWithTypeAndName & columns_to_capture, + bool is_short_circuit_argument_) : column_size(size) , function(function) + , is_short_circuit_argument(is_short_circuit_argument_) { appendArguments(columns_to_capture); } @@ -41,7 +47,7 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const for (auto & column : capture) column.column = column.column->cloneResized(size); - return ColumnFunction::create(size, function, capture); + return ColumnFunction::create(size, function, capture, is_short_circuit_argument); } ColumnPtr ColumnFunction::replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const @@ -59,7 +65,7 @@ ColumnPtr ColumnFunction::replicateRange(size_t start_row, size_t end_row, const column.column = column.column->replicateRange(start_row, end_row, offsets); size_t replicated_size = 0 == column_size ? 0 : (offsets[end_row - 1]); - return ColumnFunction::create(replicated_size, function, capture); + return ColumnFunction::create(replicated_size, function, capture, is_short_circuit_argument); } ColumnPtr ColumnFunction::cut(size_t start, size_t length) const @@ -68,7 +74,7 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const for (auto & column : capture) column.column = column.column->cut(start, length); - return ColumnFunction::create(length, function, capture); + return ColumnFunction::create(length, function, capture, is_short_circuit_argument); } ColumnPtr ColumnFunction::filter(const Filter & filter, ssize_t result_size_hint) const @@ -88,7 +94,7 @@ ColumnPtr ColumnFunction::filter(const Filter & filter, ssize_t result_size_hint else filtered_size = capture.front().column->size(); - return ColumnFunction::create(filtered_size, function, capture); + return ColumnFunction::create(filtered_size, function, capture, is_short_circuit_argument); } ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const @@ -107,7 +113,7 @@ ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const for (auto & column : capture) column.column = column.column->permute(perm, limit); - return ColumnFunction::create(limit, function, capture); + return ColumnFunction::create(limit, function, capture, is_short_circuit_argument); } std::vector ColumnFunction::scatter( @@ -138,7 +144,7 @@ std::vector ColumnFunction::scatter( { auto & capture = captures[part]; size_t s = capture.empty() ? counts[part] : capture.front().column->size(); - columns.emplace_back(ColumnFunction::create(s, function, std::move(capture))); + columns.emplace_back(ColumnFunction::create(s, function, std::move(capture), is_short_circuit_argument)); } return columns; @@ -253,7 +259,19 @@ ColumnWithTypeAndName ColumnFunction::reduce() const captured), ErrorCodes::LOGICAL_ERROR); - Block block(captured_columns); + if (is_short_circuit_argument && column_size == 0) + return {function->getReturnType()->createColumn(), function->getReturnType(), ""}; + + auto columns = captured_columns; + if (is_short_circuit_argument) + { + const size_t required_arguments = function->isShortCircuit() ? 1 : columns.size(); + for (size_t i = 0; i < required_arguments; ++i) + if (const auto * deferred = checkAndGetShortCircuitArgument(columns[i].column)) + columns[i].column = deferred->reduce().column; + } + + Block block(columns); block.insert({nullptr, function->getReturnType(), ""}); ColumnNumbers arguments(captured_columns.size()); @@ -265,4 +283,56 @@ ColumnWithTypeAndName ColumnFunction::reduce() const return block.getByPosition(captured_columns.size()); } +const ColumnFunction * checkAndGetShortCircuitArgument(const ColumnPtr & column) +{ + const auto * function = typeid_cast(column.get()); + return function && function->isShortCircuitArgument() ? function : nullptr; +} + +void maskedExecute(ColumnWithTypeAndName & column, const IColumn::Filter & mask) +{ + const auto * deferred = checkAndGetShortCircuitArgument(column.column); + if (!deferred) + return; + + RUNTIME_CHECK(column.column->size() == mask.size()); + const size_t selected = countBytesInFilter(mask); + if (selected == 0) + { + column.column = column.type->createColumnConstWithDefaultValue(mask.size()); + return; + } + if (selected == mask.size()) + { + column.column = deferred->reduce().column; + return; + } + + auto filtered = deferred->filter(mask, selected); + auto result = static_cast(*filtered).reduce().column; + if (auto materialized = result->convertToFullColumnIfConst()) + result = std::move(materialized); + RUNTIME_CHECK(result->size() == selected); + + // Use the existing bulk insertion interface instead of adding expand() to every TiFlash column type. + auto expanded = result->cloneEmpty(); + expanded->reserve(mask.size()); + size_t source = 0; + for (size_t begin = 0; begin < mask.size();) + { + size_t end = begin + 1; + while (end < mask.size() && (mask[end] != 0) == (mask[begin] != 0)) + ++end; + if (mask[begin]) + { + expanded->insertRangeFrom(*result, source, end - begin); + source += end - begin; + } + else + expanded->insertManyDefaults(end - begin); + begin = end; + } + column.column = std::move(expanded); +} + } // namespace DB diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h index b1f7b44b23e..1c30438c072 100644 --- a/dbms/src/Columns/ColumnFunction.h +++ b/dbms/src/Columns/ColumnFunction.h @@ -25,15 +25,19 @@ namespace DB class IFunctionBase; using FunctionBasePtr = std::shared_ptr; -/** A column containing a lambda expression. - * Behaves like a constant-column. Contains an expression, but not input or output data. +/** A column containing a lambda or deferred scalar expression and its captured arguments. + * A deferred scalar expression is evaluated only after its captures have been filtered. */ class ColumnFunction final : public COWPtrHelper { private: friend class COWPtrHelper; - ColumnFunction(size_t size, FunctionBasePtr function, const ColumnsWithTypeAndName & columns_to_capture); + ColumnFunction( + size_t size, + FunctionBasePtr function, + const ColumnsWithTypeAndName & columns_to_capture, + bool is_short_circuit_argument = false); public: const char * getFamilyName() const override { return "Function"; } @@ -70,6 +74,7 @@ class ColumnFunction final : public COWPtrHelper void appendArguments(const ColumnsWithTypeAndName & columns); ColumnWithTypeAndName reduce() const; + bool isShortCircuitArgument() const { return is_short_circuit_argument; } Field operator[](size_t) const override { @@ -283,8 +288,14 @@ class ColumnFunction final : public COWPtrHelper size_t column_size; FunctionBasePtr function; ColumnsWithTypeAndName captured_columns; + bool is_short_circuit_argument; void appendArgument(const ColumnWithTypeAndName & column); }; +const ColumnFunction * checkAndGetShortCircuitArgument(const ColumnPtr & column); + +/// Adapted from ClickHouse Columns/MaskOperations.cpp: filter captures, reduce, then restore row positions. +void maskedExecute(ColumnWithTypeAndName & column, const IColumn::Filter & mask); + } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp index f7e2d00d50c..1d146d16fbc 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp @@ -49,8 +49,6 @@ #include #include -#include - namespace DB { namespace ErrorCodes @@ -1003,13 +1001,6 @@ String DAGExpressionAnalyzer::buildFilterColumn( const google::protobuf::RepeatedPtrField & conditions, bool null_as_false) { - building_filter_conditions = true; - json_valid_guarded_exprs.clear(); - SCOPE_EXIT({ - building_filter_conditions = false; - json_valid_guarded_exprs.clear(); - }); - String filter_column_name; if (conditions.size() == 1) { @@ -1030,12 +1021,7 @@ String DAGExpressionAnalyzer::buildFilterColumn( { Names arg_names; for (const auto & condition : conditions) - { - auto guards_before_condition = json_valid_guarded_exprs; arg_names.push_back(getActions(condition, actions, true)); - json_valid_guarded_exprs = std::move(guards_before_condition); - recordJsonValidGuards(condition); - } // connect all the conditions by logical and // two_value_and treats null as false inside the `two_value_and` function, so the output column // will always be UInt8 type, which can save the merge step in FilterDescription @@ -1046,30 +1032,6 @@ String DAGExpressionAnalyzer::buildFilterColumn( return filter_column_name; } -void DAGExpressionAnalyzer::recordJsonValidGuards(const tipb::Expr & expr) -{ - if (!building_filter_conditions || !isScalarFunctionExpr(expr)) - return; - - if (expr.sig() == tipb::ScalarFuncSig::JsonValidStringSig && expr.children_size() == 1) - { - json_valid_guarded_exprs.emplace(exprToString(expr.children(0), getCurrentInputColumns())); - return; - } - - if (expr.sig() == tipb::ScalarFuncSig::LogicalAnd) - { - for (const auto & child : expr.children()) - recordJsonValidGuards(child); - } -} - -bool DAGExpressionAnalyzer::isJsonValidGuarded(const tipb::Expr & expr) const -{ - return building_filter_conditions - && json_valid_guarded_exprs.contains(exprToString(expr, getCurrentInputColumns())); -} - std::tuple DAGExpressionAnalyzer::buildPushDownFilter( const google::protobuf::RepeatedPtrField & conditions, bool null_as_false) diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h index aaf93284f95..39fca3d0fab 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h @@ -320,17 +320,11 @@ class DAGExpressionAnalyzer : private boost::noncopyable const std::vector & require_schema, const std::vector & output_offsets) const; - void recordJsonValidGuards(const tipb::Expr & expr); - bool isJsonValidGuarded(const tipb::Expr & expr) const; - // all columns from table scan NamesAndTypes source_columns; DAGPreparedSets prepared_sets; const Context & context; - bool building_filter_conditions = false; - std::unordered_set json_valid_guarded_exprs; - friend class DAGExpressionAnalyzerHelper; }; diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index 8f195919811..e24398e65d1 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -202,20 +202,13 @@ String DAGExpressionAnalyzerHelper::buildLogicalFunction( const ExpressionActionsPtr & actions) { const String & func_name = getFunctionName(expr); - auto guards_before_function = analyzer->json_valid_guarded_exprs; Names argument_names; for (const auto & child : expr.children()) { - auto guards_before_child = analyzer->json_valid_guarded_exprs; String name = analyzer->getActions(child, actions, true); argument_names.push_back(name); - analyzer->json_valid_guarded_exprs = std::move(guards_before_child); - if (func_name == "and" || func_name == "two_value_and") - analyzer->recordJsonValidGuards(child); } - String result = analyzer->applyFunction(func_name, argument_names, actions, getCollatorFromExpr(expr)); - analyzer->json_valid_guarded_exprs = std::move(guards_before_function); - return result; + return analyzer->applyFunction(func_name, argument_names, actions, getCollatorFromExpr(expr)); } // left(str,len) = substrUTF8(str,1,len) @@ -306,12 +299,7 @@ String DAGExpressionAnalyzerHelper::buildSingleParamJsonRelatedFunctions( const auto & input_expr = expr.children(0); String arg = analyzer->getActions(input_expr, actions); const auto & collator = getCollatorFromExpr(expr); - const bool ignore_invalid_json - = func_name == FunctionCastStringAsJson::name && analyzer->isJsonValidGuarded(input_expr); String result_name = genFuncString(func_name, {arg}, {collator}, {&input_expr.field_type(), &expr.field_type()}); - // Guarded and strict casts can coexist in different logical branches and must not share an action. - if (ignore_invalid_json) - result_name += "_json_valid_guarded"; if (actions->getSampleBlock().has(result_name)) return result_name; @@ -330,7 +318,6 @@ String DAGExpressionAnalyzerHelper::buildSingleParamJsonRelatedFunctions( { function_cast_string_as_json->setInputTiDBFieldType(input_expr.field_type()); function_cast_string_as_json->setOutputTiDBFieldType(expr.field_type()); - function_cast_string_as_json->setIgnoreInvalidJson(ignore_invalid_json); } else if (auto * function_cast_time_as_json = dynamic_cast(function_impl); function_cast_time_as_json) diff --git a/dbms/src/Flash/tests/gtest_filter_executor.cpp b/dbms/src/Flash/tests/gtest_filter_executor.cpp index 04688ca612d..00704b64dfa 100644 --- a/dbms/src/Flash/tests/gtest_filter_executor.cpp +++ b/dbms/src/Flash/tests/gtest_filter_executor.cpp @@ -90,6 +90,43 @@ try } CATCH +TEST_F(FilterExecutorTestRunner, ShortCircuitJsonGuard) +try +{ + context.addMockTable( + {"test_db", "json_guard"}, + {{"document", TiDB::TP::TypeString}}, + {toNullableVec("document", {"", "invalid json", R"({"a": 1})", {}, R"({"b": 2})"})}); + auto request = context.scan("test_db", "json_guard").filter(eq(col("document"), col("document"))).build(context); + auto * executor = request->has_root_executor() ? request->mutable_root_executor() + : request->mutable_executors(request->executors_size() - 1); + ASSERT_TRUE(executor->has_selection()); + auto * selection = executor->mutable_selection(); + const auto column_ref = selection->conditions(0).children(0); + auto json_valid = selection->conditions(0); + json_valid.set_sig(tipb::ScalarFuncSig::JsonValidStringSig); + json_valid.clear_children(); + *json_valid.add_children() = column_ref; + auto cast_json = json_valid; + cast_json.set_sig(tipb::ScalarFuncSig::CastStringAsJson); + cast_json.mutable_field_type()->set_tp(TiDB::TypeJSON); + cast_json.mutable_field_type()->set_flag(TiDB::ColumnFlagParseToJSON); + auto is_null = json_valid; + is_null.set_sig(tipb::ScalarFuncSig::StringIsNull); + *is_null.mutable_children(0) = cast_json; + auto is_not_null = json_valid; + is_not_null.set_sig(tipb::ScalarFuncSig::UnaryNotInt); + *is_not_null.mutable_children(0) = is_null; + selection->clear_conditions(); + *selection->add_conditions() = json_valid; + *selection->add_conditions() = is_not_null; + + WRAP_FOR_TEST_BEGIN + executeAndAssertColumnsEqual(request, {toNullableVec({R"({"a": 1})", R"({"b": 2})"})}); + WRAP_FOR_TEST_END +} +CATCH + TEST_F(FilterExecutorTestRunner, andOr) try { diff --git a/dbms/src/Functions/FunctionsConversion.h b/dbms/src/Functions/FunctionsConversion.h index 3dad20ce3b5..7dd96d00569 100644 --- a/dbms/src/Functions/FunctionsConversion.h +++ b/dbms/src/Functions/FunctionsConversion.h @@ -2650,6 +2650,8 @@ class ExecutableFunctionCast : public IExecutableFunction class FunctionCast final : public IFunctionBase { public: + bool isSuitableForShortCircuitArgumentsExecution() const override { return true; } + using WrapperType = std::function; using MonotonicityForRange = std::function; diff --git a/dbms/src/Functions/FunctionsJson.h b/dbms/src/Functions/FunctionsJson.h index a2aa587714b..092035533af 100644 --- a/dbms/src/Functions/FunctionsJson.h +++ b/dbms/src/Functions/FunctionsJson.h @@ -1473,7 +1473,6 @@ class FunctionCastStringAsJson : public IFunction void setInputTiDBFieldType(const tipb::FieldType & tidb_tp_) { input_tidb_tp = tidb_tp_; } void setOutputTiDBFieldType(const tipb::FieldType & tidb_tp_) { output_tidb_tp = tidb_tp_; } - void setIgnoreInvalidJson(bool value) { ignore_invalid_json = value; } void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -1573,18 +1572,11 @@ class FunctionCastStringAsJson : public IFunction offsets_to, input_source, column_nullable.getNullMapData(), - block.rows(), - ignore_invalid_json); + block.rows()); } else { - doExecuteForParsingJson( - data_to, - offsets_to, - input_source, - {}, - block.rows(), - ignore_invalid_json); + doExecuteForParsingJson(data_to, offsets_to, input_source, {}, block.rows()); } } else @@ -1705,8 +1697,7 @@ class FunctionCastStringAsJson : public IFunction ColumnString::Offsets & offsets_to, const std::unique_ptr & data_from, const NullMap & null_map_from, - size_t size, - bool ignore_invalid_json) + size_t size) { // json_type + size of data_from. size_t reserve_size = size + data_from->getSizeForReserve(); @@ -1727,32 +1718,14 @@ class FunctionCastStringAsJson : public IFunction const auto & slice = data_from->getWhole(); if (unlikely(slice.size == 0)) - { - if (!ignore_invalid_json) - throw Exception("Invalid JSON text: The document is empty."); - JsonBinary::appendNull(write_buffer); - writeChar(0, write_buffer); - offsets_to[i] = write_buffer.count(); - data_from->next(); - continue; - } + throw Exception("Invalid JSON text: The document is empty."); const auto & json_elem = parser.parse(slice.data, slice.size); if (unlikely(json_elem.error())) - { - if (!ignore_invalid_json || checkJsonValid(reinterpret_cast(slice.data), slice.size)) - { - throw Exception(fmt::format( - "Invalid JSON text: The document root must not be followed by other values, details: {}", - simdjson::error_message(json_elem.error()))); - } - // Keep vectorized evaluation alive until the matching JSON_VALID conjunct filters this row. - JsonBinary::appendNull(write_buffer); - } - else - { - JsonBinary::appendSIMDJsonElem(write_buffer, json_elem.value_unsafe()); - } + throw Exception(fmt::format( + "Invalid JSON text: The document root must not be followed by other values, details: {}", + simdjson::error_message(json_elem.error()))); + JsonBinary::appendSIMDJsonElem(write_buffer, json_elem.value_unsafe()); writeChar(0, write_buffer); offsets_to[i] = write_buffer.count(); @@ -1783,7 +1756,6 @@ class FunctionCastStringAsJson : public IFunction std::optional input_tidb_tp; std::optional output_tidb_tp; TiDB::TiDBCollatorPtr collator = nullptr; - bool ignore_invalid_json = false; }; class FunctionCastTimeAsJson : public IFunction diff --git a/dbms/src/Functions/FunctionsLogical.h b/dbms/src/Functions/FunctionsLogical.h index 66db640de73..5e411b97640 100644 --- a/dbms/src/Functions/FunctionsLogical.h +++ b/dbms/src/Functions/FunctionsLogical.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include #include @@ -543,6 +544,7 @@ class FunctionAnyArityLogical : public IFunction size_t getNumberOfArguments() const override { return 0; } bool useDefaultImplementationForNulls() const override { return !special_impl_for_nulls; } + bool isShortCircuit() const override { return Impl::isSaturable; } /// Get result types by argument types. If the function does not apply to /// these arguments, throw an exception. @@ -994,8 +996,82 @@ class FunctionAnyArityLogical : public IFunction } } + void executeShortCircuit(Block & block, const ColumnNumbers & arguments, size_t result) const + { + const size_t rows = block.getByPosition(arguments.front()).column->size(); + const bool nullable = block.getByPosition(result).type->isNullable(); + auto values = ColumnUInt8::create(rows, static_cast(!Impl::isSaturatedValue(true))); + auto nulls = ColumnUInt8::create(rows, 0); + auto & data = values->getData(); + auto & null_map = nulls->getData(); + IColumn::Filter mask(rows, 1); + + for (auto position : arguments) + { + // Keep the deferred column immutable: another branch can request a different mask. + auto argument = block.getByPosition(position); + maskedExecute(argument, mask); + const IColumn * column = argument.column.get(); + const bool constant = column->isColumnConst(); + if (constant) + column = &static_cast(*column).getDataColumn(); + + const UInt8Container * argument_nulls = nullptr; + if (const auto * nullable_column = checkAndGetColumn(column)) + { + argument_nulls = &nullable_column->getNullMapData(); + column = &nullable_column->getNestedColumn(); + } + + UInt8Container converted; + const UInt8Container * argument_values = nullptr; + if (const auto * uint8_column = checkAndGetColumn(column)) + argument_values = &uint8_column->getData(); + else + { + converted.resize(constant ? 1 : rows, 0); + if (!argument.column->onlyNull()) + convertToUInt8(column, converted); + argument_values = &converted; + } + + size_t remaining = 0; + for (size_t row = 0; row < rows; ++row) + { + if (!mask[row]) + continue; + const size_t index = constant ? 0 : row; + const bool is_null = argument.column->onlyNull() || (argument_nulls && (*argument_nulls)[index]); + const bool value = (*argument_values)[index] != 0; + if constexpr (null_as_false) + data[row] = Impl::apply(data[row], !is_null && value); + else + std::tie(data[row], null_map[row]) + = Impl::applyTwoNullable(data[row], null_map[row], value, is_null); + mask[row] = !Impl::isSaturatedValue(data[row], null_map[row]); + remaining += mask[row]; + } + if (remaining == 0) + break; + } + + block.getByPosition(result).column + = nullable ? ColumnNullable::create(std::move(values), std::move(nulls)) : ColumnPtr(std::move(values)); + } + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override { + if constexpr (Impl::isSaturable) + { + for (auto argument : arguments) + { + if (checkAndGetShortCircuitArgument(block.getByPosition(argument).column)) + { + executeShortCircuit(block, arguments, result); + return; + } + } + } bool has_nullable_input_column = false; size_t num_arguments = arguments.size(); diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp index 67326c0dbbe..f2459b286b6 100644 --- a/dbms/src/Functions/FunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp @@ -279,6 +279,8 @@ class FunctionToColumnTypeName : public IFunction class FunctionDumpColumnStructure : public IFunction { public: + bool isSuitableForShortCircuitArgumentsExecution() const override { return false; } + static constexpr auto name = "dumpColumnStructure"; static FunctionPtr create(const Context &) { return std::make_shared(); } diff --git a/dbms/src/Functions/FunctionsTiDBConversion.h b/dbms/src/Functions/FunctionsTiDBConversion.h index bbb7a9d1611..35d40bcb4e2 100644 --- a/dbms/src/Functions/FunctionsTiDBConversion.h +++ b/dbms/src/Functions/FunctionsTiDBConversion.h @@ -1828,6 +1828,8 @@ template class FunctionTiDBCast final : public IFunctionBase { public: + bool isSuitableForShortCircuitArgumentsExecution() const override { return true; } + using WrapperType = std::function; diff --git a/dbms/src/Functions/IFunction.h b/dbms/src/Functions/IFunction.h index d09b45d32dd..b899a15a637 100644 --- a/dbms/src/Functions/IFunction.h +++ b/dbms/src/Functions/IFunction.h @@ -114,6 +114,12 @@ class IFunctionBase */ virtual bool isSuitableForConstantFolding() const { return true; } + /// Short-circuit functions consume deferred arguments themselves. The first argument is always required. + virtual bool isShortCircuit() const { return false; } + + /// Opt in only for row-independent functions that can execute on a filtered block. + virtual bool isSuitableForShortCircuitArgumentsExecution() const { return false; } + /** Function is called "injective" if it returns different result for different values of arguments. * Example: hex, negate, tuple... * @@ -278,6 +284,12 @@ class IFunction virtual bool isDeterministicInScopeOfQuery() const { return true; } virtual bool hasInformationAboutMonotonicity() const { return false; } + virtual bool isShortCircuit() const { return false; } + virtual bool isSuitableForShortCircuitArgumentsExecution() const + { + return isDeterministic() && isDeterministicInScopeOfQuery() && isSuitableForConstantFolding(); + } + using Monotonicity = IFunctionBase::Monotonicity; virtual Monotonicity getMonotonicityForRange( const IDataType & /*type*/, @@ -367,6 +379,12 @@ class DefaultFunctionBase final : public IFunctionBase bool isSuitableForConstantFolding() const override { return function->isSuitableForConstantFolding(); } + bool isShortCircuit() const override { return function->isShortCircuit(); } + bool isSuitableForShortCircuitArgumentsExecution() const override + { + return function->isSuitableForShortCircuitArgumentsExecution(); + } + bool isInjective(const Block & sample_block) override { return function->isInjective(sample_block); } bool isDeterministic() const override { return function->isDeterministic(); } diff --git a/dbms/src/Functions/tests/gtest_json_valid.cpp b/dbms/src/Functions/tests/gtest_json_valid.cpp index 0e1798c5bd2..978dbe8b578 100644 --- a/dbms/src/Functions/tests/gtest_json_valid.cpp +++ b/dbms/src/Functions/tests/gtest_json_valid.cpp @@ -146,6 +146,7 @@ try auto actions = std::make_shared(block.getColumnsWithTypeAndName()); DAGExpressionAnalyzer analyzer(block, *context); const auto filter_column = analyzer.buildFilterColumn(actions, conditions, true); + actions->finalize({filter_column}); actions->execute(block); return block.getByName(filter_column); }; @@ -183,6 +184,27 @@ try make_field_type(TiDB::TypeLongLong, TiDB::ColumnFlagIsBooleanFlag)); *guarded_and.add_children() = json_valid; *guarded_and.add_children() = is_not_null; + + { + Block block({createColumn({"", "invalid json", R"({"a": 1})"}, "json")}); + auto actions = std::make_shared(block.getColumnsWithTypeAndName()); + DAGExpressionAnalyzer analyzer(block, *context); + const auto result = analyzer.getActions(guarded_and, actions, true); + actions->finalize({result}); + actions->execute(block); + ASSERT_COLUMN_EQ(createColumn({0, 0, 1}), block.getByName(result)); + } + { + Block block({createColumn({"", "invalid json", R"({"a": 1})"}, "json")}); + DAGExpressionAnalyzer analyzer(block, *context); + auto [before_where, filter_column, after_where] = analyzer.buildPushDownFilter(guarded_conditions, true); + before_where->execute(block); + ASSERT_COLUMN_EQ(createColumn({0, 0, 1}), block.getByName(filter_column)); + after_where->execute(block); + ASSERT_EQ(block.columns(), 1); + ASSERT_TRUE(block.has("json")); + } + auto unguarded_or = make_scalar( tipb::ScalarFuncSig::LogicalOr, make_field_type(TiDB::TypeLongLong, TiDB::ColumnFlagIsBooleanFlag)); diff --git a/dbms/src/Functions/tests/gtest_short_circuit.cpp b/dbms/src/Functions/tests/gtest_short_circuit.cpp new file mode 100644 index 00000000000..bb0c6694c2a --- /dev/null +++ b/dbms/src/Functions/tests/gtest_short_circuit.cpp @@ -0,0 +1,399 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include + +namespace DB::tests +{ +class ShortCircuit : public FunctionTest +{ +protected: + void add(ExpressionActions & actions, const String & function, const Names & arguments, const String & result) + { + actions.add( + ExpressionAction::applyFunction(FunctionFactory::instance().get(function, *context), arguments, result)); + } + + static size_t lazyCount(const ExpressionActions & actions) + { + return std::count_if(actions.getActions().begin(), actions.getActions().end(), [](const auto & action) { + return action.is_lazy_executed; + }); + } +}; + +TEST_F(ShortCircuit, AndOrDivision) +{ + for (const String name : {"and", "or", "two_value_and"}) + { + const bool is_or = name == "or"; + Block input({ + createColumn({0, 2, 0, 4}, "denominator"), + createConstColumn(4, 12, "numerator"), + createConstColumn(4, 0, "zero"), + createConstColumn(4, 5, "five"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, is_or ? "equals" : "notEquals", {"denominator", "zero"}, "guard"); + add(actions, "intDiv", {"numerator", "denominator"}, "division"); + add(actions, "greater", {"division", "five"}, "comparison"); + add(actions, name, {"guard", "comparison"}, "result"); + actions.finalize({"result"}); + ASSERT_EQ(lazyCount(actions), 2); + + // The same immutable plan and captures can be used for multiple blocks. + for (size_t run = 0; run < 2; ++run) + { + Block block = input; + actions.execute(block); + ASSERT_COLUMN_EQ(createColumn({is_or, 1, is_or, 0}), block.getByName("result")); + for (const auto & column : block) + ASSERT_EQ(checkAndGetShortCircuitArgument(column.column), nullptr); + } + Block empty = input.cloneEmpty(); + actions.execute(empty); + ASSERT_EQ(empty.getByName("result").column->size(), 0); + } +} + +TEST_F(ShortCircuit, UnrelatedHelperColumnHasDifferentSize) +{ + for (const auto & name : {"and", "or", "two_value_and"}) + { + const bool is_or = String(name) == "or"; + Block input({ + createColumn({}, "match_helper"), + createColumn({is_or, !is_or, !is_or}, "guard"), + createColumn({0, 2, 3}, "denominator"), + createConstColumn(3, 6, "six"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"six", "denominator"}, "division"); + add(actions, name, {"guard", "division"}, "result"); + actions.finalize({"match_helper", "result"}); + ASSERT_EQ(lazyCount(actions), 1); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({is_or, 1, 1}), input.getByName("result")); + } +} + +TEST_F(ShortCircuit, RequiredDivisionStillThrows) +{ + Block input({createColumn({0, 1}, "denominator"), createConstColumn(2, 1, "one")}); + for (const Names & arguments : {Names{"one", "division"}, Names{"division", "one"}}) + { + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"one", "denominator"}, "division"); + add(actions, "and", arguments, "result"); + actions.finalize({"result"}); + Block block = input; + ASSERT_THROW(actions.execute(block), Exception); + } +} + +TEST_F(ShortCircuit, CastPreservesDeferredSubtree) +{ + for (const String name : {"CAST", "tidb_cast"}) + { + Block input({ + createColumn({0, 2, 0, 4}, "denominator"), + createColumn({0, 1, 0, 1}, "guard"), + createConstColumn(4, 12, "numerator"), + createConstColumn(4, "Float64", "type"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"numerator", "denominator"}, "division"); + add(actions, name, {"division", "type"}, "cast"); + add(actions, "and", {"guard", "cast"}, "result"); + actions.finalize({"result"}); + ASSERT_EQ(lazyCount(actions), 2); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({0, 1, 0, 1}), input.getByName("result")); + } +} + +TEST_F(ShortCircuit, NullableTruthTables) +try +{ + Block input({ + createColumn>({0, 0, 0, -0.5, -0.5, -0.5, {}, {}, {}}, "left"), + createColumn>({0, 0.5, {}, 0, 0.5, {}, 0, 0.5, {}}, "right"), + createConstColumn(9, 0, "zero"), + }); + for (const String name : {"and", "or", "two_value_and"}) + { + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "plus", {"right", "zero"}, "deferred_right"); + add(actions, name, {"left", "deferred_right"}, "result"); + actions.finalize({"result"}); + ASSERT_EQ(lazyCount(actions), 1); + Block block = input; + actions.execute(block); + ASSERT_COLUMN_EQ( + executeFunction(name, {input.getByName("left"), input.getByName("right")}, nullptr, true), + block.getByName("result")); + } +} +CATCH + +TEST_F(ShortCircuit, NullIsNotFalseForThreeValuedAnd) +{ + Block input({ + createColumn>({{}}, "null"), + createColumn({0}, "zero"), + createConstColumn(1, 1, "one"), + }); + for (const String name : {"and", "or", "two_value_and"}) + { + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"one", "zero"}, "division"); + add(actions, name, {"null", "division"}, "result"); + actions.finalize({"result"}); + Block block = input; + if (name == "two_value_and") + { + actions.execute(block); + ASSERT_COLUMN_EQ(createColumn({0}), block.getByName("result")); + } + else + ASSERT_THROW(actions.execute(block), Exception); + } +} + +TEST_F(ShortCircuit, VariadicWithOnlyNullAndConstants) +try +{ + auto null_column = createOnlyNullColumn(4); + null_column.name = "null"; + for (const UInt8 constant : {0, 1}) + { + Block input({ + createColumn({0, 1, 0, 1}, "first"), + createColumn>({{}, 0, 2, {}}, "last"), + createConstColumn(4, constant, "constant"), + createConstColumn(4, 0, "zero"), + null_column, + }); + for (const String name : {"and", "or", "two_value_and"}) + { + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "plus", {"last", "zero"}, "deferred_last"); + add(actions, name, {"first", "null", "constant", "deferred_last"}, "result"); + actions.finalize({"result"}); + Block block = input; + actions.execute(block); + auto expected = executeFunction( + name, + {input.getByName("first"), null_column, input.getByName("constant"), input.getByName("last")}, + nullptr, + true); + if (auto materialized = expected.column->convertToFullColumnIfConst()) + expected.column = std::move(materialized); + ASSERT_COLUMN_EQ(expected, block.getByName("result")); + } + } +} +CATCH + +TEST_F(ShortCircuit, AllSelectedAndAllSkipped) +{ + for (const UInt8 selected : {0, 1}) + { + Block input({ + createConstColumn(3, selected, "guard"), + createColumn({selected, selected, selected}, "denominator"), + createConstColumn(3, 1, "one"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"one", "denominator"}, "division"); + add(actions, "and", {"guard", "division"}, "result"); + actions.finalize({"result"}); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({selected, selected, selected}), input.getByName("result")); + } +} + +TEST_F(ShortCircuit, FailedConstantFoldingIsDeferred) +{ + for (const UInt8 selected : {0, 1}) + { + Block input({ + createColumn({selected, selected}, "guard"), + createConstColumn(2, 0, "zero"), + createConstColumn(2, 1, "one"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + ASSERT_NO_THROW(add(actions, "intDiv", {"one", "zero"}, "division")); + add(actions, "and", {"guard", "division"}, "result"); + actions.finalize({"result"}); + if (selected) + ASSERT_THROW(actions.execute(input), Exception); + else + { + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({0, 0}), input.getByName("result")); + } + } +} + +TEST_F(ShortCircuit, SharedExpressionUsesIndependentMasks) +{ + Block input({ + createColumn({0, 2, 3}, "denominator"), + createColumn({0, 1, 0}, "left_guard"), + createColumn({0, 0, 1}, "right_guard"), + createConstColumn(3, 6, "six"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"six", "denominator"}, "shared"); + add(actions, "and", {"left_guard", "shared"}, "left_result"); + add(actions, "and", {"right_guard", "shared"}, "right_result"); + actions.finalize({"left_result", "right_result"}); + ASSERT_EQ(lazyCount(actions), 1); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({0, 1, 0}), input.getByName("left_result")); + ASSERT_COLUMN_EQ(createColumn({0, 0, 1}), input.getByName("right_result")); +} + +TEST_F(ShortCircuit, SharedEagerConsumerAndOutputPreventDeferral) +{ + for (const bool shared_output : {false, true}) + { + Block input({ + createColumn({0, 1}, "denominator"), + createConstColumn(2, 1, "one"), + createConstColumn(2, 0, "zero"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"one", "denominator"}, "division"); + add(actions, "and", {"zero", "division"}, "guarded"); + Names outputs{"guarded"}; + if (shared_output) + outputs.push_back("division"); + else + { + add(actions, "plus", {"division", "one"}, "unguarded"); + outputs.push_back("unguarded"); + } + actions.finalize(outputs); + ASSERT_EQ(lazyCount(actions), 0); + ASSERT_THROW(actions.execute(input), Exception); + } +} + +TEST_F(ShortCircuit, NestedLogicalFunctionsAndAliases) +{ + Block input({ + createColumn({0, 2, 3, 0}, "denominator"), + createColumn({0, 1, 1, 0}, "outer_guard"), + createColumn({0, 0, 1, 0}, "inner_guard"), + createConstColumn(4, 6, "six"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"six", "denominator"}, "division"); + actions.add(ExpressionAction::copyColumn("division", "alias")); + add(actions, "or", {"inner_guard", "alias"}, "nested"); + add(actions, "and", {"outer_guard", "nested"}, "result"); + actions.finalize({"result"}); + ASSERT_EQ(lazyCount(actions), 2); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({0, 1, 1, 0}), input.getByName("result")); +} + +TEST_F(ShortCircuit, TypeErrorsRemainStrict) +{ + Block input({createColumn({"text"}, "text"), createConstColumn(1, 0, "zero")}); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + ASSERT_THROW(add(actions, "and", {"zero", "text"}, "result"), Exception); +} + +TEST_F(ShortCircuit, JsonSubtreeIsSkippedIncludingDynamicPath) +{ + for (const bool invalid_selected_path : {false, true}) + { + Block input({ + createColumn({"invalid json", R"({"a": 1})", "", R"({"b": 2})"}, "document"), + createColumn({"invalid path", invalid_selected_path ? "invalid path" : "$.a", "[", "$.a"}, "path"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "json_valid_string", {"document"}, "guard"); + add(actions, "cast_string_as_json", {"document"}, "json"); + add(actions, "json_extract", {"json", "path"}, "extracted"); + add(actions, "isNotNull", {"extracted"}, "present"); + add(actions, "and", {"guard", "present"}, "result"); + // This is a projected expression, not an analyzer filter with a JSON-specific guard. + actions.finalize({"result"}); + ASSERT_EQ(lazyCount(actions), 3); + if (invalid_selected_path) + ASSERT_THROW(actions.execute(input), Exception); + else + { + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({0, 1, 0, 0}), input.getByName("result")); + } + } +} + +TEST_F(ShortCircuit, ConstantInvalidJsonIsSkipped) +{ + Block input({ + createConstColumn(2, "invalid json", "document"), + createConstColumn(2, 0, "guard"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "cast_string_as_json", {"document"}, "json"); + add(actions, "isNotNull", {"json"}, "present"); + add(actions, "and", {"guard", "present"}, "result"); + actions.finalize({"result"}); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({0, 0}), input.getByName("result")); +} + +TEST_F(ShortCircuit, RowDependentFunctionRemainsEager) +{ + Block input({ + createColumn({10, 20, 30}, "value"), + createColumn({0, 1, 1}, "guard"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "runningDifference", {"value"}, "difference"); + add(actions, "and", {"guard", "difference"}, "result"); + actions.finalize({"result"}); + ASSERT_EQ(lazyCount(actions), 0); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({0, 1, 1}), input.getByName("result")); +} + +TEST_F(ShortCircuit, AppendActionsAfterFinalize) +{ + Block input({ + createColumn({0, 2}, "denominator"), + createColumn({0, 1}, "guard"), + createConstColumn(2, 6, "six"), + }); + ExpressionActions actions(input.getColumnsWithTypeAndName()); + add(actions, "intDiv", {"six", "denominator"}, "division"); + add(actions, "and", {"guard", "division"}, "result"); + actions.finalize({"result"}); + add(actions, "not", {"result"}, "negated"); + actions.add(ExpressionAction::project(NamesWithAliases{{"negated", "output"}})); + actions.execute(input); + ASSERT_COLUMN_EQ(createColumn({1, 0}), input.getByName("output")); +} +} // namespace DB::tests diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 1f266133df9..a2d1236f943 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -33,6 +34,7 @@ #pragma GCC diagnostic pop #include +#include #include #include @@ -47,6 +49,8 @@ extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; extern const int TOO_MANY_TEMPORARY_COLUMNS; extern const int TOO_MANY_TEMPORARY_NON_CONST_COLUMNS; +extern const int MEMORY_LIMIT_EXCEEDED; +extern const int QUERY_WAS_CANCELLED; } // namespace ErrorCodes @@ -202,7 +206,20 @@ void ExpressionAction::prepare(Block & sample_block) new_column.type = result_type; sample_block.insert(std::move(new_column)); - function->execute(sample_block, arguments, result_position); + try + { + function->execute(sample_block, arguments, result_position); + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::LOGICAL_ERROR || e.code() == ErrorCodes::MEMORY_LIMIT_EXCEEDED + || e.code() == ErrorCodes::QUERY_WAS_CANCELLED) + throw; + // A parent short-circuit function is not known yet. Leave failed constant + // evaluation to execution, where the expression may be on an unselected branch. + sample_block.getByPosition(result_position).column = nullptr; + break; + } /// If the result is not a constant, just in case, we will consider the result as unknown. ColumnWithTypeAndName & col = sample_block.safeGetByPosition(result_position); @@ -361,7 +378,19 @@ void ExpressionAction::execute(Block & block) const size_t num_columns_without_result = block.columns(); block.insert({nullptr, result_type, result_name}); - function->execute(block, arguments, num_columns_without_result); + if (is_lazy_executed) + { + ColumnsWithTypeAndName captured; + captured.reserve(arguments.size()); + for (auto argument : arguments) + captured.push_back(block.getByPosition(argument)); + // Join blocks may contain unrelated helper columns with a different row count. + const size_t rows = captured.empty() ? block.rows() : captured.front().column->size(); + block.getByPosition(num_columns_without_result).column + = ColumnFunction::create(rows, function, captured, true); + } + else + function->execute(block, arguments, num_columns_without_result); break; } @@ -537,6 +566,8 @@ void ExpressionActions::addImpl(ExpressionAction action, Names & new_names) action.prepare(sample_block); actions.push_back(action); + if (short_circuit_prepared) + prepareShortCircuitActions(); } void ExpressionActions::prependProjectInput() @@ -733,6 +764,58 @@ void ExpressionActions::finalize(const Names & output_columns, bool keep_used_in } actions.swap(new_actions); + prepareShortCircuitActions(); +} + +void ExpressionActions::prepareShortCircuitActions() +{ + short_circuit_prepared = true; + // Adapted from ClickHouse's lazy-node analysis. In the linear action representation, + // reverse traversal visits every consumer before its producer. Any eager consumer wins. + Names output_names; + for (const auto & column : sample_block) + output_names.push_back(column.name); + NameSet eager(output_names.begin(), output_names.end()); + NameSet deferred; + for (auto & action : std::views::reverse(actions)) + { + action.is_lazy_executed = false; + if (action.type == ExpressionAction::REMOVE_COLUMN) + continue; + + if (action.type == ExpressionAction::APPLY_FUNCTION) + { + action.is_lazy_executed = deferred.contains(action.result_name) && !eager.contains(action.result_name) + && action.function->isSuitableForShortCircuitArgumentsExecution(); + eager.erase(action.result_name); + deferred.erase(action.result_name); + for (size_t i = 0; i < action.argument_names.size(); ++i) + { + const bool lazy_argument = action.is_lazy_executed || (action.function->isShortCircuit() && i != 0); + (lazy_argument ? deferred : eager).insert(action.argument_names[i]); + } + } + else if (action.type == ExpressionAction::COPY_COLUMN) + { + const bool lazy = deferred.contains(action.result_name) && !eager.contains(action.result_name); + eager.erase(action.result_name); + deferred.erase(action.result_name); + (lazy ? deferred : eager).insert(action.source_name); + } + else if (action.type == ExpressionAction::ADD_COLUMN) + { + eager.erase(action.result_name); + deferred.erase(action.result_name); + } + else + { + // Do not move deferred computation across projection, join, expand or nullable conversion. + eager.insert(deferred.begin(), deferred.end()); + deferred.clear(); + for (const auto & name : action.getNeededColumns()) + eager.insert(name); + } + } } diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index 0cad5ec9caa..a2441f9ab9f 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -94,6 +94,7 @@ struct ExpressionAction FunctionBasePtr function; Names argument_names; TiDB::TiDBCollatorPtr collator = nullptr; + bool is_lazy_executed = false; /// For JOIN std::shared_ptr join; @@ -218,8 +219,10 @@ class ExpressionActions NamesAndTypesList input_columns; Actions actions; Block sample_block; + bool short_circuit_prepared = false; void addImpl(ExpressionAction action, Names & new_names); + void prepareShortCircuitActions(); }; using ExpressionActionsPtr = std::shared_ptr; diff --git a/docs/design/2026-09-07-expression-short-circuit.md b/docs/design/2026-09-07-expression-short-circuit.md new file mode 100644 index 00000000000..cff4031b057 --- /dev/null +++ b/docs/design/2026-09-07-expression-short-circuit.md @@ -0,0 +1,123 @@ +# Short-Circuit Evaluation of Logical Expressions + +## Purpose + +Replace the JSON_VALID-specific guard introduced by PR #11036 with deferred +expression evaluation. A guard must prevent execution of an unneeded expression +subtree, not change the error handling of a JSON cast inside that subtree. + +The initial consumers are `and`, `or`, and `two_value_and`. The mechanism applies +to finalized expression actions, including filters, projections, and pushed-down +filters. It does not require or recognize JSON-specific expression patterns. + +## Planning + +`ExpressionActions::finalize()` marks lazy actions after output pruning and column +lifetime analysis. A reverse traversal records eager and deferred consumers of +each column. An expression may be deferred only if all its consumers allow it and +the function can execute on a filtered block. Result columns remain eager. + +The first argument of a logical function is required. Later arguments can be +deferred; if a logical function itself is deferred by an outer guard, its first +argument can also be deferred until that outer mask has been applied. Aliases +propagate consumer requirements. Projection, join, expansion, and nullable +conversion actions are conservative barriers. + +Ordinary scalar functions are eligible if they are deterministic both globally +and within a query and suitable for constant folding. Block-dependent functions +such as `runningDifference` and `dumpColumnStructure`, random functions, and +side-effecting functions such as `sleep` remain eager. Custom `IFunctionBase` +implementations opt in explicitly. +Functions with additional row or block dependencies must override eligibility. + +Actions that have not been finalized retain eager evaluation and materialized +intermediate columns. Appending an action to a finalized plan recomputes its lazy +marks. Type checking still happens when actions are built. Failed constant value +evaluation is retried at execution, where a parent may skip the expression; +logical, memory-limit, and query-cancellation errors are not deferred. + +## Execution + +An eligible action captures its arguments in the existing `ColumnFunction`. +Its short-circuit flag distinguishes it from the existing lambda representation. +Logical functions maintain an active-row mask and evaluate arguments in action +argument order: + +- `and` stops on a known false value. +- `or` stops on a known true value. +- SQL NULL does not saturate either three-valued operation. +- `two_value_and` treats NULL as false and stops on it. + +`maskedExecute()` filters the captured inputs, reduces the expression on selected +rows, and expands its result to the original row positions. Expansion uses the +existing bulk column insertion methods rather than adding an interface to every +column implementation. Unselected result slots are placeholders used only by the +logical consumer; they are never fed through the skipped expression subtree. +An empty mask does not execute the expression; a full mask avoids filtering and +expansion. Logical functions without deferred arguments retain their existing +vectorized implementation. + +Captured expressions remain immutable. Shared deferred expressions can be +evaluated under different masks without reusing a result from the wrong branch. +An independent eager consumer forces the shared producer to remain eager; short +circuiting cannot suppress an error required by another output. + +## ClickHouse Adaptation and Boundaries + +The reference implementation is the local ClickHouse source tree, specifically +`Interpreters/ExpressionActions.cpp` lazy-node analysis, +`Columns/ColumnFunction.cpp`, `Columns/MaskOperations.cpp::maskedExecute`, and +`Functions/FunctionsLogical.cpp`. TiFlash retains its linear actions, existing +column interfaces, and TiDB-specific logical and arithmetic semantics. + +This change does not port ClickHouse's SQL setting or its cost-based distinction +between `enable` and `force_enable`. Eligible later-argument subtrees are deferred +by default. Filtering allocations and repeated execution of shared subtrees are +potential costs; a cost policy or mask-aware cache requires separate benchmarks. +This change also does not implement `if`/`multiIf` branch masks or NULL-driven +short circuiting for ordinary functions. + +There is no guarantee that textual SQL predicate order survives TiDB planning. +Short circuiting follows the expression order delivered to TiFlash and does not +reorder predicates to discover guards. Strict JSON parsing remains unchanged on +selected rows. `tidbDivide` continues returning NULL for division by zero as +required by TiDB semantics; that behavior is not the JSON guard being removed. + +## Regression Coverage + +`ShortCircuit.*` exercises throwing arithmetic chains, complete JSON subtrees +including dynamic paths, constant errors, nullable truth tables, variadic logical +arguments, nested expressions, aliases, shared consumers, repeated execution, +empty blocks, block-dependent functions, and plan extension. + +`TestJsonValid.GuardStringToJsonParsingInFilter` retains the original PR's positive +and negative cases and also exercises analyzer projection and pushed-down filter +action chains. Existing logical, JSON, arithmetic, filter executor, and projection +executor suites should be run alongside the new tests. + +## Verification + +Validated on macOS arm64 with Clang 17 in the DEBUG build: + +```bash +cmake --preset dev +cmake --build --preset unit-tests -j 8 + +LOG_LEVEL=error cmake-build-debug/dbms/gtests_dbms \ + --gtest_filter='ShortCircuit.*:Logical.*:TwoValueAnd.*:TestJson*.*:TestCastAsJson.*:TestCastJsonAsString.*:TestBinaryArithmeticFunctions.*:TestTidbConversion.*:FilterExecutorTestRunner.*:ProjectionExecutorTestRunner.*' + +LOG_LEVEL=error cmake-build-debug/dbms/gtests_dbms \ + --gtest_filter='PhysicalPlanTestRunner.*:ExpandBlockInputStreamTest.*:ExpandProjectionTest.*:JoinExecutorTestRunner.SimpleJoin:JoinExecutorTestRunner.JoinCast:JoinExecutorTestRunner.LeftJoinAggWithOtherCondition:JoinExecutorTestRunner.FullOuterJoinWithLeftAndRightConditions' +``` + +The first regression selection passed 123 tests across 20 suites, including all +16 new `ShortCircuit` tests. The second passed 16 tests across four suites. +The JSON guard filter executor test covers both legacy streams and the pipeline +engine, with multiple block sizes and concurrency levels. Formatting checks and +`git diff --check` also passed. + +This is targeted regression coverage, not a full-suite or full-stack TiDB +compatibility run. Sanitizers and performance benchmarks have not been run. +Production-like performance and end-to-end TiDB predicate-planning tests remain +necessary before rollout, particularly because eligible subtrees are deferred +by default without a cost-based policy.