diff --git a/be/src/exprs/function/function_string_misc.cpp b/be/src/exprs/function/function_string_misc.cpp index 663fa0fe018591..c818239301eb34 100644 --- a/be/src/exprs/function/function_string_misc.cpp +++ b/be/src/exprs/function/function_string_misc.cpp @@ -809,8 +809,14 @@ class FunctionNgramSearch : public IFunction { static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } size_t get_number_of_arguments() const override { return 3; } + bool use_default_implementation_for_nulls() const override { return false; } + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return std::make_shared(); + auto result = std::make_shared(); + if (std::ranges::any_of(arguments, [](const auto& type) { return type->is_nullable(); })) { + return make_nullable(result); + } + return result; } // ngram_search(text,pattern,gram_num) @@ -820,12 +826,37 @@ class FunctionNgramSearch : public IFunction { auto col_res = ColumnFloat64::create(); bool col_const[3]; ColumnPtr argument_columns[3]; + NullableColumnInfos nullable_infos(block.columns()); + bool only_null = false; for (int i = 0; i < 3; ++i) { + const auto& argument = block.get_by_position(arguments[i]); + auto& info = nullable_infos[arguments[i]]; + if (argument.type->is_nullable()) { + info = argument.get_nullable_column_info(); + only_null |= info.only_null; + } std::tie(argument_columns[i], col_const[i]) = - unpack_if_const(block.get_by_position(arguments[i]).column); + unpack_if_const(argument.unnest_nullable(info, false).column); + } + // A semantic constant can arrive as a materialized CSE slot. Validate its first value + // before propagating NULL from any argument, including text or pattern. + if (nullable_infos[arguments[2]].is_nullable && + block.get_by_position(arguments[2]).get_nullable_null_map_column()->get_data()[0]) { + return Status::InvalidArgument( + "ngram_search(text,pattern,gram_num): gram_num support const value only."); } - auto pattern = assert_cast(argument_columns[1].get())->get_data_at(0); auto gram_num = assert_cast(argument_columns[2].get())->get_element(0); + if (gram_num <= 0) { + return Status::InvalidArgument( + "ngram_search(text,pattern,gram_num): gram_num must be a positive constant."); + } + if (only_null) { + block.replace_by_position(result, + block.get_by_position(result).type->create_column_const( + input_rows_count, Field())); + return Status::OK(); + } + auto pattern = assert_cast(argument_columns[1].get())->get_data_at(0); const auto* text_col = assert_cast(argument_columns[0].get()); if (col_const[0]) { @@ -834,7 +865,12 @@ class FunctionNgramSearch : public IFunction { _execute_impl(text_col, pattern, gram_num, *col_res, input_rows_count); } - block.replace_by_position(result, std::move(col_res)); + if (block.get_by_position(result).type->is_nullable()) { + block.replace_by_position(result, wrap_in_nullable(std::move(col_res), block, arguments, + nullable_infos, input_rows_count)); + } else { + block.replace_by_position(result, std::move(col_res)); + } return Status::OK(); } diff --git a/be/test/exprs/function/function_string_test.cpp b/be/test/exprs/function/function_string_test.cpp index b908d95ee95375..e92ff84911b253 100644 --- a/be/test/exprs/function/function_string_test.cpp +++ b/be/test/exprs/function/function_string_test.cpp @@ -81,6 +81,97 @@ DataSet make_md5_varbinary_dataset(const std::vector& inputs) { } // namespace +TEST(function_string_test, ngram_search_constant_gram) { + const InputTypeSet input_types = {ConstedNotnull {TYPE_STRING}, ConstedNotnull {TYPE_STRING}, + ConstedNotnull {TYPE_INT}}; + for (int32_t gram : {1, 3, 4}) { + const double expected = gram <= 3 ? 1.0 : 0.0; + const DataSet data_set = {{{std::string("abc"), std::string("abc"), gram}, expected}}; + ASSERT_TRUE(check_function("ngram_search", input_types, data_set).ok()); + } +} + +TEST(function_string_test, ngram_search_nullable_gram) { + const InputTypeSet input_types = {TYPE_STRING, Consted {TYPE_STRING}, Consted {TYPE_INT}}; + const DataSet data_set = {{{std::string("abc"), std::string("abc"), int32_t(1)}, 1.0}, + {{std::string("ab"), std::string("abc"), int32_t(1)}, 0.8}, + {{Null(), std::string("abc"), int32_t(1)}, Null()}}; + for (const auto& row : data_set) { + ASSERT_TRUE( + (check_function("ngram_search", input_types, {row}).ok())); + } +} + +TEST(function_string_test, ngram_search_nonpositive_gram) { + const InputTypeSet input_types = {Notnull {TYPE_STRING}, ConstedNotnull {TYPE_STRING}, + ConstedNotnull {TYPE_INT}}; + const InputTypeSet nullable_types = {TYPE_STRING, Consted {TYPE_STRING}, Consted {TYPE_INT}}; + for (int32_t gram : {0, -1}) { + const DataSet data_set = {{{std::string("abc"), std::string("abc"), gram}, 0.0}}; + auto st = check_function("ngram_search", input_types, data_set, -1, -1, + true); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("gram_num must be a positive constant"), std::string::npos); + st = check_function("ngram_search", nullable_types, data_set, -1, -1, + true); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("gram_num must be a positive constant"), std::string::npos); + } +} + +TEST(function_string_test, ngram_search_materialized_gram) { + // CSE can materialize a constant gram expression into an intermediate slot. + const InputTypeSet input_types = {Notnull {TYPE_STRING}, ConstedNotnull {TYPE_STRING}, + Notnull {TYPE_INT}}; + ASSERT_TRUE(check_function( + "ngram_search", input_types, + {{{std::string("ab"), std::string("abc"), int32_t(1)}, 0.8}}) + .ok()); + const auto st = check_function( + "ngram_search", input_types, + {{{std::string("ab"), std::string("abc"), int32_t(0)}, 0.0}}, -1, -1, true); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("gram_num must be a positive constant"), std::string::npos); +} + +TEST(function_string_test, ngram_search_null_gram) { + for (const auto& input_types : + {InputTypeSet {TYPE_STRING, Consted {TYPE_STRING}, Consted {TYPE_INT}}, + InputTypeSet {TYPE_STRING, Consted {TYPE_STRING}, TYPE_INT}}) { + for (const auto& text : {AnyType(std::string("abc")), AnyType(Null())}) { + const auto st = check_function( + "ngram_search", input_types, {{{text, std::string("abc"), Null()}, Null()}}, -1, + -1, true); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("gram_num support const value only"), std::string::npos); + } + } +} + +TEST(function_string_test, ngram_search_invalid_gram_with_null_text) { + for (const auto& input_types : + {InputTypeSet {Consted {TYPE_STRING}, Consted {TYPE_STRING}, Consted {TYPE_INT}}, + InputTypeSet {TYPE_STRING, Consted {TYPE_STRING}, TYPE_INT}}) { + for (int32_t gram : {0, -1}) { + const auto st = check_function( + "ngram_search", input_types, {{{Null(), std::string("abc"), gram}, Null()}}, -1, + -1, true); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("gram_num must be a positive constant"), + std::string::npos); + } + } +} + +TEST(function_string_test, ngram_search_nullable_multirow) { + const InputTypeSet input_types = {TYPE_STRING, TYPE_STRING, TYPE_INT}; + const DataSet data_set = {{{std::string("abc"), std::string("abc"), int32_t(1)}, 1.0}, + {{Null(), std::string("abc"), int32_t(1)}, Null()}, + {{std::string("ab"), std::string("abc"), int32_t(1)}, 0.8}}; + ASSERT_TRUE( + (check_function("ngram_search", input_types, data_set).ok())); +} + TEST(function_string_test, parse_data_size_nullable) { const InputTypeSet input_types = {PrimitiveType::TYPE_STRING}; const DataSet data_set = {{{Null()}, Null()}, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0f87bdd2aaf54a..6ddebc911e0fe4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -174,6 +174,21 @@ public static Expression foldByBE(ExpressionMatchingContext context) return root; } + /** Evaluate a semantic constant whose value is required for argument validation. */ + public static Expression evaluateConstant(Expression expression, ConnectContext context) { + // Required evaluation must also honor exclusions such as Sleep, which can outlive + // the RPC timeout. Leave excluded expressions unevaluated for the caller to reject. + if (expression.anyMatch(e -> shouldSkipFold((Expression) e))) { + return expression; + } + Expr legacyExpr = ExpressionTranslator.translate(expression, null); + Map constants = Collections.singletonMap("0", expression); + Map thriftExpressions = Collections.singletonMap( + "0", ExprToThriftVisitor.treeToThrift(legacyExpr)); + return evalOnBE(Collections.singletonMap("0", thriftExpressions), constants, context) + .getOrDefault("0", expression); + } + private static Expression replace( Expression root, Map constMap, Map resultMap) { for (Entry entry : constMap.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NgramSearch.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NgramSearch.java index 396244657bc6e1..bdf371551aaa2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NgramSearch.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NgramSearch.java @@ -19,14 +19,20 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -57,21 +63,47 @@ private NgramSearch(ScalarFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { - if (!child(1).isConstant()) { + if (!getArgument(1).isConstant()) { throw new AnalysisException( "ngram_search(text,pattern,gram_num): pattern support const value only."); } - Expression gramNum = child(2); - if (!(gramNum instanceof IntegerLikeLiteral)) { + Expression gramNum = getArgument(2); + if (!gramNum.isConstant() || !gramNum.getDataType().isIntegralType()) { throw new AnalysisException( "ngram_search(text,pattern,gram_num): gram_num support const value only."); } - if (((IntegerLikeLiteral) gramNum).getIntValue() <= 0) { + gramNum = FoldConstantRuleOnFE.evaluateWithoutContext(gramNum); + if (gramNum instanceof NullLiteral) { + throw new AnalysisException( + "ngram_search(text,pattern,gram_num): gram_num support const value only."); + } + if (gramNum instanceof IntegerLikeLiteral && ((IntegerLikeLiteral) gramNum).getIntValue() <= 0) { throw new AnalysisException( "ngram_search(text,pattern,gram_num): gram_num must be a positive constant."); } } + /** Resolve the required constant before rewrites can discard the function call. */ + public NgramSearch withFoldedGramNumber() { + Expression gramNum = getArgument(2); + if (!gramNum.getDataType().equals(IntegerType.INSTANCE)) { + gramNum = new Cast(gramNum, IntegerType.INSTANCE); + } + gramNum = FoldConstantRuleOnFE.evaluateWithoutContext(gramNum); + // Argument validation is independent of the optional BE folding setting. Keep the + // evaluated value in the plan so CSE and execution use exactly the value we validate. + if (!(gramNum instanceof Literal)) { + gramNum = FoldConstantRuleOnBE.evaluateConstant(gramNum, ConnectContext.get()); + if (!(gramNum instanceof Literal)) { + throw new AnalysisException( + "ngram_search(text,pattern,gram_num): failed to evaluate constant gram_num."); + } + } + NgramSearch folded = withChildren(ImmutableList.of(getArgument(0), getArgument(1), gramNum)); + folded.checkLegalityBeforeTypeCoercion(); + return folded; + } + /** * withChildren. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 8341877755d7c5..65bc9710f57571 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -49,6 +49,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.NgramSearch; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -847,6 +848,10 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { return new MapLiteral(); } + if (boundFunction instanceof NgramSearch) { + boundFunction = ((NgramSearch) boundFunction).withFoldedGramNumber(); + } + // type coercion return implicitCastInputTypes(boundFunction, boundFunction.expectedInputTypes()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantRuleOnBETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantRuleOnBETest.java new file mode 100644 index 00000000000000..27cbf24d579f22 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantRuleOnBETest.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression; + +import org.apache.doris.catalog.Env; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Crc32; +import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.proto.InternalService.PConstantExprResult; +import org.apache.doris.proto.InternalService.PExprResult; +import org.apache.doris.proto.InternalService.PExprResultMap; +import org.apache.doris.proto.Types.PGenericType; +import org.apache.doris.proto.Types.PScalarType; +import org.apache.doris.proto.Types.PStatus; +import org.apache.doris.proto.Types.PTypeDesc; +import org.apache.doris.proto.Types.PTypeNode; +import org.apache.doris.proto.Types.PValues; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TFoldConstantParams; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPrimitiveType; +import org.apache.doris.thrift.TTypeNodeType; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; + +class FoldConstantRuleOnBETest { + private final ConnectContext context = new ConnectContext(); + private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class); + private MockedStatic env; + private MockedStatic backendService; + + @BeforeEach + void setUp() throws Exception { + SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class); + Mockito.when(systemInfo.getAllBackendByCurrentCluster(true)) + .thenReturn(new ArrayList<>(Collections.singletonList(1L))); + Backend backend = new Backend(1L, "127.0.0.1", 9050); + backend.setBrpcPort(8060); + Mockito.when(systemInfo.getBackend(1L)).thenReturn(backend); + env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentSystemInfo).thenReturn(systemInfo); + backendService = Mockito.mockStatic(BackendServiceProxy.class); + backendService.when(BackendServiceProxy::getInstance).thenReturn(proxy); + + PScalarType type = PScalarType.newBuilder().setType(TPrimitiveType.INT.getValue()).build(); + PExprResult result = PExprResult.newBuilder().setSuccess(true).setType(type).setContent("1") + .setTypeDesc(PTypeDesc.newBuilder().addTypes(PTypeNode.newBuilder() + .setType(TTypeNodeType.SCALAR.getValue()).setScalarType(type))) + .setResultContent(PValues.newBuilder() + .setType(PGenericType.newBuilder().setId(PGenericType.TypeId.INT32)).addInt32Value(1)) + .build(); + Mockito.when(proxy.foldConstantExpr(Mockito.any(TNetworkAddress.class), Mockito.any(TFoldConstantParams.class))) + .thenReturn(CompletableFuture.completedFuture(PConstantExprResult.newBuilder() + .setStatus(PStatus.newBuilder().setStatusCode(0)) + .putExprResultMap("0", PExprResultMap.newBuilder().putMap("0", result).build()).build())); + } + + @AfterEach + void tearDown() { + backendService.close(); + env.close(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testSkippedExpressionsDoNotSendRpc(boolean foldOnBe) { + context.getSessionVariable().setEnableFoldConstantByBe(foldOnBe); + Expression sleep = new Sleep(new IntegerLiteral(1)); + for (Expression expression : Arrays.asList(sleep, + new Add(new IntegerLiteral(1), new Cast(sleep, IntegerType.INSTANCE)), + new NonNullable(new Crc32(new StringLiteral("abc"))))) { + Expression result = FoldConstantRuleOnBE.evaluateConstant(expression, context); + Mockito.verifyNoInteractions(proxy); + Assertions.assertSame(expression, result); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testSafeConstantStillSendsRpc(boolean foldOnBe) throws Exception { + context.getSessionVariable().setEnableFoldConstantByBe(foldOnBe); + Expression expression = new Cast(new Add( + new Mod(new Crc32(new StringLiteral("abc")), new IntegerLiteral(3)), + new IntegerLiteral(1)), IntegerType.INSTANCE); + Assertions.assertEquals(new IntegerLiteral(1), FoldConstantRuleOnBE.evaluateConstant(expression, context)); + Mockito.verify(proxy).foldConstantExpr(Mockito.any(TNetworkAddress.class), + Mockito.any(TFoldConstantParams.class)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NgramSearchTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NgramSearchTest.java new file mode 100644 index 00000000000000..43a966d4e8690c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NgramSearchTest.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StringType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +class NgramSearchTest { + + @Test + void testLiteralGramNumber() { + assertValidGramNumber(new IntegerLiteral(3)); + } + + @Test + void testFoldableArithmeticGramNumber() { + assertValidGramNumber(new Add(new IntegerLiteral(1), new IntegerLiteral(2))); + } + + @Test + void testFoldableCastGramNumber() { + assertValidGramNumber(new Cast(new StringLiteral("3"), IntegerType.INSTANCE)); + } + + @Test + void testFoldableFunctionGramNumber() { + assertValidGramNumber(new Abs(new IntegerLiteral(-3))); + } + + @Test + void testBackendOnlyConstantGramNumber() { + try (MockedStatic evaluator = Mockito.mockStatic(FoldConstantRuleOnBE.class)) { + evaluator.when(() -> FoldConstantRuleOnBE.evaluateConstant(Mockito.any(), Mockito.any())) + .thenReturn(new IntegerLiteral(1)); + Expression analyzed = ExpressionAnalyzer.analyzeFunction(null, null, + new NgramSearch(new StringLiteral("abc"), new StringLiteral("abc"), backendGram())); + Expression folded = FoldConstantRuleOnFE.evaluateWithoutContext(analyzed.child(2)); + Assertions.assertEquals(new IntegerLiteral(1), folded); + evaluator.verify(() -> FoldConstantRuleOnBE.evaluateConstant( + Mockito.argThat(expression -> expression.getDataType().equals(IntegerType.INSTANCE)), + Mockito.any())); + } + } + + @Test + void testBackendNullGramNumber() { + assertInvalidBackendGramNumber(new NullLiteral(IntegerType.INSTANCE), "gram_num support const value only"); + } + + @Test + void testBackendNonPositiveGramNumber() { + assertInvalidBackendGramNumber(new IntegerLiteral(0), "gram_num must be a positive constant"); + assertInvalidBackendGramNumber(new IntegerLiteral(-1), "gram_num must be a positive constant"); + } + + @Test + void testBackendEvaluationFailure() { + try (MockedStatic evaluator = Mockito.mockStatic(FoldConstantRuleOnBE.class)) { + evaluator.when(() -> FoldConstantRuleOnBE.evaluateConstant(Mockito.any(), Mockito.any())) + .thenAnswer(invocation -> invocation.getArgument(0)); + assertInvalidGramNumber(backendGram(), "failed to evaluate constant gram_num"); + } + } + + private Expression backendGram() { + return new Add(new Mod(new Crc32(new StringLiteral("abc")), new IntegerLiteral(3)), + new IntegerLiteral(1)); + } + + private void assertInvalidBackendGramNumber(Expression value, String message) { + try (MockedStatic evaluator = Mockito.mockStatic(FoldConstantRuleOnBE.class)) { + evaluator.when(() -> FoldConstantRuleOnBE.evaluateConstant(Mockito.any(), Mockito.any())) + .thenReturn(value); + assertInvalidGramNumber(backendGram(), message); + // NULL propagation must not remove the function before validating the gram. + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> ExpressionAnalyzer.analyzeFunction(null, null, + new NgramSearch(new NullLiteral(StringType.INSTANCE), + new StringLiteral("abc"), backendGram()))); + Assertions.assertTrue(exception.getMessage().contains(message), exception.getMessage()); + } + } + + @Test + void testSkippedBackendGramNumber() { + assertInvalidGramNumber(new Add(new IntegerLiteral(1), + new Cast(new Sleep(new IntegerLiteral(1)), IntegerType.INSTANCE)), + "failed to evaluate constant gram_num"); + } + + @Test + void testNonPositiveGramNumber() { + assertInvalidGramNumber(new IntegerLiteral(0), "gram_num must be a positive constant"); + assertInvalidGramNumber(new IntegerLiteral(-1), "gram_num must be a positive constant"); + assertInvalidGramNumber(new Subtract(new IntegerLiteral(1), new IntegerLiteral(1)), + "gram_num must be a positive constant"); + assertInvalidGramNumber(new Subtract(new IntegerLiteral(1), new IntegerLiteral(2)), + "gram_num must be a positive constant"); + } + + @Test + void testNonConstantGramNumber() { + SlotReference gram = SlotReference.of("gram", IntegerType.INSTANCE); + assertInvalidGramNumber(gram, "gram_num support const value only"); + assertInvalidGramNumber(new Add(gram, new IntegerLiteral(1)), "gram_num support const value only"); + assertInvalidGramNumber(new Cast(new Random(), IntegerType.INSTANCE), "gram_num support const value only"); + assertInvalidGramNumber(new ConnectionId(), "gram_num support const value only"); + } + + @Test + void testNonIntegerGramNumber() { + assertInvalidGramNumber(new StringLiteral("3"), "gram_num support const value only"); + assertInvalidGramNumber(new DoubleLiteral(3.0), "gram_num support const value only"); + assertInvalidGramNumber(new NullLiteral(), "gram_num support const value only"); + assertInvalidGramNumber(new Cast(new NullLiteral(), IntegerType.INSTANCE), + "gram_num support const value only"); + } + + @Test + void testNonConstantPattern() { + NgramSearch function = new NgramSearch(new StringLiteral("abc"), + SlotReference.of("pattern", StringType.INSTANCE), new IntegerLiteral(3)); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> ExpressionAnalyzer.analyzeFunction(null, null, function)); + Assertions.assertTrue(exception.getMessage().contains("pattern support const value only")); + } + + private void assertValidGramNumber(Expression gram) { + Expression analyzed = ExpressionAnalyzer.analyzeFunction(null, null, + new NgramSearch(new StringLiteral("abc"), new StringLiteral("abc"), gram)); + Assertions.assertEquals(new IntegerLiteral(3), + FoldConstantRuleOnFE.evaluateWithoutContext(analyzed.child(2))); + } + + private void assertInvalidGramNumber(Expression gram, String message) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> ExpressionAnalyzer.analyzeFunction(null, null, + new NgramSearch(new StringLiteral("abc"), new StringLiteral("abc"), gram))); + Assertions.assertTrue(exception.getMessage().contains(message), exception.getMessage()); + } +} diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_ngram_search_foldable_gram.out b/regression-test/data/query_p0/sql_functions/string_functions/test_ngram_search_foldable_gram.out new file mode 100644 index 00000000000000..e2bec2ab216c2d --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_ngram_search_foldable_gram.out @@ -0,0 +1,112 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !literal -- +1 + +-- !arithmetic -- +1 + +-- !cast -- +1 + +-- !function -- +1 + +-- !nested -- +1 + +-- !be_function -- +1 + +-- !be_cast -- +1 + +-- !nullable_text -- +\N + +-- !be_nullable_text -- +\N + +-- !be_nullable_pattern -- +\N + +-- !be_empty -- + +-- !be_cse -- +891568578 \N +891568578 \N +891568578 0.8 +891568578 0.8 +891568578 1 +891568578 1 + +-- !rows -- +0 1 1 +1 0 1 +2 \N 1 +3 1 1 +4 0 1 +5 \N 1 + +-- !be_rows -- +0 1 1 +1 0.8 1 +2 \N 1 +3 1 1 +4 0.8 1 +5 \N 1 + +-- !literal -- +1 + +-- !arithmetic -- +1 + +-- !cast -- +1 + +-- !function -- +1 + +-- !nested -- +1 + +-- !be_function -- +1 + +-- !be_cast -- +1 + +-- !nullable_text -- +\N + +-- !be_nullable_text -- +\N + +-- !be_nullable_pattern -- +\N + +-- !be_empty -- + +-- !be_cse -- +891568578 \N +891568578 \N +891568578 0.8 +891568578 0.8 +891568578 1 +891568578 1 + +-- !rows -- +0 1 1 +1 0 1 +2 \N 1 +3 1 1 +4 0 1 +5 \N 1 + +-- !be_rows -- +0 1 1 +1 0.8 1 +2 \N 1 +3 1 1 +4 0.8 1 +5 \N 1 diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_ngram_search_foldable_gram.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_ngram_search_foldable_gram.groovy new file mode 100644 index 00000000000000..2d6a9f94d45c30 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_ngram_search_foldable_gram.groovy @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ngram_search_foldable_gram") { + for (def foldOnBe in [false, true]) { + sql "set enable_fold_constant_by_be = ${foldOnBe}" + + qt_literal "select ngram_search('abc', 'abc', 3)" + qt_arithmetic "select ngram_search('abc', 'abc', 1 + 2)" + qt_cast "select ngram_search('abc', 'abc', cast('3' as int))" + qt_function "select ngram_search('abc', 'abc', abs(-3))" + qt_nested "select ngram_search('abc', 'abc', cast(abs(-2) + 1 as int))" + qt_be_function "select ngram_search('abc', 'abc', crc32('abc') % 3 + 1)" + qt_be_cast "select ngram_search('abc', 'abc', cast(crc32('abc') % 3 + 1 as int))" + qt_nullable_text "select ngram_search(cast(null as string), 'abc', 1 + 2)" + + qt_be_nullable_text "select ngram_search(cast(null as string), 'abc', crc32('abc') % 3 + 1)" + qt_be_nullable_pattern "select ngram_search('abc', cast(null as string), crc32('abc') % 3 + 1)" + order_qt_be_empty """ + select ngram_search(cast(number as string), 'abc', crc32('abc') % 3 + 1) + from numbers("number" = "0") + """ + order_qt_be_cse """ + select crc32('abc'), + ngram_search(case number % 3 when 0 then 'abc' when 1 then 'ab' else null end, + 'abc', crc32('abc') % 3 + 1) + from numbers("number" = "6") + """ + + order_qt_rows """ + select number, + ngram_search(case number % 3 when 0 then 'abc' when 1 then 'ab' else null end, + 'abc', 1 + 2), + ngram_search('abc', 'abc', cast('3' as int)) + from numbers("number" = "6") + """ + + order_qt_be_rows """ + select number, + ngram_search(case number % 3 when 0 then 'abc' when 1 then 'ab' else null end, + 'abc', crc32('abc') % 3 + 1), + ngram_search('abc', 'abc', crc32('abc') % 3 + 1) + from numbers("number" = "6") + """ + + for (def gram in ["0", "-1", "1 - 1", "1 - 2", "cast('0' as int)", + "crc32('abc') % 3", "crc32('abc') % 3 - 1"]) { + test { + sql "select ngram_search('abc', 'abc', ${gram})" + exception "gram_num must be a positive constant" + } + test { + sql """select crc32('abc'), ngram_search(cast(number as string), 'abc', ${gram}) + from numbers("number" = "3")""" + exception "gram_num must be a positive constant" + } + } + // Argument validation must precede NULL propagation, CSE and empty-plan rewrites. + for (def gram in ["crc32('abc') % 3", "crc32('abc') % 3 - 1", "crc32('abc') % 0"]) { + def error = gram.endsWith("% 0") ? "gram_num support const value only" : + "gram_num must be a positive constant" + for (def text in ["'abc'", "cast(null as string)"]) { + test { + sql "select ngram_search(${text}, 'abc', ${gram})" + exception error + } + } + for (def source in ['numbers("number" = "0")', 'numbers("number" = "3")', + 'numbers("number" = "3") where false', 'numbers("number" = "3") limit 0']) { + test { + sql """select ngram_search(cast(number as string), 'abc', ${gram}) from ${source}""" + exception error + } + test { + sql """select crc32('abc'), + ngram_search(cast(number as string), 'abc', ${gram}) from ${source}""" + exception error + } + } + } + // Required gram evaluation must not dispatch expressions excluded from BE folding. + for (def prefix in ["select", "explain select"]) { + for (def suffix in ["", " where false"]) { + test { + sql "${prefix} ngram_search('abc', 'abc', 1 + cast(sleep(0) as int))${suffix}" + exception "failed to evaluate constant gram_num" + } + } + } + test { + sql "select ngram_search(cast(null as string), 'abc', 1 + cast(sleep(0) as int))" + exception "failed to evaluate constant gram_num" + } + for (def gram in ["'3'", "3.5", "null", "cast(null as int)", "cast(rand() as int)"]) { + test { + sql "select ngram_search('abc', 'abc', ${gram})" + exception "gram_num support const value only" + } + } + test { + sql "select ngram_search('abc', 'abc', cast(number as int)) from numbers(\"number\" = \"3\")" + exception "gram_num support const value only" + } + test { + sql "select ngram_search('abc', cast(number as string), 3) from numbers(\"number\" = \"3\")" + exception "pattern support const value only" + } + } +}