diff --git a/be/benchmark/benchmark_character_encoding.hpp b/be/benchmark/benchmark_character_encoding.hpp new file mode 100644 index 00000000000000..f0f27b8a23bd10 --- /dev/null +++ b/be/benchmark/benchmark_character_encoding.hpp @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/column/column_const.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/function_context.h" +#include "util/defer_op.h" + +namespace doris { +namespace { + +struct CharacterEncodingData { + std::string input; + std::string expected; + std::string charset; +}; + +template +CharacterEncodingData make_character_encoding_data(size_t length) { + CharacterEncodingData data; + auto& [input, expected, charset] = data; + if constexpr (Scenario == 0) { + charset = "UTF-16BE"; + input.assign(length, 'A'); + for (size_t i = 0; i < length; ++i) { + expected.append("\0A", 2); + } + } else if constexpr (Scenario == 1) { + charset = "ISO-8859-1"; + input.assign(length, '\xE9'); + for (size_t i = 0; i < length; ++i) { + expected += "é"; + } + } else if constexpr (Scenario == 2) { + charset = "UTF-8"; + input.assign(length - 7, 'A'); + input += "中😀"; + expected = input; + } else { + charset = "UTF-16BE"; + for (size_t i = 0; i < length / 2; ++i) { + input += "N-"; // The UTF-16BE byte pair for 中. + expected += "中"; + } + } + return data; +} + +// Benchmark actual block execution, including result allocation and converter setup, not just +// ICU calls. Inputs are materialized columns so constant folding cannot eliminate conversion. +// The charset is constant by contract. Large rows use smaller blocks to bound working memory. +template +void BM_character_encoding(benchmark::State& state) { + const size_t length = state.range(0); + const size_t rows = std::min(4096, (1 << 20) / length); + const auto [input, expected, charset] = make_character_encoding_data(length); + DataTypePtr string_type = std::make_shared(); + DataTypePtr binary_type = std::make_shared(); + DataTypePtr input_type = Encode ? string_type : binary_type; + DataTypePtr result_type = Encode ? binary_type : string_type; + auto values = input_type->create_column(); + for (size_t i = 0; i < rows; ++i) { + values->insert_data(input.data(), input.size()); + } + auto charsets = ColumnString::create(); + charsets->insert_data(charset.data(), charset.size()); + ColumnPtr charset_column = ColumnConst::create(std::move(charsets), rows); + Block block {{std::move(values), input_type, "input"}, + {std::move(charset_column), string_type, "charset"}}; + auto function = SimpleFunctionFactory::instance().get_function( + Encode ? "encode" : "decode", block.get_columns_with_type_and_name(), result_type); + if (function == nullptr) { + state.SkipWithError("Character encoding function not registered"); + return; + } + auto context = FunctionContext::create_context(nullptr, result_type, {input_type, string_type}); + auto status = function->open(context.get(), FunctionContext::FRAGMENT_LOCAL); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + return; + } + Defer close_fragment {[&] { + auto close_status = function->close(context.get(), FunctionContext::FRAGMENT_LOCAL); + if (!close_status.ok()) { + state.SkipWithError(close_status.to_string()); + } + }}; + status = function->open(context.get(), FunctionContext::THREAD_LOCAL); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + return; + } + Defer close_thread {[&] { + auto close_status = function->close(context.get(), FunctionContext::THREAD_LOCAL); + if (!close_status.ok()) { + state.SkipWithError(close_status.to_string()); + } + }}; + block.insert({nullptr, result_type, "result"}); + + // Verify every output outside the measured loop. + status = function->execute(context.get(), block, {0, 1}, 2, rows); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + return; + } + for (size_t i = 0; i < rows; ++i) { + auto actual = block.get_by_position(2).column->get_data_at(i); + if (std::string_view(actual.data, actual.size) != expected) { + state.SkipWithError("Character encoding result mismatch"); + return; + } + } + for (auto _ : state) { + status = function->execute(context.get(), block, {0, 1}, 2, rows); + if (!status.ok()) { + state.SkipWithError(status.to_string()); + break; + } + benchmark::DoNotOptimize(block.get_by_position(2).column); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * rows); + state.SetBytesProcessed(state.iterations() * rows * input.size()); +} + +BENCHMARK_TEMPLATE(BM_character_encoding, true, 0) + ->Name("encode_utf16be_ascii") + ->ArgsProduct({{15, 63, 1023, 65535}}); +BENCHMARK_TEMPLATE(BM_character_encoding, false, 1) + ->Name("decode_latin1_nonascii") + ->ArgsProduct({{15, 63, 1023, 65535}}); +BENCHMARK_TEMPLATE(BM_character_encoding, true, 2) + ->Name("encode_utf8_mixed") + ->ArgsProduct({{15, 63, 1023, 65535}}); +BENCHMARK_TEMPLATE(BM_character_encoding, false, 2) + ->Name("decode_utf8_mixed") + ->ArgsProduct({{15, 63, 1023, 65535}}); +BENCHMARK_TEMPLATE(BM_character_encoding, false, 3) + ->Name("decode_utf16be_cjk") + ->ArgsProduct({{16, 64, 1024, 65536}}); + +} // namespace +} // namespace doris diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index d68509f785b23e..55a08b1fddd3e0 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -27,6 +27,7 @@ #include "benchmark_binary_arithmetic.hpp" #include "benchmark_bit_pack.hpp" #include "benchmark_case_expr.hpp" +#include "benchmark_character_encoding.hpp" #include "benchmark_column_array_view.hpp" #include "benchmark_column_array_view_distance.hpp" #include "benchmark_fastunion.hpp" diff --git a/be/src/exprs/function/function_character_encoding.cpp b/be/src/exprs/function/function_character_encoding.cpp new file mode 100644 index 00000000000000..3e317db4740b7d --- /dev/null +++ b/be/src/exprs/function/function_character_encoding.cpp @@ -0,0 +1,422 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/string_ref.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" + +namespace doris { +namespace { + +enum class CharacterSet : uint8_t { + US_ASCII, + ISO_8859_1, + UTF_8, + UTF_16BE, + UTF_16LE, + UTF_16, + SIZE, +}; + +constexpr std::array(CharacterSet::SIZE)> + SUPPORTED_CHARACTER_SETS = {"US-ASCII", "ISO-8859-1", "UTF-8", + "UTF-16BE", "UTF-16LE", "UTF-16"}; + +bool equals_ignore_case(StringRef value, std::string_view expected) { + if (value.size != expected.size()) { + return false; + } + for (size_t i = 0; i < value.size; ++i) { + const char current = value.data[i] >= 'a' && value.data[i] <= 'z' + ? value.data[i] - ('a' - 'A') + : value.data[i]; + if (current != expected[i]) { + return false; + } + } + return true; +} + +Status parse_character_set(StringRef value, CharacterSet& character_set) { + for (size_t i = 0; i < SUPPORTED_CHARACTER_SETS.size(); ++i) { + if (equals_ignore_case(value, SUPPORTED_CHARACTER_SETS[i])) { + character_set = static_cast(i); + return Status::OK(); + } + } + return Status::InvalidArgument( + "Unsupported character set '{}'. Supported character sets are US-ASCII, " + "ISO-8859-1, UTF-8, UTF-16BE, UTF-16LE, and UTF-16", + std::string(value.data, value.size)); +} + +Status conversion_error(std::string_view character_set_name, simdutf::error_code error) { + return Status::InvalidArgument("Character conversion using '{}' failed: {}", character_set_name, + simdutf::error_to_string(error)); +} + +Status reject_too_large(std::string_view character_set_name) { + return Status::InvalidArgument("Input is too large for character conversion using '{}'", + character_set_name); +} + +// Grow the byte buffer, then return the address of the newly reserved range. +char* reserve_output(ColumnString::Chars& output, size_t extra) { + const size_t start = output.size(); + ColumnString::check_chars_length(start + extra, 0); + output.resize(start + extra); + return reinterpret_cast(output.data() + start); +} + +Status copy_validated(StringRef input, std::string_view character_set_name, + simdutf::result validation, ColumnString::Chars& output) { + if (validation.error != simdutf::SUCCESS) { + return conversion_error(character_set_name, validation.error); + } + memcpy(reserve_output(output, input.size), input.data, input.size); + return Status::OK(); +} + +// Doris string bytes are not guaranteed to be char16_t-aligned. +const char16_t* utf16_units(StringRef input, std::vector& aligned) { + if (reinterpret_cast(input.data) % alignof(char16_t) == 0) { + return reinterpret_cast(input.data); + } + const size_t units = input.size / 2; + aligned.resize(units); + memcpy(aligned.data(), input.data, input.size); + return aligned.data(); +} + +template +class FunctionCharacterEncoding : public IFunction { +public: + static constexpr auto name = Encode ? "encode" : "decode"; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 2; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + DataTypePtr result_type; + if constexpr (Encode) { + result_type = std::make_shared(); + } else { + result_type = std::make_shared(); + } + return have_nullable(arguments) ? make_nullable(result_type) : result_type; + } + + ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; } + + bool use_default_implementation_for_nulls() const override { return false; } + + Status execute_impl(FunctionContext* /*context*/, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + auto [input_column, input_is_const] = + unpack_if_const(block.get_by_position(arguments[0]).column); + auto [character_set_column, character_set_is_const] = + unpack_if_const(block.get_by_position(arguments[1]).column); + DCHECK(character_set_is_const); + const auto* input_nullable = check_and_get_column(input_column.get()); + const auto* character_set_nullable = + check_and_get_column(character_set_column.get()); + const IColumn* input_nested = + input_nullable ? &input_nullable->get_nested_column() : input_column.get(); + const IColumn* character_set_nested = character_set_nullable + ? &character_set_nullable->get_nested_column() + : character_set_column.get(); + const auto& character_sets = assert_cast(*character_set_nested); + const NullMap* input_null_map = + input_nullable ? &input_nullable->get_null_map_data() : nullptr; + const NullMap* character_set_null_map = + character_set_nullable ? &character_set_nullable->get_null_map_data() : nullptr; + const bool has_nullable = input_null_map != nullptr || character_set_null_map != nullptr; + auto result_column = create_result_column(); + if constexpr (Encode) { + result_column->get_data().reserve(input_rows_count); + } else { + result_column->reserve(input_rows_count); + } + ColumnUInt8::MutablePtr result_null_column; + if (has_nullable) { + result_null_column = ColumnUInt8::create(input_rows_count, 0); + } + // Varbinary owns out-of-line values in an arena and inlines small values. Share one + // tracked scratch buffer across rows so that its capacity is not retained per row. + ColumnString::Chars scratch; + std::vector utf16_scratch; + CharacterSet constant_character_set = CharacterSet::UTF_8; + if (character_set_is_const && input_rows_count != 0 && + !(character_set_null_map && (*character_set_null_map)[0])) { + RETURN_IF_ERROR( + parse_character_set(character_sets.get_data_at(0), constant_character_set)); + } + + for (size_t row = 0; row < input_rows_count; ++row) { + const size_t input_index = index_check_const(row, input_is_const); + const bool input_is_null = input_null_map && (*input_null_map)[input_index]; + const bool character_set_is_null = + character_set_null_map && (*character_set_null_map)[0]; + if (input_is_null || character_set_is_null) { + result_column->insert_default(); + result_null_column->get_data()[row] = 1; + continue; + } + + const StringRef input = input_nested->get_data_at(input_index); + if constexpr (Encode) { + scratch.clear(); + RETURN_IF_ERROR( + convert_input(input, constant_character_set, utf16_scratch, scratch)); + result_column->insert_data(reinterpret_cast(scratch.data()), + scratch.size()); + } else { + // Write straight into the result column, including when the buffer grows. + auto& chars = result_column->get_chars(); + RETURN_IF_ERROR(convert_input(input, constant_character_set, utf16_scratch, chars)); + result_column->get_offsets().push_back(chars.size()); + } + } + + if (has_nullable) { + block.replace_by_position(result, + ColumnNullable::create(std::move(result_column), + std::move(result_null_column))); + } else { + block.replace_by_position(result, std::move(result_column)); + } + return Status::OK(); + } + +private: + using ResultColumn = std::conditional_t; + + static typename ResultColumn::MutablePtr create_result_column() { + return ResultColumn::create(); + } + + static std::string_view charset_name(CharacterSet character_set) { + return SUPPORTED_CHARACTER_SETS[static_cast(character_set)]; + } + + static Status convert_input(StringRef input, CharacterSet character_set, + std::vector& utf16_scratch, + ColumnString::Chars& converted) { + if (input.size == 0) { + return Status::OK(); + } + const std::string_view character_set_name = charset_name(character_set); + if (input.size > static_cast(std::numeric_limits::max())) { + return reject_too_large(character_set_name); + } + if constexpr (Encode) { + return encode_input(input, character_set, character_set_name, utf16_scratch, converted); + } else { + return decode_input(input, character_set, character_set_name, utf16_scratch, converted); + } + } + + static Status encode_input(StringRef input, CharacterSet character_set, + std::string_view character_set_name, + std::vector& utf16_scratch, ColumnString::Chars& output) { + switch (character_set) { + case CharacterSet::US_ASCII: + return copy_validated(input, character_set_name, + simdutf::validate_ascii_with_errors(input.data, input.size), + output); + case CharacterSet::UTF_8: + return copy_validated(input, character_set_name, + simdutf::validate_utf8_with_errors(input.data, input.size), + output); + case CharacterSet::ISO_8859_1: + return encode_latin1(input, character_set_name, output); + case CharacterSet::UTF_16BE: + return encode_utf16(input, character_set_name, false, false, utf16_scratch, output); + case CharacterSet::UTF_16LE: + return encode_utf16(input, character_set_name, true, false, utf16_scratch, output); + case CharacterSet::UTF_16: + // Java's UTF-16 encoder always emits a big-endian BOM. + return encode_utf16(input, character_set_name, false, true, utf16_scratch, output); + default: + return Status::InvalidArgument("Unsupported character set '{}'", character_set_name); + } + } + + static Status decode_input(StringRef input, CharacterSet character_set, + std::string_view character_set_name, + std::vector& utf16_scratch, ColumnString::Chars& output) { + switch (character_set) { + case CharacterSet::US_ASCII: + return copy_validated(input, character_set_name, + simdutf::validate_ascii_with_errors(input.data, input.size), + output); + case CharacterSet::UTF_8: + return copy_validated(input, character_set_name, + simdutf::validate_utf8_with_errors(input.data, input.size), + output); + case CharacterSet::ISO_8859_1: + return decode_latin1(input, character_set_name, output); + case CharacterSet::UTF_16BE: + return decode_utf16(input, character_set_name, false, utf16_scratch, output); + case CharacterSet::UTF_16LE: + return decode_utf16(input, character_set_name, true, utf16_scratch, output); + case CharacterSet::UTF_16: + return decode_utf16_with_bom(input, character_set_name, utf16_scratch, output); + default: + return Status::InvalidArgument("Unsupported character set '{}'", character_set_name); + } + } + + static Status encode_latin1(StringRef input, std::string_view character_set_name, + ColumnString::Chars& output) { + const size_t start = output.size(); + char* dest = reserve_output(output, input.size); + const size_t written = simdutf::convert_utf8_to_latin1(input.data, input.size, dest); + if (written == 0) { + const simdutf::result detail = + simdutf::convert_utf8_to_latin1_with_errors(input.data, input.size, dest); + output.resize(start); + const simdutf::error_code error = + detail.error == simdutf::SUCCESS ? simdutf::OTHER : detail.error; + return conversion_error(character_set_name, error); + } + output.resize(start + written); + return Status::OK(); + } + + static Status decode_latin1(StringRef input, std::string_view character_set_name, + ColumnString::Chars& output) { + const size_t need = simdutf::utf8_length_from_latin1(input.data, input.size); + char* dest = reserve_output(output, need); + const size_t written = simdutf::convert_latin1_to_utf8(input.data, input.size, dest); + if (written != need) { + output.resize(output.size() - need); + return conversion_error(character_set_name, simdutf::OTHER); + } + return Status::OK(); + } + + static Status encode_utf16(StringRef input, std::string_view character_set_name, + bool little_endian, bool write_bom, + std::vector& utf16_scratch, ColumnString::Chars& output) { + utf16_scratch.resize(input.size); + const simdutf::result result = + little_endian ? simdutf::convert_utf8_to_utf16le_with_errors(input.data, input.size, + utf16_scratch.data()) + : simdutf::convert_utf8_to_utf16be_with_errors(input.data, input.size, + utf16_scratch.data()); + if (result.error != simdutf::SUCCESS) { + return conversion_error(character_set_name, result.error); + } + const size_t payload_bytes = result.count * sizeof(char16_t); + char* dest = reserve_output(output, payload_bytes + (write_bom ? 2 : 0)); + if (write_bom) { + auto* bytes = reinterpret_cast(dest); + bytes[0] = 0xFE; + bytes[1] = 0xFF; + dest += 2; + } + memcpy(dest, utf16_scratch.data(), payload_bytes); + return Status::OK(); + } + + // Java's UTF-16 decoder honors either BOM and defaults to big endian without one. + static Status decode_utf16_with_bom(StringRef input, std::string_view character_set_name, + std::vector& utf16_scratch, + ColumnString::Chars& output) { + if (input.size < 2) { + return conversion_error(character_set_name, simdutf::TOO_SHORT); + } + const auto first = static_cast(input.data[0]); + const auto second = static_cast(input.data[1]); + bool little_endian = false; + if (first == 0xFE && second == 0xFF) { + input = input.substring(2); + } else if (first == 0xFF && second == 0xFE) { + input = input.substring(2); + little_endian = true; + } + if (input.size == 0) { + return Status::OK(); + } + return decode_utf16(input, character_set_name, little_endian, utf16_scratch, output); + } + + static Status decode_utf16(StringRef input, std::string_view character_set_name, + bool little_endian, std::vector& utf16_scratch, + ColumnString::Chars& output) { + if (input.size % 2 != 0) { + return conversion_error(character_set_name, simdutf::TOO_SHORT); + } + const size_t units = input.size / 2; + if (units > (std::numeric_limits::max() / 3)) { + return reject_too_large(character_set_name); + } + const char16_t* units_ptr = utf16_units(input, utf16_scratch); + const size_t start = output.size(); + char* dest = reserve_output(output, units * 3); + const simdutf::result result = + little_endian + ? simdutf::convert_utf16le_to_utf8_with_errors(units_ptr, units, dest) + : simdutf::convert_utf16be_to_utf8_with_errors(units_ptr, units, dest); + if (result.error != simdutf::SUCCESS) { + output.resize(start); + return conversion_error(character_set_name, result.error); + } + output.resize(start + result.count); + return Status::OK(); + } +}; + +using FunctionEncode = FunctionCharacterEncoding; +using FunctionDecode = FunctionCharacterEncoding; + +} // namespace + +void register_function_character_encoding(SimpleFunctionFactory& factory) { + factory.register_function(); + factory.register_function(); +} + +} // namespace doris diff --git a/be/src/exprs/function/simple_function_factory.h b/be/src/exprs/function/simple_function_factory.h index 5dd09e0847dba5..60d5bb14827758 100644 --- a/be/src/exprs/function/simple_function_factory.h +++ b/be/src/exprs/function/simple_function_factory.h @@ -123,6 +123,7 @@ void register_function_binary(SimpleFunctionFactory& factory); void register_function_levenshtein(SimpleFunctionFactory& factory); void register_function_hamming_distance(SimpleFunctionFactory& factory); void register_function_soundex(SimpleFunctionFactory& factory); +void register_function_character_encoding(SimpleFunctionFactory& factory); #if defined(BE_TEST) && !defined(BE_BENCHMARK) void register_function_throw_exception(SimpleFunctionFactory& factory); @@ -366,6 +367,7 @@ class SimpleFunctionFactory { register_function_levenshtein(instance); register_function_hamming_distance(instance); register_function_soundex(instance); + register_function_character_encoding(instance); register_function_json_transform(instance); register_function_json_hash(instance); #if defined(BE_TEST) && !defined(BE_BENCHMARK) diff --git a/be/test/exprs/function/function_character_encoding_test.cpp b/be/test/exprs/function/function_character_encoding_test.cpp new file mode 100644 index 00000000000000..be88acac23d39d --- /dev/null +++ b/be/test/exprs/function/function_character_encoding_test.cpp @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/function/function_test_util.h" + +namespace doris { + +using namespace ut_type; + +template +void check_character_encoding(const std::string& function_name, PrimitiveType input_type, + const DataSet& data_set) { + for (bool input_is_const : {false, true}) { + for (const auto& line : data_set) { + InputTypeSet input_types; + if (input_is_const) { + input_types.emplace_back(Consted {input_type}); + } else { + input_types.emplace_back(input_type); + } + input_types.emplace_back(Consted {PrimitiveType::TYPE_VARCHAR}); + ASSERT_TRUE( + (check_function(function_name, input_types, {line}).ok())); + } + } +} + +TEST(function_character_encoding_test, encode_supported_charsets) { + // The UTF-16 byte pairs 0x4E2D and 0x2D4E are "N-" and "-N" as raw bytes. + DataSet data_set = { + {{std::string("A"), std::string("US-ASCII")}, VARBINARY("A")}, + {{std::string("é"), std::string("ISO-8859-1")}, VARBINARY("\xE9")}, + {{std::string("中"), std::string("UTF-8")}, VARBINARY("\xE4\xB8\xAD")}, + {{std::string("中"), std::string("UTF-16BE")}, VARBINARY("N-")}, + {{std::string("中"), std::string("UTF-16LE")}, VARBINARY("-N")}, + {{std::string("中"), std::string("UTF-16")}, VARBINARY("\xFE\xFF\x4E\x2D")}, + {{std::string("😀"), std::string("utf-16be")}, + VARBINARY(std::string_view("\xD8\x3D\xDE\x00", 4))}, + {{std::string("A\0中", 5), std::string("UTF-8")}, + VARBINARY(std::string_view("A\0\xE4\xB8\xAD", 5))}, + {{std::string(""), std::string("UTF-16")}, VARBINARY("")}, + {{Null(), std::string("UTF-8")}, Null()}, + {{std::string("text"), Null()}, Null()}, + }; + + check_character_encoding("encode", PrimitiveType::TYPE_VARCHAR, data_set); +} + +TEST(function_character_encoding_test, decode_supported_charsets) { + // The UTF-16 byte pairs 0x4E2D and 0x2D4E are "N-" and "-N" as raw bytes. + DataSet data_set = { + {{VARBINARY("A"), std::string("US-ASCII")}, std::string("A")}, + {{VARBINARY("\xE9"), std::string("ISO-8859-1")}, std::string("é")}, + {{VARBINARY("\xE4\xB8\xAD"), std::string("UTF-8")}, std::string("中")}, + {{VARBINARY("N-"), std::string("UTF-16BE")}, std::string("中")}, + {{VARBINARY("-N"), std::string("UTF-16LE")}, std::string("中")}, + {{VARBINARY("\xFE\xFF\x4E\x2D"), std::string("UTF-16")}, std::string("中")}, + {{VARBINARY("\xFF\xFE\x2D\x4E"), std::string("utf-16")}, std::string("中")}, + {{VARBINARY("N-"), std::string("UTF-16")}, std::string("中")}, + {{VARBINARY("\xFE\xFF"), std::string("UTF-16")}, std::string("")}, + {{VARBINARY(std::string_view("\xD8\x3D\xDE\x00", 4)), std::string("UTF-16BE")}, + std::string("😀")}, + {{VARBINARY(std::string_view("A\0\xE4\xB8\xAD", 5)), std::string("UTF-8")}, + std::string("A\0中", 5)}, + {{VARBINARY(""), std::string("UTF-16")}, std::string("")}, + {{Null(), std::string("UTF-8")}, Null()}, + {{VARBINARY("text"), Null()}, Null()}, + }; + + check_character_encoding("decode", PrimitiveType::TYPE_VARBINARY, data_set); +} + +TEST(function_character_encoding_test, rejects_invalid_conversions) { + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, + Consted {PrimitiveType::TYPE_VARCHAR}}; + DataSet data_set = { + {{std::string("text"), std::string("GBK")}, VARBINARY("")}, + }; + + Status status = check_function("encode", input_types, data_set, -1, + -1, true); + ASSERT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Unsupported character set"), std::string::npos); + } + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, + Consted {PrimitiveType::TYPE_VARCHAR}}; + DataSet data_set = { + {{std::string("中"), std::string("US-ASCII")}, VARBINARY("")}, + }; + + Status status = check_function("encode", input_types, data_set, -1, + -1, true); + ASSERT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Character conversion using 'US-ASCII' failed"), + std::string::npos); + } + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, + Consted {PrimitiveType::TYPE_VARCHAR}}; + DataSet data_set = { + {{VARBINARY("\xE4\xB8"), std::string("UTF-8")}, std::string("")}, + }; + + Status status = + check_function("decode", input_types, data_set, -1, -1, true); + ASSERT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Character conversion using 'UTF-8' failed"), + std::string::npos); + } +} + +TEST(function_character_encoding_test, invalid_constant_character_set_precedes_null_input) { + DataSet encode_data = { + {{Null(), std::string("GBK")}, Null()}, + }; + Status encode_status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, Consted {PrimitiveType::TYPE_VARCHAR}}, + encode_data, -1, -1, true); + ASSERT_TRUE(encode_status.is()) << encode_status; + EXPECT_NE(encode_status.to_string().find("Unsupported character set"), std::string::npos); + + DataSet decode_data = { + {{Null(), std::string("GBK")}, Null()}, + }; + Status decode_status = check_function( + "decode", {PrimitiveType::TYPE_VARBINARY, Consted {PrimitiveType::TYPE_VARCHAR}}, + decode_data, -1, -1, true); + ASSERT_TRUE(decode_status.is()) << decode_status; + EXPECT_NE(decode_status.to_string().find("Unsupported character set"), std::string::npos); +} + +TEST(function_character_encoding_test, requires_constant_character_set) { + DataSet encode_data = { + {{std::string("A"), std::string("UTF-8")}, VARBINARY("A")}, + }; + Status encode_status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}, encode_data, -1, + -1, true); + ASSERT_TRUE(encode_status.is()) << encode_status; + EXPECT_NE(encode_status.to_string().find("must be constant"), std::string::npos); + + DataSet decode_data = { + {{VARBINARY("A"), std::string("UTF-8")}, std::string("A")}, + }; + Status decode_status = check_function( + "decode", {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARCHAR}, decode_data, -1, + -1, true); + ASSERT_TRUE(decode_status.is()) << decode_status; + EXPECT_NE(decode_status.to_string().find("must be constant"), std::string::npos); +} + +TEST(function_character_encoding_test, streaming_boundaries_and_row_reuse) { + // Cross the pivot boundary and force output expansion, including pending surrogate pairs. + for (size_t length : {1, 15, 1023, 1024, 1025, 65535}) { + std::string ascii(length, 'A'); + std::string utf16; + for (size_t i = 0; i < length; ++i) { + utf16.append("\0A", 2); + } + const std::string utf16_bom = std::string("\xFE\xFF", 2) + utf16; + const std::string supplementary = ascii + "😀"; + const std::string supplementary_utf16 = utf16 + std::string("\xD8\x3D\xDE\0", 4); + DataSet encoded = { + {{ascii, std::string("UTF-16BE")}, VARBINARY(utf16)}, + {{supplementary, std::string("UTF-16BE")}, VARBINARY(supplementary_utf16)}, + {{Null(), std::string("UTF-16BE")}, Null()}, + {{std::string(""), std::string("UTF-16BE")}, VARBINARY("")}, + {{ascii, std::string("UTF-16")}, VARBINARY(utf16_bom)}, + {{std::string("A"), std::string("UTF-16BE")}, + VARBINARY(std::string_view("\0A", 2))}, + }; + check_character_encoding("encode", PrimitiveType::TYPE_VARCHAR, encoded); + + std::string latin1(length, '\xE9'); + std::string expanded; + for (size_t i = 0; i < length; ++i) { + expanded += "é"; + } + DataSet decoded = { + {{VARBINARY(latin1), std::string("ISO-8859-1")}, expanded}, + {{VARBINARY(supplementary_utf16), std::string("UTF-16BE")}, supplementary}, + {{Null(), std::string("ISO-8859-1")}, Null()}, + {{VARBINARY(""), std::string("ISO-8859-1")}, std::string("")}, + {{VARBINARY(utf16_bom), std::string("UTF-16")}, ascii}, + {{VARBINARY("\xFF\xFE\x2D\x4E"), std::string("UTF-16")}, std::string("中")}, + {{VARBINARY("\xFE\xFF"), std::string("UTF-16")}, std::string("")}, + {{VARBINARY("\xE9"), std::string("ISO-8859-1")}, std::string("é")}, + }; + check_character_encoding("decode", PrimitiveType::TYPE_VARBINARY, decoded); + } +} + +TEST(function_character_encoding_test, rejects_invalid_input_after_streaming) { + const std::string invalid_utf8 = std::string(4096, 'A') + "\xE4\xB8"; + const std::string unrepresentable = std::string(4096, 'A') + "中"; + for (const auto& input : {invalid_utf8, unrepresentable}) { + DataSet data_set = {{{input, std::string("US-ASCII")}, VARBINARY("")}}; + Status status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, Consted {PrimitiveType::TYPE_VARCHAR}}, + data_set, -1, -1, true); + ASSERT_TRUE(status.is()) << status; + } + const std::string expanding_invalid_utf8 = std::string(50000, 'A') + "\xE4\xB8"; + DataSet invalid_encode = {{{expanding_invalid_utf8, std::string("UTF-16BE")}, VARBINARY("")}}; + Status encode_status = check_function( + "encode", {PrimitiveType::TYPE_VARCHAR, Consted {PrimitiveType::TYPE_VARCHAR}}, + invalid_encode, -1, -1, true); + ASSERT_TRUE(encode_status.is()) << encode_status; + + std::string invalid_utf16; + for (size_t i = 0; i < 25000; ++i) { + invalid_utf16 += "N-"; // The UTF-16BE byte pair for 中. + } + invalid_utf16.append("\xD8\x3D", 2); // An unpaired high surrogate after several pivot fills. + DataSet data_set = {{{VARBINARY(invalid_utf16), std::string("UTF-16BE")}, std::string("")}}; + Status status = check_function( + "decode", {PrimitiveType::TYPE_VARBINARY, Consted {PrimitiveType::TYPE_VARCHAR}}, + data_set, -1, -1, true); + ASSERT_TRUE(status.is()) << status; +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 6ccaead81c4c2f..68479546b3509a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -183,6 +183,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; import org.apache.doris.nereids.trees.expressions.functions.scalar.Dceil; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Decode; import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; import org.apache.doris.nereids.trees.expressions.functions.scalar.DeduplicateMap; import org.apache.doris.nereids.trees.expressions.functions.scalar.Degrees; @@ -200,6 +201,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.E; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Elt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Encode; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsBigInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsLargeInt; @@ -780,6 +782,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(DaysDiff.class, "days_diff"), scalar(DaysSub.class, "days_sub", "date_sub", "subdate"), scalar(Dceil.class, "dceil"), + scalar(Decode.class, "decode"), scalar(DecodeAsVarchar.class, "decode_as_varchar"), scalar(DeduplicateMap.class, "deduplicate_map"), scalar(Degrees.class, "degrees"), @@ -799,6 +802,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(ElementAt.class, "element_at", "struct_element"), scalar(Elt.class, "elt"), scalar(Embed.class, "embed"), + scalar(Encode.class, "encode"), scalar(EncodeAsSmallInt.class, "encode_as_smallint"), scalar(EncodeAsInt.class, "encode_as_int"), scalar(EncodeAsBigInt.class, "encode_as_bigint"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java index c2679c1f862c8a..641bc0412275c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java @@ -40,6 +40,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.ArrayType; @@ -52,7 +53,9 @@ import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -70,6 +73,8 @@ */ public class StringArithmetic { private static final long MAX_DAMERAU_LEVENSHTEIN_MATRIX_CELLS = 16L * 1024L * 1024L; + private static final List SUPPORTED_CHARACTER_SETS = ImmutableList.of( + "US-ASCII", "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16"); private static Literal castStringLikeLiteral(StringLikeLiteral first, String value) { if (first instanceof StringLiteral) { @@ -1151,6 +1156,67 @@ public static Expression urlEncode(StringLikeLiteral first) { } } + /** + * Executable arithmetic function encode + */ + @ExecFunction(name = "encode") + public static Expression encode(StringLikeLiteral source, StringLikeLiteral characterSet) { + Charset charset = supportedCharacterSet(characterSet.getValue()); + try { + ByteBuffer encoded = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .encode(CharBuffer.wrap(source.getValue())); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + return new VarBinaryLiteral(bytes); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Failed to encode value using " + characterSet.getValue(), e); + } + } + + /** + * Executable arithmetic function decode + */ + @ExecFunction(name = "decode") + public static Expression decode(VarBinaryLiteral binary, StringLikeLiteral characterSet) { + Charset charset = supportedCharacterSet(characterSet.getValue()); + try { + CharBuffer decoded = charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap((byte[]) binary.getValue())); + return new StringLiteral(decoded.toString()); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Failed to decode value using " + characterSet.getValue(), e); + } + } + + private static Charset supportedCharacterSet(String name) { + for (String supportedCharacterSet : SUPPORTED_CHARACTER_SETS) { + if (equalsIgnoreAsciiCase(name, supportedCharacterSet)) { + return Charset.forName(supportedCharacterSet); + } + } + throw new IllegalArgumentException("Unsupported character set: " + name); + } + + private static boolean equalsIgnoreAsciiCase(String value, String expected) { + if (value.length() != expected.length()) { + return false; + } + for (int i = 0; i < value.length(); i++) { + char current = value.charAt(i); + if (current >= 'a' && current <= 'z') { + current -= 'a' - 'A'; + } + if (current != expected.charAt(i)) { + return false; + } + } + return true; + } + /** * Executable arithmetic functions append_trailing_char_if_absent */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CharacterSetLiterals.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CharacterSetLiterals.java new file mode 100644 index 00000000000000..9354feba7d9b5c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CharacterSetLiterals.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** Character sets accepted by encode and decode. */ +final class CharacterSetLiterals { + private static final List SUPPORTED = ImmutableList.of( + "US-ASCII", "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16"); + + private CharacterSetLiterals() { + } + + static void checkSecondArgument(ScalarFunction function) { + Expression characterSet = function.getArgument(1); + if (!characterSet.isLiteral()) { + throw new AnalysisException("the second argument of function " + + function.getName() + " must be a literal: " + function.toSql()); + } + if (characterSet.isNullLiteral()) { + return; + } + if (!(characterSet instanceof StringLikeLiteral)) { + throw new AnalysisException("the second argument of function " + + function.getName() + " must be a string literal: " + function.toSql()); + } + String value = ((StringLikeLiteral) characterSet).getValue(); + for (String supported : SUPPORTED) { + if (equalsIgnoreAsciiCase(value, supported)) { + return; + } + } + throw new AnalysisException("Unsupported character set '" + value + + "'. Supported character sets are US-ASCII, ISO-8859-1, UTF-8, " + + "UTF-16BE, UTF-16LE, and UTF-16"); + } + + private static boolean equalsIgnoreAsciiCase(String value, String expected) { + if (value.length() != expected.length()) { + return false; + } + for (int i = 0; i < value.length(); i++) { + char current = value.charAt(i); + if (current >= 'a' && current <= 'z') { + current -= 'a' - 'A'; + } + if (current != expected.charAt(i)) { + return false; + } + } + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java new file mode 100644 index 00000000000000..4cb1296e6dcabf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Decode.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'decode'. This class is generated by GenerateFunction. + */ +public class Decode extends ScalarFunction + implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(StringType.INSTANCE).args(VarBinaryType.INSTANCE, StringType.INSTANCE) + ); + + /** + * constructor with 2 arguments. + */ + public Decode(Expression binary, Expression characterSet) { + super("decode", binary, characterSet); + } + + /** constructor for withChildren and reuse signature */ + private Decode(ScalarFunctionParams functionParams) { + super(functionParams); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + CharacterSetLiterals.checkSecondArgument(this); + } + + @Override + public void checkLegalityAfterRewrite() { + checkLegalityBeforeTypeCoercion(); + } + + /** + * withChildren. + */ + @Override + public Decode withChildren(List children) { + Preconditions.checkArgument(children.size() == 2); + return new Decode(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitDecode(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java new file mode 100644 index 00000000000000..1bab3f65ed517b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Encode.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'encode'. This class is generated by GenerateFunction. + */ +public class Encode extends ScalarFunction + implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(VarBinaryType.INSTANCE).args(StringType.INSTANCE, StringType.INSTANCE) + ); + + /** + * constructor with 2 arguments. + */ + public Encode(Expression source, Expression characterSet) { + super("encode", source, characterSet); + } + + /** constructor for withChildren and reuse signature */ + private Encode(ScalarFunctionParams functionParams) { + super(functionParams); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + CharacterSetLiterals.checkSecondArgument(this); + } + + @Override + public void checkLegalityAfterRewrite() { + checkLegalityBeforeTypeCoercion(); + } + + /** + * withChildren. + */ + @Override + public Encode withChildren(List children) { + Preconditions.checkArgument(children.size() == 2); + return new Encode(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitEncode(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 246eea0567114a..f22005f5112be0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -200,6 +200,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; import org.apache.doris.nereids.trees.expressions.functions.scalar.Dceil; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Decode; import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; import org.apache.doris.nereids.trees.expressions.functions.scalar.Degrees; import org.apache.doris.nereids.trees.expressions.functions.scalar.Dexp; @@ -216,6 +217,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.E; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Elt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Encode; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsBigInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsInt; import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsLargeInt; @@ -1087,6 +1089,10 @@ default R visitCutToFirstSignificantSubdomain(CutToFirstSignificantSubdomain cut return visitScalarFunction(cutToFirstSignificantSubdomain, context); } + default R visitEncode(Encode encode, C context) { + return visitScalarFunction(encode, context); + } + default R visitEncodeAsSmallInt(EncodeAsSmallInt encode, C context) { return visitScalarFunction(encode, context); } @@ -1292,6 +1298,10 @@ default R visitDigitalMasking(DigitalMasking digitalMasking, C context) { return visitScalarFunction(digitalMasking, context); } + default R visitDecode(Decode decode, C context) { + return visitScalarFunction(decode, context); + } + default R visitDecodeAsVarchar(DecodeAsVarchar decode, C context) { return visitScalarFunction(decode, context); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java index fda0a6f31b6e17..cf143f7a226e24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java @@ -19,6 +19,8 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.ExpressionEvaluator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Decode; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Encode; import org.apache.doris.nereids.trees.expressions.functions.scalar.UrlDecode; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; @@ -27,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -96,11 +99,73 @@ void testUrlDecodeStillFoldsValidUtf8() { assertUrlDecodeValue("%EF%BF%BD", "�"); } + @Test + void testEncodeFoldsSupportedCharsets() { + assertEncodeValue("A", "US-ASCII", new byte[] {0x41}); + assertEncodeValue("é", "ISO-8859-1", new byte[] {(byte) 0xE9}); + assertEncodeValue("中", "UTF-8", new byte[] {(byte) 0xE4, (byte) 0xB8, (byte) 0xAD}); + assertEncodeValue("中", "UTF-16BE", new byte[] {0x4E, 0x2D}); + assertEncodeValue("中", "UTF-16LE", new byte[] {0x2D, 0x4E}); + assertEncodeValue("中", "utf-16", new byte[] {(byte) 0xFE, (byte) 0xFF, 0x4E, 0x2D}); + assertEncodeValue("😀", "UTF-16BE", new byte[] {(byte) 0xD8, 0x3D, (byte) 0xDE, 0x00}); + assertEncodeValue("", "UTF-16", new byte[] {}); + } + + @Test + void testDecodeFoldsSupportedCharsets() { + assertDecodeValue(new byte[] {0x41}, "US-ASCII", "A"); + assertDecodeValue(new byte[] {(byte) 0xE9}, "ISO-8859-1", "é"); + assertDecodeValue(new byte[] {(byte) 0xE4, (byte) 0xB8, (byte) 0xAD}, "UTF-8", "中"); + assertDecodeValue(new byte[] {0x4E, 0x2D}, "UTF-16BE", "中"); + assertDecodeValue(new byte[] {0x2D, 0x4E}, "UTF-16LE", "中"); + assertDecodeValue(new byte[] {(byte) 0xFE, (byte) 0xFF, 0x4E, 0x2D}, "UTF-16", "中"); + assertDecodeValue(new byte[] {(byte) 0xFF, (byte) 0xFE, 0x2D, 0x4E}, "utf-16", "中"); + assertDecodeValue(new byte[] {0x4E, 0x2D}, "UTF-16", "中"); + assertDecodeValue(new byte[] {(byte) 0xFE, (byte) 0xFF}, "UTF-16", ""); + assertDecodeValue(new byte[] {(byte) 0xD8, 0x3D, (byte) 0xDE, 0x00}, "UTF-16BE", "😀"); + assertDecodeValue(new byte[] {}, "UTF-16", ""); + } + + @Test + void testInvalidCharacterConversionDoesNotFold() { + Encode unmappable = new Encode(new StringLiteral("中"), new StringLiteral("US-ASCII")); + Decode malformed = new Decode(new VarBinaryLiteral(new byte[] {(byte) 0xE4, (byte) 0xB8}), + new StringLiteral("UTF-8")); + Encode unsupported = new Encode(new StringLiteral("text"), new StringLiteral("GBK")); + + Assertions.assertSame(unmappable, ExpressionEvaluator.INSTANCE.eval(unmappable)); + Assertions.assertSame(malformed, ExpressionEvaluator.INSTANCE.eval(malformed)); + Assertions.assertSame(unsupported, ExpressionEvaluator.INSTANCE.eval(unsupported)); + } + + @Test + void testUnicodeCaseFoldedCharacterSetDoesNotFold() { + String unicodeCaseFoldedCharset = "U\u017F-ASCII"; // U+017F LATIN SMALL LETTER LONG S + Encode encode = new Encode(new StringLiteral("A"), new StringLiteral(unicodeCaseFoldedCharset)); + Decode decode = new Decode(new VarBinaryLiteral(new byte[] {0x41}), + new StringLiteral(unicodeCaseFoldedCharset)); + + Assertions.assertSame(encode, ExpressionEvaluator.INSTANCE.eval(encode)); + Assertions.assertSame(decode, ExpressionEvaluator.INSTANCE.eval(decode)); + } + private void assertUrlDecodeValue(String encoded, String expected) { Expression result = ExpressionEvaluator.INSTANCE.eval(new UrlDecode(new StringLiteral(encoded))); Assertions.assertEquals(expected, ((StringLikeLiteral) result).getValue()); } + private void assertEncodeValue(String value, String characterSet, byte[] expected) { + Expression result = ExpressionEvaluator.INSTANCE.eval( + new Encode(new StringLiteral(value), new StringLiteral(characterSet))); + Assertions.assertArrayEquals(expected, (byte[]) ((VarBinaryLiteral) result).getValue()); + } + + private void assertDecodeValue(byte[] value, String characterSet, String expected) { + Expression result = ExpressionEvaluator.INSTANCE.eval( + new Decode(new VarBinaryLiteral(value), new StringLiteral(characterSet))); + Assertions.assertEquals(expected, ((StringLikeLiteral) result).getValue()); + } + @Test void testParseUrlQueryStopsAtFragment() { // The only '?' is inside the fragment, so the url has no query component. diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodePlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodePlannerTest.java new file mode 100644 index 00000000000000..253b4077efa009 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodePlannerTest.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class EncodeDecodePlannerTest extends TestWithFeService implements MemoPatternMatchSupported { + + @Test + void testLiteralCallsFoldDuringPlanRewrite() { + VarBinaryLiteral encoded = new VarBinaryLiteral(new byte[] {0x4E, 0x2D}); + PlanChecker.from(connectContext) + .analyze("select encode('中', 'UTF-16BE')") + .rewrite() + .matches(logicalResultSink( + logicalOneRowRelation().when(oneRow -> + oneRow.getProjects().get(0).child(0).equals(encoded)))); + + StringLiteral decoded = new StringLiteral("中"); + PlanChecker.from(connectContext) + .analyze("select decode(X'E4B8AD', 'UTF-8')") + .rewrite() + .matches(logicalResultSink( + logicalOneRowRelation().when(oneRow -> + oneRow.getProjects().get(0).child(0).equals(decoded)))); + } + + @Test + void testInvalidCharsetRejectedBeforeNullFolding() { + AnalysisException encodeError = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze("select encode(NULL, 'GBK')")); + Assertions.assertTrue(encodeError.getMessage().contains("Unsupported character set")); + + AnalysisException decodeError = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze("select decode(NULL, 'GBK')")); + Assertions.assertTrue(decodeError.getMessage().contains("Unsupported character set")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java new file mode 100644 index 00000000000000..5b28389add3d7f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeDecodeTest.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class EncodeDecodeTest { + + @Test + public void testEncodeExpressionContract() { + StringLiteral source = new StringLiteral("hello"); + StringLiteral characterSet = new StringLiteral("UTF-8"); + Encode encode = new Encode(source, characterSet); + + Assertions.assertEquals("encode", encode.getName()); + Assertions.assertEquals(2, encode.arity()); + Assertions.assertSame(source, encode.child(0)); + Assertions.assertSame(characterSet, encode.child(1)); + + FunctionSignature signature = encode.getSignatures().get(0); + Assertions.assertEquals(VarBinaryType.INSTANCE, signature.returnType); + Assertions.assertEquals(StringType.INSTANCE, signature.getArgType(0)); + Assertions.assertEquals(StringType.INSTANCE, signature.getArgType(1)); + + StringLiteral replacementSource = new StringLiteral("world"); + StringLiteral replacementCharacterSet = new StringLiteral("UTF-16"); + Encode rewritten = encode.withChildren( + ImmutableList.of(replacementSource, replacementCharacterSet)); + Assertions.assertNotSame(encode, rewritten); + Assertions.assertSame(replacementSource, rewritten.child(0)); + Assertions.assertSame(replacementCharacterSet, rewritten.child(1)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> encode.withChildren(ImmutableList.of(replacementSource))); + } + + @Test + public void testDecodeExpressionContract() { + VarBinaryLiteral binary = new VarBinaryLiteral(new byte[] {0x68, 0x69}); + StringLiteral characterSet = new StringLiteral("UTF-8"); + Decode decode = new Decode(binary, characterSet); + + Assertions.assertEquals("decode", decode.getName()); + Assertions.assertEquals(2, decode.arity()); + Assertions.assertSame(binary, decode.child(0)); + Assertions.assertSame(characterSet, decode.child(1)); + + FunctionSignature signature = decode.getSignatures().get(0); + Assertions.assertEquals(StringType.INSTANCE, signature.returnType); + Assertions.assertEquals(VarBinaryType.INSTANCE, signature.getArgType(0)); + Assertions.assertEquals(StringType.INSTANCE, signature.getArgType(1)); + + VarBinaryLiteral replacementBinary = new VarBinaryLiteral(new byte[] {0x41}); + StringLiteral replacementCharacterSet = new StringLiteral("US-ASCII"); + Decode rewritten = decode.withChildren( + ImmutableList.of(replacementBinary, replacementCharacterSet)); + Assertions.assertNotSame(decode, rewritten); + Assertions.assertSame(replacementBinary, rewritten.child(0)); + Assertions.assertSame(replacementCharacterSet, rewritten.child(1)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> decode.withChildren(ImmutableList.of(replacementBinary))); + } + + @Test + public void testVisitorDelegatesToScalarFunction() { + Encode encode = new Encode(new StringLiteral("hello"), new StringLiteral("UTF-8")); + Decode decode = new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + new StringLiteral("UTF-8")); + ExpressionVisitor visitor = new ExpressionVisitor() { + @Override + public Expression visit(Expression expression, Void context) { + return expression; + } + }; + + Assertions.assertSame(encode, encode.accept(visitor, null)); + Assertions.assertSame(decode, decode.accept(visitor, null)); + } + + @Test + public void testCharacterSetMustBeConstant() { + SlotReference characterSetColumn = new SlotReference("charset", StringType.INSTANCE); + Encode encode = new Encode(new StringLiteral("hello"), characterSetColumn); + Decode decode = new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + characterSetColumn); + + AnalysisException encodeException = Assertions.assertThrows( + AnalysisException.class, encode::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(encodeException.getMessage().contains( + "second argument of function encode must be a literal")); + AnalysisException decodeException = Assertions.assertThrows( + AnalysisException.class, decode::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(decodeException.getMessage().contains( + "second argument of function decode must be a literal")); + + AnalysisException encodeUpper = Assertions.assertThrows(AnalysisException.class, + new Encode(new StringLiteral("hello"), new Upper(new StringLiteral("utf-8"))) + ::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(encodeUpper.getMessage().contains("must be a literal")); + + Encode literalEncode = new Encode(new StringLiteral("hello"), new StringLiteral("utf-8")); + Decode literalDecode = new Decode(new VarBinaryLiteral(new byte[] {0x68, 0x69}), + new StringLiteral("UTF-8")); + Assertions.assertDoesNotThrow(literalEncode::checkLegalityBeforeTypeCoercion); + Assertions.assertDoesNotThrow(literalEncode::checkLegalityAfterRewrite); + Assertions.assertDoesNotThrow(literalDecode::checkLegalityBeforeTypeCoercion); + Assertions.assertDoesNotThrow(literalDecode::checkLegalityAfterRewrite); + + Encode nullCharset = new Encode(new StringLiteral("hello"), new NullLiteral(StringType.INSTANCE)); + Assertions.assertDoesNotThrow(nullCharset::checkLegalityBeforeTypeCoercion); + + AnalysisException unsupported = Assertions.assertThrows(AnalysisException.class, + new Encode(new StringLiteral("hello"), new StringLiteral("GBK")) + ::checkLegalityBeforeTypeCoercion); + Assertions.assertTrue(unsupported.getMessage().contains("Unsupported character set")); + } +} diff --git a/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out new file mode 100644 index 00000000000000..fa44a7072e384e --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/binary_functions/test_encode_decode.out @@ -0,0 +1,35 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !encode_supported_charsets -- +1 41 +10 FEFF4E2D +11 \N +12 \N +2 E9 +3 E4B8AD +4 4E2D +5 2D4E +6 FEFF4E2D +7 D83DDE00 +8 +9 FEFF4E2D + +-- !decode_supported_charsets -- +1 A +10 中 +11 \N +12 \N +2 é +3 中 +4 中 +5 中 +6 中 +7 😀 +8 +9 中 + +-- !encode_null_valid_charset -- +\N + +-- !decode_null_valid_charset -- +\N + diff --git a/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy new file mode 100644 index 00000000000000..1b31ee85286f51 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/binary_functions/test_encode_decode.groovy @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_encode_decode") { + sql "drop table if exists test_encode_decode" + sql """ + create table test_encode_decode ( + id int, + plain_text string, + binary_value string, + charset varchar(32) + ) duplicate key(id) + distributed by hash(id) buckets 1 + properties ("replication_num" = "1") + """ + + sql """ + insert into test_encode_decode values + (1, 'A', unhex('41'), 'US-ASCII'), + (2, 'é', unhex('E9'), 'ISO-8859-1'), + (3, '中', unhex('E4B8AD'), 'UTF-8'), + (4, '中', unhex('4E2D'), 'UTF-16BE'), + (5, '中', unhex('2D4E'), 'UTF-16LE'), + (6, '中', unhex('FEFF4E2D'), 'UTF-16'), + (7, '😀', unhex('D83DDE00'), 'UTF-16BE'), + (8, '', unhex(''), 'UTF-16'), + (9, '中', unhex('FFFE2D4E'), 'utf-16'), + (10, '中', unhex('4E2D'), 'UTF-16'), + (11, null, null, 'UTF-8'), + (12, 'text', unhex('74657874'), null) + """ + + // Vectorized CASE evaluates every branch for every row, so mixed-charset + // CASE encode/decode queries fail on strict conversion. Filter each + // constant charset down to compatible rows instead. + order_qt_encode_supported_charsets """ + select * from ( + select id, hex(encode(plain_text, 'US-ASCII')) as encoded from test_encode_decode where id = 1 + union all + select id, hex(encode(plain_text, 'ISO-8859-1')) from test_encode_decode where id = 2 + union all + select id, hex(encode(plain_text, 'UTF-8')) from test_encode_decode where id = 3 + union all + select id, hex(encode(plain_text, 'UTF-16BE')) from test_encode_decode where id = 4 + union all + select id, hex(encode(plain_text, 'UTF-16LE')) from test_encode_decode where id = 5 + union all + select id, hex(encode(plain_text, 'UTF-16')) from test_encode_decode where id = 6 + union all + select id, hex(encode(plain_text, 'UTF-16BE')) from test_encode_decode where id = 7 + union all + select id, hex(encode(plain_text, 'UTF-16')) from test_encode_decode where id = 8 + union all + select id, hex(encode(plain_text, 'utf-16')) from test_encode_decode where id = 9 + union all + select id, hex(encode(plain_text, 'UTF-16')) from test_encode_decode where id = 10 + union all + select id, hex(encode(plain_text, 'UTF-8')) from test_encode_decode where id = 11 + union all + select id, hex(encode(plain_text, null)) from test_encode_decode where id = 12 + ) t + order by id + """ + + order_qt_decode_supported_charsets """ + select * from ( + select id, decode(cast(binary_value as varbinary), 'US-ASCII') as decoded from test_encode_decode where id = 1 + union all + select id, decode(cast(binary_value as varbinary), 'ISO-8859-1') from test_encode_decode where id = 2 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-8') from test_encode_decode where id = 3 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16BE') from test_encode_decode where id = 4 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16LE') from test_encode_decode where id = 5 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16') from test_encode_decode where id = 6 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16BE') from test_encode_decode where id = 7 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16') from test_encode_decode where id = 8 + union all + select id, decode(cast(binary_value as varbinary), 'utf-16') from test_encode_decode where id = 9 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-16') from test_encode_decode where id = 10 + union all + select id, decode(cast(binary_value as varbinary), 'UTF-8') from test_encode_decode where id = 11 + union all + select id, decode(cast(binary_value as varbinary), null) from test_encode_decode where id = 12 + ) t + order by id + """ + + qt_encode_null_valid_charset "select encode(null, 'UTF-8')" + qt_decode_null_valid_charset "select decode(null, 'UTF-8')" + + test { + sql "select encode('text', 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select encode('中', 'US-ASCII')" + exception "Character conversion using 'US-ASCII' failed" + } + + test { + sql "select decode(X'E4B8', 'UTF-8')" + exception "Character conversion using 'UTF-8' failed" + } + + test { + sql "select encode('A', 'Uſ-ASCII')" + exception "Unsupported character set" + } + + test { + sql "select encode(plain_text, charset) from test_encode_decode where id = 1" + exception "second argument of function encode must be a literal" + } + + test { + sql "select decode(cast(binary_value as varbinary), charset) from test_encode_decode where id = 1" + exception "second argument of function decode must be a literal" + } + + test { + sql "select hex(encode('中', upper('utf-8')))" + exception "must be a literal" + } + + test { + sql "select decode(X'E4B8AD', upper('utf-8'))" + exception "must be a literal" + } + + test { + sql "select encode(null, 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select encode(cast(null as string), 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select encode(plain_text, 'GBK') from test_encode_decode where id = 11" + exception "Unsupported character set" + } + + test { + sql "select decode(null, 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select decode(cast(null as varbinary), 'GBK')" + exception "Unsupported character set" + } + + test { + sql "select decode(cast(binary_value as varbinary), 'GBK') from test_encode_decode where id = 11" + exception "Unsupported character set" + } +}