diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index e15d81db4f..abc3682c51 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -2,7 +2,7 @@ name: Build and Test on Linux on: push: - branches: [ master ] + branches: [ master, urma_transport_dev ] paths-ignore: - '**.md' pull_request: @@ -85,6 +85,37 @@ jobs: -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. make -j ${{env.proc_num}} && make clean + cmake-unittest-urma-mock: + # No URMA hardware is available in CI, so this builds against brpc's + # URMA link-time mock (WITH_URMA_MOCK=ON) instead of a real liburma, and + # actually runs UrmaTransport's unit tests against it. + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-all-dependencies + - name: install gperftools + # test/CMakeLists.txt links ${GPERFTOOLS_LIBRARIES} into every + # brpc_*_unittest target unconditionally, and FindGperftools.cmake + # leaves it as "...-NOTFOUND" when the library is absent, which then + # reaches the linker verbatim. The existing unittest jobs build via + # test/Makefile and get gperftools from init-ut-make-config; this is + # the first CMake-based unittest job, so install it explicitly. + run: sudo apt-get install -y libgoogle-perftools-dev + - name: configure + run: | + mkdir build && cd build + cmake -DBUILD_UNIT_TESTS=ON -DDOWNLOAD_GTEST=ON \ + -DWITH_URMA=ON -DWITH_URMA_MOCK=ON \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. + - name: build + run: | + cd build + make -j ${{env.proc_num}} brpc_urma_unittest + - name: run tests + run: | + cd build + ctest -R brpc_urma_unittest --output-on-failure + gcc-compile-with-make-protobuf: runs-on: ubuntu-22.04 steps: diff --git a/BUILD.bazel b/BUILD.bazel index 8df080c4bb..cb94044461 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -54,6 +54,9 @@ DEFINES = [ }) + select({ "//bazel/config:brpc_with_urma": ["BRPC_WITH_URMA=1"], "//conditions:default": [], + }) + select({ + "//bazel/config:brpc_with_urma_mock": ["BRPC_WITH_URMA_MOCK=1"], + "//conditions:default": [], }) + select({ "//bazel/config:brpc_with_debug_bthread_sche_safety": ["BRPC_DEBUG_BTHREAD_SCHE_SAFETY=1"], "//conditions:default": ["BRPC_DEBUG_BTHREAD_SCHE_SAFETY=0"], @@ -100,6 +103,16 @@ LINKOPTS = [ "-libverbs", ], "//conditions:default": [], +}) + select({ + # brpc_with_urma_mock is a specialization of brpc_with_urma (see + # bazel/config/BUILD.bazel), so it wins this select whenever both defines + # are set: mock builds compile against brpc's link-time mock instead and + # must not also link the real liburma, or urma_* symbols would clash. + "//bazel/config:brpc_with_urma_mock": [], + "//bazel/config:brpc_with_urma": [ + "-lurma", + ], + "//conditions:default": [], }) + select({ "//bazel/config:brpc_with_asan": ["-fsanitize=address"], "//conditions:default": [], diff --git a/CMakeLists.txt b/CMakeLists.txt index 10e9052dcb..d1be49c8ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,9 @@ option(WITH_URMA "With URMA (openEuler Unified Remote Memory Access)" OFF) option(DOWNLOAD_URMA_HEADERS "Download UMDK headers when WITH_URMA is enabled and headers are absent" ON) +option(WITH_URMA_MOCK + "Explicitly allow linking brpc's URMA link-time mock when liburma is not found (WITH_URMA only). The mock cannot talk to real URMA hardware, so this must be opted into rather than silently substituted." + OFF) option(WITH_UBRING "With UB" OFF) option(WITH_DEBUG_BTHREAD_SCHE_SAFETY "With debugging bthread sche safety" OFF) option(WITH_DEBUG_LOCK "With debugging lock" OFF) @@ -371,11 +374,19 @@ if(WITH_URMA) if(URMA_LIB) message(STATUS "Found URMA library: ${URMA_LIB}") set(URMA_USE_MOCK 0) - else() + elseif(WITH_URMA_MOCK) message(STATUS - "liburma not found; building with the URMA link-time mock") + "liburma not found; WITH_URMA_MOCK=ON, building with the URMA " + "link-time mock") set(URMA_USE_MOCK 1) + else() + message(FATAL_ERROR + "Fail to find liburma. Install liburma, set URMA_ROOT, or " + "explicitly opt into brpc's link-time mock with " + "-DWITH_URMA_MOCK=ON (the mock cannot talk to real URMA " + "hardware; only enable it for CI/tests without URMA hardware).") endif() + list(APPEND BRPC_COMMON_DEFINITIONS BRPC_WITH_URMA_MOCK=${URMA_USE_MOCK}) endif() find_library(PROTOC_LIB NAMES protoc) diff --git a/MODULE.bazel b/MODULE.bazel index 391b8c2784..12e7f6fad0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -53,13 +53,10 @@ git_override( commit = '1e08f8e0507b6b6b1f4416a9a22cf5c28beaba93', # Jun 28, 2024 ) -git_repository = use_repo_rule( - '@bazel_tools//tools/build_defs/repo:git.bzl', - 'git_repository', +umdk_repository = use_repo_rule( + '//bazel/third_party/umdk:repositories.bzl', + 'umdk_repository', ) -git_repository( +umdk_repository( name = 'umdk', - build_file = '//bazel/third_party/umdk:umdk.BUILD', - remote = 'https://atomgit.com/openeuler/umdk.git', - commit = '564ee727a55523d4351a8fb3c94292b388ebb924', # v26.06.0_CAM ) diff --git a/WORKSPACE b/WORKSPACE index 22fc411b32..37ff3875e1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -19,6 +19,7 @@ workspace(name = "com_github_brpc_brpc") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("//bazel/third_party/umdk:repositories.bzl", "umdk_repository") # # Constants @@ -279,11 +280,8 @@ http_archive( urls = ["https://archive.apache.org/dist/thrift/0.15.0/thrift-0.15.0.tar.gz"], ) -git_repository( +umdk_repository( name = "umdk", - build_file = "//bazel/third_party/umdk:umdk.BUILD", - remote = "https://atomgit.com/openeuler/umdk.git", - commit = "564ee727a55523d4351a8fb3c94292b388ebb924", # v26.06.0_CAM ) # Header-only JSON library used by iobuf_unittest's IOBuf<->std::iostream diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel index 98e801de34..d5ae2c8552 100644 --- a/bazel/config/BUILD.bazel +++ b/bazel/config/BUILD.bazel @@ -116,6 +116,21 @@ config_setting( visibility = ["//visibility:public"], ) +# A specialization of :brpc_with_urma (its define_values is a superset), so +# Bazel resolves select()s using both to this setting whenever it matches. +# Passing --define BRPC_WITH_URMA=true alone builds against a real liburma; +# adding --define BRPC_WITH_URMA_MOCK=true switches to brpc's URMA link-time +# mock instead. The mock cannot talk to real URMA hardware, so it must be +# opted into rather than silently substituted. +config_setting( + name = "brpc_with_urma_mock", + define_values = { + "BRPC_WITH_URMA": "true", + "BRPC_WITH_URMA_MOCK": "true", + }, + visibility = ["//visibility:public"], +) + config_setting( name = "brpc_with_boringssl", define_values = {"BRPC_WITH_BORINGSSL": "true"}, diff --git a/bazel/third_party/umdk/umdk.BUILD b/bazel/third_party/umdk/BUILD.bazel similarity index 70% rename from bazel/third_party/umdk/umdk.BUILD rename to bazel/third_party/umdk/BUILD.bazel index 410e6a2736..e7578fc37e 100644 --- a/bazel/third_party/umdk/umdk.BUILD +++ b/bazel/third_party/umdk/BUILD.bazel @@ -12,17 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "urma_headers", - hdrs = glob([ - "src/urma/lib/urma/bond/include/*.h", - "src/urma/lib/urma/core/include/*.h", - ]), - includes = [ - "src/urma/lib/urma/bond/include", - "src/urma/lib/urma/core/include", - ], -) +# +# This empty BUILD.bazel file is required to make Bazel treat +# this directory as a package. diff --git a/bazel/third_party/umdk/repositories.bzl b/bazel/third_party/umdk/repositories.bzl new file mode 100644 index 0000000000..6cf2fad9ac --- /dev/null +++ b/bazel/third_party/umdk/repositories.bzl @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_UMDK_REMOTE = "https://atomgit.com/openeuler/umdk.git" +_UMDK_COMMIT = "564ee727a55523d4351a8fb3c94292b388ebb924" # v26.06.0_CAM + +_BUILD_FILE = """ +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "urma_headers", + hdrs = glob([ + "src/urma/lib/urma/bond/include/*.h", + "src/urma/lib/urma/core/include/*.h", + ]), + includes = [ + "src/urma/lib/urma/bond/include", + "src/urma/lib/urma/core/include", + ], +) +""" + +def _is_false(value): + return value.lower() in ["0", "false", "no", "off"] + +def _first_existing_file(ctx, root, relpaths): + for relpath in relpaths: + # get_child() joins one segment at a time; passing a relative path + # containing separators is not portable across Bazel versions. + path = ctx.path(root) + for segment in relpath.split("/"): + path = path.get_child(segment) + if path.exists: + return relpath + return None + +def _write_build_file(ctx): + ctx.file("BUILD.bazel", _BUILD_FILE) + +def _use_local_umdk(ctx, urma_root): + core_include = _first_existing_file(ctx, urma_root, [ + "urma_api.h", + "ub/umdk/urma/urma_api.h", + "umdk/urma/urma_api.h", + "urma/urma_api.h", + # CMake's find_path() implicitly also searches an include/ prefix + # under HINTS, so URMA_ROOT=/usr resolves there. Probe the same + # layouts here to keep URMA_ROOT meaning the same in both builds. + "include/ub/umdk/urma/urma_api.h", + "include/umdk/urma/urma_api.h", + "include/urma/urma_api.h", + "include/urma_api.h", + "src/urma/lib/urma/core/include/urma_api.h", + ]) + if not core_include: + fail("URMA_ROOT is set to '%s', but no urma_api.h was found under it" % + urma_root) + + core_include = core_include[:-len("/urma_api.h")] if "/" in core_include else "." + bond_include = _first_existing_file(ctx, urma_root, [ + "urma_ubagg.h", + "ub/umdk/urma/urma_ubagg.h", + "umdk/urma/urma_ubagg.h", + "urma/urma_ubagg.h", + "include/ub/umdk/urma/urma_ubagg.h", + "include/umdk/urma/urma_ubagg.h", + "include/urma/urma_ubagg.h", + "include/urma_ubagg.h", + "src/urma/lib/urma/bond/include/urma_ubagg.h", + ]) + if not bond_include: + fail(("URMA_ROOT is set to '%s' and urma_api.h was found under it, " + + "but no urma_ubagg.h was. brpc needs both the core and the bond " + + "UMDK headers; a partial tree would only fail later with a " + + "missing-include error far from its cause.") % urma_root) + bond_include = bond_include[:-len("/urma_ubagg.h")] if "/" in bond_include else "." + + ctx.symlink( + ctx.path(urma_root).get_child(core_include), + "src/urma/lib/urma/core/include", + ) + ctx.symlink( + ctx.path(urma_root).get_child(bond_include), + "src/urma/lib/urma/bond/include", + ) + _write_build_file(ctx) + +def _run(ctx, args): + result = ctx.execute(args, quiet = False) + if result.return_code != 0: + fail("Failed to run '%s'\nstdout:\n%s\nstderr:\n%s" % + (" ".join(args), result.stdout, result.stderr)) + +def _download_umdk(ctx): + checkout = "umdk_checkout" + _run(ctx, ["git", "init", checkout]) + _run(ctx, ["git", "-C", checkout, "remote", "add", "origin", _UMDK_REMOTE]) + _run(ctx, ["git", "-C", checkout, "fetch", "--depth", "1", "origin", _UMDK_COMMIT]) + _run(ctx, ["git", "-C", checkout, "checkout", "--detach", "FETCH_HEAD"]) + ctx.symlink( + ctx.path(checkout).get_child("src/urma/lib/urma/core/include"), + "src/urma/lib/urma/core/include", + ) + ctx.symlink( + ctx.path(checkout).get_child("src/urma/lib/urma/bond/include"), + "src/urma/lib/urma/bond/include", + ) + _write_build_file(ctx) + +def _umdk_repository_impl(ctx): + urma_root = ctx.os.environ.get("URMA_ROOT", "") + if urma_root: + _use_local_umdk(ctx, urma_root) + return + + download = ctx.os.environ.get("BRPC_DOWNLOAD_URMA_HEADERS", "true") + if _is_false(download): + fail("Failed to find urma_api.h. Set URMA_ROOT to an installed UMDK " + + "tree or allow downloading with --repo_env=BRPC_DOWNLOAD_URMA_HEADERS=true.") + + _download_umdk(ctx) + +umdk_repository = repository_rule( + implementation = _umdk_repository_impl, + environ = [ + "BRPC_DOWNLOAD_URMA_HEADERS", + "URMA_ROOT", + ], +) diff --git a/config_brpc.sh b/config_brpc.sh index 2c1394e840..525f3b709f 100755 --- a/config_brpc.sh +++ b/config_brpc.sh @@ -54,11 +54,12 @@ else LDD=ldd fi -TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-urma,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` +TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-urma,with-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` WITH_GLOG=0 WITH_THRIFT=0 WITH_RDMA=0 WITH_URMA=0 +WITH_URMA_MOCK=0 WITH_MESALINK=0 WITH_BTHREAD_TRACER=0 WITH_ASAN=0 @@ -92,6 +93,7 @@ while true; do --with-thrift) WITH_THRIFT=1; shift 1 ;; --with-rdma) WITH_RDMA=1; shift 1 ;; --with-urma) WITH_URMA=1; shift 1 ;; + --with-urma-mock) WITH_URMA_MOCK=1; shift 1 ;; --with-mesalink) WITH_MESALINK=1; shift 1 ;; --with-bthread-tracer) WITH_BTHREAD_TRACER=1; shift 1 ;; --with-debug-bthread-sche-safety ) BRPC_DEBUG_BTHREAD_SCHE_SAFETY=1; shift 1 ;; @@ -554,9 +556,14 @@ if [ $WITH_URMA != 0 ]; then append_to_output_libs "$URMA_LIB" append_to_output "DYNAMIC_LINKINGS+=-lurma" append_to_output "URMA_USE_MOCK=0" - else + CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_URMA_MOCK=0" + elif [ $WITH_URMA_MOCK != 0 ]; then append_to_output "URMA_USE_MOCK=1" - print_info "liburma not found; using URMA link-time mock" + CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_URMA_MOCK=1" + print_info "liburma not found; --with-urma-mock given, using URMA link-time mock" + else + >&2 $ECHO "Fail to find liburma. Install liburma, or explicitly opt into brpc's link-time mock with --with-urma-mock (the mock cannot talk to real URMA hardware; only use it for CI/tests without URMA hardware)." + exit 1 fi fi diff --git a/docs/cn/urma.md b/docs/cn/urma.md index a9be634ec7..dd1120b303 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -20,7 +20,7 @@ WR。完成事件既可由 JFC 忙轮询获取,也可通过 JFCE 事件 fd 获 ### CMake 编译 ```bash -# 带 URMA 支持编译 brpc +# 带 URMA 支持编译 brpc(需要 liburma;无硬件/CI 场景见下方 mock 说明) cmake -B build -DWITH_URMA=ON make -C build -j$(nproc) @@ -30,11 +30,66 @@ cmake -B build make -C build -j$(nproc) ``` +未安装 `liburma` 时(例如 CI 环境),需显式开启链接期 mock,而不是依赖 +隐式回退: + +```bash +cmake -B build -DWITH_URMA=ON -DWITH_URMA_MOCK=ON +make -C build -j$(nproc) +``` + `WITH_URMA=ON` 使用上游 UMDK 头文件进行编译。CMake 优先使用系统安装的 SDK;找不到头文件时,会参照 Mooncake 的 mock 构建方式下载固定版本的 UMDK,可通过 `DOWNLOAD_URMA_HEADERS=OFF` 禁止下载。找到 `liburma` 时使用 -真实硬件数据通路,否则链接 brpc 的 mock,使 URMA 代码和测试仍可在无硬件 -环境编译。 +真实硬件数据通路;否则默认直接报错终止构建,避免静默回退到 mock 而产出 +一个看似支持 URMA、实际无法访问真实硬件的产物。需要在无硬件环境(例如 +CI)编译和测试 URMA 代码时,显式传入 `-DWITH_URMA_MOCK=ON` +(Make 对应 `config_brpc.sh --with-urma-mock`)以主动选择链接 brpc 的 +mock。 + +### Bazel 编译 + +```bash +# 带 URMA 支持编译 brpc(链接真实 liburma,需系统已安装) +bazel build --define BRPC_WITH_URMA=true //:brpc + +# 无硬件/CI 场景:显式选择链接期 mock(不链接 liburma) +bazel build --define BRPC_WITH_URMA=true --define BRPC_WITH_URMA_MOCK=true //:brpc +``` + +`bazel/config:brpc_with_urma_mock` 是 `bazel/config:brpc_with_urma` 的 +一个特化(`define_values` 是后者的超集),因此同时传入两个 `--define` +时 Bazel 会按更具体的设置解析:编译期定义 `BRPC_WITH_URMA_MOCK=1` +并跳过 `-lurma` 链接;只传 `BRPC_WITH_URMA=true` 时才会链接真实 +`liburma`。这与 CMake/Make 侧「找不到 liburma 就必须显式加 +`--with-urma-mock`,否则直接报错」的语义等价,只是 Bazel 没有 +`find_library` 式的自动探测,需要由调用方显式指定其中一个。 + +Bazel 使用的 `@umdk` 仓库与 CMake 的头文件查找顺序保持一致: + +1. 如果设置了 `URMA_ROOT`,Bazel 会优先使用这个本地 UMDK 安装/源码树中的 + 头文件。支持系统安装布局(例如 + `$URMA_ROOT/ub/umdk/urma/urma_api.h`)和 UMDK 源码布局(例如 + `$URMA_ROOT/src/urma/lib/urma/core/include/urma_api.h`)。 +2. 如果未设置 `URMA_ROOT`,Bazel 才会从 + `https://atomgit.com/openeuler/umdk.git` 下载固定版本 + `564ee727a55523d4351a8fb3c94292b388ebb924`(即 `v26.06.0_CAM`)。 +3. 离线或 hermetic CI 中若要禁止该下载兜底,传入 + `--repo_env=BRPC_DOWNLOAD_URMA_HEADERS=false`;此时 `URMA_ROOT` 不可用会 + 在仓库解析阶段直接报错。 + +如果要用一个准备好的本地仓库或内部镜像直接替换 `@umdk`,使用 Bazel 的 +仓库覆盖。被覆盖的仓库必须提供兼容的 `@umdk//:urma_headers` target;原始 +UMDK 安装目录或源码 checkout 更适合走 `URMA_ROOT`。 + +```bash +bazel build --define BRPC_WITH_URMA=true \ + --override_repository=umdk=/path/to/local/umdk \ + //:brpc +``` + +`--distdir` 只服务于 `http_archive` / `http_file` 这类走 Bazel downloader +的规则;它不会满足 `@umdk` 的 git 下载兜底。 ## 使用 diff --git a/docs/en/urma.md b/docs/en/urma.md index 964f648bd2..3432ab9613 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -20,7 +20,7 @@ from a JFC either by busy polling or through a JFCE event fd. ### Build with CMake ```bash -# Build brpc with URMA support +# Build brpc with URMA support (requires liburma; see below for CI/mock builds) cmake -B build -DWITH_URMA=ON make -C build -j$(nproc) @@ -30,13 +30,72 @@ cmake -B build make -C build -j$(nproc) ``` +Without `liburma` installed (e.g. in CI), explicitly opt into the link-time +mock instead of relying on an implicit fallback: + +```bash +cmake -B build -DWITH_URMA=ON -DWITH_URMA_MOCK=ON +make -C build -j$(nproc) +``` + `WITH_URMA=ON` compiles against upstream UMDK headers. CMake prefers an installed SDK and, following Mooncake's mock setup, downloads a pinned UMDK release when the headers are unavailable. Set `DOWNLOAD_URMA_HEADERS=OFF` to disable downloading. -When `liburma` is found it is linked for the hardware data path. Otherwise, -brpc uses its link-time mock so URMA code and tests can still be built without -hardware. +When `liburma` is found it is linked for the hardware data path. Otherwise +the build fails by default, since silently falling back to the mock could +mask a broken environment and ship a binary that looks URMA-capable but +cannot reach real hardware. Pass `-DWITH_URMA_MOCK=ON` +(`config_brpc.sh --with-urma-mock`) to explicitly opt into brpc's +link-time mock so URMA code and tests can still be built without hardware +(e.g. in CI). + +### Build with Bazel + +```bash +# Build brpc with URMA support (links the real liburma; must be installed) +bazel build --define BRPC_WITH_URMA=true //:brpc + +# No hardware / CI: explicitly select the link-time mock (does not link liburma) +bazel build --define BRPC_WITH_URMA=true --define BRPC_WITH_URMA_MOCK=true //:brpc +``` + +`bazel/config:brpc_with_urma_mock` is a specialization of +`bazel/config:brpc_with_urma` (its `define_values` is a superset), so when +both `--define`s are passed Bazel resolves selects to the more specific +setting: it compiles with `BRPC_WITH_URMA_MOCK=1` and skips the `-lurma` +link. Passing only `BRPC_WITH_URMA=true` links the real `liburma`. This +mirrors the CMake/Make behavior of failing the build unless `liburma` is +found or the mock is explicitly requested — Bazel just has no +`find_library`-style auto-detection, so the caller must pick one of the two +`--define`s explicitly. + +The `@umdk` repository used by Bazel mirrors CMake's header lookup order: + +1. If `URMA_ROOT` is set, Bazel uses headers from that local UMDK install/tree. + Supported layouts include an installed SDK such as + `$URMA_ROOT/ub/umdk/urma/urma_api.h` and a UMDK source checkout such as + `$URMA_ROOT/src/urma/lib/urma/core/include/urma_api.h`. +2. Otherwise Bazel downloads the pinned UMDK revision + `564ee727a55523d4351a8fb3c94292b388ebb924` (`v26.06.0_CAM`) from + `https://atomgit.com/openeuler/umdk.git`. +3. To forbid that fallback in offline or hermetic CI, pass + `--repo_env=BRPC_DOWNLOAD_URMA_HEADERS=false`; repository resolution will + fail if `URMA_ROOT` is not usable. + +For a prepared local repository or internal mirror that should replace `@umdk` +directly, use Bazel's repository override instead. The override target must +expose a compatible `@umdk//:urma_headers` target; use `URMA_ROOT` for a raw +UMDK install or source checkout. + +```bash +bazel build --define BRPC_WITH_URMA=true \ + --override_repository=umdk=/path/to/local/umdk \ + //:brpc +``` + +`--distdir` only feeds Bazel's downloader-based rules such as `http_archive` +and `http_file`; it does not satisfy the git fallback used by `@umdk`. ## Usage diff --git a/example/cmake/BrpcExample.cmake b/example/cmake/BrpcExample.cmake index 6b2c7850ff..9f6baaffcf 100644 --- a/example/cmake/BrpcExample.cmake +++ b/example/cmake/BrpcExample.cmake @@ -86,6 +86,7 @@ macro(brpc_example_find_common_deps out_libs) ) endif() + # Search for libthrift* by best effort. If it is not found and brpc is # compiled with thrift protocol enabled, a link error would be reported. find_library(THRIFT_LIB NAMES thrift) @@ -123,6 +124,14 @@ macro(brpc_example_find_common_deps out_libs) find_package(OpenSSL REQUIRED) + # brpc built with -DWITH_URMA=ON carries undefined urma_* symbols, so every + # example has to link liburma. Search by best effort: when brpc was built + # without URMA the symbols are absent and the library is not needed. + find_library(URMA_LIB NAMES urma) + if(NOT URMA_LIB) + set(URMA_LIB "") + endif() + set(_common_libs Threads::Threads ${GFLAGS_LIBRARY} @@ -132,6 +141,7 @@ macro(brpc_example_find_common_deps out_libs) ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY} ${THRIFT_LIB} + ${URMA_LIB} dl ) diff --git a/example/urma_performance/CMakeLists.txt b/example/urma_performance/CMakeLists.txt index 154970fbd3..eae472fe90 100644 --- a/example/urma_performance/CMakeLists.txt +++ b/example/urma_performance/CMakeLists.txt @@ -26,13 +26,7 @@ brpc_example_find_common_deps(DYNAMIC_LIB) protobuf_generate_cpp(PROTO_SRC PROTO_HEADER test.proto) set(BRPC_EXAMPLE_WITH_URMA ON) -find_library(URMA_LIB NAMES urma) -if(URMA_LIB) - list(APPEND DYNAMIC_LIB ${URMA_LIB}) -else() - message(STATUS - "liburma not found; using the URMA implementation linked into brpc") -endif() + add_executable(urma_performance_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) brpc_example_configure_target(urma_performance_client) diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index 266c08b19a..715c9f0cb6 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -32,7 +32,7 @@ // - Device-name contract: device->name == "mock_urma_device" so tests can // match it with --urma_device=mock_urma_device. -#if BRPC_WITH_URMA +#if BRPC_WITH_URMA && BRPC_WITH_URMA_MOCK #include "urma_api.h" @@ -269,6 +269,17 @@ urma_context_t *urma_create_context(urma_device_t *device, uint32_t eid_index) { if (!device) { return nullptr; } +// The bonding provider extension. brpc only calls this with +// BONDP_USER_CTL_SET_BONDING_MODE, and only for a device whose name marks it +// as a bonding device -- which the mock never reports. Accept the call so the +// symbol resolves in mock builds where the UMDK tree does ship urma_ubagg.h. +urma_status_t urma_user_ctl(urma_context_t *ctx, urma_user_ctl_in_t *in, + urma_user_ctl_out_t *out) { + if (ctx == nullptr || in == nullptr || out == nullptr) { + return URMA_EINVAL; + } + return URMA_SUCCESS; +} urma_context_t *ctx = new urma_context_t; ctx->async_fd = 0; ctx->dev = device; @@ -710,4 +721,4 @@ void urma_ack_jfc(urma_jfc_t*[], uint32_t[], uint32_t) { } // extern "C" -#endif // BRPC_WITH_URMA +#endif // BRPC_WITH_URMA && BRPC_WITH_URMA_MOCK diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 78aa225989..6e6634e990 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -148,10 +148,13 @@ UrmaEndpoint::UrmaEndpoint(Socket* s) _state(UNINIT), _handshake_version(0), _resource(nullptr) { - _sq_size = static_cast( - std::max(16, std::min(4096, static_cast(FLAGS_urma_sq_size)))); - _rq_size = static_cast( - std::max(16, std::min(4096, static_cast(FLAGS_urma_rq_size)))); + // FLAGS_urma_sq_size / FLAGS_urma_rq_size are range-checked to [16, 4096] + // once in GlobalUrmaInitializeImpl() (urma_helper.cpp), which every path + // that can construct a UrmaEndpoint (UrmaTransport::Init(), gated by + // ContextInitOrDie()) runs before any endpoint exists. No need to + // re-clamp here. + _sq_size = static_cast(FLAGS_urma_sq_size); + _rq_size = static_cast(FLAGS_urma_rq_size); _read_butex = bthread::butex_create_checked>(); _read_butex->store(0, butil::memory_order_relaxed); } @@ -670,6 +673,11 @@ class UrmaIOBuf : private butil::IOBuf { } }; +// urma_jfs_cfg_t::max_sge is uint8_t (see urma_helper.cpp), so +// GetUrmaMaxSge() never exceeds this. Bound the on-stack SGE array by it +// instead of alloca()-ing a runtime-controlled size. +static constexpr int kUrmaMaxSgePerWr = 255; + ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { if (!_resource || !_resource->jetty || !_resource->remote_jetty) { errno = ENOTCONN; @@ -678,14 +686,11 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { int max_sge = GetUrmaMaxSge(); if (max_sge < 1) { max_sge = 1; + } else if (max_sge > kUrmaMaxSgePerWr) { + max_sge = kUrmaMaxSgePerWr; } - urma_sge_t* sglist = static_cast( - alloca(sizeof(urma_sge_t) * max_sge)); - if (!sglist) { - errno = ENOMEM; - return -1; - } + urma_sge_t sglist[kUrmaMaxSgePerWr]; size_t current = 0; ssize_t total_len = 0; @@ -1216,52 +1221,51 @@ void UrmaEndpoint::OnNewDataFromTcp(Socket* m) { InputMessenger::OnNewMessages(m); return; } - int progress = 0; - while (true) { - const State state = - ep->_state.load(butil::memory_order_acquire); - if (state == UNINIT) { - if (!m->CreatedByConnect()) { - // Server side: kick off the handshake bthread. - if (!IsUrmaAvailable()) { - ep->_state = FALLBACK_TCP; - tp->_urma_state = UrmaTransport::URMA_OFF; - InputMessenger::OnNewMessages(m); - return; - } - SocketUniquePtr s; - m->ReAddress(&s); - ep->_state = S_HELLO_WAIT; - bthread_t tid; - bthread_attr_t attr = BTHREAD_ATTR_NORMAL; - bthread_attr_set_name(&attr, "UrmaServerHandshake"); - if (bthread_start_background(&tid, &attr, - ProcessHandshakeAtServer, ep) != 0) { - ep->_state = UNINIT; - LOG(FATAL) << "Fail to start UrmaServerHandshake bthread"; - } else { - s.release(); - } + const State state = ep->_state.load(butil::memory_order_acquire); + if (state == UNINIT) { + if (!m->CreatedByConnect()) { + // Server side: kick off the handshake bthread. + if (!IsUrmaAvailable()) { + ep->_state = FALLBACK_TCP; + tp->_urma_state = UrmaTransport::URMA_OFF; + InputMessenger::OnNewMessages(m); return; } - // Client side: handled by ProcessHandshakeAtClient. - return; - } else if (state < ESTABLISHED) { - // During handshake: wake the handshake bthread parked in ReadFromFd. - ep->_read_butex->fetch_add(1, butil::memory_order_release); - bthread::butex_wake(ep->_read_butex); - return; - } else if (state == FALLBACK_TCP) { - InputMessenger::OnNewMessages(m); - return; - } else if (state == ESTABLISHED) { - TryReadOnTcpDuringUrmaEst(m); + SocketUniquePtr s; + m->ReAddress(&s); + ep->_state = S_HELLO_WAIT; + bthread_t tid; + bthread_attr_t attr = BTHREAD_ATTR_NORMAL; + bthread_attr_set_name(&attr, "UrmaServerHandshake"); + if (bthread_start_background(&tid, &attr, + ProcessHandshakeAtServer, ep) != 0) { + ep->_state = UNINIT; + LOG(FATAL) << "Fail to start UrmaServerHandshake bthread"; + } else { + s.release(); + } return; } - if (!m->MoreReadEvents(&progress)) { - break; - } + // Client side: handled by ProcessHandshakeAtClient. + return; + } + if (state < ESTABLISHED) { + // During handshake: wake the handshake bthread parked in ReadFromFd. + ep->_read_butex->fetch_add(1, butil::memory_order_release); + bthread::butex_wake(ep->_read_butex); + return; + } + if (state == FALLBACK_TCP) { + InputMessenger::OnNewMessages(m); + return; + } + if (state == ESTABLISHED) { + TryReadOnTcpDuringUrmaEst(m); + return; } + // state == FAILED: FailHandshake() already called _socket->SetFailed(), + // which tears the socket down through the normal Socket path. There is + // nothing left for the edge-trigger dispatcher to do. } inline void UrmaEndpoint::TryReadOnTcp() { diff --git a/src/brpc/urma_transport.cpp b/src/brpc/urma_transport.cpp index 311786e8eb..5c76b64006 100644 --- a/src/brpc/urma_transport.cpp +++ b/src/brpc/urma_transport.cpp @@ -58,6 +58,12 @@ void UrmaTransport::Init(Socket* socket, const SocketOptions& options) { _default_connect = options.app_connect; _on_edge_trigger = options.on_edge_triggered_events; if (options.need_on_edge_trigger && _on_edge_trigger == nullptr) { + // Unlike RDMA (which only wires OnNewDataFromTcp for client sockets + // and relies on InputMessenger::OnNewMessages / ParseRdmaHandshake + // for the server side), URMA uses OnNewDataFromTcp for both roles: + // its UNINIT branch itself dispatches on m->CreatedByConnect(), + // starting the server handshake bthread directly instead of going + // through a protocol-level handshake message. _on_edge_trigger = urma::UrmaEndpoint::OnNewDataFromTcp; } _tcp_transport = std::make_shared(); @@ -87,8 +93,14 @@ std::shared_ptr UrmaTransport::Connect() { } int UrmaTransport::CutFromIOBuf(butil::IOBuf* buf) { + // Only send over the URMA channel once the handshake has NEGOTIATED it + // (URMA_ON). While the state is still URMA_UNKNOWN (handshake in + // progress, or a server connection that turned out to be plain TCP and + // never handshook) or URMA_OFF (fell back), _resource is not yet set up + // and everything must go over the TCP fd. Mirrors the URMA_ON check in + // WaitEpollOut() and RDMA's equivalent check. if (_urma_ep && - _urma_state.load(butil::memory_order_acquire) != URMA_OFF) { + _urma_state.load(butil::memory_order_acquire) == URMA_ON) { butil::IOBuf* data_arr[1] = {buf}; return _urma_ep->CutFromIOBufList(data_arr, 1); } else { @@ -98,7 +110,7 @@ int UrmaTransport::CutFromIOBuf(butil::IOBuf* buf) { ssize_t UrmaTransport::CutFromIOBufList(butil::IOBuf** buf, size_t ndata) { if (_urma_ep && - _urma_state.load(butil::memory_order_acquire) != URMA_OFF) { + _urma_state.load(butil::memory_order_acquire) == URMA_ON) { return _urma_ep->CutFromIOBufList(buf, ndata); } else { return _tcp_transport->CutFromIOBufList(buf, ndata);