From bc977094c956da9527d67151e631a0635a5a656f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 13 Jul 2026 12:11:36 -0700 Subject: [PATCH 01/33] test(netty-4.1): Add pipelining test --- .../server/NettyHttp11PipeliningTest.java | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java new file mode 100644 index 00000000000..610fa95ddbe --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -0,0 +1,223 @@ +package datadog.trace.instrumentation.netty41.server; + +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TraceAssertions.SORT_BY_START_TIME; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.agent.test.assertions.TraceMatcher; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpServerCodec; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.regex.Pattern; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class NettyHttp11PipeliningTest extends AbstractInstrumentationTest { + + private static final String FIRST_PATH = "/pipelined/first"; + private static final String SECOND_PATH = "/pipelined/second"; + private static final String THIRD_PATH = "/pipelined/third"; + + private EventLoopGroup eventLoopGroup; + private PipeliningHandler handler; + private int port; + + @BeforeAll + void startServer() throws Exception { + eventLoopGroup = new NioEventLoopGroup(); + handler = new PipeliningHandler(3); + ServerBootstrap bootstrap = + new ServerBootstrap() + .group(eventLoopGroup) + .channel(NioServerSocketChannel.class) + .childHandler( + new ChannelInitializer() { + @Override + protected void initChannel(Channel ch) { + ch.pipeline().addLast(new HttpServerCodec()); + ch.pipeline().addLast(new HttpObjectAggregator(65536)); + ch.pipeline().addLast(handler); + } + }); + Channel channel = bootstrap.bind(0).sync().channel(); + port = ((InetSocketAddress) channel.localAddress()).getPort(); + } + + @AfterAll + void stopServer() { + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully(); + } + } + + @Test + void createsServerSpanForEachPipelinedRequest() throws Exception { + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(pipelinedRequests().getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + handler.awaitAllRequestsReceived(), + "server did not receive all pipelined requests before responding"); + + handler.writeResponses(); + + assertEquals("response " + FIRST_PATH, readHttpResponseBody(socket.getInputStream())); + assertEquals("response " + SECOND_PATH, readHttpResponseBody(socket.getInputStream())); + assertEquals("response " + THIRD_PATH, readHttpResponseBody(socket.getInputStream())); + } + + assertTraces( + SORT_BY_START_TIME, + serverTrace(FIRST_PATH), + serverTrace(SECOND_PATH), + serverTrace(THIRD_PATH)); + } + + private static String pipelinedRequests() { + return "GET " + + FIRST_PATH + + " HTTP/1.1\r\nHost: localhost\r\n\r\n" + + "GET " + + SECOND_PATH + + " HTTP/1.1\r\nHost: localhost\r\n\r\n" + + "GET " + + THIRD_PATH + + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; + } + + private static TraceMatcher serverTrace(String path) { + return trace( + span() + .root() + .operationName(Pattern.compile("netty\\.request")) + .resourceName(Pattern.compile("GET " + Pattern.quote(path))) + .type("web")); + } + + private static String readHttpResponseBody(InputStream in) throws IOException { + String headers = readHeaders(in); + assertTrue(headers.startsWith("HTTP/1.1 200 "), "unexpected response: " + headers); + int contentLength = contentLength(headers); + byte[] body = new byte[contentLength]; + int read = 0; + while (read < contentLength) { + int count = in.read(body, read, contentLength - read); + if (count == -1) { + throw new EOFException("response ended before body was complete"); + } + read += count; + } + return new String(body, UTF_8); + } + + private static String readHeaders(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int state = 0; + while (state < 4) { + int b = in.read(); + if (b == -1) { + throw new EOFException("response ended before headers were complete"); + } + out.write(b); + if ((state == 0 || state == 2) && b == '\r') { + state++; + } else if ((state == 1 || state == 3) && b == '\n') { + state++; + } else { + state = b == '\r' ? 1 : 0; + } + } + return out.toString(US_ASCII.name()); + } + + private static int contentLength(String headers) { + for (String line : headers.split("\r\n")) { + int separator = line.indexOf(':'); + if (separator > 0 && "content-length".equalsIgnoreCase(line.substring(0, separator))) { + return Integer.parseInt(line.substring(separator + 1).trim()); + } + } + throw new AssertionError("missing content-length header: " + headers); + } + + private static final class PipeliningHandler + extends SimpleChannelInboundHandler { + private final CountDownLatch receivedRequests; + private final List paths = new ArrayList<>(); + private volatile ChannelHandlerContext context; + + private PipeliningHandler(int expectedRequests) { + receivedRequests = new CountDownLatch(expectedRequests); + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) { + context = ctx; + synchronized (paths) { + paths.add(request.uri()); + } + receivedRequests.countDown(); + } + + private boolean awaitAllRequestsReceived() throws InterruptedException { + return receivedRequests.await(5, SECONDS); + } + + private void writeResponses() { + ChannelHandlerContext responseContext = context; + if (responseContext == null) { + throw new IllegalStateException("no request context captured"); + } + List responsePaths; + synchronized (paths) { + responsePaths = new ArrayList<>(paths); + } + responseContext + .executor() + .execute( + () -> { + for (String path : responsePaths) { + byte[] body = ("response " + path).getBytes(UTF_8); + DefaultFullHttpResponse response = + new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.wrappedBuffer(body)); + response.headers().set(CONTENT_LENGTH, body.length); + responseContext.write(response); + } + responseContext.flush(); + }); + } + } +} From 7f7fdf417c2bc8e60c646e038b8040aa78132211 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 13 Jul 2026 15:21:23 -0700 Subject: [PATCH 02/33] fix(netty-4.1): pipelined response context tracking --- .../ChannelFutureListenerInstrumentation.java | 1 + ...yChannelHandlerContextInstrumentation.java | 6 +- .../NettyChannelPipelineInstrumentation.java | 1 + .../HttpServerRequestTracingHandler.java | 30 +++-- .../HttpServerResponseTracingHandler.java | 16 ++- .../server/MaybeBlockResponseHandler.java | 34 ++--- .../netty41/AttributeKeys.java | 12 +- .../netty41/ServerRequestContext.java | 121 ++++++++++++++++++ 8 files changed, 162 insertions(+), 59 deletions(-) create mode 100644 dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java index 85fe3902fc0..26754d69a6f 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java @@ -46,6 +46,7 @@ public ElementMatcher hierarchyMatcher() { public String[] helperClassNames() { return new String[] { packageName + ".AttributeKeys", + packageName + ".ServerRequestContext", // client helpers packageName + ".client.NettyHttpClientDecorator", packageName + ".client.NettyResponseInjectAdapter", diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java index 62363fda0cc..b73e4e0fbf7 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java @@ -14,7 +14,6 @@ import static net.bytebuddy.matcher.ElementMatchers.isPublic; import com.google.auto.service.AutoService; -import datadog.context.Context; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.instrumentation.api.AgentScope; @@ -48,6 +47,7 @@ public ElementMatcher hierarchyMatcher() { public String[] helperClassNames() { return new String[] { packageName + ".AttributeKeys", + packageName + ".ServerRequestContext", packageName + ".client.NettyHttpClientDecorator", packageName + ".server.ResponseExtractAdapter", packageName + ".server.NettyHttpServerDecorator", @@ -71,8 +71,8 @@ public void methodAdvice(MethodTransformer transformer) { public static class FireAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static AgentScope scopeSpan(@Advice.This final ChannelHandlerContext ctx) { - final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); - final AgentSpan channelSpan = spanFromContext(storedContext); + final AgentSpan channelSpan = + spanFromContext(ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get()); if (channelSpan == null || channelSpan == activeSpan()) { // don't modify the scope return noopScope(); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java index 7fce2fd12eb..668077b87a6 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java @@ -72,6 +72,7 @@ public ElementMatcher hierarchyMatcher() { public String[] helperClassNames() { return new String[] { packageName + ".AttributeKeys", + packageName + ".ServerRequestContext", // client helpers packageName + ".client.NettyHttpClientDecorator", packageName + ".client.NettyResponseInjectAdapter", diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index ce58fac2ded..fbf2c689360 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -1,22 +1,21 @@ package datadog.trace.instrumentation.netty41.server; -import static datadog.trace.instrumentation.netty41.AttributeKeys.ANALYZED_RESPONSE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.BLOCKED_RESPONSE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.PARENT_CONTEXT_ATTRIBUTE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.REQUEST_HEADERS_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; import datadog.context.Context; import datadog.context.ContextScope; import datadog.trace.api.gateway.Flow; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpRequest; +import java.util.Deque; @ChannelHandler.Sharable public class HttpServerRequestTracingHandler extends ChannelInboundHandlerAdapter { @@ -49,11 +48,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { DECORATE.afterStart(span); DECORATE.onRequest(span, channel, request, parentContext); - channel.attr(ANALYZED_RESPONSE_KEY).set(null); - channel.attr(BLOCKED_RESPONSE_KEY).set(null); - - channel.attr(CONTEXT_ATTRIBUTE_KEY).set(context); - channel.attr(REQUEST_HEADERS_ATTRIBUTE_KEY).set(request.headers()); + final ServerRequestContext serverContext = + ServerRequestContext.add(channel, context, request.headers()); Flow.Action.RequestBlockingAction rba = span.getRequestBlockingAction(); if (rba != null) { @@ -76,7 +72,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { DECORATE.onError(span, throwable); DECORATE.beforeFinish(ignored.context()); span.finish(); // Finish the span manually since finishSpanOnClose was false - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).remove(); + ServerRequestContext.remove(ctx.channel(), serverContext); throw throwable; } } @@ -88,11 +84,17 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); } finally { try { - final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).getAndRemove(); - final AgentSpan span = AgentSpan.fromContext(storedContext); - if (span != null && span.phasedFinish()) { - // at this point we can just publish this span to avoid loosing the rest of the trace - span.publish(); + final Deque storedContexts = + ServerRequestContext.removeAll(ctx.channel()); + if (storedContexts != null) { + ServerRequestContext storedContext; + while ((storedContext = storedContexts.pollFirst()) != null) { + final AgentSpan span = AgentSpan.fromContext(storedContext.tracingContext()); + if (span != null && span.phasedFinish()) { + // at this point we can just publish this span to avoid losing the rest of the trace + span.publish(); + } + } } } catch (final Throwable ignored) { } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 04d7f8893fc..11ea5475e1e 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -1,6 +1,5 @@ package datadog.trace.instrumentation.netty41.server; -import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.WEBSOCKET_SENDER_HANDLER_CONTEXT; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; @@ -8,6 +7,7 @@ import datadog.context.ContextScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.websocket.HandlerContext; +import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandlerAdapter; @@ -22,10 +22,16 @@ public class HttpServerResponseTracingHandler extends ChannelOutboundHandlerAdap @Override public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise prm) { - final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); + if (!(msg instanceof HttpResponse)) { + ctx.write(msg, prm); + return; + } + + final ServerRequestContext serverContext = ServerRequestContext.nextResponse(ctx.channel()); + final Context storedContext = serverContext == null ? null : serverContext.tracingContext(); final AgentSpan span = AgentSpan.fromContext(storedContext); - if (span == null || !(msg instanceof HttpResponse)) { + if (span == null) { ctx.write(msg, prm); return; } @@ -39,7 +45,7 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann DECORATE.onError(span, throwable); span.setHttpStatusCode(500); span.finish(); // Finish the span manually since finishSpanOnClose was false - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).remove(); + ServerRequestContext.remove(ctx.channel(), serverContext); throw throwable; } final boolean isWebsocketUpgrade = @@ -55,7 +61,7 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann DECORATE.onResponse(span, response); DECORATE.beforeFinish(scope.context()); span.finish(); // Finish the span manually since finishSpanOnClose was false - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).remove(); + ServerRequestContext.remove(ctx.channel(), serverContext); } } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java index 4f79bd6aea6..172c838b52f 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java @@ -1,9 +1,5 @@ package datadog.trace.instrumentation.netty41.server; -import static datadog.trace.instrumentation.netty41.AttributeKeys.ANALYZED_RESPONSE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.BLOCKED_RESPONSE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.REQUEST_HEADERS_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; import static io.netty.handler.codec.http.HttpHeaders.setContentLength; @@ -13,6 +9,7 @@ import datadog.trace.api.gateway.RequestContext; import datadog.trace.bootstrap.blocking.BlockingActionHelper; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; @@ -36,27 +33,12 @@ public class MaybeBlockResponseHandler extends ChannelOutboundHandlerAdapter { private MaybeBlockResponseHandler() {} - private static boolean isAnalyzedResponse(Channel ch) { - return ch.attr(ANALYZED_RESPONSE_KEY).get() != null; - } - - private static void markAnalyzedResponse(Channel ch) { - ch.attr(ANALYZED_RESPONSE_KEY).set(Boolean.TRUE); - } - - private static boolean isBlockedResponse(Channel ch) { - return ch.attr(BLOCKED_RESPONSE_KEY).get() != null; - } - - private static void markBlockedResponse(Channel ch) { - ch.attr(BLOCKED_RESPONSE_KEY).set(Boolean.TRUE); - } - @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) throws Exception { Channel channel = ctx.channel(); - Context storedContext = channel.attr(CONTEXT_ATTRIBUTE_KEY).get(); + ServerRequestContext serverContext = ServerRequestContext.nextResponse(channel); + Context storedContext = serverContext == null ? null : serverContext.tracingContext(); AgentSpan span = AgentSpan.fromContext(storedContext); RequestContext requestContext; if (span == null || (requestContext = span.getRequestContext()) == null) { @@ -64,8 +46,8 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr return; } - if (isAnalyzedResponse(channel)) { - if (isBlockedResponse(channel)) { + if (serverContext.isResponseAnalyzed()) { + if (serverContext.isResponseBlocked()) { // block further writes log.debug("Write suppressed, msg {} dropped", msg); ReferenceCountUtil.release(msg); @@ -88,14 +70,14 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr Flow flow = DECORATE.callIGCallbackResponseAndHeaders( span, origResponse, origResponse.getStatus().code(), ResponseExtractAdapter.GETTER); - markAnalyzedResponse(channel); + serverContext.markResponseAnalyzed(); Flow.Action action = flow.getAction(); if (!(action instanceof Flow.Action.RequestBlockingAction)) { super.write(ctx, msg, prm); return; } - markBlockedResponse(channel); + serverContext.markResponseBlocked(); Flow.Action.RequestBlockingAction rba = (Flow.Action.RequestBlockingAction) action; int httpCode = BlockingActionHelper.getHttpCode(rba.getStatusCode()); HttpResponseStatus httpResponseStatus = HttpResponseStatus.valueOf(httpCode); @@ -112,7 +94,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr BlockingContentType bct = rba.getBlockingContentType(); if (bct != BlockingContentType.NONE) { - HttpHeaders reqHeaders = ctx.attr(REQUEST_HEADERS_ATTRIBUTE_KEY).get(); + HttpHeaders reqHeaders = serverContext.requestHeaders(); String acceptHeader = reqHeaders != null ? reqHeaders.get("accept") : null; BlockingActionHelper.TemplateType type = BlockingActionHelper.determineTemplateType(bct, acceptHeader); diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java index 24e34d48ab5..307002a68fb 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java @@ -7,7 +7,6 @@ import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.websocket.HandlerContext; -import io.netty.handler.codec.http.HttpHeaders; import io.netty.util.AttributeKey; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -33,15 +32,6 @@ public final class AttributeKeys { public static final AttributeKey PARENT_CONTEXT_ATTRIBUTE_KEY = attributeKey("datadog.server.parent-context"); - public static final AttributeKey REQUEST_HEADERS_ATTRIBUTE_KEY = - attributeKey("datadog.server.request.headers"); - - public static final AttributeKey ANALYZED_RESPONSE_KEY = - attributeKey("datadog.server.analyzed_response"); - - public static final AttributeKey BLOCKED_RESPONSE_KEY = - attributeKey("datadog.server.blocked_response"); - public static final AttributeKey WEBSOCKET_SENDER_HANDLER_CONTEXT = attributeKey("datadog.server.websocket.sender.handler_context"); @@ -56,7 +46,7 @@ public final class AttributeKeys { * cassandra driver. */ @SuppressWarnings("unchecked") - private static AttributeKey attributeKey(final String key) { + static AttributeKey attributeKey(final String key) { ConcurrentMap> map = MAPS.get(AttributeKey.class); AttributeKey attributeKey = (AttributeKey) map.get(key); if (null == attributeKey) { diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java new file mode 100644 index 00000000000..1a780668312 --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -0,0 +1,121 @@ +package datadog.trace.instrumentation.netty41; + +import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; + +import datadog.context.Context; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.util.AttributeKey; +import io.netty.util.AttributeMap; +import java.util.ArrayDeque; +import java.util.Deque; + +/** Per-request server state stored on the channel until the matching response is written. */ +public final class ServerRequestContext { + + /** Pending server request contexts for a channel. */ + private static final AttributeKey> + SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY = + AttributeKeys.attributeKey("datadog.server.request.contexts"); + + /** Adds a request context to the queue tail. */ + public static ServerRequestContext add( + final AttributeMap attributes, final Context context, final HttpHeaders requestHeaders) { + final ServerRequestContext serverContext = new ServerRequestContext(context, requestHeaders); + getOrCreate(attributes).addLast(serverContext); + // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a + // legacy mirror of the current inbound request used by generic fire* activation. + attributes.attr(CONTEXT_ATTRIBUTE_KEY).set(context); + return serverContext; + } + + /** Returns the server request context for the next response. */ + public static ServerRequestContext nextResponse(final AttributeMap attributes) { + final Deque contexts = + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); + // HTTP/1.1 responses are written in request order, including when requests are pipelined on one + // connection. + return contexts == null ? null : contexts.peekFirst(); + } + + /** Removes a completed or failed request context. */ + public static void remove( + final AttributeMap attributes, final ServerRequestContext serverContext) { + if (serverContext == null) { + return; + } + final Deque contexts = + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); + if (contexts != null) { + if (contexts.peekFirst() == serverContext) { + // Response completion consumes the queue head. + contexts.pollFirst(); + } else { + // Request-side failures normally remove the tail. Remove by value to cover later cleanup + // after additional pipelined requests were queued. + contexts.remove(serverContext); + } + final ServerRequestContext currentContext = contexts.peekLast(); + if (currentContext == null) { + attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); + } else { + // Keep the legacy mirror pointed at the current inbound request after removing an older + // response context. + attributes.attr(CONTEXT_ATTRIBUTE_KEY).set(currentContext.tracingContext()); + } + } + } + + /** Removes all pending request contexts on channel close. */ + public static Deque removeAll(final AttributeMap attributes) { + // The legacy mirror must not outlive the authoritative request queue. + attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); + return attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove(); + } + + /** Creates the per-channel server request context queue. */ + private static Deque getOrCreate(final AttributeMap attributes) { + Deque contexts = + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); + if (contexts == null) { + // Netty serializes handler callbacks for a channel on its EventLoop, so this queue does not + // need the allocation and atomic overhead of a concurrent collection. + contexts = new ArrayDeque<>(); + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).set(contexts); + } + return contexts; + } + + private final Context tracingContext; + private final HttpHeaders requestHeaders; + private boolean responseAnalyzed; + private boolean responseBlocked; + + public Context tracingContext() { + return tracingContext; + } + + public HttpHeaders requestHeaders() { + return requestHeaders; + } + + public boolean isResponseAnalyzed() { + return responseAnalyzed; + } + + public void markResponseAnalyzed() { + responseAnalyzed = true; + } + + public boolean isResponseBlocked() { + return responseBlocked; + } + + public void markResponseBlocked() { + responseBlocked = true; + } + + private ServerRequestContext(final Context tracingContext, final HttpHeaders requestHeaders) { + this.tracingContext = tracingContext; + this.requestHeaders = requestHeaders; + } +} From 57f9d65709b10c0ec26e7b39e0ee6a8463a6e8bc Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 13 Jul 2026 19:32:26 -0700 Subject: [PATCH 03/33] Disable Netty server tracing on context queue overflow --- .../HttpServerRequestTracingHandler.java | 20 ++--- .../netty41/ServerRequestContextTest.java | 28 ++++++ .../netty41/ServerRequestContext.java | 86 ++++++++++++++++--- 3 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index fbf2c689360..7e7dfcb1e89 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -15,7 +15,6 @@ import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpRequest; -import java.util.Deque; @ChannelHandler.Sharable public class HttpServerRequestTracingHandler extends ChannelInboundHandlerAdapter { @@ -37,6 +36,12 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } final HttpRequest request = (HttpRequest) msg; + if (!ServerRequestContext.canTrackRequest(channel)) { + channel.attr(PARENT_CONTEXT_ATTRIBUTE_KEY).remove(); + ctx.fireChannelRead(msg); + return; + } + final HttpHeaders headers = request.headers(); final Context storedParentContext = channel.attr(PARENT_CONTEXT_ATTRIBUTE_KEY).getAndRemove(); final Context parentContext = @@ -84,18 +89,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); } finally { try { - final Deque storedContexts = - ServerRequestContext.removeAll(ctx.channel()); - if (storedContexts != null) { - ServerRequestContext storedContext; - while ((storedContext = storedContexts.pollFirst()) != null) { - final AgentSpan span = AgentSpan.fromContext(storedContext.tracingContext()); - if (span != null && span.phasedFinish()) { - // at this point we can just publish this span to avoid losing the rest of the trace - span.publish(); - } - } - } + ServerRequestContext.closeAll(ctx.channel()); } catch (final Throwable ignored) { } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java new file mode 100644 index 00000000000..cc2896bde03 --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -0,0 +1,28 @@ +package datadog.trace.instrumentation.netty41; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.context.Context; +import io.netty.handler.codec.http.DefaultHttpHeaders; +import io.netty.util.DefaultAttributeMap; +import org.junit.jupiter.api.Test; + +class ServerRequestContextTest { + private static final int PIPELINING_LIMIT = 1000; + + @Test + void disablesTrackingWhenPipeliningLimitIsExceeded() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + + for (int i = 0; i < PIPELINING_LIMIT; i++) { + assertNotNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); + } + + assertFalse(ServerRequestContext.canTrackRequest(attributes)); + assertNull(ServerRequestContext.nextResponse(attributes)); + assertNull(attributes.attr(AttributeKeys.CONTEXT_ATTRIBUTE_KEY).get()); + assertNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); + } +} diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 1a780668312..4a8ea3e59e7 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -3,25 +3,33 @@ import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import datadog.context.Context; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import io.netty.handler.codec.http.HttpHeaders; import io.netty.util.AttributeKey; import io.netty.util.AttributeMap; import java.util.ArrayDeque; import java.util.Deque; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Per-request server state stored on the channel until the matching response is written. */ public final class ServerRequestContext { - - /** Pending server request contexts for a channel. */ - private static final AttributeKey> - SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY = - AttributeKeys.attributeKey("datadog.server.request.contexts"); + /** Returns whether a new server request can be tracked on this channel. */ + public static boolean canTrackRequest(final AttributeMap attributes) { + final Deque contexts = + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); + return contexts == null || canAdd(attributes, contexts); + } /** Adds a request context to the queue tail. */ public static ServerRequestContext add( final AttributeMap attributes, final Context context, final HttpHeaders requestHeaders) { + final Deque contexts = getOrCreate(attributes); + if (!canAdd(attributes, contexts)) { + return null; + } final ServerRequestContext serverContext = new ServerRequestContext(context, requestHeaders); - getOrCreate(attributes).addLast(serverContext); + contexts.addLast(serverContext); // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a // legacy mirror of the current inbound request used by generic fire* activation. attributes.attr(CONTEXT_ATTRIBUTE_KEY).set(context); @@ -34,7 +42,7 @@ public static ServerRequestContext nextResponse(final AttributeMap attributes) { attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); // HTTP/1.1 responses are written in request order, including when requests are pipelined on one // connection. - return contexts == null ? null : contexts.peekFirst(); + return contexts == null || isPoisoned(contexts) ? null : contexts.peekFirst(); } /** Removes a completed or failed request context. */ @@ -46,6 +54,9 @@ public static void remove( final Deque contexts = attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); if (contexts != null) { + if (isPoisoned(contexts)) { + return; + } if (contexts.peekFirst() == serverContext) { // Response completion consumes the queue head. contexts.pollFirst(); @@ -65,13 +76,24 @@ public static void remove( } } - /** Removes all pending request contexts on channel close. */ - public static Deque removeAll(final AttributeMap attributes) { + /** Closes all pending request contexts on channel close. */ + public static void closeAll(final AttributeMap attributes) { // The legacy mirror must not outlive the authoritative request queue. attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); - return attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove(); + close(attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove()); } + private static final int PIPELINING_LIMIT = 1000; + + private static final Logger log = LoggerFactory.getLogger(ServerRequestContext.class); + + /** Pending server request contexts for a channel. */ + private static final AttributeKey> + SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY = + AttributeKeys.attributeKey("datadog.server.request.contexts"); + + private static final Deque POISONED_CONTEXTS = new ArrayDeque<>(0); + /** Creates the per-channel server request context queue. */ private static Deque getOrCreate(final AttributeMap attributes) { Deque contexts = @@ -85,6 +107,50 @@ private static Deque getOrCreate(final AttributeMap attrib return contexts; } + private static void close(final Deque contexts) { + if (contexts == null || isPoisoned(contexts)) { + return; + } + ServerRequestContext context; + while ((context = contexts.pollFirst()) != null) { + try { + final AgentSpan span = AgentSpan.fromContext(context.tracingContext()); + if (span != null && span.phasedFinish()) { + // These contexts no longer have a response handler path that can finish the span. + span.publish(); + } + } catch (final Throwable ignored) { + } + } + } + + private static boolean canAdd( + final AttributeMap attributes, final Deque contexts) { + if (isPoisoned(contexts)) { + return false; + } + // If this limit is exceeded, stop tracing on the channel and drain the deque. This suggests + // contexts are not being removed, for example because the server stopped writing responses. + if (contexts.size() >= PIPELINING_LIMIT) { + final int pendingContexts = contexts.size(); + close(contexts); + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).set(POISONED_CONTEXTS); + attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); + log.error( + "Too many pending Netty server request contexts on a channel; " + + "closing {} contexts and disabling Netty server tracing on that channel " + + "(limit: {})", + pendingContexts, + PIPELINING_LIMIT); + return false; + } + return true; + } + + private static boolean isPoisoned(final Deque contexts) { + return contexts == POISONED_CONTEXTS; + } + private final Context tracingContext; private final HttpHeaders requestHeaders; private boolean responseAnalyzed; From 6173156b9e8ef9d1178709e58c206d12dfbdbc0a Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 16 Jul 2026 12:32:09 -0700 Subject: [PATCH 04/33] Fix Netty 4.1 response context handling Track server request contexts per pipelined response, snapshot only the Accept header, and keep AppSec response-block state channel-level so late streaming chunks are dropped until close. --- .../HttpServerRequestTracingHandler.java | 2 +- .../server/MaybeBlockResponseHandler.java | 21 ++++----- .../netty41/ServerRequestContextTest.java | 34 +++++++++++++- .../server/MaybeBlockResponseHandlerTest.java | 30 ++++++++++++ .../netty41/ServerRequestContext.java | 47 ++++++++++++------- 5 files changed, 102 insertions(+), 32 deletions(-) create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 7e7dfcb1e89..0d934772d56 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -54,7 +54,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { DECORATE.onRequest(span, channel, request, parentContext); final ServerRequestContext serverContext = - ServerRequestContext.add(channel, context, request.headers()); + ServerRequestContext.add(channel, context, headers.get("accept")); Flow.Action.RequestBlockingAction rba = span.getRequestBlockingAction(); if (rba != null) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java index 172c838b52f..5dab095e4e0 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java @@ -37,6 +37,13 @@ private MaybeBlockResponseHandler() {} public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) throws Exception { Channel channel = ctx.channel(); + if (ServerRequestContext.isResponseBlocked(channel)) { + // block further writes while the blocking response close is still asynchronous + log.debug("Write suppressed, msg {} dropped", msg); + ReferenceCountUtil.release(msg); + return; + } + ServerRequestContext serverContext = ServerRequestContext.nextResponse(channel); Context storedContext = serverContext == null ? null : serverContext.tracingContext(); AgentSpan span = AgentSpan.fromContext(storedContext); @@ -47,13 +54,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr } if (serverContext.isResponseAnalyzed()) { - if (serverContext.isResponseBlocked()) { - // block further writes - log.debug("Write suppressed, msg {} dropped", msg); - ReferenceCountUtil.release(msg); - } else { - super.write(ctx, msg, prm); - } + super.write(ctx, msg, prm); return; } @@ -77,7 +78,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr return; } - serverContext.markResponseBlocked(); + ServerRequestContext.markResponseBlocked(channel); Flow.Action.RequestBlockingAction rba = (Flow.Action.RequestBlockingAction) action; int httpCode = BlockingActionHelper.getHttpCode(rba.getStatusCode()); HttpResponseStatus httpResponseStatus = HttpResponseStatus.valueOf(httpCode); @@ -94,10 +95,8 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr BlockingContentType bct = rba.getBlockingContentType(); if (bct != BlockingContentType.NONE) { - HttpHeaders reqHeaders = serverContext.requestHeaders(); - String acceptHeader = reqHeaders != null ? reqHeaders.get("accept") : null; BlockingActionHelper.TemplateType type = - BlockingActionHelper.determineTemplateType(bct, acceptHeader); + BlockingActionHelper.determineTemplateType(bct, serverContext.acceptHeader()); headers.set("Content-type", BlockingActionHelper.getContentType(type)); byte[] template = BlockingActionHelper.getTemplate(type, rba.getSecurityResponseId()); setContentLength(response, template.length); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index cc2896bde03..4cac8e6c1e8 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -1,8 +1,10 @@ package datadog.trace.instrumentation.netty41; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.context.Context; import io.netty.handler.codec.http.DefaultHttpHeaders; @@ -17,12 +19,40 @@ void disablesTrackingWhenPipeliningLimitIsExceeded() { DefaultAttributeMap attributes = new DefaultAttributeMap(); for (int i = 0; i < PIPELINING_LIMIT; i++) { - assertNotNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); + assertNotNull(ServerRequestContext.add(attributes, Context.root(), null)); } assertFalse(ServerRequestContext.canTrackRequest(attributes)); assertNull(ServerRequestContext.nextResponse(attributes)); assertNull(attributes.attr(AttributeKeys.CONTEXT_ATTRIBUTE_KEY).get()); - assertNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); + assertNull(ServerRequestContext.add(attributes, Context.root(), null)); + } + + @Test + void capturesAcceptHeaderValue() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + DefaultHttpHeaders headers = new DefaultHttpHeaders(); + headers.set("accept", "text/html"); + + ServerRequestContext serverContext = + ServerRequestContext.add(attributes, Context.root(), headers.get("accept")); + headers.set("accept", "application/json"); + + assertEquals("text/html", serverContext.acceptHeader()); + } + + @Test + void tracksBlockedResponseUntilChannelClose() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + ServerRequestContext serverContext = ServerRequestContext.add(attributes, Context.root(), null); + + ServerRequestContext.markResponseBlocked(attributes); + ServerRequestContext.remove(attributes, serverContext); + + assertTrue(ServerRequestContext.isResponseBlocked(attributes)); + + ServerRequestContext.closeAll(attributes); + + assertFalse(ServerRequestContext.isResponseBlocked(attributes)); } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java new file mode 100644 index 00000000000..7ba3a2aabf1 --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java @@ -0,0 +1,30 @@ +package datadog.trace.instrumentation.netty41.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.context.Context; +import datadog.trace.instrumentation.netty41.ServerRequestContext; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import org.junit.jupiter.api.Test; + +class MaybeBlockResponseHandlerTest { + + @Test + void dropsWritesAfterBlockedContextHasBeenRemoved() { + EmbeddedChannel channel = new EmbeddedChannel(MaybeBlockResponseHandler.INSTANCE); + ServerRequestContext serverContext = ServerRequestContext.add(channel, Context.root(), null); + ServerRequestContext.markResponseBlocked(channel); + ServerRequestContext.remove(channel, serverContext); + ByteBuf lateResponseChunk = Unpooled.buffer().writeByte(1); + + assertFalse(channel.writeOutbound(lateResponseChunk)); + + assertEquals(0, lateResponseChunk.refCnt()); + assertNull(channel.readOutbound()); + channel.finishAndReleaseAll(); + } +} diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 4a8ea3e59e7..5819f940090 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -4,7 +4,6 @@ import datadog.context.Context; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import io.netty.handler.codec.http.HttpHeaders; import io.netty.util.AttributeKey; import io.netty.util.AttributeMap; import java.util.ArrayDeque; @@ -14,7 +13,10 @@ /** Per-request server state stored on the channel until the matching response is written. */ public final class ServerRequestContext { - /** Returns whether a new server request can be tracked on this channel. */ + /** + * Returns whether a new server request can be tracked on this channel (and may disable server + * tracing for the channel if the pending-context limit is exceeded). + */ public static boolean canTrackRequest(final AttributeMap attributes) { final Deque contexts = attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); @@ -23,12 +25,12 @@ public static boolean canTrackRequest(final AttributeMap attributes) { /** Adds a request context to the queue tail. */ public static ServerRequestContext add( - final AttributeMap attributes, final Context context, final HttpHeaders requestHeaders) { + final AttributeMap attributes, final Context context, final String acceptHeader) { final Deque contexts = getOrCreate(attributes); if (!canAdd(attributes, contexts)) { return null; } - final ServerRequestContext serverContext = new ServerRequestContext(context, requestHeaders); + final ServerRequestContext serverContext = new ServerRequestContext(context, acceptHeader); contexts.addLast(serverContext); // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a // legacy mirror of the current inbound request used by generic fire* activation. @@ -45,6 +47,16 @@ public static ServerRequestContext nextResponse(final AttributeMap attributes) { return contexts == null || isPoisoned(contexts) ? null : contexts.peekFirst(); } + /** Returns whether the channel is closing after an AppSec response block. */ + public static boolean isResponseBlocked(final AttributeMap attributes) { + return attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).get() != null; + } + + /** Marks the channel as closing after an AppSec response block. */ + public static void markResponseBlocked(final AttributeMap attributes) { + attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).set(Boolean.TRUE); + } + /** Removes a completed or failed request context. */ public static void remove( final AttributeMap attributes, final ServerRequestContext serverContext) { @@ -68,6 +80,10 @@ public static void remove( final ServerRequestContext currentContext = contexts.peekLast(); if (currentContext == null) { attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); + // No request-matching state remains. Drop the empty per-channel queue so idle + // keep-alive or upgraded channels do not retain it after the last HTTP request; + // getOrCreate will recreate it lazily if another request arrives. + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).remove(); } else { // Keep the legacy mirror pointed at the current inbound request after removing an older // response context. @@ -80,6 +96,7 @@ public static void remove( public static void closeAll(final AttributeMap attributes) { // The legacy mirror must not outlive the authoritative request queue. attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); + attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).remove(); close(attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove()); } @@ -92,6 +109,9 @@ public static void closeAll(final AttributeMap attributes) { SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY = AttributeKeys.attributeKey("datadog.server.request.contexts"); + private static final AttributeKey BLOCKED_RESPONSE_ATTRIBUTE_KEY = + AttributeKeys.attributeKey("datadog.server.blocked_response"); + private static final Deque POISONED_CONTEXTS = new ArrayDeque<>(0); /** Creates the per-channel server request context queue. */ @@ -152,16 +172,15 @@ private static boolean isPoisoned(final Deque contexts) { } private final Context tracingContext; - private final HttpHeaders requestHeaders; + private final String acceptHeader; private boolean responseAnalyzed; - private boolean responseBlocked; public Context tracingContext() { return tracingContext; } - public HttpHeaders requestHeaders() { - return requestHeaders; + public String acceptHeader() { + return acceptHeader; } public boolean isResponseAnalyzed() { @@ -172,16 +191,8 @@ public void markResponseAnalyzed() { responseAnalyzed = true; } - public boolean isResponseBlocked() { - return responseBlocked; - } - - public void markResponseBlocked() { - responseBlocked = true; - } - - private ServerRequestContext(final Context tracingContext, final HttpHeaders requestHeaders) { + private ServerRequestContext(final Context tracingContext, final String acceptHeader) { this.tracingContext = tracingContext; - this.requestHeaders = requestHeaders; + this.acceptHeader = acceptHeader; } } From 793775770aaec7660b93d9be80708d2a1e5460fb Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 13 Jul 2026 15:56:21 -0700 Subject: [PATCH 05/33] Add Netty chunked response span timing tests --- .../server/NettyChunkedResponseSpanTest.java | 417 ++++++++++++++++++ .../server/NettyHttp11PipeliningTest.java | 99 +---- .../server/NettyHttpServerTestSupport.java | 159 +++++++ 3 files changed, 584 insertions(+), 91 deletions(-) create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java new file mode 100644 index 00000000000..a48a9b2e9d2 --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -0,0 +1,417 @@ +package datadog.trace.instrumentation.netty41.server; + +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpHeaderNames.TRANSFER_ENCODING; +import static io.netty.handler.codec.http.HttpHeaderNames.UPGRADE; +import static io.netty.handler.codec.http.HttpHeaderValues.CHUNKED; +import static io.netty.handler.codec.http.HttpResponseStatus.CONTINUE; +import static io.netty.handler.codec.http.HttpResponseStatus.NOT_MODIFIED; +import static io.netty.handler.codec.http.HttpResponseStatus.NO_CONTENT; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpResponseStatus.RESET_CONTENT; +import static io.netty.handler.codec.http.HttpResponseStatus.SWITCHING_PROTOCOLS; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.test.assertions.SpanMatcher; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultHttpResponse; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpServerCodec; +import io.netty.handler.codec.http.LastHttpContent; +import java.net.Socket; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.regex.Pattern; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class NettyChunkedResponseSpanTest extends NettyHttpServerTestSupport { + + private static final String PATH = "/chunked"; + private static final String CONTINUE_PATH = "/chunked/continue"; + private static final String NO_CONTENT_PATH = "/bodyless/no-content"; + private static final String RESET_CONTENT_PATH = "/bodyless/reset-content"; + private static final String NOT_MODIFIED_PATH = "/bodyless/not-modified"; + private static final String CONTENT_LENGTH_ZERO_PATH = "/bodyless/content-length-zero"; + private static final String HEAD_PATH = "/bodyless/head"; + private static final String WEBSOCKET_PATH = "/websocket"; + private static final String UPGRADE_PATH = "/upgrade"; + private static final String CONNECT_AUTHORITY = "example.com:443"; + private static final String EARLY_HINTS_PATH = "/chunked/early-hints"; + private static final HttpResponseStatus EARLY_HINTS = new HttpResponseStatus(103, "Early Hints"); + + private final ChunkedResponseHandler handler = new ChunkedResponseHandler(); + + @Override + protected void configurePipeline(Channel ch) { + ch.pipeline().addLast(new HttpServerCodec()); + ch.pipeline().addLast(handler); + } + + @Test + void keepsServerSpanOpenUntilLastResponseChunk() throws Exception { + boolean reportedBeforeLastChunk; + try (Socket socket = connect()) { + socket.getOutputStream().write(request().getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitFirstChunkWritten(); + reportedBeforeLastChunk = writer.waitForTracesMax(1, 1); + + writeLastChunk(responseContext); + assertEquals("first", readChunkedHttpResponseBody(socket.getInputStream())); + } + + assertFalse( + reportedBeforeLastChunk, "server span should not be reported before LastHttpContent"); + assertTraces(trace(serverSpan(PATH))); + } + + @Test + void keepsServerSpanOpenUntilConnectionDrops() throws Exception { + boolean reportedBeforeConnectionDrop; + try (Socket socket = connect()) { + socket.getOutputStream().write(request().getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitFirstChunkWritten(); + reportedBeforeConnectionDrop = writer.waitForTracesMax(1, 1); + + closeChannel(responseContext); + } + + assertFalse( + reportedBeforeConnectionDrop, "server span should not be reported before connection drop"); + assertTraces(trace(serverSpan(PATH).error())); + } + + @Test + void keepsServerSpansSeparateForSequentialKeepAliveChunkedResponses() throws Exception { + boolean firstReportedBeforeLastChunk; + boolean secondReportedBeforeLastChunk; + try (Socket socket = connect()) { + socket.getOutputStream().write(request().getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext firstResponseContext = handler.awaitFirstChunkWritten(); + firstReportedBeforeLastChunk = writer.waitForTracesMax(1, 1); + + writeLastChunk(firstResponseContext); + assertEquals("first", readChunkedHttpResponseBody(socket.getInputStream())); + assertTrue(writer.waitForTracesMax(1, 5), "first keep-alive response was not reported"); + + socket.getOutputStream().write(request().getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext secondResponseContext = handler.awaitFirstChunkWritten(); + secondReportedBeforeLastChunk = writer.waitForTracesMax(2, 1); + + writeLastChunk(secondResponseContext); + assertEquals("first", readChunkedHttpResponseBody(socket.getInputStream())); + } + + assertFalse( + firstReportedBeforeLastChunk, + "first server span should not be reported before LastHttpContent"); + assertFalse( + secondReportedBeforeLastChunk, + "second server span should not be reported before LastHttpContent"); + assertTraces(trace(serverSpan(PATH)), trace(serverSpan(PATH))); + } + + @Test + void keepsServerSpanOpenAfter100ContinueUntilLastResponseChunk() throws Exception { + boolean reportedAfter100Continue; + boolean reportedBeforeLastChunk; + try (Socket socket = connect()) { + socket.getOutputStream().write(request(CONTINUE_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.await100ContinueWritten(); + assertTrue( + readHeaders(socket.getInputStream()).startsWith("HTTP/1.1 100 "), + "server did not write 100 Continue"); + reportedAfter100Continue = writer.waitForTracesMax(1, 1); + + writeFirstChunk(responseContext); + responseContext = handler.awaitFirstChunkWritten(); + reportedBeforeLastChunk = writer.waitForTracesMax(1, 1); + + writeLastChunk(responseContext); + assertEquals("first", readChunkedHttpResponseBody(socket.getInputStream())); + } + + assertFalse(reportedAfter100Continue, "server span should not be reported after 100 Continue"); + assertFalse( + reportedBeforeLastChunk, "server span should not be reported before LastHttpContent"); + assertTraces(trace(serverSpan(CONTINUE_PATH))); + } + + @Test + void keepsServerSpanOpenAfter103EarlyHintsUntilLastResponseChunk() throws Exception { + boolean reportedAfterEarlyHints; + boolean reportedBeforeLastChunk; + try (Socket socket = connect()) { + socket.getOutputStream().write(request(EARLY_HINTS_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitEarlyHintsWritten(); + assertResponseStatus(readHeaders(socket.getInputStream()), EARLY_HINTS); + reportedAfterEarlyHints = writer.waitForTracesMax(1, 1); + + writeFirstChunk(responseContext); + responseContext = handler.awaitFirstChunkWritten(); + reportedBeforeLastChunk = writer.waitForTracesMax(1, 1); + + writeLastChunk(responseContext); + assertEquals("first", readChunkedHttpResponseBody(socket.getInputStream())); + } + + assertFalse(reportedAfterEarlyHints, "server span should not be reported after 103"); + assertFalse( + reportedBeforeLastChunk, "server span should not be reported before LastHttpContent"); + assertTraces(trace(serverSpan(EARLY_HINTS_PATH))); + } + + @Test + void finishesServerSpanForHeaderOnlyNoContentResponse() throws Exception { + assertHeaderOnlyResponseFinishes(NO_CONTENT_PATH, NO_CONTENT); + } + + @Test + void finishesServerSpanForHeaderOnlyResetContentResponse() throws Exception { + assertHeaderOnlyResponseFinishes(RESET_CONTENT_PATH, RESET_CONTENT); + } + + @Test + void finishesServerSpanForHeaderOnlyNotModifiedResponse() throws Exception { + assertHeaderOnlyResponseFinishes(NOT_MODIFIED_PATH, NOT_MODIFIED); + } + + @Test + void finishesServerSpanForHeaderOnlyContentLengthZeroResponse() throws Exception { + assertHeaderOnlyResponseFinishes(CONTENT_LENGTH_ZERO_PATH, OK); + } + + @Test + void finishesServerSpanForHeaderOnlyHeadResponse() throws Exception { + assertHeaderOnlyResponseFinishes("HEAD", HEAD_PATH, OK); + } + + @Test + void finishesServerSpanForHeaderOnlyConnectResponse() throws Exception { + assertHeaderOnlyResponseFinishes("CONNECT", CONNECT_AUTHORITY, OK, "/"); + } + + @Test + void finishesServerSpanForHeaderOnlyWebSocketUpgrade() throws Exception { + assertHeaderOnlyResponseFinishes(WEBSOCKET_PATH, SWITCHING_PROTOCOLS); + } + + @Test + void finishesServerSpanForHeaderOnlyProtocolUpgrade() throws Exception { + assertHeaderOnlyResponseFinishes(UPGRADE_PATH, SWITCHING_PROTOCOLS); + } + + private void assertHeaderOnlyResponseFinishes(String path, HttpResponseStatus status) + throws Exception { + assertHeaderOnlyResponseFinishes("GET", path, status, path); + } + + private void assertHeaderOnlyResponseFinishes( + String method, String path, HttpResponseStatus status) throws Exception { + assertHeaderOnlyResponseFinishes(method, path, status, path); + } + + private void assertHeaderOnlyResponseFinishes( + String method, String path, HttpResponseStatus status, String resourcePath) throws Exception { + try (Socket socket = connect()) { + socket.getOutputStream().write(request(method, path).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + handler.awaitHeaderOnlyResponseWritten(path); + assertResponseStatus(readHeaders(socket.getInputStream()), status); + assertTrue( + writer.waitForTracesMax(1, 5), + "server span should be reported after header-only response " + status.code()); + } + + assertTraces(trace(serverSpan(method, resourcePath))); + } + + private static String request() { + return request(PATH); + } + + private static String request(String path) { + return request("GET", path); + } + + private static String request(String method, String path) { + String headers = method + " " + path + " HTTP/1.1\r\nHost: localhost\r\n"; + if (WEBSOCKET_PATH.equals(path)) { + headers += + "Connection: Upgrade\r\n" + + "Upgrade: websocket\r\n" + + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + + "Sec-WebSocket-Version: 13\r\n"; + } else if (UPGRADE_PATH.equals(path)) { + headers += "Connection: Upgrade\r\n" + "Upgrade: h2c\r\n"; + } + return headers + "\r\n"; + } + + private static SpanMatcher serverSpan(String path) { + return serverSpan("GET", path); + } + + private static SpanMatcher serverSpan(String method, String path) { + return span() + .root() + .operationName(Pattern.compile("netty\\.request")) + .resourceName(Pattern.compile(method + " " + Pattern.quote(path))) + .type("web"); + } + + private static void assertResponseStatus(String headers, HttpResponseStatus status) { + assertTrue( + headers.startsWith("HTTP/1.1 " + status.code() + " "), "unexpected response: " + headers); + } + + private void writeFirstChunk(ChannelHandlerContext responseContext) { + responseContext.executor().execute(() -> handler.writeChunkedResponse(responseContext)); + } + + private static void writeLastChunk(ChannelHandlerContext responseContext) { + responseContext + .executor() + .execute(() -> responseContext.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT)); + } + + private static void closeChannel(ChannelHandlerContext responseContext) { + responseContext.executor().execute(() -> responseContext.channel().close()); + } + + @ChannelHandler.Sharable + private static final class ChunkedResponseHandler + extends SimpleChannelInboundHandler { + private final BlockingQueue continueWrites = new LinkedBlockingQueue<>(); + private final BlockingQueue earlyHintsWrites = + new LinkedBlockingQueue<>(); + private final BlockingQueue firstChunkWrites = + new LinkedBlockingQueue<>(); + private final BlockingQueue headerOnlyWrites = new LinkedBlockingQueue<>(); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { + if (CONTINUE_PATH.equals(request.uri())) { + ctx.writeAndFlush(new DefaultFullHttpResponse(HTTP_1_1, CONTINUE)) + .addListener(future -> continueWrites.offer(ctx)); + } else if (EARLY_HINTS_PATH.equals(request.uri())) { + ctx.writeAndFlush(new DefaultFullHttpResponse(HTTP_1_1, EARLY_HINTS)) + .addListener(future -> earlyHintsWrites.offer(ctx)); + } else if (NO_CONTENT_PATH.equals(request.uri())) { + writeHeaderOnlyResponse(ctx, request.uri(), NO_CONTENT); + } else if (RESET_CONTENT_PATH.equals(request.uri())) { + writeHeaderOnlyResponse(ctx, request.uri(), RESET_CONTENT); + } else if (NOT_MODIFIED_PATH.equals(request.uri())) { + writeHeaderOnlyResponse(ctx, request.uri(), NOT_MODIFIED); + } else if (CONTENT_LENGTH_ZERO_PATH.equals(request.uri())) { + writeContentLengthZeroResponse(ctx, request.uri()); + } else if (HEAD_PATH.equals(request.uri())) { + writeHeaderOnlyResponse(ctx, request.uri(), OK); + } else if (CONNECT_AUTHORITY.equals(request.uri())) { + writeHeaderOnlyResponse(ctx, request.uri(), OK); + } else if (WEBSOCKET_PATH.equals(request.uri())) { + writeHeaderOnlyWebSocketUpgrade(ctx, request.uri()); + } else if (UPGRADE_PATH.equals(request.uri())) { + writeHeaderOnlyProtocolUpgrade(ctx, request.uri()); + } else { + writeChunkedResponse(ctx); + } + } + + private void writeChunkedResponse(ChannelHandlerContext ctx) { + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(TRANSFER_ENCODING, CHUNKED); + ctx.write(response); + ctx.writeAndFlush(new DefaultHttpContent(Unpooled.copiedBuffer("first", UTF_8))) + .addListener(future -> firstChunkWrites.offer(ctx)); + } + + private void writeHeaderOnlyResponse( + ChannelHandlerContext ctx, String path, HttpResponseStatus status) { + ctx.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, status)) + .addListener(future -> headerOnlyWrites.offer(path)); + } + + private void writeContentLengthZeroResponse(ChannelHandlerContext ctx, String path) { + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONTENT_LENGTH, 0); + ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); + } + + private void writeHeaderOnlyWebSocketUpgrade(ChannelHandlerContext ctx, String path) { + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, SWITCHING_PROTOCOLS); + response.headers().set(UPGRADE, "WebSocket"); + response.headers().set(CONNECTION, "Upgrade"); + ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); + } + + private void writeHeaderOnlyProtocolUpgrade(ChannelHandlerContext ctx, String path) { + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, SWITCHING_PROTOCOLS); + response.headers().set(UPGRADE, "h2c"); + response.headers().set(CONNECTION, "Upgrade"); + ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); + } + + private ChannelHandlerContext awaitFirstChunkWritten() throws InterruptedException { + ChannelHandlerContext responseContext = firstChunkWrites.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not write the first chunk"); + } + return responseContext; + } + + private ChannelHandlerContext await100ContinueWritten() throws InterruptedException { + ChannelHandlerContext responseContext = continueWrites.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not write 100 Continue"); + } + return responseContext; + } + + private ChannelHandlerContext awaitEarlyHintsWritten() throws InterruptedException { + ChannelHandlerContext responseContext = earlyHintsWrites.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not write 103 Early Hints"); + } + return responseContext; + } + + private void awaitHeaderOnlyResponseWritten(String path) throws InterruptedException { + String responsePath = headerOnlyWrites.poll(5, SECONDS); + if (responsePath == null) { + throw new AssertionError("server did not write the header-only response"); + } + assertEquals(path, responsePath, "server wrote header-only response for unexpected path"); + } + } +} diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java index 610fa95ddbe..d40cbed8270 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -12,79 +12,42 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.agent.test.assertions.TraceMatcher; -import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpServerCodec; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetSocketAddress; import java.net.Socket; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.regex.Pattern; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class NettyHttp11PipeliningTest extends AbstractInstrumentationTest { +public class NettyHttp11PipeliningTest extends NettyHttpServerTestSupport { private static final String FIRST_PATH = "/pipelined/first"; private static final String SECOND_PATH = "/pipelined/second"; private static final String THIRD_PATH = "/pipelined/third"; - private EventLoopGroup eventLoopGroup; - private PipeliningHandler handler; - private int port; - - @BeforeAll - void startServer() throws Exception { - eventLoopGroup = new NioEventLoopGroup(); - handler = new PipeliningHandler(3); - ServerBootstrap bootstrap = - new ServerBootstrap() - .group(eventLoopGroup) - .channel(NioServerSocketChannel.class) - .childHandler( - new ChannelInitializer() { - @Override - protected void initChannel(Channel ch) { - ch.pipeline().addLast(new HttpServerCodec()); - ch.pipeline().addLast(new HttpObjectAggregator(65536)); - ch.pipeline().addLast(handler); - } - }); - Channel channel = bootstrap.bind(0).sync().channel(); - port = ((InetSocketAddress) channel.localAddress()).getPort(); - } + private final PipeliningHandler handler = new PipeliningHandler(3); - @AfterAll - void stopServer() { - if (eventLoopGroup != null) { - eventLoopGroup.shutdownGracefully(); - } + @Override + protected void configurePipeline(Channel ch) { + ch.pipeline().addLast(new HttpServerCodec()); + ch.pipeline().addLast(new HttpObjectAggregator(65536)); + ch.pipeline().addLast(handler); } @Test void createsServerSpanForEachPipelinedRequest() throws Exception { - try (Socket socket = new Socket("localhost", port)) { - socket.setSoTimeout(5000); + try (Socket socket = connect()) { socket.getOutputStream().write(pipelinedRequests().getBytes(US_ASCII)); socket.getOutputStream().flush(); @@ -127,52 +90,6 @@ private static TraceMatcher serverTrace(String path) { .type("web")); } - private static String readHttpResponseBody(InputStream in) throws IOException { - String headers = readHeaders(in); - assertTrue(headers.startsWith("HTTP/1.1 200 "), "unexpected response: " + headers); - int contentLength = contentLength(headers); - byte[] body = new byte[contentLength]; - int read = 0; - while (read < contentLength) { - int count = in.read(body, read, contentLength - read); - if (count == -1) { - throw new EOFException("response ended before body was complete"); - } - read += count; - } - return new String(body, UTF_8); - } - - private static String readHeaders(InputStream in) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int state = 0; - while (state < 4) { - int b = in.read(); - if (b == -1) { - throw new EOFException("response ended before headers were complete"); - } - out.write(b); - if ((state == 0 || state == 2) && b == '\r') { - state++; - } else if ((state == 1 || state == 3) && b == '\n') { - state++; - } else { - state = b == '\r' ? 1 : 0; - } - } - return out.toString(US_ASCII.name()); - } - - private static int contentLength(String headers) { - for (String line : headers.split("\r\n")) { - int separator = line.indexOf(':'); - if (separator > 0 && "content-length".equalsIgnoreCase(line.substring(0, separator))) { - return Integer.parseInt(line.substring(separator + 1).trim()); - } - } - throw new AssertionError("missing content-length header: " + headers); - } - private static final class PipeliningHandler extends SimpleChannelInboundHandler { private final CountDownLatch receivedRequests; diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java new file mode 100644 index 00000000000..f946f5cb01b --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java @@ -0,0 +1,159 @@ +package datadog.trace.instrumentation.netty41.server; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.test.AbstractInstrumentationTest; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +abstract class NettyHttpServerTestSupport extends AbstractInstrumentationTest { + + private EventLoopGroup eventLoopGroup; + private Channel serverChannel; + private int port; + + @BeforeAll + void startServer() throws Exception { + eventLoopGroup = new NioEventLoopGroup(); + ServerBootstrap bootstrap = + new ServerBootstrap() + .group(eventLoopGroup) + .channel(NioServerSocketChannel.class) + .childHandler( + new ChannelInitializer() { + @Override + protected void initChannel(Channel ch) { + configurePipeline(ch); + } + }); + serverChannel = bootstrap.bind(0).sync().channel(); + port = ((InetSocketAddress) serverChannel.localAddress()).getPort(); + } + + @AfterAll + void stopServer() { + if (serverChannel != null) { + serverChannel.close(); + } + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully(); + } + } + + protected abstract void configurePipeline(Channel ch); + + protected Socket connect() throws IOException { + Socket socket = new Socket("localhost", port); + socket.setSoTimeout(5000); + return socket; + } + + protected static String readHttpResponseBody(InputStream in) throws IOException { + String headers = readHeaders(in); + assertOkResponse(headers); + byte[] body = new byte[contentLength(headers)]; + readFully(in, body); + return new String(body, UTF_8); + } + + protected static String readChunkedHttpResponseBody(InputStream in) throws IOException { + String headers = readHeaders(in); + assertOkResponse(headers); + + ByteArrayOutputStream body = new ByteArrayOutputStream(); + while (true) { + int chunkSize = Integer.parseInt(readLine(in), 16); + if (chunkSize == 0) { + while (!readLine(in).isEmpty()) {} + return body.toString(UTF_8.name()); + } + byte[] chunk = new byte[chunkSize]; + readFully(in, chunk); + body.write(chunk); + assertEquals("", readLine(in), "chunk was not followed by CRLF"); + } + } + + private static void assertOkResponse(String headers) { + assertTrue(headers.startsWith("HTTP/1.1 200 "), "unexpected response: " + headers); + } + + protected static String readHeaders(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int state = 0; + while (state < 4) { + int b = in.read(); + if (b == -1) { + throw new EOFException("response ended before headers were complete"); + } + out.write(b); + if ((state == 0 || state == 2) && b == '\r') { + state++; + } else if ((state == 1 || state == 3) && b == '\n') { + state++; + } else { + state = b == '\r' ? 1 : 0; + } + } + return out.toString(US_ASCII.name()); + } + + private static String readLine(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + boolean seenCarriageReturn = false; + while (true) { + int b = in.read(); + if (b == -1) { + throw new EOFException("response ended before line was complete"); + } + if (seenCarriageReturn && b == '\n') { + return out.toString(US_ASCII.name()); + } + if (seenCarriageReturn) { + out.write('\r'); + seenCarriageReturn = false; + } + if (b == '\r') { + seenCarriageReturn = true; + } else { + out.write(b); + } + } + } + + private static int contentLength(String headers) { + for (String line : headers.split("\r\n")) { + int separator = line.indexOf(':'); + if (separator > 0 && "content-length".equalsIgnoreCase(line.substring(0, separator))) { + return Integer.parseInt(line.substring(separator + 1).trim()); + } + } + throw new AssertionError("missing content-length header: " + headers); + } + + private static void readFully(InputStream in, byte[] bytes) throws IOException { + int read = 0; + while (read < bytes.length) { + int count = in.read(bytes, read, bytes.length - read); + if (count == -1) { + throw new EOFException("response ended before body was complete"); + } + read += count; + } + } +} From 5be33eb209e518bf3c4a05f93e18ef4c60fc9d68 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 13 Jul 2026 22:10:45 -0700 Subject: [PATCH 06/33] Fix Netty 4.1 server span completion for streaming responses Keep server spans open until terminal response writes, finish incomplete responses on channel close, and use queued request contexts to avoid keep-alive/pipelining misattribution. Cover chunked, bodyless, HEAD, CONNECT, interim, and WebSocket upgrade response cases. --- .../HttpServerRequestTracingHandler.java | 35 +++++- .../HttpServerResponseTracingHandler.java | 100 ++++++++++++++---- .../server/NettyChunkedResponseSpanTest.java | 46 +++++--- .../netty41/ServerRequestContext.java | 66 ++++++++++-- 4 files changed, 201 insertions(+), 46 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 0d934772d56..74fb3755581 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -15,10 +15,13 @@ import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpRequest; +import java.util.Deque; @ChannelHandler.Sharable public class HttpServerRequestTracingHandler extends ChannelInboundHandlerAdapter { public static HttpServerRequestTracingHandler INSTANCE = new HttpServerRequestTracingHandler(); + private static final String INCOMPLETE_RESPONSE_MESSAGE = + "Channel closed before response completed"; @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { @@ -54,7 +57,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { DECORATE.onRequest(span, channel, request, parentContext); final ServerRequestContext serverContext = - ServerRequestContext.add(channel, context, headers.get("accept")); + ServerRequestContext.add(channel, context, request); Flow.Action.RequestBlockingAction rba = span.getRequestBlockingAction(); if (rba != null) { @@ -89,9 +92,37 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); } finally { try { - ServerRequestContext.closeAll(ctx.channel()); + final Deque storedContexts = + ServerRequestContext.removeAll(ctx.channel()); + if (storedContexts != null) { + ServerRequestContext storedContext; + while ((storedContext = storedContexts.pollFirst()) != null) { + if (storedContext.isResponseStarted()) { + finishSpanOnIncompleteResponse(storedContext.tracingContext()); + } else { + publishSpanOnChannelClose(storedContext.tracingContext()); + } + } + } } catch (final Throwable ignored) { } } } + + private static void finishSpanOnIncompleteResponse(final Context storedContext) { + final AgentSpan span = AgentSpan.fromContext(storedContext); + if (span != null) { + DECORATE.onError(span, new IllegalStateException(INCOMPLETE_RESPONSE_MESSAGE)); + DECORATE.beforeFinish(storedContext); + span.finish(); + } + } + + private static void publishSpanOnChannelClose(final Context storedContext) { + final AgentSpan span = AgentSpan.fromContext(storedContext); + if (span != null && span.phasedFinish()) { + // At this point we can just publish this span to avoid losing the rest of the trace. + span.publish(); + } + } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 11ea5475e1e..dd30238c810 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -13,8 +13,12 @@ import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.concurrent.Future; @ChannelHandler.Sharable public class HttpServerResponseTracingHandler extends ChannelOutboundHandlerAdapter { @@ -22,11 +26,6 @@ public class HttpServerResponseTracingHandler extends ChannelOutboundHandlerAdap @Override public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise prm) { - if (!(msg instanceof HttpResponse)) { - ctx.write(msg, prm); - return; - } - final ServerRequestContext serverContext = ServerRequestContext.nextResponse(ctx.channel()); final Context storedContext = serverContext == null ? null : serverContext.tracingContext(); final AgentSpan span = AgentSpan.fromContext(storedContext); @@ -36,33 +35,94 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann return; } - try (final ContextScope scope = storedContext.attach()) { - final HttpResponse response = (HttpResponse) msg; + if (!(msg instanceof HttpResponse) && !serverContext.isResponseStarted()) { + ctx.write(msg, prm); + return; + } + try (final ContextScope ignored = storedContext.attach()) { + final boolean terminalResponse = isTerminalResponse(ctx, span, serverContext, msg); + final ChannelPromise writePromise = terminalResponse && prm.isVoid() ? ctx.newPromise() : prm; try { - ctx.write(msg, prm); + if (terminalResponse) { + ServerRequestContext.remove(ctx.channel(), serverContext); + writePromise.addListener(future -> finishSpan(storedContext, span, future)); + } + ctx.write(msg, writePromise); } catch (final Throwable throwable) { DECORATE.onError(span, throwable); span.setHttpStatusCode(500); - span.finish(); // Finish the span manually since finishSpanOnClose was false - ServerRequestContext.remove(ctx.channel(), serverContext); + if (!terminalResponse) { + ServerRequestContext.remove(ctx.channel(), serverContext); + } + finishSpan(storedContext, span); throw throwable; } - final boolean isWebsocketUpgrade = - response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS - && "websocket".equals(response.headers().get(HttpHeaderNames.UPGRADE)); + } + } + + private static boolean isTerminalResponse( + final ChannelHandlerContext ctx, + final AgentSpan span, + final ServerRequestContext serverContext, + final Object msg) { + if (msg instanceof HttpResponse) { + final HttpResponse response = (HttpResponse) msg; + + final boolean isWebsocketUpgrade = isWebsocketUpgrade(response); + if (isInformationalResponse(response) && !isWebsocketUpgrade) { + return false; + } + if (isWebsocketUpgrade) { ctx.channel() .attr(WEBSOCKET_SENDER_HANDLER_CONTEXT) .set(new HandlerContext.Sender(span, ctx.channel().id().asShortText())); } - if (response.status() != HttpResponseStatus.CONTINUE - && (response.status() != HttpResponseStatus.SWITCHING_PROTOCOLS || isWebsocketUpgrade)) { - DECORATE.onResponse(span, response); - DECORATE.beforeFinish(scope.context()); - span.finish(); // Finish the span manually since finishSpanOnClose was false - ServerRequestContext.remove(ctx.channel(), serverContext); - } + DECORATE.onResponse(span, response); + serverContext.markResponseStarted(); + return msg instanceof LastHttpContent + || isBodylessResponse(serverContext, response) + || isWebsocketUpgrade; } + return serverContext.isResponseStarted() && msg instanceof LastHttpContent; + } + + private static boolean isInformationalResponse(final HttpResponse response) { + final int statusCode = response.status().code(); + return statusCode >= 100 && statusCode < 200; + } + + private static boolean isWebsocketUpgrade(final HttpResponse response) { + return response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS + && response + .headers() + .containsValue(HttpHeaderNames.UPGRADE, HttpHeaderValues.WEBSOCKET, true); + } + + private static boolean isBodylessResponse( + final ServerRequestContext serverContext, final HttpResponse response) { + final int statusCode = response.status().code(); + return serverContext.isHeadRequest() + || statusCode == 204 + || statusCode == 205 + || statusCode == 304 + || (serverContext.isConnectRequest() && statusCode >= 200 && statusCode < 300) + || (HttpUtil.getContentLength(response, -1) == 0 + && !HttpUtil.isTransferEncodingChunked(response)); + } + + private static void finishSpan( + final Context storedContext, final AgentSpan span, final Future future) { + if (!future.isSuccess()) { + DECORATE.onError(span, future.cause()); + span.setHttpStatusCode(500); + } + finishSpan(storedContext, span); + } + + private static void finishSpan(final Context storedContext, final AgentSpan span) { + DECORATE.beforeFinish(storedContext); + span.finish(); // Finish the span manually since finishSpanOnClose was false } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index a48a9b2e9d2..75d0eb1a69b 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -51,8 +51,8 @@ public class NettyChunkedResponseSpanTest extends NettyHttpServerTestSupport { private static final String NOT_MODIFIED_PATH = "/bodyless/not-modified"; private static final String CONTENT_LENGTH_ZERO_PATH = "/bodyless/content-length-zero"; private static final String HEAD_PATH = "/bodyless/head"; + private static final String NO_RESPONSE_PATH = "/no-response"; private static final String WEBSOCKET_PATH = "/websocket"; - private static final String UPGRADE_PATH = "/upgrade"; private static final String CONNECT_AUTHORITY = "example.com:443"; private static final String EARLY_HINTS_PATH = "/chunked/early-hints"; private static final HttpResponseStatus EARLY_HINTS = new HttpResponseStatus(103, "Early Hints"); @@ -102,6 +102,22 @@ void keepsServerSpanOpenUntilConnectionDrops() throws Exception { assertTraces(trace(serverSpan(PATH).error())); } + @Test + void closesServerSpanWithoutErrorWhenConnectionDropsBeforeResponseStarts() throws Exception { + boolean reportedBeforeConnectionDrop; + try (Socket socket = connect()) { + socket.getOutputStream().write(request(NO_RESPONSE_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + handler.awaitRequestWithoutResponse(); + reportedBeforeConnectionDrop = writer.waitForTracesMax(1, 1); + } + + assertFalse( + reportedBeforeConnectionDrop, "server span should not be reported before connection drop"); + assertTraces(trace(serverSpan(NO_RESPONSE_PATH))); + } + @Test void keepsServerSpansSeparateForSequentialKeepAliveChunkedResponses() throws Exception { boolean firstReportedBeforeLastChunk; @@ -225,11 +241,6 @@ void finishesServerSpanForHeaderOnlyWebSocketUpgrade() throws Exception { assertHeaderOnlyResponseFinishes(WEBSOCKET_PATH, SWITCHING_PROTOCOLS); } - @Test - void finishesServerSpanForHeaderOnlyProtocolUpgrade() throws Exception { - assertHeaderOnlyResponseFinishes(UPGRADE_PATH, SWITCHING_PROTOCOLS); - } - private void assertHeaderOnlyResponseFinishes(String path, HttpResponseStatus status) throws Exception { assertHeaderOnlyResponseFinishes("GET", path, status, path); @@ -272,8 +283,6 @@ private static String request(String method, String path) { + "Upgrade: websocket\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + "Sec-WebSocket-Version: 13\r\n"; - } else if (UPGRADE_PATH.equals(path)) { - headers += "Connection: Upgrade\r\n" + "Upgrade: h2c\r\n"; } return headers + "\r\n"; } @@ -318,6 +327,8 @@ private static final class ChunkedResponseHandler private final BlockingQueue firstChunkWrites = new LinkedBlockingQueue<>(); private final BlockingQueue headerOnlyWrites = new LinkedBlockingQueue<>(); + private final BlockingQueue requestsWithoutResponse = + new LinkedBlockingQueue<>(); @Override protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { @@ -339,10 +350,10 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { writeHeaderOnlyResponse(ctx, request.uri(), OK); } else if (CONNECT_AUTHORITY.equals(request.uri())) { writeHeaderOnlyResponse(ctx, request.uri(), OK); + } else if (NO_RESPONSE_PATH.equals(request.uri())) { + requestsWithoutResponse.offer(ctx); } else if (WEBSOCKET_PATH.equals(request.uri())) { writeHeaderOnlyWebSocketUpgrade(ctx, request.uri()); - } else if (UPGRADE_PATH.equals(request.uri())) { - writeHeaderOnlyProtocolUpgrade(ctx, request.uri()); } else { writeChunkedResponse(ctx); } @@ -375,13 +386,6 @@ private void writeHeaderOnlyWebSocketUpgrade(ChannelHandlerContext ctx, String p ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); } - private void writeHeaderOnlyProtocolUpgrade(ChannelHandlerContext ctx, String path) { - DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, SWITCHING_PROTOCOLS); - response.headers().set(UPGRADE, "h2c"); - response.headers().set(CONNECTION, "Upgrade"); - ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); - } - private ChannelHandlerContext awaitFirstChunkWritten() throws InterruptedException { ChannelHandlerContext responseContext = firstChunkWrites.poll(5, SECONDS); if (responseContext == null) { @@ -406,6 +410,14 @@ private ChannelHandlerContext awaitEarlyHintsWritten() throws InterruptedExcepti return responseContext; } + private ChannelHandlerContext awaitRequestWithoutResponse() throws InterruptedException { + ChannelHandlerContext responseContext = requestsWithoutResponse.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not receive request without response"); + } + return responseContext; + } + private void awaitHeaderOnlyResponseWritten(String path) throws InterruptedException { String responsePath = headerOnlyWrites.poll(5, SECONDS); if (responsePath == null) { diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 5819f940090..29b7df5ced4 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -4,6 +4,9 @@ import datadog.context.Context; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpRequest; import io.netty.util.AttributeKey; import io.netty.util.AttributeMap; import java.util.ArrayDeque; @@ -25,12 +28,27 @@ public static boolean canTrackRequest(final AttributeMap attributes) { /** Adds a request context to the queue tail. */ public static ServerRequestContext add( - final AttributeMap attributes, final Context context, final String acceptHeader) { + final AttributeMap attributes, final Context context, final HttpHeaders requestHeaders) { + return add(attributes, context, requestHeaders, null); + } + + /** Adds a request context to the queue tail. */ + public static ServerRequestContext add( + final AttributeMap attributes, final Context context, final HttpRequest request) { + return add(attributes, context, request.headers(), request.method()); + } + + private static ServerRequestContext add( + final AttributeMap attributes, + final Context context, + final HttpHeaders requestHeaders, + final HttpMethod requestMethod) { final Deque contexts = getOrCreate(attributes); if (!canAdd(attributes, contexts)) { return null; } - final ServerRequestContext serverContext = new ServerRequestContext(context, acceptHeader); + final ServerRequestContext serverContext = + new ServerRequestContext(context, requestHeaders, requestMethod); contexts.addLast(serverContext); // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a // legacy mirror of the current inbound request used by generic fire* activation. @@ -94,10 +112,14 @@ public static void remove( /** Closes all pending request contexts on channel close. */ public static void closeAll(final AttributeMap attributes) { + close(removeAll(attributes)); + } + + /** Removes all pending request contexts. */ + public static Deque removeAll(final AttributeMap attributes) { // The legacy mirror must not outlive the authoritative request queue. attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); - attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).remove(); - close(attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove()); + return attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove(); } private static final int PIPELINING_LIMIT = 1000; @@ -172,7 +194,9 @@ private static boolean isPoisoned(final Deque contexts) { } private final Context tracingContext; - private final String acceptHeader; + private final HttpHeaders requestHeaders; + private final HttpMethod requestMethod; + private boolean responseStarted; private boolean responseAnalyzed; public Context tracingContext() { @@ -183,6 +207,22 @@ public String acceptHeader() { return acceptHeader; } + public boolean isHeadRequest() { + return HttpMethod.HEAD.equals(requestMethod); + } + + public boolean isConnectRequest() { + return HttpMethod.CONNECT.equals(requestMethod); + } + + public boolean isResponseStarted() { + return responseStarted; + } + + public void markResponseStarted() { + responseStarted = true; + } + public boolean isResponseAnalyzed() { return responseAnalyzed; } @@ -191,8 +231,20 @@ public void markResponseAnalyzed() { responseAnalyzed = true; } - private ServerRequestContext(final Context tracingContext, final String acceptHeader) { + public boolean isResponseBlocked() { + return responseBlocked; + } + + public void markResponseBlocked() { + responseBlocked = true; + } + + private ServerRequestContext( + final Context tracingContext, + final HttpHeaders requestHeaders, + final HttpMethod requestMethod) { this.tracingContext = tracingContext; - this.acceptHeader = acceptHeader; + this.requestHeaders = requestHeaders; + this.requestMethod = requestMethod; } } From aaaf77d791f65576db31e6285ada9bc04a38dc62 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 14 Jul 2026 08:57:24 -0700 Subject: [PATCH 07/33] Fix Netty server span lifecycle for streaming responses Keep Netty 4.1 server spans open until terminal response writes while preserving request-ended callback timing for AppSec/IAST. Track one-shot beforeFinish execution separately from span finish so streaming spans get correct duration without dropping request-end security tags. --- .../HttpServerRequestTracingHandler.java | 10 +++++-- .../HttpServerResponseTracingHandler.java | 28 +++++++++++++++---- .../netty41/ServerRequestContext.java | 9 ++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 74fb3755581..fc4efadda99 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -98,7 +98,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { ServerRequestContext storedContext; while ((storedContext = storedContexts.pollFirst()) != null) { if (storedContext.isResponseStarted()) { - finishSpanOnIncompleteResponse(storedContext.tracingContext()); + finishSpanOnIncompleteResponse(storedContext); } else { publishSpanOnChannelClose(storedContext.tracingContext()); } @@ -109,11 +109,15 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { } } - private static void finishSpanOnIncompleteResponse(final Context storedContext) { + private static void finishSpanOnIncompleteResponse(final ServerRequestContext serverContext) { + final Context storedContext = serverContext.tracingContext(); final AgentSpan span = AgentSpan.fromContext(storedContext); if (span != null) { DECORATE.onError(span, new IllegalStateException(INCOMPLETE_RESPONSE_MESSAGE)); - DECORATE.beforeFinish(storedContext); + if (!serverContext.isBeforeFinishCalled()) { + serverContext.markBeforeFinishCalled(); + DECORATE.beforeFinish(storedContext); + } span.finish(); } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index dd30238c810..14ca128f350 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -46,16 +46,20 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann try { if (terminalResponse) { ServerRequestContext.remove(ctx.channel(), serverContext); - writePromise.addListener(future -> finishSpan(storedContext, span, future)); + writePromise.addListener( + future -> finishSpan(serverContext, storedContext, span, future)); } ctx.write(msg, writePromise); + if (serverContext.isResponseStarted()) { + beforeFinish(serverContext, storedContext); + } } catch (final Throwable throwable) { DECORATE.onError(span, throwable); span.setHttpStatusCode(500); if (!terminalResponse) { ServerRequestContext.remove(ctx.channel(), serverContext); } - finishSpan(storedContext, span); + finishSpan(serverContext, storedContext, span); throw throwable; } } @@ -113,16 +117,28 @@ private static boolean isBodylessResponse( } private static void finishSpan( - final Context storedContext, final AgentSpan span, final Future future) { + final ServerRequestContext serverContext, + final Context storedContext, + final AgentSpan span, + final Future future) { if (!future.isSuccess()) { DECORATE.onError(span, future.cause()); span.setHttpStatusCode(500); } - finishSpan(storedContext, span); + finishSpan(serverContext, storedContext, span); } - private static void finishSpan(final Context storedContext, final AgentSpan span) { - DECORATE.beforeFinish(storedContext); + private static void finishSpan( + final ServerRequestContext serverContext, final Context storedContext, final AgentSpan span) { + beforeFinish(serverContext, storedContext); span.finish(); // Finish the span manually since finishSpanOnClose was false } + + private static void beforeFinish( + final ServerRequestContext serverContext, final Context storedContext) { + if (!serverContext.isBeforeFinishCalled()) { + serverContext.markBeforeFinishCalled(); + DECORATE.beforeFinish(storedContext); + } + } } diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 29b7df5ced4..68d6e8641e9 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -197,6 +197,7 @@ private static boolean isPoisoned(final Deque contexts) { private final HttpHeaders requestHeaders; private final HttpMethod requestMethod; private boolean responseStarted; + private boolean beforeFinishCalled; private boolean responseAnalyzed; public Context tracingContext() { @@ -223,6 +224,14 @@ public void markResponseStarted() { responseStarted = true; } + public boolean isBeforeFinishCalled() { + return beforeFinishCalled; + } + + public void markBeforeFinishCalled() { + beforeFinishCalled = true; + } + public boolean isResponseAnalyzed() { return responseAnalyzed; } From 372e9a81ed7fd14a44d727898285d93f28b8325d Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 14 Jul 2026 16:49:28 -0700 Subject: [PATCH 08/33] Fix Netty 4.1 server span edge cases from PR review - Defer beforeFinish until the response actually completes. - Finish close-delimited responses without error. - Guard Content-Length parsing. - Skip AppSec response analysis for all interim 1xx responses in MaybeBlockResponseHandler, not just 100 Continue. Add a regression test for close-delimited responses. --- .../HttpServerRequestTracingHandler.java | 22 +++++++----- .../HttpServerResponseTracingHandler.java | 36 +++++++++++++++---- .../server/MaybeBlockResponseHandler.java | 9 ++++- .../server/NettyChunkedResponseSpanTest.java | 34 ++++++++++++++++++ .../netty41/ServerRequestContext.java | 9 +++++ 5 files changed, 94 insertions(+), 16 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index fc4efadda99..1e2ffa28ba1 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -98,7 +98,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { ServerRequestContext storedContext; while ((storedContext = storedContexts.pollFirst()) != null) { if (storedContext.isResponseStarted()) { - finishSpanOnIncompleteResponse(storedContext); + finishSpanOnChannelClose(storedContext); } else { publishSpanOnChannelClose(storedContext.tracingContext()); } @@ -109,17 +109,23 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { } } - private static void finishSpanOnIncompleteResponse(final ServerRequestContext serverContext) { + private static void finishSpanOnChannelClose(final ServerRequestContext serverContext) { final Context storedContext = serverContext.tracingContext(); final AgentSpan span = AgentSpan.fromContext(storedContext); - if (span != null) { + if (span == null) { + return; + } + if (!serverContext.isResponseCloseDelimited()) { + // The response declared a body via Content-Length or chunked encoding, but the channel closed + // before that body completed. Close-delimited responses, in contrast, end normally when the + // connection closes. DECORATE.onError(span, new IllegalStateException(INCOMPLETE_RESPONSE_MESSAGE)); - if (!serverContext.isBeforeFinishCalled()) { - serverContext.markBeforeFinishCalled(); - DECORATE.beforeFinish(storedContext); - } - span.finish(); } + if (!serverContext.isBeforeFinishCalled()) { + serverContext.markBeforeFinishCalled(); + DECORATE.beforeFinish(storedContext); + } + span.finish(); } private static void publishSpanOnChannelClose(final Context storedContext) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 14ca128f350..e11b814be28 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -50,9 +50,6 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann future -> finishSpan(serverContext, storedContext, span, future)); } ctx.write(msg, writePromise); - if (serverContext.isResponseStarted()) { - beforeFinish(serverContext, storedContext); - } } catch (final Throwable throwable) { DECORATE.onError(span, throwable); span.setHttpStatusCode(500); @@ -85,9 +82,18 @@ private static boolean isTerminalResponse( } DECORATE.onResponse(span, response); serverContext.markResponseStarted(); - return msg instanceof LastHttpContent + if (msg instanceof LastHttpContent || isBodylessResponse(serverContext, response) - || isWebsocketUpgrade; + || isWebsocketUpgrade) { + return true; + } + // A response with neither a Content-Length nor chunked transfer-encoding is delimited by the + // connection closing, so a later channel close is the normal end of this response rather than + // an incomplete one. + if (!hasKnownBodyLength(response)) { + serverContext.markResponseCloseDelimited(); + } + return false; } return serverContext.isResponseStarted() && msg instanceof LastHttpContent; } @@ -112,8 +118,24 @@ private static boolean isBodylessResponse( || statusCode == 205 || statusCode == 304 || (serverContext.isConnectRequest() && statusCode >= 200 && statusCode < 300) - || (HttpUtil.getContentLength(response, -1) == 0 - && !HttpUtil.isTransferEncodingChunked(response)); + || (contentLength(response) == 0 && !HttpUtil.isTransferEncodingChunked(response)); + } + + private static boolean hasKnownBodyLength(final HttpResponse response) { + return contentLength(response) >= 0 || HttpUtil.isTransferEncodingChunked(response); + } + + /** + * Returns the response {@code Content-Length}, or {@code -1} when it is absent or malformed. A + * malformed value is left for Netty's encoder to reject rather than failing the write from the + * tracing handler. + */ + private static long contentLength(final HttpResponse response) { + try { + return HttpUtil.getContentLength(response, -1L); + } catch (final NumberFormatException e) { + return -1L; + } } private static void finishSpan( diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java index 5dab095e4e0..c1ab0de230f 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java @@ -63,7 +63,14 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr return; } HttpResponse origResponse = (HttpResponse) msg; - if (origResponse.status().code() == HttpResponseStatus.CONTINUE.code()) { + int statusCode = origResponse.status().code(); + // Interim 1xx responses (e.g. 100 Continue, 103 Early Hints) precede the final response. + // Analyzing one here would consume the one-shot response analysis before the final response is + // written, so its status and headers would never be inspected. Switching Protocols (101) is + // terminal, so it is still analyzed. + if (statusCode >= 100 + && statusCode < 200 + && statusCode != HttpResponseStatus.SWITCHING_PROTOCOLS.code()) { super.write(ctx, msg, prm); return; } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index 75d0eb1a69b..d37cb0bac2f 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -52,6 +52,7 @@ public class NettyChunkedResponseSpanTest extends NettyHttpServerTestSupport { private static final String CONTENT_LENGTH_ZERO_PATH = "/bodyless/content-length-zero"; private static final String HEAD_PATH = "/bodyless/head"; private static final String NO_RESPONSE_PATH = "/no-response"; + private static final String CLOSE_DELIMITED_PATH = "/close-delimited"; private static final String WEBSOCKET_PATH = "/websocket"; private static final String CONNECT_AUTHORITY = "example.com:443"; private static final String EARLY_HINTS_PATH = "/chunked/early-hints"; @@ -102,6 +103,27 @@ void keepsServerSpanOpenUntilConnectionDrops() throws Exception { assertTraces(trace(serverSpan(PATH).error())); } + @Test + void finishesServerSpanWithoutErrorForCloseDelimitedResponse() throws Exception { + boolean reportedBeforeConnectionClose; + try (Socket socket = connect()) { + socket.getOutputStream().write(request(CLOSE_DELIMITED_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitFirstChunkWritten(); + reportedBeforeConnectionClose = writer.waitForTracesMax(1, 1); + + closeChannel(responseContext); + } + + assertFalse( + reportedBeforeConnectionClose, + "server span should not be reported before the connection closes"); + // The response has neither Content-Length nor chunked encoding, so closing the connection is + // its normal completion and the span must not be flagged as an error. + assertTraces(trace(serverSpan(CLOSE_DELIMITED_PATH))); + } + @Test void closesServerSpanWithoutErrorWhenConnectionDropsBeforeResponseStarts() throws Exception { boolean reportedBeforeConnectionDrop; @@ -352,6 +374,8 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { writeHeaderOnlyResponse(ctx, request.uri(), OK); } else if (NO_RESPONSE_PATH.equals(request.uri())) { requestsWithoutResponse.offer(ctx); + } else if (CLOSE_DELIMITED_PATH.equals(request.uri())) { + writeCloseDelimitedResponse(ctx); } else if (WEBSOCKET_PATH.equals(request.uri())) { writeHeaderOnlyWebSocketUpgrade(ctx, request.uri()); } else { @@ -367,6 +391,16 @@ private void writeChunkedResponse(ChannelHandlerContext ctx) { .addListener(future -> firstChunkWrites.offer(ctx)); } + private void writeCloseDelimitedResponse(ChannelHandlerContext ctx) { + // No Content-Length and no chunked transfer-encoding: the body is delimited by the connection + // closing. + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONNECTION, "close"); + ctx.write(response); + ctx.writeAndFlush(new DefaultHttpContent(Unpooled.copiedBuffer("first", UTF_8))) + .addListener(future -> firstChunkWrites.offer(ctx)); + } + private void writeHeaderOnlyResponse( ChannelHandlerContext ctx, String path, HttpResponseStatus status) { ctx.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, status)) diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 68d6e8641e9..69bcb8762b0 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -197,6 +197,7 @@ private static boolean isPoisoned(final Deque contexts) { private final HttpHeaders requestHeaders; private final HttpMethod requestMethod; private boolean responseStarted; + private boolean responseCloseDelimited; private boolean beforeFinishCalled; private boolean responseAnalyzed; @@ -224,6 +225,14 @@ public void markResponseStarted() { responseStarted = true; } + public boolean isResponseCloseDelimited() { + return responseCloseDelimited; + } + + public void markResponseCloseDelimited() { + responseCloseDelimited = true; + } + public boolean isBeforeFinishCalled() { return beforeFinishCalled; } From 0a6269fbd9d8323d31ffb897eb20aa25bdc9df79 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 14 Jul 2026 17:49:52 -0700 Subject: [PATCH 09/33] Restore request-ended callbacks for terminal Netty responses --- .../netty41/server/HttpServerResponseTracingHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index e11b814be28..366d8508455 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -50,6 +50,10 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann future -> finishSpan(serverContext, storedContext, span, future)); } ctx.write(msg, writePromise); + if (terminalResponse) { + // Run the request-ended callbacks (AppSec/IAST) here, while the context is attached + beforeFinish(serverContext, storedContext); + } } catch (final Throwable throwable) { DECORATE.onError(span, throwable); span.setHttpStatusCode(500); From 344cbb46cf54d5163c8b98cddd3333f27dad5d3c Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 16 Jul 2026 14:26:11 -0700 Subject: [PATCH 10/33] Resolved: terminal responses no longer call `beforeFinish(...)` immediately after `ctx.write(...)`. The write-promise listener is now the only normal terminal completion path, so `finishSpan(...)` runs after the write completes and invokes `beforeFinish(...)` there. The exception path is also guarded with `writePromise.isDone()`: if the terminal write promise has already completed, the listener has already handled finishing, so the catch block just rethrows and avoids a second finish/decorator pass. --- .../server/HttpServerResponseTracingHandler.java | 16 +++++++--------- .../netty41/ServerRequestContextTest.java | 9 +++++---- .../server/MaybeBlockResponseHandlerTest.java | 4 +++- .../netty41/ServerRequestContext.java | 6 ++++-- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 366d8508455..1a2f0fe2dc4 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -50,17 +50,15 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann future -> finishSpan(serverContext, storedContext, span, future)); } ctx.write(msg, writePromise); - if (terminalResponse) { - // Run the request-ended callbacks (AppSec/IAST) here, while the context is attached - beforeFinish(serverContext, storedContext); - } } catch (final Throwable throwable) { - DECORATE.onError(span, throwable); - span.setHttpStatusCode(500); - if (!terminalResponse) { - ServerRequestContext.remove(ctx.channel(), serverContext); + if (!terminalResponse || !writePromise.isDone()) { + DECORATE.onError(span, throwable); + span.setHttpStatusCode(500); + if (!terminalResponse) { + ServerRequestContext.remove(ctx.channel(), serverContext); + } + finishSpan(serverContext, storedContext, span); } - finishSpan(serverContext, storedContext, span); throw throwable; } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index 4cac8e6c1e8..f51ad0fa8bb 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -19,13 +19,13 @@ void disablesTrackingWhenPipeliningLimitIsExceeded() { DefaultAttributeMap attributes = new DefaultAttributeMap(); for (int i = 0; i < PIPELINING_LIMIT; i++) { - assertNotNull(ServerRequestContext.add(attributes, Context.root(), null)); + assertNotNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); } assertFalse(ServerRequestContext.canTrackRequest(attributes)); assertNull(ServerRequestContext.nextResponse(attributes)); assertNull(attributes.attr(AttributeKeys.CONTEXT_ATTRIBUTE_KEY).get()); - assertNull(ServerRequestContext.add(attributes, Context.root(), null)); + assertNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); } @Test @@ -35,7 +35,7 @@ void capturesAcceptHeaderValue() { headers.set("accept", "text/html"); ServerRequestContext serverContext = - ServerRequestContext.add(attributes, Context.root(), headers.get("accept")); + ServerRequestContext.add(attributes, Context.root(), headers); headers.set("accept", "application/json"); assertEquals("text/html", serverContext.acceptHeader()); @@ -44,7 +44,8 @@ void capturesAcceptHeaderValue() { @Test void tracksBlockedResponseUntilChannelClose() { DefaultAttributeMap attributes = new DefaultAttributeMap(); - ServerRequestContext serverContext = ServerRequestContext.add(attributes, Context.root(), null); + ServerRequestContext serverContext = + ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders()); ServerRequestContext.markResponseBlocked(attributes); ServerRequestContext.remove(attributes, serverContext); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java index 7ba3a2aabf1..658acc3f94d 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java @@ -9,6 +9,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultHttpHeaders; import org.junit.jupiter.api.Test; class MaybeBlockResponseHandlerTest { @@ -16,7 +17,8 @@ class MaybeBlockResponseHandlerTest { @Test void dropsWritesAfterBlockedContextHasBeenRemoved() { EmbeddedChannel channel = new EmbeddedChannel(MaybeBlockResponseHandler.INSTANCE); - ServerRequestContext serverContext = ServerRequestContext.add(channel, Context.root(), null); + ServerRequestContext serverContext = + ServerRequestContext.add(channel, Context.root(), new DefaultHttpHeaders()); ServerRequestContext.markResponseBlocked(channel); ServerRequestContext.remove(channel, serverContext); ByteBuf lateResponseChunk = Unpooled.buffer().writeByte(1); diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 69bcb8762b0..9836d1112d3 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -112,6 +112,7 @@ public static void remove( /** Closes all pending request contexts on channel close. */ public static void closeAll(final AttributeMap attributes) { + attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).remove(); close(removeAll(attributes)); } @@ -194,12 +195,13 @@ private static boolean isPoisoned(final Deque contexts) { } private final Context tracingContext; - private final HttpHeaders requestHeaders; + private final String acceptHeader; private final HttpMethod requestMethod; private boolean responseStarted; private boolean responseCloseDelimited; private boolean beforeFinishCalled; private boolean responseAnalyzed; + private boolean responseBlocked; public Context tracingContext() { return tracingContext; @@ -262,7 +264,7 @@ private ServerRequestContext( final HttpHeaders requestHeaders, final HttpMethod requestMethod) { this.tracingContext = tracingContext; - this.requestHeaders = requestHeaders; + this.acceptHeader = requestHeaders == null ? null : requestHeaders.get("accept"); this.requestMethod = requestMethod; } } From c2e0b3fb7b1dfe2d245a9e4fdc346bec7d79d966 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 16 Jul 2026 14:35:59 -0700 Subject: [PATCH 11/33] Changed HttpServerResponseTracingHandler.java so unknown-length responses are marked close-delimited before `LastHttpContent` can make them terminal. Close-delimited contexts now wait for `channelInactive`. Added a regression in NettyChunkedResponseSpanTest.java for `DefaultFullHttpResponse` with no `Content-Length` and no chunked encoding. --- .../HttpServerResponseTracingHandler.java | 12 +++-- .../server/NettyChunkedResponseSpanTest.java | 45 +++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 1a2f0fe2dc4..ec68f6726fe 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -84,9 +84,7 @@ private static boolean isTerminalResponse( } DECORATE.onResponse(span, response); serverContext.markResponseStarted(); - if (msg instanceof LastHttpContent - || isBodylessResponse(serverContext, response) - || isWebsocketUpgrade) { + if (isBodylessResponse(serverContext, response) || isWebsocketUpgrade) { return true; } // A response with neither a Content-Length nor chunked transfer-encoding is delimited by the @@ -94,10 +92,16 @@ private static boolean isTerminalResponse( // an incomplete one. if (!hasKnownBodyLength(response)) { serverContext.markResponseCloseDelimited(); + return false; + } + if (msg instanceof LastHttpContent) { + return true; } return false; } - return serverContext.isResponseStarted() && msg instanceof LastHttpContent; + return serverContext.isResponseStarted() + && !serverContext.isResponseCloseDelimited() + && msg instanceof LastHttpContent; } private static boolean isInformationalResponse(final HttpResponse response) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index d37cb0bac2f..bf2e72f3a7e 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -53,6 +53,7 @@ public class NettyChunkedResponseSpanTest extends NettyHttpServerTestSupport { private static final String HEAD_PATH = "/bodyless/head"; private static final String NO_RESPONSE_PATH = "/no-response"; private static final String CLOSE_DELIMITED_PATH = "/close-delimited"; + private static final String CLOSE_DELIMITED_FULL_RESPONSE_PATH = "/close-delimited/full"; private static final String WEBSOCKET_PATH = "/websocket"; private static final String CONNECT_AUTHORITY = "example.com:443"; private static final String EARLY_HINTS_PATH = "/chunked/early-hints"; @@ -124,6 +125,27 @@ void finishesServerSpanWithoutErrorForCloseDelimitedResponse() throws Exception assertTraces(trace(serverSpan(CLOSE_DELIMITED_PATH))); } + @Test + void waitsForChannelCloseForCloseDelimitedFullResponse() throws Exception { + boolean reportedBeforeConnectionClose; + try (Socket socket = connect()) { + socket + .getOutputStream() + .write(request(CLOSE_DELIMITED_FULL_RESPONSE_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitCloseDelimitedFullResponseWritten(); + reportedBeforeConnectionClose = writer.waitForTracesMax(1, 1); + + closeChannel(responseContext); + } + + assertFalse( + reportedBeforeConnectionClose, + "server span should not be reported before the connection closes"); + assertTraces(trace(serverSpan(CLOSE_DELIMITED_FULL_RESPONSE_PATH))); + } + @Test void closesServerSpanWithoutErrorWhenConnectionDropsBeforeResponseStarts() throws Exception { boolean reportedBeforeConnectionDrop; @@ -348,6 +370,8 @@ private static final class ChunkedResponseHandler new LinkedBlockingQueue<>(); private final BlockingQueue firstChunkWrites = new LinkedBlockingQueue<>(); + private final BlockingQueue closeDelimitedFullResponseWrites = + new LinkedBlockingQueue<>(); private final BlockingQueue headerOnlyWrites = new LinkedBlockingQueue<>(); private final BlockingQueue requestsWithoutResponse = new LinkedBlockingQueue<>(); @@ -376,6 +400,8 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { requestsWithoutResponse.offer(ctx); } else if (CLOSE_DELIMITED_PATH.equals(request.uri())) { writeCloseDelimitedResponse(ctx); + } else if (CLOSE_DELIMITED_FULL_RESPONSE_PATH.equals(request.uri())) { + writeCloseDelimitedFullResponse(ctx); } else if (WEBSOCKET_PATH.equals(request.uri())) { writeHeaderOnlyWebSocketUpgrade(ctx, request.uri()); } else { @@ -401,6 +427,16 @@ private void writeCloseDelimitedResponse(ChannelHandlerContext ctx) { .addListener(future -> firstChunkWrites.offer(ctx)); } + private void writeCloseDelimitedFullResponse(ChannelHandlerContext ctx) { + // No Content-Length and no chunked transfer-encoding: the full response still has a + // close-delimited body on the wire. + DefaultFullHttpResponse response = + new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.copiedBuffer("first", UTF_8)); + response.headers().set(CONNECTION, "close"); + ctx.writeAndFlush(response) + .addListener(future -> closeDelimitedFullResponseWrites.offer(ctx)); + } + private void writeHeaderOnlyResponse( ChannelHandlerContext ctx, String path, HttpResponseStatus status) { ctx.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, status)) @@ -444,6 +480,15 @@ private ChannelHandlerContext awaitEarlyHintsWritten() throws InterruptedExcepti return responseContext; } + private ChannelHandlerContext awaitCloseDelimitedFullResponseWritten() + throws InterruptedException { + ChannelHandlerContext responseContext = closeDelimitedFullResponseWrites.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not write the full close-delimited response"); + } + return responseContext; + } + private ChannelHandlerContext awaitRequestWithoutResponse() throws InterruptedException { ChannelHandlerContext responseContext = requestsWithoutResponse.poll(5, SECONDS); if (responseContext == null) { From 4f7d9245eecceac1ec5a776bc5f58fb007936741 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 16 Jul 2026 14:52:56 -0700 Subject: [PATCH 12/33] Restrict close-delimited response handling to HTTP/1.x only. The Netty HTTP/2 server path now caches when `Http2StreamFrameToHttpObjectCodec` is present and avoids marking no-length responses as close-delimited there. This keeps HTTP/2 responses dependent on `LastHttpContent`/END_STREAM, so an early stream close is still reported as an incomplete-response error. --- .../NettyChannelPipelineInstrumentation.java | 2 ++ .../server/HttpServerResponseTracingHandler.java | 15 +++++++++++---- .../instrumentation/netty41/AttributeKeys.java | 3 +++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java index 668077b87a6..52aa8aa36ef 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java @@ -8,6 +8,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.noopContinuation; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONNECT_PARENT_CONTINUATION_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.HTTP2_CONNECTION_CODEC_ATTRIBUTE_KEY; +import static datadog.trace.instrumentation.netty41.AttributeKeys.HTTP2_STREAM_CODEC_ATTRIBUTE_KEY; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.returns; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -237,6 +238,7 @@ else if (handler instanceof HttpClientCodec) { pipeline, handler, HttpClientResponseTracingHandler.INSTANCE); } else if (NettyHttp2Helper.isHttp2FrameCodec(handler)) { if (NettyHttp2Helper.isServer(handler)) { + pipeline.channel().attr(HTTP2_STREAM_CODEC_ATTRIBUTE_KEY).set(Boolean.TRUE); NettyPipelineHelper.addHandlerAfter( pipeline, handler, diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index ec68f6726fe..069205f7f67 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.netty41.server; +import static datadog.trace.instrumentation.netty41.AttributeKeys.HTTP2_STREAM_CODEC_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.WEBSOCKET_SENDER_HANDLER_CONTEXT; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; @@ -87,10 +88,10 @@ private static boolean isTerminalResponse( if (isBodylessResponse(serverContext, response) || isWebsocketUpgrade) { return true; } - // A response with neither a Content-Length nor chunked transfer-encoding is delimited by the - // connection closing, so a later channel close is the normal end of this response rather than - // an incomplete one. - if (!hasKnownBodyLength(response)) { + // HTTP/1.x responses with neither a Content-Length nor chunked transfer-encoding are + // delimited by the connection closing. HTTP/2 streams translated through + // Http2StreamFrameToHttpObjectCodec are delimited by END_STREAM/LastHttpContent instead. + if (isCloseDelimitedHttp1Response(ctx, response)) { serverContext.markResponseCloseDelimited(); return false; } @@ -131,6 +132,12 @@ private static boolean hasKnownBodyLength(final HttpResponse response) { return contentLength(response) >= 0 || HttpUtil.isTransferEncodingChunked(response); } + private static boolean isCloseDelimitedHttp1Response( + final ChannelHandlerContext ctx, final HttpResponse response) { + return !hasKnownBodyLength(response) + && !Boolean.TRUE.equals(ctx.channel().attr(HTTP2_STREAM_CODEC_ATTRIBUTE_KEY).get()); + } + /** * Returns the response {@code Content-Length}, or {@code -1} when it is absent or malformed. A * malformed value is left for Netty's encoder to reject rather than failing the write from the diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java index 307002a68fb..b8790de68ef 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java @@ -29,6 +29,9 @@ public final class AttributeKeys { public static final AttributeKey HTTP2_CONNECTION_CODEC_ATTRIBUTE_KEY = attributeKey("datadog.http2.connection.codec"); + public static final AttributeKey HTTP2_STREAM_CODEC_ATTRIBUTE_KEY = + attributeKey("datadog.http2.stream.codec"); + public static final AttributeKey PARENT_CONTEXT_ATTRIBUTE_KEY = attributeKey("datadog.server.parent-context"); From 69ec7b0498b24abc836d74776c8454ba7471e5ee Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 16 Jul 2026 15:06:17 -0700 Subject: [PATCH 13/33] The Netty 4.1 websocket upgrade check is now value-based for status `101`, and the regression test for a distinct `HttpResponseStatus(101, ...)` passes. --- .../HttpServerResponseTracingHandler.java | 2 +- .../server/NettyChunkedResponseSpanTest.java | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 069205f7f67..29ce9f444ea 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -111,7 +111,7 @@ private static boolean isInformationalResponse(final HttpResponse response) { } private static boolean isWebsocketUpgrade(final HttpResponse response) { - return response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS + return response.status().code() == HttpResponseStatus.SWITCHING_PROTOCOLS.code() && response .headers() .containsValue(HttpHeaderNames.UPGRADE, HttpHeaderValues.WEBSOCKET, true); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index bf2e72f3a7e..93ff7ae01cc 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -55,9 +55,12 @@ public class NettyChunkedResponseSpanTest extends NettyHttpServerTestSupport { private static final String CLOSE_DELIMITED_PATH = "/close-delimited"; private static final String CLOSE_DELIMITED_FULL_RESPONSE_PATH = "/close-delimited/full"; private static final String WEBSOCKET_PATH = "/websocket"; + private static final String CUSTOM_WEBSOCKET_STATUS_PATH = "/websocket/custom-status"; private static final String CONNECT_AUTHORITY = "example.com:443"; private static final String EARLY_HINTS_PATH = "/chunked/early-hints"; private static final HttpResponseStatus EARLY_HINTS = new HttpResponseStatus(103, "Early Hints"); + private static final HttpResponseStatus CUSTOM_SWITCHING_PROTOCOLS = + new HttpResponseStatus(SWITCHING_PROTOCOLS.code(), "Switching Protocols"); private final ChunkedResponseHandler handler = new ChunkedResponseHandler(); @@ -285,6 +288,12 @@ void finishesServerSpanForHeaderOnlyWebSocketUpgrade() throws Exception { assertHeaderOnlyResponseFinishes(WEBSOCKET_PATH, SWITCHING_PROTOCOLS); } + @Test + void finishesServerSpanForHeaderOnlyWebSocketUpgradeWithDistinctStatusInstance() + throws Exception { + assertHeaderOnlyResponseFinishes(CUSTOM_WEBSOCKET_STATUS_PATH, SWITCHING_PROTOCOLS); + } + private void assertHeaderOnlyResponseFinishes(String path, HttpResponseStatus status) throws Exception { assertHeaderOnlyResponseFinishes("GET", path, status, path); @@ -321,7 +330,7 @@ private static String request(String path) { private static String request(String method, String path) { String headers = method + " " + path + " HTTP/1.1\r\nHost: localhost\r\n"; - if (WEBSOCKET_PATH.equals(path)) { + if (isWebSocketPath(path)) { headers += "Connection: Upgrade\r\n" + "Upgrade: websocket\r\n" @@ -331,6 +340,10 @@ private static String request(String method, String path) { return headers + "\r\n"; } + private static boolean isWebSocketPath(String path) { + return WEBSOCKET_PATH.equals(path) || CUSTOM_WEBSOCKET_STATUS_PATH.equals(path); + } + private static SpanMatcher serverSpan(String path) { return serverSpan("GET", path); } @@ -403,7 +416,9 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { } else if (CLOSE_DELIMITED_FULL_RESPONSE_PATH.equals(request.uri())) { writeCloseDelimitedFullResponse(ctx); } else if (WEBSOCKET_PATH.equals(request.uri())) { - writeHeaderOnlyWebSocketUpgrade(ctx, request.uri()); + writeHeaderOnlyWebSocketUpgrade(ctx, request.uri(), SWITCHING_PROTOCOLS); + } else if (CUSTOM_WEBSOCKET_STATUS_PATH.equals(request.uri())) { + writeHeaderOnlyWebSocketUpgrade(ctx, request.uri(), CUSTOM_SWITCHING_PROTOCOLS); } else { writeChunkedResponse(ctx); } @@ -449,8 +464,9 @@ private void writeContentLengthZeroResponse(ChannelHandlerContext ctx, String pa ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); } - private void writeHeaderOnlyWebSocketUpgrade(ChannelHandlerContext ctx, String path) { - DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, SWITCHING_PROTOCOLS); + private void writeHeaderOnlyWebSocketUpgrade( + ChannelHandlerContext ctx, String path, HttpResponseStatus status) { + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, status); response.headers().set(UPGRADE, "WebSocket"); response.headers().set(CONNECTION, "Upgrade"); ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); From cde82d0068ff84666c3efe2547a932a540dee649 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 16 Jul 2026 15:16:42 -0700 Subject: [PATCH 14/33] It now treats known-length responses as complete once the declared body bytes are successfully written, so closing the connection afterward no longer marks the span as an incomplete-response error. Chunked/incomplete known-length closes still error as expected. Tests passed for `NettyChunkedResponseSpanTest`; changes are not committed. --- .../HttpServerResponseTracingHandler.java | 32 +++- .../server/NettyChunkedResponseSpanTest.java | 144 +++++++++++++++++- .../netty41/ServerRequestContext.java | 19 +++ 3 files changed, 190 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 29ce9f444ea..58b3fe9fc23 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -9,10 +9,13 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.websocket.HandlerContext; import datadog.trace.instrumentation.netty41.ServerRequestContext; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufHolder; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; +import io.netty.channel.FileRegion; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaderValues; import io.netty.handler.codec.http.HttpResponse; @@ -43,19 +46,23 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann try (final ContextScope ignored = storedContext.attach()) { final boolean terminalResponse = isTerminalResponse(ctx, span, serverContext, msg); - final ChannelPromise writePromise = terminalResponse && prm.isVoid() ? ctx.newPromise() : prm; + final boolean knownLengthResponseComplete = + !terminalResponse && serverContext.recordResponseBodyBytes(responseBodyBytes(msg)); + final boolean finishResponseOnWrite = terminalResponse || knownLengthResponseComplete; + final ChannelPromise writePromise = + finishResponseOnWrite && prm.isVoid() ? ctx.newPromise() : prm; try { - if (terminalResponse) { + if (finishResponseOnWrite) { ServerRequestContext.remove(ctx.channel(), serverContext); writePromise.addListener( future -> finishSpan(serverContext, storedContext, span, future)); } ctx.write(msg, writePromise); } catch (final Throwable throwable) { - if (!terminalResponse || !writePromise.isDone()) { + if (!finishResponseOnWrite || !writePromise.isDone()) { DECORATE.onError(span, throwable); span.setHttpStatusCode(500); - if (!terminalResponse) { + if (!finishResponseOnWrite) { ServerRequestContext.remove(ctx.channel(), serverContext); } finishSpan(serverContext, storedContext, span); @@ -88,6 +95,10 @@ private static boolean isTerminalResponse( if (isBodylessResponse(serverContext, response) || isWebsocketUpgrade) { return true; } + final long contentLength = contentLength(response); + if (contentLength > 0 && !HttpUtil.isTransferEncodingChunked(response)) { + serverContext.setResponseContentLength(contentLength); + } // HTTP/1.x responses with neither a Content-Length nor chunked transfer-encoding are // delimited by the connection closing. HTTP/2 streams translated through // Http2StreamFrameToHttpObjectCodec are delimited by END_STREAM/LastHttpContent instead. @@ -138,6 +149,19 @@ private static boolean isCloseDelimitedHttp1Response( && !Boolean.TRUE.equals(ctx.channel().attr(HTTP2_STREAM_CODEC_ATTRIBUTE_KEY).get()); } + private static long responseBodyBytes(final Object msg) { + if (msg instanceof ByteBuf) { + return ((ByteBuf) msg).readableBytes(); + } + if (msg instanceof ByteBufHolder) { + return ((ByteBufHolder) msg).content().readableBytes(); + } + if (msg instanceof FileRegion) { + return ((FileRegion) msg).count(); + } + return 0; + } + /** * Returns the response {@code Content-Length}, or {@code -1} when it is absent or malformed. A * malformed value is left for Netty's encoder to reject rather than failing the write from the diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index 93ff7ae01cc..da6547148e4 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -26,6 +26,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.DefaultFileRegion; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.DefaultHttpContent; @@ -34,7 +35,12 @@ import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpServerCodec; import io.netty.handler.codec.http.LastHttpContent; +import java.io.IOException; import java.net.Socket; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.regex.Pattern; @@ -54,6 +60,9 @@ public class NettyChunkedResponseSpanTest extends NettyHttpServerTestSupport { private static final String NO_RESPONSE_PATH = "/no-response"; private static final String CLOSE_DELIMITED_PATH = "/close-delimited"; private static final String CLOSE_DELIMITED_FULL_RESPONSE_PATH = "/close-delimited/full"; + private static final String KNOWN_LENGTH_BYTE_BUF_PATH = "/known-length/bytebuf"; + private static final String KNOWN_LENGTH_FILE_REGION_PATH = "/known-length/file-region"; + private static final String INCOMPLETE_KNOWN_LENGTH_PATH = "/known-length/incomplete"; private static final String WEBSOCKET_PATH = "/websocket"; private static final String CUSTOM_WEBSOCKET_STATUS_PATH = "/websocket/custom-status"; private static final String CONNECT_AUTHORITY = "example.com:443"; @@ -149,6 +158,50 @@ void waitsForChannelCloseForCloseDelimitedFullResponse() throws Exception { assertTraces(trace(serverSpan(CLOSE_DELIMITED_FULL_RESPONSE_PATH))); } + @Test + void finishesServerSpanWithoutErrorForKnownLengthByteBufResponseClosedAfterBody() + throws Exception { + try (Socket socket = connect()) { + socket.getOutputStream().write(request(KNOWN_LENGTH_BYTE_BUF_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitKnownLengthByteBufWritten(); + closeChannel(responseContext); + assertEquals("first", readHttpResponseBody(socket.getInputStream())); + } + + assertTraces(trace(serverSpan(KNOWN_LENGTH_BYTE_BUF_PATH))); + } + + @Test + void finishesServerSpanWithoutErrorForKnownLengthFileRegionResponseClosedAfterBody() + throws Exception { + try (Socket socket = connect()) { + socket.getOutputStream().write(request(KNOWN_LENGTH_FILE_REGION_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitKnownLengthFileRegionWritten(); + closeChannel(responseContext); + assertEquals("first", readHttpResponseBody(socket.getInputStream())); + } + + assertTraces(trace(serverSpan(KNOWN_LENGTH_FILE_REGION_PATH))); + } + + @Test + void marksKnownLengthResponseErrorWhenConnectionClosesBeforeDeclaredBodyComplete() + throws Exception { + try (Socket socket = connect()) { + socket.getOutputStream().write(request(INCOMPLETE_KNOWN_LENGTH_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + ChannelHandlerContext responseContext = handler.awaitIncompleteKnownLengthWritten(); + closeChannel(responseContext); + } + + assertTraces(trace(serverSpan(INCOMPLETE_KNOWN_LENGTH_PATH).error())); + } + @Test void closesServerSpanWithoutErrorWhenConnectionDropsBeforeResponseStarts() throws Exception { boolean reportedBeforeConnectionDrop; @@ -385,12 +438,18 @@ private static final class ChunkedResponseHandler new LinkedBlockingQueue<>(); private final BlockingQueue closeDelimitedFullResponseWrites = new LinkedBlockingQueue<>(); + private final BlockingQueue knownLengthByteBufWrites = + new LinkedBlockingQueue<>(); + private final BlockingQueue knownLengthFileRegionWrites = + new LinkedBlockingQueue<>(); + private final BlockingQueue incompleteKnownLengthWrites = + new LinkedBlockingQueue<>(); private final BlockingQueue headerOnlyWrites = new LinkedBlockingQueue<>(); private final BlockingQueue requestsWithoutResponse = new LinkedBlockingQueue<>(); @Override - protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { + protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) throws Exception { if (CONTINUE_PATH.equals(request.uri())) { ctx.writeAndFlush(new DefaultFullHttpResponse(HTTP_1_1, CONTINUE)) .addListener(future -> continueWrites.offer(ctx)); @@ -415,6 +474,12 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { writeCloseDelimitedResponse(ctx); } else if (CLOSE_DELIMITED_FULL_RESPONSE_PATH.equals(request.uri())) { writeCloseDelimitedFullResponse(ctx); + } else if (KNOWN_LENGTH_BYTE_BUF_PATH.equals(request.uri())) { + writeKnownLengthByteBufResponse(ctx); + } else if (KNOWN_LENGTH_FILE_REGION_PATH.equals(request.uri())) { + writeKnownLengthFileRegionResponse(ctx); + } else if (INCOMPLETE_KNOWN_LENGTH_PATH.equals(request.uri())) { + writeIncompleteKnownLengthResponse(ctx); } else if (WEBSOCKET_PATH.equals(request.uri())) { writeHeaderOnlyWebSocketUpgrade(ctx, request.uri(), SWITCHING_PROTOCOLS); } else if (CUSTOM_WEBSOCKET_STATUS_PATH.equals(request.uri())) { @@ -452,6 +517,45 @@ private void writeCloseDelimitedFullResponse(ChannelHandlerContext ctx) { .addListener(future -> closeDelimitedFullResponseWrites.offer(ctx)); } + private void writeKnownLengthByteBufResponse(ChannelHandlerContext ctx) { + byte[] body = "first".getBytes(UTF_8); + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONNECTION, "close"); + response.headers().set(CONTENT_LENGTH, body.length); + ctx.write(response); + ctx.writeAndFlush(Unpooled.wrappedBuffer(body)) + .addListener(future -> knownLengthByteBufWrites.offer(ctx)); + } + + private void writeKnownLengthFileRegionResponse(ChannelHandlerContext ctx) throws IOException { + byte[] body = "first".getBytes(UTF_8); + Path path = Files.createTempFile("netty-known-length", ".body"); + Files.write(path, body); + FileChannel fileChannel = FileChannel.open(path, StandardOpenOption.READ); + + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONNECTION, "close"); + response.headers().set(CONTENT_LENGTH, body.length); + ctx.write(response); + ctx.writeAndFlush(new DefaultFileRegion(fileChannel, 0, body.length)) + .addListener( + future -> { + close(fileChannel); + delete(path); + knownLengthFileRegionWrites.offer(ctx); + }); + } + + private void writeIncompleteKnownLengthResponse(ChannelHandlerContext ctx) { + byte[] body = "first".getBytes(UTF_8); + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONNECTION, "close"); + response.headers().set(CONTENT_LENGTH, body.length + 1); + ctx.write(response); + ctx.writeAndFlush(Unpooled.wrappedBuffer(body)) + .addListener(future -> incompleteKnownLengthWrites.offer(ctx)); + } + private void writeHeaderOnlyResponse( ChannelHandlerContext ctx, String path, HttpResponseStatus status) { ctx.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, status)) @@ -505,6 +609,30 @@ private ChannelHandlerContext awaitCloseDelimitedFullResponseWritten() return responseContext; } + private ChannelHandlerContext awaitKnownLengthByteBufWritten() throws InterruptedException { + ChannelHandlerContext responseContext = knownLengthByteBufWrites.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not write the known-length ByteBuf response"); + } + return responseContext; + } + + private ChannelHandlerContext awaitKnownLengthFileRegionWritten() throws InterruptedException { + ChannelHandlerContext responseContext = knownLengthFileRegionWrites.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not write the known-length FileRegion response"); + } + return responseContext; + } + + private ChannelHandlerContext awaitIncompleteKnownLengthWritten() throws InterruptedException { + ChannelHandlerContext responseContext = incompleteKnownLengthWrites.poll(5, SECONDS); + if (responseContext == null) { + throw new AssertionError("server did not write the incomplete known-length response"); + } + return responseContext; + } + private ChannelHandlerContext awaitRequestWithoutResponse() throws InterruptedException { ChannelHandlerContext responseContext = requestsWithoutResponse.poll(5, SECONDS); if (responseContext == null) { @@ -520,5 +648,19 @@ private void awaitHeaderOnlyResponseWritten(String path) throws InterruptedExcep } assertEquals(path, responsePath, "server wrote header-only response for unexpected path"); } + + private static void close(FileChannel fileChannel) { + try { + fileChannel.close(); + } catch (IOException ignored) { + } + } + + private static void delete(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + } + } } } diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 9836d1112d3..997d59abfc2 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -202,6 +202,8 @@ private static boolean isPoisoned(final Deque contexts) { private boolean beforeFinishCalled; private boolean responseAnalyzed; private boolean responseBlocked; + private long responseContentLength = -1; + private long responseBodyBytes; public Context tracingContext() { return tracingContext; @@ -235,6 +237,23 @@ public void markResponseCloseDelimited() { responseCloseDelimited = true; } + public void setResponseContentLength(final long contentLength) { + responseContentLength = contentLength; + responseBodyBytes = 0; + } + + public boolean recordResponseBodyBytes(final long bodyBytes) { + if (responseContentLength < 0 || bodyBytes <= 0) { + return false; + } + if (Long.MAX_VALUE - responseBodyBytes < bodyBytes) { + responseBodyBytes = Long.MAX_VALUE; + } else { + responseBodyBytes += bodyBytes; + } + return responseBodyBytes >= responseContentLength; + } + public boolean isBeforeFinishCalled() { return beforeFinishCalled; } From 3ebe9c3de3fb2dcc83d44e2c7a6839054ac84638 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 16 Jul 2026 22:54:39 -0700 Subject: [PATCH 15/33] Fix Netty IAST reporting on async response completion Reactivate the stored request context while finishing Netty server spans so IAST request-ended callbacks run with the request span active. This lets sink reports attach to the request span when response write promises complete asynchronously or on channel close. Add Netty regressions covering IAST request-ended callbacks for known-length full responses and delayed write-promise completion. --- .../HttpServerRequestTracingHandler.java | 22 ++- .../HttpServerResponseTracingHandler.java | 6 +- .../server/NettyChunkedResponseSpanTest.java | 187 ++++++++++++++++++ 3 files changed, 203 insertions(+), 12 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 1e2ffa28ba1..0f93dec3101 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -115,17 +115,19 @@ private static void finishSpanOnChannelClose(final ServerRequestContext serverCo if (span == null) { return; } - if (!serverContext.isResponseCloseDelimited()) { - // The response declared a body via Content-Length or chunked encoding, but the channel closed - // before that body completed. Close-delimited responses, in contrast, end normally when the - // connection closes. - DECORATE.onError(span, new IllegalStateException(INCOMPLETE_RESPONSE_MESSAGE)); - } - if (!serverContext.isBeforeFinishCalled()) { - serverContext.markBeforeFinishCalled(); - DECORATE.beforeFinish(storedContext); + try (final ContextScope ignored = storedContext.attach()) { + if (!serverContext.isResponseCloseDelimited()) { + // The response declared a body via Content-Length or chunked encoding, but the channel + // closed before that body completed. Close-delimited responses, in contrast, end normally + // when the connection closes. + DECORATE.onError(span, new IllegalStateException(INCOMPLETE_RESPONSE_MESSAGE)); + } + if (!serverContext.isBeforeFinishCalled()) { + serverContext.markBeforeFinishCalled(); + DECORATE.beforeFinish(storedContext); + } + span.finish(); } - span.finish(); } private static void publishSpanOnChannelClose(final Context storedContext) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 58b3fe9fc23..78e1cf7bfd8 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -189,8 +189,10 @@ private static void finishSpan( private static void finishSpan( final ServerRequestContext serverContext, final Context storedContext, final AgentSpan span) { - beforeFinish(serverContext, storedContext); - span.finish(); // Finish the span manually since finishSpanOnClose was false + try (final ContextScope ignored = storedContext.attach()) { + beforeFinish(serverContext, storedContext); + span.finish(); // Finish the span manually since finishSpanOnClose was false + } } private static void beforeFinish( diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index da6547148e4..c2952606c41 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -22,19 +22,35 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.agent.test.assertions.SpanMatcher; +import datadog.trace.api.function.TriConsumer; +import datadog.trace.api.gateway.Events; +import datadog.trace.api.gateway.Flow; +import datadog.trace.api.gateway.IGSpanInfo; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.gateway.Subscription; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.TagContext; +import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelOutboundHandlerAdapter; +import io.netty.channel.ChannelPromise; import io.netty.channel.DefaultFileRegion; import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultHttpHeaders; import io.netty.handler.codec.http.DefaultHttpResponse; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpServerCodec; import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.ReferenceCountUtil; import java.io.IOException; import java.net.Socket; import java.nio.channels.FileChannel; @@ -43,6 +59,10 @@ import java.nio.file.StandardOpenOption; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Supplier; import java.util.regex.Pattern; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -60,6 +80,7 @@ public class NettyChunkedResponseSpanTest extends NettyHttpServerTestSupport { private static final String NO_RESPONSE_PATH = "/no-response"; private static final String CLOSE_DELIMITED_PATH = "/close-delimited"; private static final String CLOSE_DELIMITED_FULL_RESPONSE_PATH = "/close-delimited/full"; + private static final String KNOWN_LENGTH_FULL_RESPONSE_PATH = "/known-length/full"; private static final String KNOWN_LENGTH_BYTE_BUF_PATH = "/known-length/bytebuf"; private static final String KNOWN_LENGTH_FILE_REGION_PATH = "/known-length/file-region"; private static final String INCOMPLETE_KNOWN_LENGTH_PATH = "/known-length/incomplete"; @@ -173,6 +194,132 @@ void finishesServerSpanWithoutErrorForKnownLengthByteBufResponseClosedAfterBody( assertTraces(trace(serverSpan(KNOWN_LENGTH_BYTE_BUF_PATH))); } + @Test + void callsIastRequestEndedForKnownLengthFullResponse() throws Exception { + Object iastRequestData = new Object(); + AtomicReference authorization = new AtomicReference<>(); + AtomicBoolean requestEnded = new AtomicBoolean(); + AtomicBoolean requestSpanActive = new AtomicBoolean(); + + Events events = Events.get(); + Subscription requestStarted = + AgentTracer.get() + .getSubscriptionService(RequestContextSlot.IAST) + .registerCallback( + events.requestStarted(), + new Supplier>() { + @Override + public Flow get() { + return new Flow.ResultFlow<>(iastRequestData); + } + }); + Subscription requestHeader = + AgentTracer.get() + .getSubscriptionService(RequestContextSlot.IAST) + .registerCallback( + events.requestHeader(), + new TriConsumer() { + @Override + public void accept(RequestContext context, String key, String value) { + if (iastRequestData == context.getData(RequestContextSlot.IAST) + && "authorization".equalsIgnoreCase(key)) { + authorization.set(value); + } + } + }); + Subscription requestEnd = + AgentTracer.get() + .getSubscriptionService(RequestContextSlot.IAST) + .registerCallback( + events.requestEnded(), + new BiFunction>() { + @Override + public Flow apply(RequestContext context, IGSpanInfo span) { + if (iastRequestData == context.getData(RequestContextSlot.IAST)) { + requestEnded.set(true); + AgentSpan activeSpan = AgentTracer.activeSpan(); + requestSpanActive.set( + activeSpan != null && activeSpan.getRequestContext() == context); + } + return Flow.ResultFlow.empty(); + } + }); + + try { + try (Socket socket = connect()) { + socket + .getOutputStream() + .write( + requestWithAuthorization(KNOWN_LENGTH_FULL_RESPONSE_PATH, "Basic dXNlcjpwYXNz") + .getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertEquals("ok", readHttpResponseBody(socket.getInputStream())); + } + + assertTraces(trace(serverSpan(KNOWN_LENGTH_FULL_RESPONSE_PATH))); + assertEquals("Basic dXNlcjpwYXNz", authorization.get()); + assertTrue(requestEnded.get(), "IAST requestEnded callback was not called"); + assertTrue(requestSpanActive.get(), "request span was not active during requestEnded"); + } finally { + requestEnd.cancel(); + requestHeader.cancel(); + requestStarted.cancel(); + } + } + + @Test + void activatesRequestSpanWhenWritePromiseCompletesAfterWriteReturns() { + Object iastRequestData = new Object(); + AtomicBoolean requestSpanActive = new AtomicBoolean(); + + Events events = Events.get(); + Subscription requestEnd = + AgentTracer.get() + .getSubscriptionService(RequestContextSlot.IAST) + .registerCallback( + events.requestEnded(), + new BiFunction>() { + @Override + public Flow apply(RequestContext context, IGSpanInfo span) { + if (iastRequestData == context.getData(RequestContextSlot.IAST)) { + AgentSpan activeSpan = AgentTracer.activeSpan(); + requestSpanActive.set( + activeSpan != null && activeSpan.getRequestContext() == context); + } + return Flow.ResultFlow.empty(); + } + }); + + HoldingOutboundHandler holdingHandler = new HoldingOutboundHandler(); + EmbeddedChannel channel = + new EmbeddedChannel(holdingHandler, HttpServerResponseTracingHandler.INSTANCE); + AgentSpan span = + AgentTracer.get() + .startSpan( + "netty", + "netty.request", + new TagContext().withRequestContextDataIast(iastRequestData)); + ServerRequestContext.add(channel, span, new DefaultHttpHeaders()); + + try { + DefaultFullHttpResponse response = + new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.copiedBuffer("ok", UTF_8)); + response.headers().set(CONTENT_LENGTH, response.content().readableBytes()); + + channel.writeOutbound(response); + assertFalse(requestSpanActive.get(), "write promise completed synchronously"); + + holdingHandler.completeWrite(); + + assertTrue(requestSpanActive.get(), "request span was not active during requestEnded"); + } finally { + holdingHandler.completeWrite(); + requestEnd.cancel(); + channel.finishAndReleaseAll(); + } + } + @Test void finishesServerSpanWithoutErrorForKnownLengthFileRegionResponseClosedAfterBody() throws Exception { @@ -381,6 +528,14 @@ private static String request(String path) { return request("GET", path); } + private static String requestWithAuthorization(String path, String authorization) { + return "GET " + + path + + " HTTP/1.1\r\nHost: localhost\r\nAuthorization: " + + authorization + + "\r\n\r\n"; + } + private static String request(String method, String path) { String headers = method + " " + path + " HTTP/1.1\r\nHost: localhost\r\n"; if (isWebSocketPath(path)) { @@ -474,6 +629,8 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) thro writeCloseDelimitedResponse(ctx); } else if (CLOSE_DELIMITED_FULL_RESPONSE_PATH.equals(request.uri())) { writeCloseDelimitedFullResponse(ctx); + } else if (KNOWN_LENGTH_FULL_RESPONSE_PATH.equals(request.uri())) { + writeKnownLengthFullResponse(ctx); } else if (KNOWN_LENGTH_BYTE_BUF_PATH.equals(request.uri())) { writeKnownLengthByteBufResponse(ctx); } else if (KNOWN_LENGTH_FILE_REGION_PATH.equals(request.uri())) { @@ -517,6 +674,13 @@ private void writeCloseDelimitedFullResponse(ChannelHandlerContext ctx) { .addListener(future -> closeDelimitedFullResponseWrites.offer(ctx)); } + private void writeKnownLengthFullResponse(ChannelHandlerContext ctx) { + DefaultFullHttpResponse response = + new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.copiedBuffer("ok", UTF_8)); + response.headers().set(CONTENT_LENGTH, response.content().readableBytes()); + ctx.writeAndFlush(response); + } + private void writeKnownLengthByteBufResponse(ChannelHandlerContext ctx) { byte[] body = "first".getBytes(UTF_8); DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); @@ -663,4 +827,27 @@ private static void delete(Path path) { } } } + + private static final class HoldingOutboundHandler extends ChannelOutboundHandlerAdapter { + private Object msg; + private ChannelPromise promise; + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) { + this.msg = msg; + this.promise = promise; + } + + private void completeWrite() { + if (promise == null || promise.isDone()) { + return; + } + try { + promise.setSuccess(); + } finally { + ReferenceCountUtil.release(msg); + msg = null; + } + } + } } From 6c6c9fc649256b6af502e03c0f367ff8a664bf97 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 21 Jul 2026 15:49:14 -0700 Subject: [PATCH 16/33] Preserve Netty HTTP/2 stream contexts Fallback to the channel context mirror when no queued ServerRequestContext exists, so HTTP/2 multiplex child-channel responses still decorate and finish propagated server spans. Add regression coverage for the no-queue context-mirror path and document why CONTEXT_ATTRIBUTE_KEY is maintained. --- .../HttpServerResponseTracingHandler.java | 22 +++++++++-- .../HttpServerResponseTracingHandlerTest.java | 37 +++++++++++++++++++ .../netty41/ServerRequestContext.java | 10 +++-- 3 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index 11ea5475e1e..ded2ef0490c 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.netty41.server; +import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.WEBSOCKET_SENDER_HANDLER_CONTEXT; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; @@ -28,7 +29,13 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann } final ServerRequestContext serverContext = ServerRequestContext.nextResponse(ctx.channel()); - final Context storedContext = serverContext == null ? null : serverContext.tracingContext(); + final Context storedContext = + serverContext == null + // HTTP/2 multiplex stream channels only inherit the mirrored context attribute from + // Http2MultiplexHandlerStreamChannelInstrumentation.PropagateContextAdvice, without a + // per-stream request queue. + ? ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get() + : serverContext.tracingContext(); final AgentSpan span = AgentSpan.fromContext(storedContext); if (span == null) { @@ -45,7 +52,7 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann DECORATE.onError(span, throwable); span.setHttpStatusCode(500); span.finish(); // Finish the span manually since finishSpanOnClose was false - ServerRequestContext.remove(ctx.channel(), serverContext); + removeServerContext(ctx, serverContext); throw throwable; } final boolean isWebsocketUpgrade = @@ -61,8 +68,17 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann DECORATE.onResponse(span, response); DECORATE.beforeFinish(scope.context()); span.finish(); // Finish the span manually since finishSpanOnClose was false - ServerRequestContext.remove(ctx.channel(), serverContext); + removeServerContext(ctx, serverContext); } } } + + private static void removeServerContext( + final ChannelHandlerContext ctx, final ServerRequestContext serverContext) { + if (serverContext == null) { + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).remove(); + } else { + ServerRequestContext.remove(ctx.channel(), serverContext); + } + } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java new file mode 100644 index 00000000000..0a31db6c483 --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java @@ -0,0 +1,37 @@ +package datadog.trace.instrumentation.netty41.server; + +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpResponse; +import org.junit.jupiter.api.Test; + +class HttpServerResponseTracingHandlerTest extends AbstractInstrumentationTest { + + @Test + void finishesMirroredContextWhenRequestQueueIsAbsent() { + EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); + AgentSpan span = startSpan("netty", "mirrored-http2-server"); + channel.attr(CONTEXT_ATTRIBUTE_KEY).set(span); + + assertTrue(channel.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK))); + + FullHttpResponse response = channel.readOutbound(); + assertNotNull(response); + response.release(); + assertNull(channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); + channel.finishAndReleaseAll(); + assertTraces(trace(span().root().operationName("mirrored-http2-server"))); + } +} diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 5819f940090..90912c7d6a0 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -33,7 +33,9 @@ public static ServerRequestContext add( final ServerRequestContext serverContext = new ServerRequestContext(context, acceptHeader); contexts.addLast(serverContext); // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a - // legacy mirror of the current inbound request used by generic fire* activation. + // context mirror of the current inbound request used by + // NettyChannelHandlerContextInstrumentation.FireAdvice and copied to HTTP/2 stream channels by + // Http2MultiplexHandlerStreamChannelInstrumentation. attributes.attr(CONTEXT_ATTRIBUTE_KEY).set(context); return serverContext; } @@ -85,8 +87,8 @@ public static void remove( // getOrCreate will recreate it lazily if another request arrives. attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).remove(); } else { - // Keep the legacy mirror pointed at the current inbound request after removing an older - // response context. + // Keep the context mirror pointed at the current inbound request after removing an older + // response context. It may still be copied to a new HTTP/2 stream channel. attributes.attr(CONTEXT_ATTRIBUTE_KEY).set(currentContext.tracingContext()); } } @@ -94,7 +96,7 @@ public static void remove( /** Closes all pending request contexts on channel close. */ public static void closeAll(final AttributeMap attributes) { - // The legacy mirror must not outlive the authoritative request queue. + // The context mirror must not outlive the authoritative request queue. attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).remove(); close(attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove()); From ba51b12a50ea2e1b81c30d0333e9b98e96ee82ac Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 21 Jul 2026 17:29:21 -0700 Subject: [PATCH 17/33] Fix deferred AppSec block responses for Netty pipelining --- .../ChannelFutureListenerInstrumentation.java | 2 + ...yChannelHandlerContextInstrumentation.java | 2 + .../NettyChannelPipelineInstrumentation.java | 2 + .../server/BlockingResponseHandler.java | 169 +++++++++++++----- .../HttpServerRequestTracingHandler.java | 3 +- .../HttpServerResponseTracingHandler.java | 2 + .../server/NettyHttp11PipeliningTest.java | 112 +++++++++++- .../netty41/ServerRequestContext.java | 9 + 8 files changed, 247 insertions(+), 54 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java index 26754d69a6f..1684d51c85d 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java @@ -58,7 +58,9 @@ public String[] helperClassNames() { packageName + ".server.NettyHttpServerDecorator", packageName + ".server.NettyHttpServerDecorator$NettyBlockResponseFunction", packageName + ".server.BlockingResponseHandler", + packageName + ".server.BlockingResponseHandler$1", packageName + ".server.BlockingResponseHandler$IgnoreAllWritesHandler", + packageName + ".server.BlockingResponseHandler$PendingBlockResponse", packageName + ".server.HttpServerRequestTracingHandler", packageName + ".server.HttpServerResponseTracingHandler", packageName + ".server.HttpServerTracingHandler" diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java index b73e4e0fbf7..c989344f43c 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java @@ -53,7 +53,9 @@ public String[] helperClassNames() { packageName + ".server.NettyHttpServerDecorator", packageName + ".server.NettyHttpServerDecorator$NettyBlockResponseFunction", packageName + ".server.BlockingResponseHandler", + packageName + ".server.BlockingResponseHandler$1", packageName + ".server.BlockingResponseHandler$IgnoreAllWritesHandler", + packageName + ".server.BlockingResponseHandler$PendingBlockResponse", packageName + ".server.HttpServerRequestTracingHandler", packageName + ".server.HttpServerResponseTracingHandler", packageName + ".server.HttpServerTracingHandler" diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java index 668077b87a6..33c8642b1ff 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java @@ -84,7 +84,9 @@ public String[] helperClassNames() { packageName + ".server.NettyHttpServerDecorator", packageName + ".server.NettyHttpServerDecorator$NettyBlockResponseFunction", packageName + ".server.BlockingResponseHandler", + packageName + ".server.BlockingResponseHandler$1", packageName + ".server.BlockingResponseHandler$IgnoreAllWritesHandler", + packageName + ".server.BlockingResponseHandler$PendingBlockResponse", packageName + ".server.HttpServerContextTrackingHandler", packageName + ".server.HttpServerRequestTracingHandler", packageName + ".server.HttpServerResponseTracingHandler", diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java index 1c49fceae2f..50412d8b698 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java @@ -4,6 +4,7 @@ import datadog.trace.api.gateway.Flow; import datadog.trace.api.internal.TraceSegment; import datadog.trace.bootstrap.blocking.BlockingActionHelper; +import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; @@ -15,6 +16,7 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; import io.netty.util.ReferenceCountUtil; import java.util.Map; import org.slf4j.Logger; @@ -22,6 +24,9 @@ public class BlockingResponseHandler extends ChannelInboundHandlerAdapter { public static final Logger log = LoggerFactory.getLogger(BlockingResponseHandler.class); + private static final String IGNORE_ALL_WRITES_HANDLER = "ignore_all_writes_handler"; + private static final String MISSING_RESPONSE_TRACING_HANDLER_MESSAGE = + "Unable to block because HttpServerResponseTracingHandler was not found on the pipeline"; private static volatile boolean HAS_WARNED; private final TraceSegment segment; @@ -29,6 +34,7 @@ public class BlockingResponseHandler extends ChannelInboundHandlerAdapter { private final BlockingContentType bct; private final Map extraHeaders; private final String securityResponseId; + private final ServerRequestContext serverContext; private boolean hasBlockedAlready; @@ -43,15 +49,19 @@ public BlockingResponseHandler( this.bct = bct; this.extraHeaders = extraHeaders; this.securityResponseId = securityResponseId; + this.serverContext = null; } - public BlockingResponseHandler(TraceSegment segment, Flow.Action.RequestBlockingAction rba) { - this( - segment, - rba.getStatusCode(), - rba.getBlockingContentType(), - rba.getExtraHeaders(), - rba.getSecurityResponseId()); + public BlockingResponseHandler( + TraceSegment segment, + Flow.Action.RequestBlockingAction rba, + ServerRequestContext serverContext) { + this.segment = segment; + this.statusCode = rba.getStatusCode(); + this.bct = rba.getBlockingContentType(); + this.extraHeaders = rba.getExtraHeaders(); + this.securityResponseId = rba.getSecurityResponseId(); + this.serverContext = serverContext; } @Override @@ -73,75 +83,138 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } if (ctxForDownstream == null) { - if (HAS_WARNED) { - log.debug( - "Unable to block because HttpServerResponseTracingHandler was not found on the pipeline"); - } else { - log.warn( - "Unable to block because HttpServerResponseTracingHandler was not found on the pipeline"); - HAS_WARNED = true; - } + logMissingResponseTracingHandler(); ctx.fireChannelRead(msg); return; } HttpRequest request = (HttpRequest) msg; - int httpCode = BlockingActionHelper.getHttpCode(statusCode); - HttpResponseStatus httpResponseStatus = HttpResponseStatus.valueOf(httpCode); - FullHttpResponse response = - new DefaultFullHttpResponse(request.protocolVersion(), httpResponseStatus); + this.hasBlockedAlready = true; - HttpHeaders headers = response.headers(); - headers.set("Connection", "close"); + PendingBlockResponse pendingBlockResponse = + new PendingBlockResponse( + segment, + statusCode, + bct, + extraHeaders, + securityResponseId, + request.protocolVersion(), + request.headers().get("accept")); + ReferenceCountUtil.release(msg); - for (Map.Entry h : this.extraHeaders.entrySet()) { - headers.set(h.getKey(), h.getValue()); + if (serverContext != null + && ServerRequestContext.nextResponse(ctx.channel()) != serverContext) { + serverContext.deferBlockResponse(pendingBlockResponse); + return; } - if (bct != BlockingContentType.NONE) { - String acceptHeader = request.headers().get("accept"); - BlockingActionHelper.TemplateType type = - BlockingActionHelper.determineTemplateType(bct, acceptHeader); - headers.set("Content-type", BlockingActionHelper.getContentType(type)); + writeBlockResponse(ctxForDownstream, pendingBlockResponse); + } - byte[] template = BlockingActionHelper.getTemplate(type, this.securityResponseId); - HttpUtil.setContentLength(response, template.length); - response.content().writeBytes(template); + private static void logMissingResponseTracingHandler() { + if (HAS_WARNED) { + log.debug(MISSING_RESPONSE_TRACING_HANDLER_MESSAGE); + } else { + log.warn(MISSING_RESPONSE_TRACING_HANDLER_MESSAGE); + HAS_WARNED = true; } + } - this.hasBlockedAlready = true; - - ReferenceCountUtil.release(msg); + static boolean maybeWriteDeferredBlockResponse( + ChannelHandlerContext ctx, ServerRequestContext serverContext) { + if (serverContext == null) { + return false; + } + Object deferredBlockResponse = serverContext.deferredBlockResponse(); + if (!(deferredBlockResponse instanceof PendingBlockResponse)) { + return false; + } + serverContext.deferBlockResponse(null); + writeBlockResponse(ctx, (PendingBlockResponse) deferredBlockResponse); + return true; + } + private static void writeBlockResponse( + ChannelHandlerContext ctxForDownstream, PendingBlockResponse pendingBlockResponse) { // write starts in the handler before the one associated with ctx // so add one that will be skipped (but that will prevent any writes later coming from later // handlers). // We do not want to start from the end of the // pipeline because there is an increased risk of hitting duplex handlers that // expect to have seen a request before processing the response - ctxForDownstream = - ctxForDownstream - .pipeline() - .addAfter( - ctxForDownstream.name(), - "ignore_all_writes_handler", - IgnoreAllWritesHandler.INSTANCE) - .context("ignore_all_writes_handler"); - - segment.effectivelyBlocked(); - - ctxForDownstream - .writeAndFlush(response) + if (ctxForDownstream.pipeline().get(IGNORE_ALL_WRITES_HANDLER) == null) { + ctxForDownstream + .pipeline() + .addAfter( + ctxForDownstream.name(), IGNORE_ALL_WRITES_HANDLER, IgnoreAllWritesHandler.INSTANCE); + } + ChannelHandlerContext writeContext = + ctxForDownstream.pipeline().context(IGNORE_ALL_WRITES_HANDLER); + + writeContext + .writeAndFlush(pendingBlockResponse.toResponse()) .addListener( fut -> { if (!fut.isSuccess()) { log.warn("Write of blocking response failed", fut.cause()); } - ctx.channel().close(); + writeContext.channel().close(); }); } + private static class PendingBlockResponse { + private final TraceSegment segment; + private final int statusCode; + private final BlockingContentType bct; + private final Map extraHeaders; + private final String securityResponseId; + private final HttpVersion protocolVersion; + private final String acceptHeader; + + private PendingBlockResponse( + TraceSegment segment, + int statusCode, + BlockingContentType bct, + Map extraHeaders, + String securityResponseId, + HttpVersion protocolVersion, + String acceptHeader) { + this.segment = segment; + this.statusCode = statusCode; + this.bct = bct; + this.extraHeaders = extraHeaders; + this.securityResponseId = securityResponseId; + this.protocolVersion = protocolVersion; + this.acceptHeader = acceptHeader; + } + + private FullHttpResponse toResponse() { + int httpCode = BlockingActionHelper.getHttpCode(statusCode); + HttpResponseStatus httpResponseStatus = HttpResponseStatus.valueOf(httpCode); + FullHttpResponse response = new DefaultFullHttpResponse(protocolVersion, httpResponseStatus); + + HttpHeaders headers = response.headers(); + headers.set("Connection", "close"); + + for (Map.Entry h : extraHeaders.entrySet()) { + headers.set(h.getKey(), h.getValue()); + } + + if (bct != BlockingContentType.NONE) { + BlockingActionHelper.TemplateType type = + BlockingActionHelper.determineTemplateType(bct, acceptHeader); + headers.set("Content-type", BlockingActionHelper.getContentType(type)); + + byte[] template = BlockingActionHelper.getTemplate(type, securityResponseId); + HttpUtil.setContentLength(response, template.length); + response.content().writeBytes(template); + } + segment.effectivelyBlocked(); + return response; + } + } + @ChannelHandler.Sharable public static class IgnoreAllWritesHandler extends ChannelOutboundHandlerAdapter { public static final IgnoreAllWritesHandler INSTANCE = new IgnoreAllWritesHandler(); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 0d934772d56..2da4b2c318d 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -62,7 +62,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { .addAfter( ctx.name(), "blocking_handler", - new BlockingResponseHandler(span.getRequestContext().getTraceSegment(), rba)); + new BlockingResponseHandler( + span.getRequestContext().getTraceSegment(), rba, serverContext)); } try { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index ded2ef0490c..f9d0fe034fa 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -69,6 +69,8 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann DECORATE.beforeFinish(scope.context()); span.finish(); // Finish the span manually since finishSpanOnClose was false removeServerContext(ctx, serverContext); + BlockingResponseHandler.maybeWriteDeferredBlockResponse( + ctx, ServerRequestContext.nextResponse(ctx.channel())); } } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java index 610fa95ddbe..8c09c7f9f64 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -3,6 +3,7 @@ import static datadog.trace.agent.test.assertions.SpanMatcher.span; import static datadog.trace.agent.test.assertions.TraceAssertions.SORT_BY_START_TIME; import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.api.gateway.Events.EVENTS; import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; import static io.netty.handler.codec.http.HttpResponseStatus.OK; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; @@ -12,11 +13,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.appsec.api.blocking.BlockingContentType; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.agent.test.assertions.TraceMatcher; +import datadog.trace.api.function.TriFunction; +import datadog.trace.api.gateway.Flow; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.gateway.SubscriptionService; +import datadog.trace.bootstrap.ActiveSubsystems; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.URIDataAdapter; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; import io.netty.channel.EventLoopGroup; @@ -36,8 +47,10 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.function.Supplier; import java.util.regex.Pattern; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -52,11 +65,13 @@ public class NettyHttp11PipeliningTest extends AbstractInstrumentationTest { private EventLoopGroup eventLoopGroup; private PipeliningHandler handler; private int port; + private Object appSecSubscriptions; + private boolean originalAppSecActive; @BeforeAll void startServer() throws Exception { eventLoopGroup = new NioEventLoopGroup(); - handler = new PipeliningHandler(3); + handler = new PipeliningHandler(); ServerBootstrap bootstrap = new ServerBootstrap() .group(eventLoopGroup) @@ -81,8 +96,18 @@ void stopServer() { } } + @AfterEach + void resetAppSec() { + if (appSecSubscriptions != null) { + ((SubscriptionService) appSecSubscriptions).reset(); + appSecSubscriptions = null; + ActiveSubsystems.APPSEC_ACTIVE = originalAppSecActive; + } + } + @Test void createsServerSpanForEachPipelinedRequest() throws Exception { + handler.expectRequests(3); try (Socket socket = new Socket("localhost", port)) { socket.setSoTimeout(5000); socket.getOutputStream().write(pipelinedRequests().getBytes(US_ASCII)); @@ -106,6 +131,33 @@ void createsServerSpanForEachPipelinedRequest() throws Exception { serverTrace(THIRD_PATH)); } + @Test + void requestBlockOnLaterPipelinedRequestDoesNotOvertakeEarlierResponse() throws Exception { + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(request(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + CountDownLatch blockedRequestSeen = enableAppSecRequestBlockingFor(SECOND_PATH); + socket.getOutputStream().write(request(SECOND_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); + + handler.writeResponses(); + + assertEquals("response " + FIRST_PATH, readHttpResponseBody(socket.getInputStream())); + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), + "second response should be the deferred blocking response"); + } + } + private static String pipelinedRequests() { return "GET " + FIRST_PATH @@ -118,6 +170,47 @@ private static String pipelinedRequests() { + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; } + private static String request(String path) { + return "GET " + path + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; + } + + private CountDownLatch enableAppSecRequestBlockingFor(String blockedPath) { + CountDownLatch blockedRequestSeen = new CountDownLatch(1); + SubscriptionService subscriptions = + (SubscriptionService) AgentTracer.get().getSubscriptionService(RequestContextSlot.APPSEC); + appSecSubscriptions = subscriptions; + originalAppSecActive = ActiveSubsystems.APPSEC_ACTIVE; + ActiveSubsystems.APPSEC_ACTIVE = true; + + subscriptions.registerCallback( + EVENTS.requestStarted(), + new Supplier>() { + @Override + public Flow get() { + return new Flow.ResultFlow<>(new Object()); + } + }); + subscriptions.registerCallback( + EVENTS.requestMethodUriRaw(), + new TriFunction>() { + @Override + public Flow apply( + RequestContext requestContext, String method, URIDataAdapter uri) { + if (!blockedPath.equals(uri.path())) { + return Flow.ResultFlow.empty(); + } + blockedRequestSeen.countDown(); + return new Flow.ResultFlow(null) { + @Override + public Action getAction() { + return new Action.RequestBlockingAction(403, BlockingContentType.NONE); + } + }; + } + }); + return blockedRequestSeen; + } + private static TraceMatcher serverTrace(String path) { return trace( span() @@ -128,7 +221,7 @@ private static TraceMatcher serverTrace(String path) { } private static String readHttpResponseBody(InputStream in) throws IOException { - String headers = readHeaders(in); + String headers = readHttpResponseHeaders(in); assertTrue(headers.startsWith("HTTP/1.1 200 "), "unexpected response: " + headers); int contentLength = contentLength(headers); byte[] body = new byte[contentLength]; @@ -143,7 +236,7 @@ private static String readHttpResponseBody(InputStream in) throws IOException { return new String(body, UTF_8); } - private static String readHeaders(InputStream in) throws IOException { + private static String readHttpResponseHeaders(InputStream in) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); int state = 0; while (state < 4) { @@ -173,14 +266,23 @@ private static int contentLength(String headers) { throw new AssertionError("missing content-length header: " + headers); } + @ChannelHandler.Sharable private static final class PipeliningHandler extends SimpleChannelInboundHandler { - private final CountDownLatch receivedRequests; + private volatile CountDownLatch receivedRequests; private final List paths = new ArrayList<>(); private volatile ChannelHandlerContext context; - private PipeliningHandler(int expectedRequests) { + private PipeliningHandler() { + expectRequests(0); + } + + private void expectRequests(int expectedRequests) { receivedRequests = new CountDownLatch(expectedRequests); + context = null; + synchronized (paths) { + paths.clear(); + } } @Override diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 90912c7d6a0..1349188bab1 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -176,6 +176,7 @@ private static boolean isPoisoned(final Deque contexts) { private final Context tracingContext; private final String acceptHeader; private boolean responseAnalyzed; + private Object deferredBlockResponse; public Context tracingContext() { return tracingContext; @@ -193,6 +194,14 @@ public void markResponseAnalyzed() { responseAnalyzed = true; } + public Object deferredBlockResponse() { + return deferredBlockResponse; + } + + public void deferBlockResponse(final Object deferredBlockResponse) { + this.deferredBlockResponse = deferredBlockResponse; + } + private ServerRequestContext(final Context tracingContext, final String acceptHeader) { this.tracingContext = tracingContext; this.acceptHeader = acceptHeader; From 88da30a8cd277fbe4c5f737f62e2752ce29017fc Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 21 Jul 2026 17:31:44 -0700 Subject: [PATCH 18/33] Avoid logging dropped Netty outbound messages --- .../netty40/server/MaybeBlockResponseHandler.java | 2 +- .../netty41/server/MaybeBlockResponseHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.0/src/main/java/datadog/trace/instrumentation/netty40/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.0/src/main/java/datadog/trace/instrumentation/netty40/server/MaybeBlockResponseHandler.java index d7ec1c109ef..6f6c1891e88 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.0/src/main/java/datadog/trace/instrumentation/netty40/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.0/src/main/java/datadog/trace/instrumentation/netty40/server/MaybeBlockResponseHandler.java @@ -70,7 +70,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr if (isAnalyzedResponse(channel)) { if (isBlockedResponse(channel)) { // block further writes - log.debug("Write suppressed, msg {} dropped", msg); + log.debug("Write suppressed; dropped outbound message"); ReferenceCountUtil.release(msg); } else { super.write(ctx, msg, prm); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java index 5dab095e4e0..068c55932e1 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java @@ -39,7 +39,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr if (ServerRequestContext.isResponseBlocked(channel)) { // block further writes while the blocking response close is still asynchronous - log.debug("Write suppressed, msg {} dropped", msg); + log.debug("Write suppressed; dropped outbound message"); ReferenceCountUtil.release(msg); return; } From 67b731ae589062fd96cca3d7344e2cac8fa92465 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 21 Jul 2026 18:11:23 -0700 Subject: [PATCH 19/33] Use computed Netty 4.1 response status code --- .../netty41/server/MaybeBlockResponseHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java index c1ab0de230f..2ae56acc57d 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java @@ -77,7 +77,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr Flow flow = DECORATE.callIGCallbackResponseAndHeaders( - span, origResponse, origResponse.getStatus().code(), ResponseExtractAdapter.GETTER); + span, origResponse, statusCode, ResponseExtractAdapter.GETTER); serverContext.markResponseAnalyzed(); Flow.Action action = flow.getAction(); if (!(action instanceof Flow.Action.RequestBlockingAction)) { From d9c5d1defba67921b4ccbd9db6b4d93b85bad36e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 21 Jul 2026 18:17:57 -0700 Subject: [PATCH 20/33] Use single-thread Netty event loop in tests --- .../server/NettyHttpServerTestSupport.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java index f946f5cb01b..d75bab18d14 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerTestSupport.java @@ -29,7 +29,7 @@ abstract class NettyHttpServerTestSupport extends AbstractInstrumentationTest { @BeforeAll void startServer() throws Exception { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = new NioEventLoopGroup(1); ServerBootstrap bootstrap = new ServerBootstrap() .group(eventLoopGroup) @@ -47,11 +47,14 @@ protected void initChannel(Channel ch) { @AfterAll void stopServer() { - if (serverChannel != null) { - serverChannel.close(); - } - if (eventLoopGroup != null) { - eventLoopGroup.shutdownGracefully(); + try { + if (serverChannel != null) { + serverChannel.close().syncUninterruptibly(); + } + } finally { + if (eventLoopGroup != null) { + eventLoopGroup.shutdownGracefully().syncUninterruptibly(); + } } } From e7046aaaf33f57f171357fd59b515295dabde8a8 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 22 Jul 2026 10:27:26 -0700 Subject: [PATCH 21/33] Avoid synthetic accessors in Netty blocking response handler --- .../netty41/ChannelFutureListenerInstrumentation.java | 1 - .../netty41/NettyChannelHandlerContextInstrumentation.java | 1 - .../netty41/NettyChannelPipelineInstrumentation.java | 1 - .../netty41/server/BlockingResponseHandler.java | 7 +++++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java index 1684d51c85d..e78dc733915 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/ChannelFutureListenerInstrumentation.java @@ -58,7 +58,6 @@ public String[] helperClassNames() { packageName + ".server.NettyHttpServerDecorator", packageName + ".server.NettyHttpServerDecorator$NettyBlockResponseFunction", packageName + ".server.BlockingResponseHandler", - packageName + ".server.BlockingResponseHandler$1", packageName + ".server.BlockingResponseHandler$IgnoreAllWritesHandler", packageName + ".server.BlockingResponseHandler$PendingBlockResponse", packageName + ".server.HttpServerRequestTracingHandler", diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java index c989344f43c..9f756293740 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelHandlerContextInstrumentation.java @@ -53,7 +53,6 @@ public String[] helperClassNames() { packageName + ".server.NettyHttpServerDecorator", packageName + ".server.NettyHttpServerDecorator$NettyBlockResponseFunction", packageName + ".server.BlockingResponseHandler", - packageName + ".server.BlockingResponseHandler$1", packageName + ".server.BlockingResponseHandler$IgnoreAllWritesHandler", packageName + ".server.BlockingResponseHandler$PendingBlockResponse", packageName + ".server.HttpServerRequestTracingHandler", diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java index 33c8642b1ff..2a471afc311 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java @@ -84,7 +84,6 @@ public String[] helperClassNames() { packageName + ".server.NettyHttpServerDecorator", packageName + ".server.NettyHttpServerDecorator$NettyBlockResponseFunction", packageName + ".server.BlockingResponseHandler", - packageName + ".server.BlockingResponseHandler$1", packageName + ".server.BlockingResponseHandler$IgnoreAllWritesHandler", packageName + ".server.BlockingResponseHandler$PendingBlockResponse", packageName + ".server.HttpServerContextTrackingHandler", diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java index 50412d8b698..91b49f5d63b 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java @@ -172,7 +172,10 @@ private static class PendingBlockResponse { private final HttpVersion protocolVersion; private final String acceptHeader; - private PendingBlockResponse( + // Prevent the generation of BlockingResponseHandler$1 by making this constructor + // package-private to allow the BlockingResponseHandler to call this. This module emits Java 8 + // bytecode, so it cannot use Java 11 nested access. + PendingBlockResponse( TraceSegment segment, int statusCode, BlockingContentType bct, @@ -189,7 +192,7 @@ private PendingBlockResponse( this.acceptHeader = acceptHeader; } - private FullHttpResponse toResponse() { + FullHttpResponse toResponse() { int httpCode = BlockingActionHelper.getHttpCode(statusCode); HttpResponseStatus httpResponseStatus = HttpResponseStatus.valueOf(httpCode); FullHttpResponse response = new DefaultFullHttpResponse(protocolVersion, httpResponseStatus); From aa33b42d65bc40c05b67ce9decef9d2cdb627414 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 22 Jul 2026 11:12:51 -0700 Subject: [PATCH 22/33] Fix Netty block function pipelining deferral --- .../server/BlockingResponseHandler.java | 5 +- .../HttpServerRequestTracingHandler.java | 5 +- .../server/NettyHttpServerDecorator.java | 21 +++- .../server/NettyHttp11PipeliningTest.java | 102 +++++++++++++++--- .../netty41/ServerRequestContext.java | 7 ++ 5 files changed, 118 insertions(+), 22 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java index 91b49f5d63b..3903380a810 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java @@ -43,13 +43,14 @@ public BlockingResponseHandler( int statusCode, BlockingContentType bct, Map extraHeaders, - String securityResponseId) { + String securityResponseId, + ServerRequestContext serverContext) { this.segment = segment; this.statusCode = statusCode; this.bct = bct; this.extraHeaders = extraHeaders; this.securityResponseId = securityResponseId; - this.serverContext = null; + this.serverContext = serverContext; } public BlockingResponseHandler( diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 2da4b2c318d..bfa8c9b8338 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -47,15 +47,14 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { final Context parentContext = storedParentContext != null ? storedParentContext : DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); + final ServerRequestContext serverContext = + ServerRequestContext.add(channel, context, headers.get("accept")); try (final ContextScope ignored = context.attach()) { final AgentSpan span = AgentSpan.fromContext(context); DECORATE.afterStart(span); DECORATE.onRequest(span, channel, request, parentContext); - final ServerRequestContext serverContext = - ServerRequestContext.add(channel, context, headers.get("accept")); - Flow.Action.RequestBlockingAction rba = span.getRequestBlockingAction(); if (rba != null) { ctx.pipeline() diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java index 101b35bc4ac..a66fda5f11c 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java @@ -10,6 +10,7 @@ import datadog.trace.bootstrap.instrumentation.api.URIDefaultDataAdapter; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.HttpServerDecorator; +import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; @@ -109,17 +110,24 @@ protected boolean isAppSecOnResponseSeparate() { @Override protected BlockResponseFunction createBlockResponseFunction( HttpRequest httpRequest, Channel channel) { - return new NettyBlockResponseFunction(channel.pipeline(), httpRequest); + return new NettyBlockResponseFunction( + channel.pipeline(), httpRequest, ServerRequestContext.currentRequest(channel)); } public static class NettyBlockResponseFunction implements BlockResponseFunction { - private final ChannelPipeline pipeline; public static final Logger log = LoggerFactory.getLogger(NettyBlockResponseFunction.class); + + private final ChannelPipeline pipeline; private final HttpRequest httpRequestMessage; + private final ServerRequestContext serverContext; - public NettyBlockResponseFunction(ChannelPipeline pipeline, HttpRequest httpRequestMessage) { + public NettyBlockResponseFunction( + ChannelPipeline pipeline, + HttpRequest httpRequestMessage, + ServerRequestContext serverContext) { this.pipeline = pipeline; this.httpRequestMessage = httpRequestMessage; + this.serverContext = serverContext; } @Override @@ -145,7 +153,12 @@ public boolean tryCommitBlockingResponse( pipeline.context(handlerBefore).name(), "blocking_handler", new BlockingResponseHandler( - segment, statusCode, templateType, extraHeaders, securityResponseId)) + segment, + statusCode, + templateType, + extraHeaders, + securityResponseId, + serverContext)) .addBefore( "blocking_handler", "before_blocking_handler", new ChannelInboundHandlerAdapter()); } catch (RuntimeException rte) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java index 8c09c7f9f64..eae7b48697b 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -9,6 +9,7 @@ import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyMap; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,11 +18,13 @@ import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.agent.test.assertions.TraceMatcher; import datadog.trace.api.function.TriFunction; +import datadog.trace.api.gateway.BlockResponseFunction; import datadog.trace.api.gateway.Flow; import datadog.trace.api.gateway.RequestContext; import datadog.trace.api.gateway.RequestContextSlot; import datadog.trace.api.gateway.SubscriptionService; import datadog.trace.bootstrap.ActiveSubsystems; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.URIDataAdapter; import io.netty.bootstrap.ServerBootstrap; @@ -38,6 +41,7 @@ import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpServerCodec; +import io.netty.util.ReferenceCountUtil; import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.IOException; @@ -103,6 +107,7 @@ void resetAppSec() { appSecSubscriptions = null; ActiveSubsystems.APPSEC_ACTIVE = originalAppSecActive; } + handler.clearBlockResponseFunctionBlocking(); } @Test @@ -158,6 +163,35 @@ void requestBlockOnLaterPipelinedRequestDoesNotOvertakeEarlierResponse() throws } } + @Test + void blockResponseFunctionOnLaterPipelinedRequestDoesNotOvertakeEarlierResponse() + throws Exception { + enableAppSec(); + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(request(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + CountDownLatch blockedRequestSeen = handler.blockWithResponseFunctionFor(SECOND_PATH); + socket.getOutputStream().write(request(SECOND_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); + + handler.writeResponses(); + + assertEquals("response " + FIRST_PATH, readHttpResponseBody(socket.getInputStream())); + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), + "second response should be the deferred blocking response"); + } + } + private static String pipelinedRequests() { return "GET " + FIRST_PATH @@ -175,21 +209,9 @@ private static String request(String path) { } private CountDownLatch enableAppSecRequestBlockingFor(String blockedPath) { + SubscriptionService subscriptions = (SubscriptionService) enableAppSec(); CountDownLatch blockedRequestSeen = new CountDownLatch(1); - SubscriptionService subscriptions = - (SubscriptionService) AgentTracer.get().getSubscriptionService(RequestContextSlot.APPSEC); - appSecSubscriptions = subscriptions; - originalAppSecActive = ActiveSubsystems.APPSEC_ACTIVE; - ActiveSubsystems.APPSEC_ACTIVE = true; - subscriptions.registerCallback( - EVENTS.requestStarted(), - new Supplier>() { - @Override - public Flow get() { - return new Flow.ResultFlow<>(new Object()); - } - }); subscriptions.registerCallback( EVENTS.requestMethodUriRaw(), new TriFunction>() { @@ -211,6 +233,24 @@ public Action getAction() { return blockedRequestSeen; } + private Object enableAppSec() { + SubscriptionService subscriptions = + (SubscriptionService) AgentTracer.get().getSubscriptionService(RequestContextSlot.APPSEC); + appSecSubscriptions = subscriptions; + originalAppSecActive = ActiveSubsystems.APPSEC_ACTIVE; + ActiveSubsystems.APPSEC_ACTIVE = true; + + subscriptions.registerCallback( + EVENTS.requestStarted(), + new Supplier>() { + @Override + public Flow get() { + return new Flow.ResultFlow<>(new Object()); + } + }); + return subscriptions; + } + private static TraceMatcher serverTrace(String path) { return trace( span() @@ -272,8 +312,11 @@ private static final class PipeliningHandler private volatile CountDownLatch receivedRequests; private final List paths = new ArrayList<>(); private volatile ChannelHandlerContext context; + private volatile String blockResponseFunctionPath; + private volatile CountDownLatch blockResponseFunctionRequestSeen; private PipeliningHandler() { + super(false); expectRequests(0); } @@ -285,13 +328,46 @@ private void expectRequests(int expectedRequests) { } } + private CountDownLatch blockWithResponseFunctionFor(String path) { + blockResponseFunctionPath = path; + blockResponseFunctionRequestSeen = new CountDownLatch(1); + return blockResponseFunctionRequestSeen; + } + + private void clearBlockResponseFunctionBlocking() { + blockResponseFunctionPath = null; + blockResponseFunctionRequestSeen = null; + } + @Override protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) { context = ctx; + boolean blockingResponseCommitted = false; synchronized (paths) { paths.add(request.uri()); } + if (request.uri().equals(blockResponseFunctionPath)) { + AgentSpan span = AgentTracer.activeSpan(); + RequestContext requestContext = span == null ? null : span.getRequestContext(); + BlockResponseFunction blockResponseFunction = + requestContext == null ? null : requestContext.getBlockResponseFunction(); + if (blockResponseFunction != null) { + blockingResponseCommitted = + blockResponseFunction.tryCommitBlockingResponse( + requestContext.getTraceSegment(), + 403, + BlockingContentType.NONE, + emptyMap(), + null); + } + if (blockingResponseCommitted && blockResponseFunctionRequestSeen != null) { + blockResponseFunctionRequestSeen.countDown(); + } + } receivedRequests.countDown(); + if (!blockingResponseCommitted) { + ReferenceCountUtil.release(request); + } } private boolean awaitAllRequestsReceived() throws InterruptedException { diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 1349188bab1..a8927cad9ea 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -49,6 +49,13 @@ public static ServerRequestContext nextResponse(final AttributeMap attributes) { return contexts == null || isPoisoned(contexts) ? null : contexts.peekFirst(); } + /** Returns the server request context for the current inbound request. */ + public static ServerRequestContext currentRequest(final AttributeMap attributes) { + final Deque contexts = + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); + return contexts == null || isPoisoned(contexts) ? null : contexts.peekLast(); + } + /** Returns whether the channel is closing after an AppSec response block. */ public static boolean isResponseBlocked(final AttributeMap attributes) { return attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).get() != null; From e7897a3c8d95b134031aab66b4007c9be9406e47 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 22 Jul 2026 13:03:14 -0700 Subject: [PATCH 23/33] Fix Netty pipelined block response ordering Defer blocked responses until the active HTTP/1.1 response is complete, including chunked/streaming responses that finish on LastHttpContent. Handle header-only responses such as HEAD, 204, 205, 304, and explicit Content-Length: 0 without requiring an application LastHttpContent. Also keep interim 1xx responses from completing the server span and add coverage for chunked, header-only, HEAD, interim, and malformed Content-Length cases. --- .../HttpServerRequestTracingHandler.java | 4 +- .../HttpServerResponseTracingHandler.java | 80 ++++- .../netty41/ServerRequestContextTest.java | 10 + .../HttpServerResponseTracingHandlerTest.java | 38 +++ .../server/NettyHttp11PipeliningTest.java | 313 ++++++++++++++++++ .../netty41/ServerRequestContext.java | 30 +- 6 files changed, 458 insertions(+), 17 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index bfa8c9b8338..71cf78d6a87 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -14,6 +14,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; @ChannelHandler.Sharable @@ -48,7 +49,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { storedParentContext != null ? storedParentContext : DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); final ServerRequestContext serverContext = - ServerRequestContext.add(channel, context, headers.get("accept")); + ServerRequestContext.add( + channel, context, headers.get("accept"), HttpMethod.HEAD.equals(request.method())); try (final ContextScope ignored = context.attach()) { final AgentSpan span = AgentSpan.fromContext(context); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index f9d0fe034fa..b45c3767066 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -9,13 +9,18 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.websocket.HandlerContext; import datadog.trace.instrumentation.netty41.ServerRequestContext; +import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.DefaultLastHttpContent; +import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.LastHttpContent; @ChannelHandler.Sharable public class HttpServerResponseTracingHandler extends ChannelOutboundHandlerAdapter { @@ -23,12 +28,18 @@ public class HttpServerResponseTracingHandler extends ChannelOutboundHandlerAdap @Override public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise prm) { - if (!(msg instanceof HttpResponse)) { + final boolean isResponse = msg instanceof HttpResponse; + if (!isResponse && !(msg instanceof LastHttpContent)) { ctx.write(msg, prm); return; } final ServerRequestContext serverContext = ServerRequestContext.nextResponse(ctx.channel()); + if (!isResponse && serverContext != null && !serverContext.isResponseStarted()) { + ctx.write(msg, prm); + return; + } + final Context storedContext = serverContext == null // HTTP/2 multiplex stream channels only inherit the mirrored context attribute from @@ -44,7 +55,8 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann } try (final ContextScope scope = storedContext.attach()) { - final HttpResponse response = (HttpResponse) msg; + final HttpResponse response = isResponse ? (HttpResponse) msg : null; + final boolean headerOnly = response != null && isHeaderOnly(response, serverContext); try { ctx.write(msg, prm); @@ -55,22 +67,36 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann removeServerContext(ctx, serverContext); throw throwable; } - final boolean isWebsocketUpgrade = - response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS - && "websocket".equals(response.headers().get(HttpHeaderNames.UPGRADE)); - if (isWebsocketUpgrade) { - ctx.channel() - .attr(WEBSOCKET_SENDER_HANDLER_CONTEXT) - .set(new HandlerContext.Sender(span, ctx.channel().id().asShortText())); - } - if (response.status() != HttpResponseStatus.CONTINUE - && (response.status() != HttpResponseStatus.SWITCHING_PROTOCOLS || isWebsocketUpgrade)) { + final boolean responseComplete; + if (response == null) { + responseComplete = true; + } else { + final boolean isWebsocketUpgrade = + response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS + && "websocket".equals(response.headers().get(HttpHeaderNames.UPGRADE)); + if (isWebsocketUpgrade) { + ctx.channel() + .attr(WEBSOCKET_SENDER_HANDLER_CONTEXT) + .set(new HandlerContext.Sender(span, ctx.channel().id().asShortText())); + } + if (isInformational(response) && !isWebsocketUpgrade) { + return; + } + if (serverContext != null) { + serverContext.markResponseStarted(); + } DECORATE.onResponse(span, response); + responseComplete = isWebsocketUpgrade || msg instanceof FullHttpResponse || headerOnly; + } + if (responseComplete) { DECORATE.beforeFinish(scope.context()); span.finish(); // Finish the span manually since finishSpanOnClose was false removeServerContext(ctx, serverContext); - BlockingResponseHandler.maybeWriteDeferredBlockResponse( - ctx, ServerRequestContext.nextResponse(ctx.channel())); + final ServerRequestContext nextResponse = ServerRequestContext.nextResponse(ctx.channel()); + if (headerOnly && !(msg instanceof FullHttpResponse) && nextResponse != null) { + ctx.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER), ctx.voidPromise()); + } + BlockingResponseHandler.maybeWriteDeferredBlockResponse(ctx, nextResponse); } } } @@ -83,4 +109,30 @@ private static void removeServerContext( ServerRequestContext.remove(ctx.channel(), serverContext); } } + + private static boolean isHeaderOnly( + final HttpResponse response, final ServerRequestContext serverContext) { + final int statusCode = response.status().code(); + return (serverContext != null && serverContext.isHeadRequest()) + || statusCode == HttpResponseStatus.NO_CONTENT.code() + || statusCode == HttpResponseStatus.RESET_CONTENT.code() + || statusCode == HttpResponseStatus.NOT_MODIFIED.code() + || (hasZeroContentLength(response) && !HttpUtil.isTransferEncodingChunked(response)); + } + + private static boolean isInformational(final HttpResponse response) { + final int statusCode = response.status().code(); + return statusCode >= 100 && statusCode < 200; + } + + private static boolean hasZeroContentLength(final HttpResponse response) { + if (!HttpUtil.isContentLengthSet(response)) { + return false; + } + try { + return HttpUtil.getContentLength(response) == 0; + } catch (final NumberFormatException ignored) { + return false; + } + } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index 4cac8e6c1e8..684de404891 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -41,6 +41,16 @@ void capturesAcceptHeaderValue() { assertEquals("text/html", serverContext.acceptHeader()); } + @Test + void capturesHeadRequest() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + + ServerRequestContext serverContext = + ServerRequestContext.add(attributes, Context.root(), null, true); + + assertTrue(serverContext.isHeadRequest()); + } + @Test void tracksBlockedResponseUntilChannelClose() { DefaultAttributeMap attributes = new DefaultAttributeMap(); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java index 0a31db6c483..cb204d7ef51 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java @@ -4,17 +4,24 @@ import static datadog.trace.agent.test.assertions.TraceMatcher.trace; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; import static io.netty.handler.codec.http.HttpResponseStatus.OK; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.context.Context; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.LastHttpContent; import org.junit.jupiter.api.Test; class HttpServerResponseTracingHandlerTest extends AbstractInstrumentationTest { @@ -34,4 +41,35 @@ void finishesMirroredContextWhenRequestQueueIsAbsent() { channel.finishAndReleaseAll(); assertTraces(trace(span().root().operationName("mirrored-http2-server"))); } + + @Test + void forwardsLastContentBeforeFinalResponseWhenItWasNotSyntheticTerminator() { + EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); + ServerRequestContext.add(channel, Context.root(), null); + LastHttpContent lastContent = new DefaultLastHttpContent(); + + assertTrue(channel.writeOutbound(lastContent)); + + LastHttpContent forwarded = channel.readOutbound(); + assertSame(lastContent, forwarded); + forwarded.release(); + channel.finishAndReleaseAll(); + } + + @Test + void doesNotThrowOnMalformedContentLength() { + EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); + AgentSpan span = startSpan("netty", "malformed-content-length-server"); + ServerRequestContext.add(channel, span, null); + FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, OK); + response.headers().set(CONTENT_LENGTH, "malformed"); + + assertDoesNotThrow(() -> assertTrue(channel.writeOutbound(response))); + + FullHttpResponse forwarded = channel.readOutbound(); + assertSame(response, forwarded); + forwarded.release(); + channel.finishAndReleaseAll(); + assertTraces(trace(span().root().operationName("malformed-content-length-server"))); + } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java index eae7b48697b..05d678fc99f 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -4,7 +4,13 @@ import static datadog.trace.agent.test.assertions.TraceAssertions.SORT_BY_START_TIME; import static datadog.trace.agent.test.assertions.TraceMatcher.trace; import static datadog.trace.api.gateway.Events.EVENTS; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpHeaderNames.TRANSFER_ENCODING; +import static io.netty.handler.codec.http.HttpHeaderValues.CHUNKED; +import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE; +import static io.netty.handler.codec.http.HttpResponseStatus.CONTINUE; +import static io.netty.handler.codec.http.HttpResponseStatus.NO_CONTENT; import static io.netty.handler.codec.http.HttpResponseStatus.OK; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; import static java.nio.charset.StandardCharsets.US_ASCII; @@ -38,8 +44,12 @@ import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultHttpResponse; +import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpServerCodec; import io.netty.util.ReferenceCountUtil; import java.io.ByteArrayOutputStream; @@ -65,6 +75,7 @@ public class NettyHttp11PipeliningTest extends AbstractInstrumentationTest { private static final String FIRST_PATH = "/pipelined/first"; private static final String SECOND_PATH = "/pipelined/second"; private static final String THIRD_PATH = "/pipelined/third"; + private static final HttpResponseStatus EARLY_HINTS = new HttpResponseStatus(103, "Early Hints"); private EventLoopGroup eventLoopGroup; private PipeliningHandler handler; @@ -163,6 +174,143 @@ void requestBlockOnLaterPipelinedRequestDoesNotOvertakeEarlierResponse() throws } } + @Test + void requestBlockOnLaterPipelinedRequestWaitsForEarlierChunkedResponseCompletion() + throws Exception { + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(request(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + CountDownLatch blockedRequestSeen = enableAppSecRequestBlockingFor(SECOND_PATH); + socket.getOutputStream().write(request(SECOND_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); + + handler.writeChunkedResponse(); + + assertEquals("response " + FIRST_PATH, readHttpChunkedResponseBody(socket.getInputStream())); + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), + "second response should be the deferred blocking response"); + } + } + + @Test + void requestBlockOnLaterPipelinedRequestFollowsEarlierHeaderOnlyResponse() throws Exception { + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(request(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + CountDownLatch blockedRequestSeen = enableAppSecRequestBlockingFor(SECOND_PATH); + socket.getOutputStream().write(request(SECOND_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); + + handler.writeHeaderOnlyResponse(); + + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 204 "), + "first response should be the header-only response"); + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), + "second response should be the deferred blocking response"); + } + } + + @Test + void requestBlockOnLaterPipelinedRequestFollowsEarlierHeadResponse() throws Exception { + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(headRequest(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + CountDownLatch blockedRequestSeen = enableAppSecRequestBlockingFor(SECOND_PATH); + socket.getOutputStream().write(request(SECOND_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); + + handler.writeHeadResponse(); + + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 200 "), + "first response should be the HEAD response"); + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), + "second response should be the deferred blocking response"); + } + } + + @Test + void lastContentAfterInterimResponseDoesNotCompleteServerSpan() throws Exception { + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(request(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + handler.writeInterimResponseWithTerminatorThenResponse(); + + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 100 "), + "first response should be the interim response"); + assertEquals("response " + FIRST_PATH, readHttpResponseBody(socket.getInputStream())); + } + } + + @Test + void requestBlockOnLaterPipelinedRequestWaitsForEarlierEarlyHintsResponseCompletion() + throws Exception { + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(request(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + CountDownLatch blockedRequestSeen = enableAppSecRequestBlockingFor(SECOND_PATH); + socket.getOutputStream().write(request(SECOND_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); + + handler.writeEarlyHintsWithTerminatorThenResponse(); + + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 103 "), + "first response should be the early hints response"); + assertEquals("response " + FIRST_PATH, readHttpResponseBody(socket.getInputStream())); + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), + "second response should be the deferred blocking response"); + } + } + @Test void blockResponseFunctionOnLaterPipelinedRequestDoesNotOvertakeEarlierResponse() throws Exception { @@ -208,6 +356,10 @@ private static String request(String path) { return "GET " + path + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; } + private static String headRequest(String path) { + return "HEAD " + path + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; + } + private CountDownLatch enableAppSecRequestBlockingFor(String blockedPath) { SubscriptionService subscriptions = (SubscriptionService) enableAppSec(); CountDownLatch blockedRequestSeen = new CountDownLatch(1); @@ -276,6 +428,36 @@ private static String readHttpResponseBody(InputStream in) throws IOException { return new String(body, UTF_8); } + private static String readHttpChunkedResponseBody(InputStream in) throws IOException { + String headers = readHttpResponseHeaders(in); + assertTrue(headers.startsWith("HTTP/1.1 200 "), "unexpected response: " + headers); + + ByteArrayOutputStream body = new ByteArrayOutputStream(); + while (true) { + String chunkSizeLine = readHttpLine(in); + int chunkSize = Integer.parseInt(chunkSizeLine, 16); + if (chunkSize == 0) { + String trailer; + do { + trailer = readHttpLine(in); + } while (!trailer.isEmpty()); + return body.toString(UTF_8.name()); + } + byte[] chunk = new byte[chunkSize]; + int read = 0; + while (read < chunkSize) { + int count = in.read(chunk, read, chunkSize - read); + if (count == -1) { + throw new EOFException("response ended before chunk was complete"); + } + read += count; + } + body.write(chunk); + String chunkTerminator = readHttpLine(in); + assertEquals("", chunkTerminator, "chunk was not followed by CRLF"); + } + } + private static String readHttpResponseHeaders(InputStream in) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); int state = 0; @@ -296,6 +478,23 @@ private static String readHttpResponseHeaders(InputStream in) throws IOException return out.toString(US_ASCII.name()); } + private static String readHttpLine(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int previous = -1; + while (true) { + int current = in.read(); + if (current == -1) { + throw new EOFException("response ended before line was complete"); + } + if (previous == '\r' && current == '\n') { + byte[] line = out.toByteArray(); + return new String(line, 0, line.length - 1, US_ASCII); + } + out.write(current); + previous = current; + } + } + private static int contentLength(String headers) { for (String line : headers.split("\r\n")) { int separator = line.indexOf(':'); @@ -397,5 +596,119 @@ private void writeResponses() { responseContext.flush(); }); } + + private void writeChunkedResponse() { + ChannelHandlerContext responseContext = context; + if (responseContext == null) { + throw new IllegalStateException("no request context captured"); + } + String path; + synchronized (paths) { + path = paths.get(0); + } + responseContext + .executor() + .execute( + () -> { + byte[] body = ("response " + path).getBytes(UTF_8); + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(TRANSFER_ENCODING, CHUNKED); + responseContext.write(response); + responseContext.write(new DefaultHttpContent(Unpooled.wrappedBuffer(body))); + responseContext.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER)); + responseContext.flush(); + }); + } + + private void writeHeaderOnlyResponse() { + ChannelHandlerContext responseContext = context; + if (responseContext == null) { + throw new IllegalStateException("no request context captured"); + } + responseContext + .executor() + .execute( + () -> { + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT); + response.headers().set(CONNECTION, KEEP_ALIVE); + responseContext.write(response); + responseContext.flush(); + }); + } + + private void writeHeadResponse() { + ChannelHandlerContext responseContext = context; + if (responseContext == null) { + throw new IllegalStateException("no request context captured"); + } + String path; + synchronized (paths) { + path = paths.get(0); + } + responseContext + .executor() + .execute( + () -> { + byte[] body = ("response " + path).getBytes(UTF_8); + DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONTENT_LENGTH, body.length); + responseContext.write(response); + responseContext.flush(); + }); + } + + private void writeInterimResponseWithTerminatorThenResponse() { + ChannelHandlerContext responseContext = context; + if (responseContext == null) { + throw new IllegalStateException("no request context captured"); + } + String path; + synchronized (paths) { + path = paths.get(0); + } + responseContext + .executor() + .execute( + () -> { + responseContext.write(new DefaultHttpResponse(HTTP_1_1, CONTINUE)); + responseContext.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER)); + + byte[] body = ("response " + path).getBytes(UTF_8); + DefaultFullHttpResponse response = + new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.wrappedBuffer(body)); + response.headers().set(CONTENT_LENGTH, body.length); + responseContext.write(response); + responseContext.flush(); + }); + } + + private void writeEarlyHintsWithTerminatorThenResponse() { + writeInformationalResponseWithTerminatorThenResponse(EARLY_HINTS); + } + + private void writeInformationalResponseWithTerminatorThenResponse(HttpResponseStatus status) { + ChannelHandlerContext responseContext = context; + if (responseContext == null) { + throw new IllegalStateException("no request context captured"); + } + String path; + synchronized (paths) { + path = paths.get(0); + } + responseContext + .executor() + .execute( + () -> { + responseContext.write(new DefaultHttpResponse(HTTP_1_1, status)); + responseContext.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER)); + + byte[] body = ("response " + path).getBytes(UTF_8); + DefaultFullHttpResponse response = + new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.wrappedBuffer(body)); + response.headers().set(CONTENT_LENGTH, body.length); + responseContext.write(response); + responseContext.flush(); + }); + } } } diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index a8927cad9ea..1d504e9aced 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -26,11 +26,21 @@ public static boolean canTrackRequest(final AttributeMap attributes) { /** Adds a request context to the queue tail. */ public static ServerRequestContext add( final AttributeMap attributes, final Context context, final String acceptHeader) { + return add(attributes, context, acceptHeader, false); + } + + /** Adds a request context to the queue tail. */ + public static ServerRequestContext add( + final AttributeMap attributes, + final Context context, + final String acceptHeader, + final boolean headRequest) { final Deque contexts = getOrCreate(attributes); if (!canAdd(attributes, contexts)) { return null; } - final ServerRequestContext serverContext = new ServerRequestContext(context, acceptHeader); + final ServerRequestContext serverContext = + new ServerRequestContext(context, acceptHeader, headRequest); contexts.addLast(serverContext); // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a // context mirror of the current inbound request used by @@ -182,6 +192,8 @@ private static boolean isPoisoned(final Deque contexts) { private final Context tracingContext; private final String acceptHeader; + private final boolean headRequest; + private boolean responseStarted; private boolean responseAnalyzed; private Object deferredBlockResponse; @@ -193,6 +205,18 @@ public String acceptHeader() { return acceptHeader; } + public boolean isHeadRequest() { + return headRequest; + } + + public boolean isResponseStarted() { + return responseStarted; + } + + public void markResponseStarted() { + responseStarted = true; + } + public boolean isResponseAnalyzed() { return responseAnalyzed; } @@ -209,8 +233,10 @@ public void deferBlockResponse(final Object deferredBlockResponse) { this.deferredBlockResponse = deferredBlockResponse; } - private ServerRequestContext(final Context tracingContext, final String acceptHeader) { + private ServerRequestContext( + final Context tracingContext, final String acceptHeader, final boolean headRequest) { this.tracingContext = tracingContext; this.acceptHeader = acceptHeader; + this.headRequest = headRequest; } } From 66f3ac8d850d247fc27808fb7501945cb69a7bee Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 22 Jul 2026 13:28:22 -0700 Subject: [PATCH 24/33] Fix Netty HTTP/2 response AppSec context handling Fall back to the mirrored channel context when HTTP/2 stream channels have no request context queue. Ignore informational responses until the final response and add regression coverage for response blocking. --- .../server/MaybeBlockResponseHandler.java | 24 +++- .../server/MaybeBlockResponseHandlerTest.java | 109 +++++++++++++++++- 2 files changed, 126 insertions(+), 7 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java index 068c55932e1..83fefa57b7d 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.netty41.server; +import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; import static io.netty.handler.codec.http.HttpHeaders.setContentLength; @@ -18,6 +19,7 @@ import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; @@ -45,7 +47,10 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr } ServerRequestContext serverContext = ServerRequestContext.nextResponse(channel); - Context storedContext = serverContext == null ? null : serverContext.tracingContext(); + Context storedContext = + serverContext == null + ? channel.attr(CONTEXT_ATTRIBUTE_KEY).get() + : serverContext.tracingContext(); AgentSpan span = AgentSpan.fromContext(storedContext); RequestContext requestContext; if (span == null || (requestContext = span.getRequestContext()) == null) { @@ -53,7 +58,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr return; } - if (serverContext.isResponseAnalyzed()) { + if (serverContext != null && serverContext.isResponseAnalyzed()) { super.write(ctx, msg, prm); return; } @@ -63,15 +68,21 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr return; } HttpResponse origResponse = (HttpResponse) msg; - if (origResponse.status().code() == HttpResponseStatus.CONTINUE.code()) { + int statusCode = origResponse.status().code(); + boolean isWebsocketUpgrade = + statusCode == HttpResponseStatus.SWITCHING_PROTOCOLS.code() + && "websocket".equals(origResponse.headers().get(HttpHeaderNames.UPGRADE)); + if (statusCode >= 100 && statusCode < 200 && !isWebsocketUpgrade) { super.write(ctx, msg, prm); return; } Flow flow = DECORATE.callIGCallbackResponseAndHeaders( - span, origResponse, origResponse.getStatus().code(), ResponseExtractAdapter.GETTER); - serverContext.markResponseAnalyzed(); + span, origResponse, statusCode, ResponseExtractAdapter.GETTER); + if (serverContext != null) { + serverContext.markResponseAnalyzed(); + } Flow.Action action = flow.getAction(); if (!(action instanceof Flow.Action.RequestBlockingAction)) { super.write(ctx, msg, prm); @@ -96,7 +107,8 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr BlockingContentType bct = rba.getBlockingContentType(); if (bct != BlockingContentType.NONE) { BlockingActionHelper.TemplateType type = - BlockingActionHelper.determineTemplateType(bct, serverContext.acceptHeader()); + BlockingActionHelper.determineTemplateType( + bct, serverContext == null ? null : serverContext.acceptHeader()); headers.set("Content-type", BlockingActionHelper.getContentType(type)); byte[] template = BlockingActionHelper.getTemplate(type, rba.getSecurityResponseId()); setContentLength(response, template.length); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java index 7ba3a2aabf1..e9ac472bfe6 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java @@ -1,17 +1,88 @@ package datadog.trace.instrumentation.netty41.server; +import static datadog.trace.api.gateway.Events.EVENTS; +import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; +import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; +import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import datadog.appsec.api.blocking.BlockingContentType; import datadog.context.Context; +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.api.function.TriConsumer; +import datadog.trace.api.gateway.Flow; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.gateway.SubscriptionService; +import datadog.trace.bootstrap.ActiveSubsystems; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpHeaders; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.util.ReferenceCountUtil; +import java.util.function.Function; +import java.util.function.Supplier; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -class MaybeBlockResponseHandlerTest { +class MaybeBlockResponseHandlerTest extends AbstractInstrumentationTest { + + private static final HttpResponseStatus EARLY_HINTS = new HttpResponseStatus(103, "Early Hints"); + + private Object appSecSubscriptions; + private boolean originalAppSecActive; + + @AfterEach + void resetAppSec() { + if (appSecSubscriptions != null) { + ((SubscriptionService) appSecSubscriptions).reset(); + appSecSubscriptions = null; + ActiveSubsystems.APPSEC_ACTIVE = originalAppSecActive; + } + } + + @Test + void blocksFinalResponseUsingMirroredContextAfterInformationalResponse() { + enableAppSecResponseBlocking(); + Context context = DECORATE.startSpan(new DefaultHttpHeaders(), Context.root()); + AgentSpan span = AgentSpan.fromContext(context); + EmbeddedChannel channel = new EmbeddedChannel(MaybeBlockResponseHandler.INSTANCE); + channel.attr(CONTEXT_ATTRIBUTE_KEY).set(context); + FullHttpResponse informationalResponse = null; + FullHttpResponse response = null; + + try { + channel.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, EARLY_HINTS)); + + informationalResponse = channel.readOutbound(); + assertNotNull(informationalResponse); + assertEquals(EARLY_HINTS, informationalResponse.status()); + informationalResponse.release(); + informationalResponse = null; + + channel.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK)); + + response = channel.readOutbound(); + assertNotNull(response); + assertEquals(FORBIDDEN, response.status()); + } finally { + ReferenceCountUtil.release(informationalResponse); + ReferenceCountUtil.release(response); + channel.finishAndReleaseAll(); + span.finish(); + } + } @Test void dropsWritesAfterBlockedContextHasBeenRemoved() { @@ -27,4 +98,40 @@ void dropsWritesAfterBlockedContextHasBeenRemoved() { assertNull(channel.readOutbound()); channel.finishAndReleaseAll(); } + + private void enableAppSecResponseBlocking() { + SubscriptionService subscriptions = + (SubscriptionService) AgentTracer.get().getSubscriptionService(RequestContextSlot.APPSEC); + appSecSubscriptions = subscriptions; + originalAppSecActive = ActiveSubsystems.APPSEC_ACTIVE; + ActiveSubsystems.APPSEC_ACTIVE = true; + + subscriptions.registerCallback( + EVENTS.requestStarted(), + new Supplier>() { + @Override + public Flow get() { + return new Flow.ResultFlow<>(new Object()); + } + }); + subscriptions.registerCallback( + EVENTS.responseHeader(), + new TriConsumer() { + @Override + public void accept(RequestContext requestContext, String name, String value) {} + }); + subscriptions.registerCallback( + EVENTS.responseHeaderDone(), + new Function>() { + @Override + public Flow apply(RequestContext requestContext) { + return new Flow.ResultFlow(null) { + @Override + public Action getAction() { + return new Action.RequestBlockingAction(403, BlockingContentType.AUTO); + } + }; + } + }); + } } From e1a31f3ee52ece219006d48efacf438a511543fc Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 22 Jul 2026 13:54:04 -0700 Subject: [PATCH 25/33] Fix Netty AppSec blocked response handling --- .../server/MaybeBlockResponseHandler.java | 10 ++--- .../server/MaybeBlockResponseHandlerTest.java | 37 ++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java index 83fefa57b7d..c9e18635dfe 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandler.java @@ -19,11 +19,11 @@ import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.util.ReferenceCountUtil; +import java.nio.channels.ClosedChannelException; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +43,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr // block further writes while the blocking response close is still asynchronous log.debug("Write suppressed; dropped outbound message"); ReferenceCountUtil.release(msg); + prm.tryFailure(new ClosedChannelException()); return; } @@ -69,10 +70,9 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise prm) thr } HttpResponse origResponse = (HttpResponse) msg; int statusCode = origResponse.status().code(); - boolean isWebsocketUpgrade = - statusCode == HttpResponseStatus.SWITCHING_PROTOCOLS.code() - && "websocket".equals(origResponse.headers().get(HttpHeaderNames.UPGRADE)); - if (statusCode >= 100 && statusCode < 200 && !isWebsocketUpgrade) { + if (statusCode >= 100 + && statusCode < 200 + && statusCode != HttpResponseStatus.SWITCHING_PROTOCOLS.code()) { super.write(ctx, msg, prm); return; } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java index e9ac472bfe6..2f98851a807 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java @@ -3,13 +3,16 @@ import static datadog.trace.api.gateway.Events.EVENTS; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; +import static io.netty.handler.codec.http.HttpHeaderNames.UPGRADE; import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpResponseStatus.SWITCHING_PROTOCOLS; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.appsec.api.blocking.BlockingContentType; import datadog.context.Context; @@ -25,12 +28,14 @@ import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.DefaultHttpHeaders; import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.util.ReferenceCountUtil; +import java.nio.channels.ClosedChannelException; import java.util.function.Function; import java.util.function.Supplier; import org.junit.jupiter.api.AfterEach; @@ -84,6 +89,32 @@ void blocksFinalResponseUsingMirroredContextAfterInformationalResponse() { } } + @Test + void blocksNonWebSocketSwitchingProtocolsResponse() { + enableAppSecResponseBlocking(); + Context context = DECORATE.startSpan(new DefaultHttpHeaders(), Context.root()); + AgentSpan span = AgentSpan.fromContext(context); + EmbeddedChannel channel = new EmbeddedChannel(MaybeBlockResponseHandler.INSTANCE); + channel.attr(CONTEXT_ATTRIBUTE_KEY).set(context); + FullHttpResponse response = null; + + try { + FullHttpResponse switchingProtocols = + new DefaultFullHttpResponse(HTTP_1_1, SWITCHING_PROTOCOLS); + switchingProtocols.headers().set(UPGRADE, "h2c"); + + channel.writeOutbound(switchingProtocols); + + response = channel.readOutbound(); + assertNotNull(response); + assertEquals(FORBIDDEN, response.status()); + } finally { + ReferenceCountUtil.release(response); + channel.finishAndReleaseAll(); + span.finish(); + } + } + @Test void dropsWritesAfterBlockedContextHasBeenRemoved() { EmbeddedChannel channel = new EmbeddedChannel(MaybeBlockResponseHandler.INSTANCE); @@ -91,10 +122,14 @@ void dropsWritesAfterBlockedContextHasBeenRemoved() { ServerRequestContext.markResponseBlocked(channel); ServerRequestContext.remove(channel, serverContext); ByteBuf lateResponseChunk = Unpooled.buffer().writeByte(1); + ChannelPromise promise = channel.newPromise(); - assertFalse(channel.writeOutbound(lateResponseChunk)); + channel.pipeline().write(lateResponseChunk, promise); assertEquals(0, lateResponseChunk.refCnt()); + assertTrue(promise.isDone()); + assertFalse(promise.isSuccess()); + assertTrue(promise.cause() instanceof ClosedChannelException); assertNull(channel.readOutbound()); channel.finishAndReleaseAll(); } From 78ebedf4863f9729829bdee6bd8611c4b8211891 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 23 Jul 2026 10:31:28 -0700 Subject: [PATCH 26/33] =?UTF-8?q?Align=20request-context=20completion=20wi?= =?UTF-8?q?th=20Netty=E2=80=99s=20message=20framing=20contract.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This removes HEAD-specific state, header parsing, and synthetic outbound messages while making pipelined response attribution follow Netty’s actual response boundaries. --- .../HttpServerRequestTracingHandler.java | 4 +- .../HttpServerResponseTracingHandler.java | 36 +----- .../netty41/ServerRequestContextTest.java | 10 -- .../HttpServerResponseTracingHandlerTest.java | 111 ++++++++++++++++-- .../server/NettyHttp11PipeliningTest.java | 42 +++++++ .../netty41/ServerRequestContext.java | 21 +--- 6 files changed, 146 insertions(+), 78 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 71cf78d6a87..bfa8c9b8338 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -14,7 +14,6 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; @ChannelHandler.Sharable @@ -49,8 +48,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { storedParentContext != null ? storedParentContext : DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); final ServerRequestContext serverContext = - ServerRequestContext.add( - channel, context, headers.get("accept"), HttpMethod.HEAD.equals(request.method())); + ServerRequestContext.add(channel, context, headers.get("accept")); try (final ContextScope ignored = context.attach()) { final AgentSpan span = AgentSpan.fromContext(context); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index b45c3767066..47aa25f7015 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -9,17 +9,13 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.websocket.HandlerContext; import datadog.trace.instrumentation.netty41.ServerRequestContext; -import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http.DefaultLastHttpContent; -import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpUtil; import io.netty.handler.codec.http.LastHttpContent; @ChannelHandler.Sharable @@ -56,7 +52,7 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann try (final ContextScope scope = storedContext.attach()) { final HttpResponse response = isResponse ? (HttpResponse) msg : null; - final boolean headerOnly = response != null && isHeaderOnly(response, serverContext); + final boolean responseComplete = msg instanceof LastHttpContent; try { ctx.write(msg, prm); @@ -67,10 +63,7 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann removeServerContext(ctx, serverContext); throw throwable; } - final boolean responseComplete; - if (response == null) { - responseComplete = true; - } else { + if (response != null) { final boolean isWebsocketUpgrade = response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS && "websocket".equals(response.headers().get(HttpHeaderNames.UPGRADE)); @@ -86,16 +79,12 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann serverContext.markResponseStarted(); } DECORATE.onResponse(span, response); - responseComplete = isWebsocketUpgrade || msg instanceof FullHttpResponse || headerOnly; } if (responseComplete) { DECORATE.beforeFinish(scope.context()); span.finish(); // Finish the span manually since finishSpanOnClose was false removeServerContext(ctx, serverContext); final ServerRequestContext nextResponse = ServerRequestContext.nextResponse(ctx.channel()); - if (headerOnly && !(msg instanceof FullHttpResponse) && nextResponse != null) { - ctx.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER), ctx.voidPromise()); - } BlockingResponseHandler.maybeWriteDeferredBlockResponse(ctx, nextResponse); } } @@ -110,29 +99,8 @@ private static void removeServerContext( } } - private static boolean isHeaderOnly( - final HttpResponse response, final ServerRequestContext serverContext) { - final int statusCode = response.status().code(); - return (serverContext != null && serverContext.isHeadRequest()) - || statusCode == HttpResponseStatus.NO_CONTENT.code() - || statusCode == HttpResponseStatus.RESET_CONTENT.code() - || statusCode == HttpResponseStatus.NOT_MODIFIED.code() - || (hasZeroContentLength(response) && !HttpUtil.isTransferEncodingChunked(response)); - } - private static boolean isInformational(final HttpResponse response) { final int statusCode = response.status().code(); return statusCode >= 100 && statusCode < 200; } - - private static boolean hasZeroContentLength(final HttpResponse response) { - if (!HttpUtil.isContentLengthSet(response)) { - return false; - } - try { - return HttpUtil.getContentLength(response) == 0; - } catch (final NumberFormatException ignored) { - return false; - } - } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index 684de404891..4cac8e6c1e8 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -41,16 +41,6 @@ void capturesAcceptHeaderValue() { assertEquals("text/html", serverContext.acceptHeader()); } - @Test - void capturesHeadRequest() { - DefaultAttributeMap attributes = new DefaultAttributeMap(); - - ServerRequestContext serverContext = - ServerRequestContext.add(attributes, Context.root(), null, true); - - assertTrue(serverContext.isHeadRequest()); - } - @Test void tracksBlockedResponseUntilChannelClose() { DefaultAttributeMap attributes = new DefaultAttributeMap(); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java index cb204d7ef51..656a082fcdc 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java @@ -4,24 +4,36 @@ import static datadog.trace.agent.test.assertions.TraceMatcher.trace; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; +import static datadog.trace.instrumentation.netty41.AttributeKeys.WEBSOCKET_SENDER_HANDLER_CONTEXT; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpHeaderNames.UPGRADE; +import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE; +import static io.netty.handler.codec.http.HttpResponseStatus.NO_CONTENT; import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpResponseStatus.SWITCHING_PROTOCOLS; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.context.Context; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.instrumentation.netty41.ServerRequestContext; +import io.netty.buffer.Unpooled; import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.EncoderException; import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpResponse; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseEncoder; import io.netty.handler.codec.http.LastHttpContent; +import io.netty.util.ReferenceCountUtil; import org.junit.jupiter.api.Test; class HttpServerResponseTracingHandlerTest extends AbstractInstrumentationTest { @@ -43,13 +55,14 @@ void finishesMirroredContextWhenRequestQueueIsAbsent() { } @Test - void forwardsLastContentBeforeFinalResponseWhenItWasNotSyntheticTerminator() { + void forwardsLastContentBeforeFinalResponseWithoutCompletingContext() { EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); - ServerRequestContext.add(channel, Context.root(), null); + ServerRequestContext serverContext = ServerRequestContext.add(channel, Context.root(), null); LastHttpContent lastContent = new DefaultLastHttpContent(); assertTrue(channel.writeOutbound(lastContent)); + assertSame(serverContext, ServerRequestContext.nextResponse(channel)); LastHttpContent forwarded = channel.readOutbound(); assertSame(lastContent, forwarded); forwarded.release(); @@ -57,19 +70,93 @@ void forwardsLastContentBeforeFinalResponseWhenItWasNotSyntheticTerminator() { } @Test - void doesNotThrowOnMalformedContentLength() { + void headerOnlyResponseDoesNotCompleteContextBeforeLastContent() { EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); - AgentSpan span = startSpan("netty", "malformed-content-length-server"); - ServerRequestContext.add(channel, span, null); - FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, OK); - response.headers().set(CONTENT_LENGTH, "malformed"); + AgentSpan span = startSpan("netty", "header-only-server"); + ServerRequestContext serverContext = ServerRequestContext.add(channel, span, null); + ServerRequestContext nextServerContext = + ServerRequestContext.add(channel, Context.root(), null); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT); - assertDoesNotThrow(() -> assertTrue(channel.writeOutbound(response))); + assertTrue(channel.writeOutbound(response)); - FullHttpResponse forwarded = channel.readOutbound(); + assertSame(serverContext, ServerRequestContext.nextResponse(channel)); + HttpResponse forwarded = channel.readOutbound(); assertSame(response, forwarded); - forwarded.release(); + ReferenceCountUtil.release(forwarded); + + assertTrue(channel.writeOutbound(LastHttpContent.EMPTY_LAST_CONTENT)); + + assertSame(nextServerContext, ServerRequestContext.nextResponse(channel)); + ReferenceCountUtil.release(channel.readOutbound()); + ServerRequestContext.remove(channel, nextServerContext); + channel.finishAndReleaseAll(); + assertTraces(trace(span().root().operationName("header-only-server"))); + } + + @Test + void rawFixedLengthBodyDoesNotCompleteContextBeforeLastContent() { + EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); + AgentSpan span = startSpan("netty", "raw-body-server"); + ServerRequestContext serverContext = ServerRequestContext.add(channel, span, null); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONTENT_LENGTH, 4); + + assertTrue(channel.writeOutbound(response)); + ReferenceCountUtil.release(channel.readOutbound()); + assertTrue(channel.writeOutbound(Unpooled.wrappedBuffer(new byte[] {1, 2, 3, 4}))); + + assertSame(serverContext, ServerRequestContext.nextResponse(channel)); + ReferenceCountUtil.release(channel.readOutbound()); + + assertTrue(channel.writeOutbound(LastHttpContent.EMPTY_LAST_CONTENT)); + + assertNull(ServerRequestContext.nextResponse(channel)); + ReferenceCountUtil.release(channel.readOutbound()); + channel.finishAndReleaseAll(); + assertTraces(trace(span().root().operationName("raw-body-server"))); + } + + @Test + void nettyEncoderRequiresLastContentBeforeNextKeepAliveResponse() { + EmbeddedChannel channel = new EmbeddedChannel(new HttpResponseEncoder()); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.headers().set(CONNECTION, KEEP_ALIVE); + response.headers().set(CONTENT_LENGTH, 4); + + assertTrue(channel.writeOutbound(response)); + ReferenceCountUtil.release(channel.readOutbound()); + assertTrue(channel.writeOutbound(Unpooled.wrappedBuffer(new byte[] {1, 2, 3, 4}))); + ReferenceCountUtil.release(channel.readOutbound()); + + EncoderException exception = + assertThrows( + EncoderException.class, + () -> channel.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK))); + + assertTrue(exception.getCause() instanceof IllegalStateException); + + assertTrue(channel.writeOutbound(LastHttpContent.EMPTY_LAST_CONTENT)); + ReferenceCountUtil.release(channel.readOutbound()); + assertTrue(channel.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK))); + ReferenceCountUtil.release(channel.readOutbound()); + channel.finishAndReleaseAll(); + } + + @Test + void fullWebsocketUpgradeCompletesContextAndPreservesHandshakeSpan() { + EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); + AgentSpan span = startSpan("netty", "websocket-handshake-server"); + ServerRequestContext.add(channel, span, null); + FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, SWITCHING_PROTOCOLS); + response.headers().set(UPGRADE, "websocket"); + + assertTrue(channel.writeOutbound(response)); + + assertNull(ServerRequestContext.nextResponse(channel)); + assertNotNull(channel.attr(WEBSOCKET_SENDER_HANDLER_CONTEXT).get()); + ReferenceCountUtil.release(channel.readOutbound()); channel.finishAndReleaseAll(); - assertTraces(trace(span().root().operationName("malformed-content-length-server"))); + assertTraces(trace(span().root().operationName("websocket-handshake-server"))); } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java index 05d678fc99f..fe78fa8d1c0 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -58,6 +58,7 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.net.Socket; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -68,6 +69,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import reactor.core.publisher.Mono; +import reactor.netty.DisposableServer; +import reactor.netty.http.server.HttpServer; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class NettyHttp11PipeliningTest extends AbstractInstrumentationTest { @@ -147,6 +151,42 @@ void createsServerSpanForEachPipelinedRequest() throws Exception { serverTrace(THIRD_PATH)); } + @Test + void reactorNettyCompletesPipelinedFixedLengthResponses() throws Exception { + DisposableServer reactorServer = + HttpServer.create() + .host("localhost") + .port(0) + .handle( + (request, response) -> { + String body = "response " + request.uri(); + response.header(CONTENT_LENGTH, Integer.toString(body.getBytes(UTF_8).length)); + return Mono.delay(Duration.ofMillis(100)) + .then(Mono.defer(() -> response.sendString(Mono.just(body)).then())); + }) + .bindNow(); + + try { + try (Socket socket = new Socket("localhost", reactorServer.port())) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(pipelinedRequests().getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertEquals("response " + FIRST_PATH, readHttpResponseBody(socket.getInputStream())); + assertEquals("response " + SECOND_PATH, readHttpResponseBody(socket.getInputStream())); + assertEquals("response " + THIRD_PATH, readHttpResponseBody(socket.getInputStream())); + } + + assertTraces( + SORT_BY_START_TIME, + serverTrace(FIRST_PATH), + serverTrace(SECOND_PATH), + serverTrace(THIRD_PATH)); + } finally { + reactorServer.disposeNow(); + } + } + @Test void requestBlockOnLaterPipelinedRequestDoesNotOvertakeEarlierResponse() throws Exception { handler.expectRequests(1); @@ -632,6 +672,7 @@ private void writeHeaderOnlyResponse() { DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT); response.headers().set(CONNECTION, KEEP_ALIVE); responseContext.write(response); + responseContext.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER)); responseContext.flush(); }); } @@ -653,6 +694,7 @@ private void writeHeadResponse() { DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); response.headers().set(CONTENT_LENGTH, body.length); responseContext.write(response); + responseContext.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER)); responseContext.flush(); }); } diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 1d504e9aced..84d0794a3b6 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -26,21 +26,11 @@ public static boolean canTrackRequest(final AttributeMap attributes) { /** Adds a request context to the queue tail. */ public static ServerRequestContext add( final AttributeMap attributes, final Context context, final String acceptHeader) { - return add(attributes, context, acceptHeader, false); - } - - /** Adds a request context to the queue tail. */ - public static ServerRequestContext add( - final AttributeMap attributes, - final Context context, - final String acceptHeader, - final boolean headRequest) { final Deque contexts = getOrCreate(attributes); if (!canAdd(attributes, contexts)) { return null; } - final ServerRequestContext serverContext = - new ServerRequestContext(context, acceptHeader, headRequest); + final ServerRequestContext serverContext = new ServerRequestContext(context, acceptHeader); contexts.addLast(serverContext); // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a // context mirror of the current inbound request used by @@ -192,7 +182,6 @@ private static boolean isPoisoned(final Deque contexts) { private final Context tracingContext; private final String acceptHeader; - private final boolean headRequest; private boolean responseStarted; private boolean responseAnalyzed; private Object deferredBlockResponse; @@ -205,10 +194,6 @@ public String acceptHeader() { return acceptHeader; } - public boolean isHeadRequest() { - return headRequest; - } - public boolean isResponseStarted() { return responseStarted; } @@ -233,10 +218,8 @@ public void deferBlockResponse(final Object deferredBlockResponse) { this.deferredBlockResponse = deferredBlockResponse; } - private ServerRequestContext( - final Context tracingContext, final String acceptHeader, final boolean headRequest) { + private ServerRequestContext(final Context tracingContext, final String acceptHeader) { this.tracingContext = tracingContext; this.acceptHeader = acceptHeader; - this.headRequest = headRequest; } } From ba58dc752eeb087400a12a82d81f5d4bab6919a6 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 23 Jul 2026 12:00:52 -0700 Subject: [PATCH 27/33] Reuse Netty blocking handler for deferred pipelined requests Mark channels when a request block is committed so later pipelined requests are forwarded to the existing blocking handler and discarded. Serialize response-function handler installation on the event loop and clean up partially installed handlers on failure. --- .../server/BlockingResponseHandler.java | 8 ++ .../HttpServerRequestTracingHandler.java | 8 +- .../server/NettyHttpServerDecorator.java | 57 ++++++-- .../netty41/ServerRequestContextTest.java | 15 ++ .../server/BlockingResponseHandlerTest.java | 136 ++++++++++++++++++ .../server/NettyHttp11PipeliningTest.java | 82 ++++++++++- .../netty41/ServerRequestContext.java | 14 ++ 7 files changed, 305 insertions(+), 15 deletions(-) create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java index 3903380a810..ee70de53651 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java @@ -24,6 +24,8 @@ public class BlockingResponseHandler extends ChannelInboundHandlerAdapter { public static final Logger log = LoggerFactory.getLogger(BlockingResponseHandler.class); + static final String BEFORE_BLOCKING_HANDLER_NAME = "before_blocking_handler"; + static final String HANDLER_NAME = "blocking_handler"; private static final String IGNORE_ALL_WRITES_HANDLER = "ignore_all_writes_handler"; private static final String MISSING_RESPONSE_TRACING_HANDLER_MESSAGE = "Unable to block because HttpServerResponseTracingHandler was not found on the pipeline"; @@ -85,6 +87,11 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { if (ctxForDownstream == null) { logMissingResponseTracingHandler(); + // Do not let a failed block intercept later requests on this keep-alive connection. + if (ctx.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME) != null) { + ctx.pipeline().remove(BEFORE_BLOCKING_HANDLER_NAME); + } + ctx.pipeline().remove(this); ctx.fireChannelRead(msg); return; } @@ -92,6 +99,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { HttpRequest request = (HttpRequest) msg; this.hasBlockedAlready = true; + ServerRequestContext.markRequestBlocked(ctx.channel()); PendingBlockResponse pendingBlockResponse = new PendingBlockResponse( diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index bfa8c9b8338..edbd549fe70 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -36,6 +36,12 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } final HttpRequest request = (HttpRequest) msg; + if (ServerRequestContext.isRequestBlocked(channel)) { + // A deferred block keeps its handler in the pipeline while an earlier response completes. + // Forward later pipelined requests to that handler instead of adding another one. + ctx.fireChannelRead(msg); + return; + } if (!ServerRequestContext.canTrackRequest(channel)) { channel.attr(PARENT_CONTEXT_ATTRIBUTE_KEY).remove(); ctx.fireChannelRead(msg); @@ -60,7 +66,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { ctx.pipeline() .addAfter( ctx.name(), - "blocking_handler", + BlockingResponseHandler.HANDLER_NAME, new BlockingResponseHandler( span.getRequestContext().getTraceSegment(), rba, serverContext)); } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java index a66fda5f11c..10334eea466 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java @@ -137,6 +137,32 @@ public boolean tryCommitBlockingResponse( BlockingContentType templateType, Map extraHeaders, String securityResponseId) { + if (pipeline.channel().eventLoop().inEventLoop()) { + return commitBlockingResponse( + segment, statusCode, templateType, extraHeaders, securityResponseId); + } + + try { + pipeline + .channel() + .eventLoop() + .execute( + () -> + commitBlockingResponse( + segment, statusCode, templateType, extraHeaders, securityResponseId)); + return true; + } catch (RuntimeException rte) { + log.warn("Failed scheduling blocking handler", rte); + return false; + } + } + + private boolean commitBlockingResponse( + TraceSegment segment, + int statusCode, + BlockingContentType templateType, + Map extraHeaders, + String securityResponseId) { ChannelHandler handlerBefore = pipeline.get(HttpServerTracingHandler.class); if (handlerBefore == null) { handlerBefore = pipeline.get(HttpServerRequestTracingHandler.class); @@ -147,29 +173,38 @@ public boolean tryCommitBlockingResponse( } } + BlockingResponseHandler blockingHandler = + new BlockingResponseHandler( + segment, statusCode, templateType, extraHeaders, securityResponseId, serverContext); + ChannelInboundHandlerAdapter beforeBlockingHandler = new ChannelInboundHandlerAdapter(); try { pipeline .addAfter( pipeline.context(handlerBefore).name(), - "blocking_handler", - new BlockingResponseHandler( - segment, - statusCode, - templateType, - extraHeaders, - securityResponseId, - serverContext)) - .addBefore( - "blocking_handler", "before_blocking_handler", new ChannelInboundHandlerAdapter()); + BlockingResponseHandler.BEFORE_BLOCKING_HANDLER_NAME, + beforeBlockingHandler) + .addAfter( + BlockingResponseHandler.BEFORE_BLOCKING_HANDLER_NAME, + BlockingResponseHandler.HANDLER_NAME, + blockingHandler); } catch (RuntimeException rte) { + removeHandlerIfPresent(beforeBlockingHandler); + removeHandlerIfPresent(blockingHandler); log.warn("Failed adding blocking handler", rte); return false; } - ChannelHandlerContext context = pipeline.context("before_blocking_handler"); + ChannelHandlerContext context = + pipeline.context(BlockingResponseHandler.BEFORE_BLOCKING_HANDLER_NAME); context.fireChannelRead(httpRequestMessage); return true; } + + private void removeHandlerIfPresent(ChannelHandler handler) { + if (pipeline.context(handler) != null) { + pipeline.remove(handler); + } + } } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index 4cac8e6c1e8..fbec38bdc57 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -55,4 +55,19 @@ void tracksBlockedResponseUntilChannelClose() { assertFalse(ServerRequestContext.isResponseBlocked(attributes)); } + + @Test + void tracksBlockedRequestUntilChannelClose() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + + assertFalse(ServerRequestContext.isRequestBlocked(attributes)); + + ServerRequestContext.markRequestBlocked(attributes); + + assertTrue(ServerRequestContext.isRequestBlocked(attributes)); + + ServerRequestContext.closeAll(attributes); + + assertFalse(ServerRequestContext.isRequestBlocked(attributes)); + } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java new file mode 100644 index 00000000000..859adc2c0fc --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java @@ -0,0 +1,136 @@ +package datadog.trace.instrumentation.netty41.server; + +import static datadog.trace.instrumentation.netty41.server.BlockingResponseHandler.BEFORE_BLOCKING_HANDLER_NAME; +import static datadog.trace.instrumentation.netty41.server.BlockingResponseHandler.HANDLER_NAME; +import static io.netty.handler.codec.http.HttpMethod.GET; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static java.util.Collections.emptyMap; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.appsec.api.blocking.BlockingContentType; +import datadog.context.Context; +import datadog.trace.instrumentation.netty41.ServerRequestContext; +import datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.NettyBlockResponseFunction; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.DefaultEventLoopGroup; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.channel.local.LocalChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.util.concurrent.Future; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +class BlockingResponseHandlerTest { + + @Test + void removesResponseFunctionHandlersWhenResponseTracingIsUnavailable() { + EmbeddedChannel channel = new EmbeddedChannel(); + ServerRequestContext serverContext = ServerRequestContext.add(channel, Context.root(), null); + BlockingResponseHandler blockingHandler = + new BlockingResponseHandler( + null, 403, BlockingContentType.NONE, emptyMap(), null, serverContext); + channel + .pipeline() + .addLast(BEFORE_BLOCKING_HANDLER_NAME, new ChannelInboundHandlerAdapter()) + .addLast(HANDLER_NAME, blockingHandler); + FullHttpRequest request = new DefaultFullHttpRequest(HTTP_1_1, GET, "/"); + + assertTrue(channel.writeInbound(request)); + + assertFalse(ServerRequestContext.isRequestBlocked(channel)); + assertNull(channel.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME)); + assertNull(channel.pipeline().get(HANDLER_NAME)); + assertSame(request, channel.readInbound()); + request.release(); + channel.finishAndReleaseAll(); + } + + @Test + void doesNotInstallBlockingHandlerWhenAuxiliaryHandlerNameIsUnavailable() { + ChannelInboundHandlerAdapter existingBeforeBlockingHandler = new ChannelInboundHandlerAdapter(); + EmbeddedChannel channel = new EmbeddedChannel(HttpServerRequestTracingHandler.INSTANCE); + channel.pipeline().addLast(BEFORE_BLOCKING_HANDLER_NAME, existingBeforeBlockingHandler); + FullHttpRequest request = new DefaultFullHttpRequest(HTTP_1_1, GET, "/"); + NettyBlockResponseFunction responseFunction = + new NettyBlockResponseFunction(channel.pipeline(), request, null); + + assertFalse( + responseFunction.tryCommitBlockingResponse( + null, 403, BlockingContentType.NONE, emptyMap(), null)); + + assertSame(existingBeforeBlockingHandler, channel.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME)); + assertNull(channel.pipeline().get(HANDLER_NAME)); + request.release(); + channel.finishAndReleaseAll(); + } + + @Test + void serializesOffEventLoopHandlerInstallationAndRequestReplay() throws Exception { + DefaultEventLoopGroup eventLoopGroup = new DefaultEventLoopGroup(1); + LocalChannel channel = new LocalChannel(); + FullHttpRequest request = new DefaultFullHttpRequest(HTTP_1_1, GET, "/"); + CountDownLatch eventLoopTaskStarted = new CountDownLatch(1); + CountDownLatch releaseEventLoop = new CountDownLatch(1); + + try { + eventLoopGroup.register(channel).sync(); + AtomicReference serverContext = new AtomicReference<>(); + channel + .eventLoop() + .submit( + () -> { + channel + .pipeline() + .addLast(HttpServerRequestTracingHandler.INSTANCE) + .addLast(HttpServerResponseTracingHandler.INSTANCE); + ServerRequestContext.add(channel, Context.root(), null); + serverContext.set(ServerRequestContext.add(channel, Context.root(), null)); + }) + .sync(); + + Future blockingTask = + channel + .eventLoop() + .submit( + () -> { + eventLoopTaskStarted.countDown(); + if (!releaseEventLoop.await(5, SECONDS)) { + throw new AssertionError("event loop was not released"); + } + return null; + }); + assertTrue(eventLoopTaskStarted.await(5, SECONDS), "event loop task did not start"); + + NettyBlockResponseFunction responseFunction = + new NettyBlockResponseFunction(channel.pipeline(), request, serverContext.get()); + assertTrue( + responseFunction.tryCommitBlockingResponse( + null, 403, BlockingContentType.NONE, emptyMap(), null)); + + assertNull(channel.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME)); + assertNull(channel.pipeline().get(HANDLER_NAME)); + + releaseEventLoop.countDown(); + blockingTask.sync(); + channel.eventLoop().submit(() -> {}).sync(); + + assertTrue(ServerRequestContext.isRequestBlocked(channel)); + assertNotNull(channel.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME)); + assertNotNull(channel.pipeline().get(HANDLER_NAME)); + } finally { + releaseEventLoop.countDown(); + channel.close().sync(); + eventLoopGroup.shutdownGracefully().sync(); + if (request.refCnt() > 0) { + request.release(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java index fe78fa8d1c0..70825de6453 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -18,6 +18,7 @@ import static java.util.Collections.emptyMap; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.appsec.api.blocking.BlockingContentType; @@ -38,6 +39,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelInitializer; import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; @@ -49,6 +51,7 @@ import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpServerCodec; import io.netty.util.ReferenceCountUtil; @@ -60,6 +63,7 @@ import java.net.Socket; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.function.Supplier; @@ -82,6 +86,7 @@ public class NettyHttp11PipeliningTest extends AbstractInstrumentationTest { private static final HttpResponseStatus EARLY_HINTS = new HttpResponseStatus(103, "Early Hints"); private EventLoopGroup eventLoopGroup; + private DecodedRequestObserver decodedRequestObserver; private PipeliningHandler handler; private int port; private Object appSecSubscriptions; @@ -90,6 +95,7 @@ public class NettyHttp11PipeliningTest extends AbstractInstrumentationTest { @BeforeAll void startServer() throws Exception { eventLoopGroup = new NioEventLoopGroup(); + decodedRequestObserver = new DecodedRequestObserver(); handler = new PipeliningHandler(); ServerBootstrap bootstrap = new ServerBootstrap() @@ -99,7 +105,13 @@ void startServer() throws Exception { new ChannelInitializer() { @Override protected void initChannel(Channel ch) { - ch.pipeline().addLast(new HttpServerCodec()); + HttpServerCodec codec = new HttpServerCodec(); + ch.pipeline().addLast(codec); + ch.pipeline() + .addAfter( + ch.pipeline().context(codec).name(), + "decoded-request-observer", + decodedRequestObserver); ch.pipeline().addLast(new HttpObjectAggregator(65536)); ch.pipeline().addLast(handler); } @@ -214,6 +226,42 @@ void requestBlockOnLaterPipelinedRequestDoesNotOvertakeEarlierResponse() throws } } + @Test + void additionalPipelinedRequestsBehindDeferredBlockAreIgnored() throws Exception { + decodedRequestObserver.expectRequests(3); + handler.expectRequests(1); + + try (Socket socket = new Socket("localhost", port)) { + socket.setSoTimeout(5000); + socket.getOutputStream().write(request(FIRST_PATH).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue(handler.awaitAllRequestsReceived(), "server did not receive first request"); + + CountDownLatch blockedRequestSeen = enableAppSecRequestBlockingFor(SECOND_PATH, THIRD_PATH); + socket + .getOutputStream() + .write((request(SECOND_PATH) + request(THIRD_PATH)).getBytes(US_ASCII)); + socket.getOutputStream().flush(); + + assertTrue( + blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); + assertTrue( + decodedRequestObserver.awaitAllRequests(), + "server did not decode all three pipelined requests before responding"); + + handler.writeResponses(); + + assertEquals("response " + FIRST_PATH, readHttpResponseBody(socket.getInputStream())); + assertTrue( + readHttpResponseHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), + "second response should be the deferred blocking response"); + assertNull( + handler.inboundException, + "additional pipelined requests should be swallowed by the existing blocking handler"); + } + } + @Test void requestBlockOnLaterPipelinedRequestWaitsForEarlierChunkedResponseCompletion() throws Exception { @@ -400,7 +448,7 @@ private static String headRequest(String path) { return "HEAD " + path + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; } - private CountDownLatch enableAppSecRequestBlockingFor(String blockedPath) { + private CountDownLatch enableAppSecRequestBlockingFor(String... blockedPaths) { SubscriptionService subscriptions = (SubscriptionService) enableAppSec(); CountDownLatch blockedRequestSeen = new CountDownLatch(1); @@ -410,7 +458,7 @@ private CountDownLatch enableAppSecRequestBlockingFor(String blockedPath) { @Override public Flow apply( RequestContext requestContext, String method, URIDataAdapter uri) { - if (!blockedPath.equals(uri.path())) { + if (!Arrays.asList(blockedPaths).contains(uri.path())) { return Flow.ResultFlow.empty(); } blockedRequestSeen.countDown(); @@ -545,6 +593,27 @@ private static int contentLength(String headers) { throw new AssertionError("missing content-length header: " + headers); } + @ChannelHandler.Sharable + private static final class DecodedRequestObserver extends ChannelInboundHandlerAdapter { + private volatile CountDownLatch receivedRequests = new CountDownLatch(0); + + private void expectRequests(int expectedRequests) { + receivedRequests = new CountDownLatch(expectedRequests); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + if (msg instanceof HttpRequest) { + receivedRequests.countDown(); + } + ctx.fireChannelRead(msg); + } + + private boolean awaitAllRequests() throws InterruptedException { + return receivedRequests.await(5, SECONDS); + } + } + @ChannelHandler.Sharable private static final class PipeliningHandler extends SimpleChannelInboundHandler { @@ -553,6 +622,7 @@ private static final class PipeliningHandler private volatile ChannelHandlerContext context; private volatile String blockResponseFunctionPath; private volatile CountDownLatch blockResponseFunctionRequestSeen; + private volatile Throwable inboundException; private PipeliningHandler() { super(false); @@ -562,6 +632,7 @@ private PipeliningHandler() { private void expectRequests(int expectedRequests) { receivedRequests = new CountDownLatch(expectedRequests); context = null; + inboundException = null; synchronized (paths) { paths.clear(); } @@ -609,6 +680,11 @@ protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) } } + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + inboundException = cause; + } + private boolean awaitAllRequestsReceived() throws InterruptedException { return receivedRequests.await(5, SECONDS); } diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 84d0794a3b6..aa5f91f18b7 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -56,6 +56,16 @@ public static ServerRequestContext currentRequest(final AttributeMap attributes) return contexts == null || isPoisoned(contexts) ? null : contexts.peekLast(); } + /** Returns whether a request block is closing the channel. */ + public static boolean isRequestBlocked(final AttributeMap attributes) { + return attributes.attr(BLOCKED_REQUEST_ATTRIBUTE_KEY).get() != null; + } + + /** Marks the channel as closing after an AppSec request block. */ + public static void markRequestBlocked(final AttributeMap attributes) { + attributes.attr(BLOCKED_REQUEST_ATTRIBUTE_KEY).set(Boolean.TRUE); + } + /** Returns whether the channel is closing after an AppSec response block. */ public static boolean isResponseBlocked(final AttributeMap attributes) { return attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).get() != null; @@ -105,6 +115,7 @@ public static void remove( public static void closeAll(final AttributeMap attributes) { // The context mirror must not outlive the authoritative request queue. attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); + attributes.attr(BLOCKED_REQUEST_ATTRIBUTE_KEY).remove(); attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).remove(); close(attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).getAndRemove()); } @@ -121,6 +132,9 @@ public static void closeAll(final AttributeMap attributes) { private static final AttributeKey BLOCKED_RESPONSE_ATTRIBUTE_KEY = AttributeKeys.attributeKey("datadog.server.blocked_response"); + private static final AttributeKey BLOCKED_REQUEST_ATTRIBUTE_KEY = + AttributeKeys.attributeKey("datadog.server.blocked_request"); + private static final Deque POISONED_CONTEXTS = new ArrayDeque<>(0); /** Creates the per-channel server request context queue. */ From b8216d35e38836b114604c4e8e81ad29b6d8c6c9 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 23 Jul 2026 12:23:59 -0700 Subject: [PATCH 28/33] Fix Netty blocking response scheduling races Snapshot request metadata before scheduling blocking responses instead of retaining and replaying reference-counted HTTP requests. Reject scheduled blocking responses when their server request context has already completed, preventing stale commits from blocking keep-alive connections. --- .../server/BlockingResponseHandler.java | 34 ++++++---- .../server/NettyHttpServerDecorator.java | 24 +++++-- .../netty41/ServerRequestContextTest.java | 12 ++++ .../server/BlockingResponseHandlerTest.java | 68 ++++++++++++++++++- .../netty41/ServerRequestContext.java | 11 +++ 5 files changed, 128 insertions(+), 21 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java index ee70de53651..1ca8f74b3ae 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java @@ -79,14 +79,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { return; } - ChannelHandlerContext ctxForDownstream = - ctx.pipeline().context(HttpServerResponseTracingHandler.class); - if (ctxForDownstream == null) { - ctxForDownstream = ctx.pipeline().context(HttpServerTracingHandler.class); - } - - if (ctxForDownstream == null) { - logMissingResponseTracingHandler(); + HttpRequest request = (HttpRequest) msg; + if (!commitBlockingResponse(ctx, request.protocolVersion(), request.headers().get("accept"))) { // Do not let a failed block intercept later requests on this keep-alive connection. if (ctx.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME) != null) { ctx.pipeline().remove(BEFORE_BLOCKING_HANDLER_NAME); @@ -96,8 +90,22 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { return; } - HttpRequest request = (HttpRequest) msg; + ReferenceCountUtil.release(msg); + } + boolean commitBlockingResponse( + final ChannelHandlerContext ctx, + final HttpVersion protocolVersion, + final String acceptHeader) { + ChannelHandlerContext ctxForDownstream = + ctx.pipeline().context(HttpServerResponseTracingHandler.class); + if (ctxForDownstream == null) { + ctxForDownstream = ctx.pipeline().context(HttpServerTracingHandler.class); + } + if (ctxForDownstream == null) { + logMissingResponseTracingHandler(); + return false; + } this.hasBlockedAlready = true; ServerRequestContext.markRequestBlocked(ctx.channel()); @@ -108,17 +116,17 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { bct, extraHeaders, securityResponseId, - request.protocolVersion(), - request.headers().get("accept")); - ReferenceCountUtil.release(msg); + protocolVersion, + acceptHeader); if (serverContext != null && ServerRequestContext.nextResponse(ctx.channel()) != serverContext) { serverContext.deferBlockResponse(pendingBlockResponse); - return; + return true; } writeBlockResponse(ctxForDownstream, pendingBlockResponse); + return true; } private static void logMissingResponseTracingHandler() { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java index 10334eea466..81d94995d05 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/NettyHttpServerDecorator.java @@ -19,6 +19,7 @@ import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpVersion; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Map; @@ -118,7 +119,8 @@ public static class NettyBlockResponseFunction implements BlockResponseFunction public static final Logger log = LoggerFactory.getLogger(NettyBlockResponseFunction.class); private final ChannelPipeline pipeline; - private final HttpRequest httpRequestMessage; + private final HttpVersion protocolVersion; + private final String acceptHeader; private final ServerRequestContext serverContext; public NettyBlockResponseFunction( @@ -126,7 +128,8 @@ public NettyBlockResponseFunction( HttpRequest httpRequestMessage, ServerRequestContext serverContext) { this.pipeline = pipeline; - this.httpRequestMessage = httpRequestMessage; + this.protocolVersion = httpRequestMessage.protocolVersion(); + this.acceptHeader = httpRequestMessage.headers().get("accept"); this.serverContext = serverContext; } @@ -163,6 +166,11 @@ private boolean commitBlockingResponse( BlockingContentType templateType, Map extraHeaders, String securityResponseId) { + if (serverContext != null + && !ServerRequestContext.isPending(pipeline.channel(), serverContext)) { + return false; + } + ChannelHandler handlerBefore = pipeline.get(HttpServerTracingHandler.class); if (handlerBefore == null) { handlerBefore = pipeline.get(HttpServerRequestTracingHandler.class); @@ -194,11 +202,13 @@ private boolean commitBlockingResponse( return false; } - ChannelHandlerContext context = - pipeline.context(BlockingResponseHandler.BEFORE_BLOCKING_HANDLER_NAME); - context.fireChannelRead(httpRequestMessage); - - return true; + ChannelHandlerContext context = pipeline.context(BlockingResponseHandler.HANDLER_NAME); + if (blockingHandler.commitBlockingResponse(context, protocolVersion, acceptHeader)) { + return true; + } + removeHandlerIfPresent(beforeBlockingHandler); + removeHandlerIfPresent(blockingHandler); + return false; } private void removeHandlerIfPresent(ChannelHandler handler) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index fbec38bdc57..a532735d086 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -41,6 +41,18 @@ void capturesAcceptHeaderValue() { assertEquals("text/html", serverContext.acceptHeader()); } + @Test + void reportsOnlyQueuedContextsAsPending() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + ServerRequestContext serverContext = ServerRequestContext.add(attributes, Context.root(), null); + + assertTrue(ServerRequestContext.isPending(attributes, serverContext)); + + ServerRequestContext.remove(attributes, serverContext); + + assertFalse(ServerRequestContext.isPending(attributes, serverContext)); + } + @Test void tracksBlockedResponseUntilChannelClose() { DefaultAttributeMap attributes = new DefaultAttributeMap(); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java index 859adc2c0fc..8600402e5b9 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandlerTest.java @@ -72,7 +72,7 @@ void doesNotInstallBlockingHandlerWhenAuxiliaryHandlerNameIsUnavailable() { } @Test - void serializesOffEventLoopHandlerInstallationAndRequestReplay() throws Exception { + void commitsOffEventLoopAfterOriginalRequestIsReleased() throws Exception { DefaultEventLoopGroup eventLoopGroup = new DefaultEventLoopGroup(1); LocalChannel channel = new LocalChannel(); FullHttpRequest request = new DefaultFullHttpRequest(HTTP_1_1, GET, "/"); @@ -113,6 +113,7 @@ void serializesOffEventLoopHandlerInstallationAndRequestReplay() throws Exceptio assertTrue( responseFunction.tryCommitBlockingResponse( null, 403, BlockingContentType.NONE, emptyMap(), null)); + request.release(); assertNull(channel.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME)); assertNull(channel.pipeline().get(HANDLER_NAME)); @@ -133,4 +134,69 @@ void serializesOffEventLoopHandlerInstallationAndRequestReplay() throws Exceptio } } } + + @Test + void doesNotBlockChannelWhenScheduledRequestContextCompletesFirst() throws Exception { + DefaultEventLoopGroup eventLoopGroup = new DefaultEventLoopGroup(1); + LocalChannel channel = new LocalChannel(); + FullHttpRequest request = new DefaultFullHttpRequest(HTTP_1_1, GET, "/"); + CountDownLatch eventLoopTaskStarted = new CountDownLatch(1); + CountDownLatch releaseEventLoop = new CountDownLatch(1); + + try { + eventLoopGroup.register(channel).sync(); + AtomicReference serverContext = new AtomicReference<>(); + channel + .eventLoop() + .submit( + () -> { + channel + .pipeline() + .addLast(HttpServerRequestTracingHandler.INSTANCE) + .addLast(HttpServerResponseTracingHandler.INSTANCE); + serverContext.set(ServerRequestContext.add(channel, Context.root(), null)); + }) + .sync(); + + Future blockingTask = + channel + .eventLoop() + .submit( + () -> { + eventLoopTaskStarted.countDown(); + if (!releaseEventLoop.await(5, SECONDS)) { + throw new AssertionError("event loop was not released"); + } + return null; + }); + assertTrue(eventLoopTaskStarted.await(5, SECONDS), "event loop task did not start"); + + Future completionTask = + channel + .eventLoop() + .submit(() -> ServerRequestContext.remove(channel, serverContext.get())); + NettyBlockResponseFunction responseFunction = + new NettyBlockResponseFunction(channel.pipeline(), request, serverContext.get()); + assertTrue( + responseFunction.tryCommitBlockingResponse( + null, 403, BlockingContentType.NONE, emptyMap(), null)); + request.release(); + + releaseEventLoop.countDown(); + blockingTask.sync(); + completionTask.sync(); + channel.eventLoop().submit(() -> {}).sync(); + + assertFalse(ServerRequestContext.isRequestBlocked(channel)); + assertNull(channel.pipeline().get(BEFORE_BLOCKING_HANDLER_NAME)); + assertNull(channel.pipeline().get(HANDLER_NAME)); + } finally { + releaseEventLoop.countDown(); + channel.close().sync(); + eventLoopGroup.shutdownGracefully().sync(); + if (request.refCnt() > 0) { + request.release(); + } + } + } } diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index aa5f91f18b7..0f8c8a43869 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -56,6 +56,17 @@ public static ServerRequestContext currentRequest(final AttributeMap attributes) return contexts == null || isPoisoned(contexts) ? null : contexts.peekLast(); } + /** Returns whether the request context is still awaiting a response on this channel. */ + public static boolean isPending( + final AttributeMap attributes, final ServerRequestContext serverContext) { + if (serverContext == null) { + return false; + } + final Deque contexts = + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).get(); + return contexts != null && !isPoisoned(contexts) && contexts.contains(serverContext); + } + /** Returns whether a request block is closing the channel. */ public static boolean isRequestBlocked(final AttributeMap attributes) { return attributes.attr(BLOCKED_REQUEST_ATTRIBUTE_KEY).get() != null; From 7dd79a76df8a3a29a733405770e6b63ebea9b210 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 23 Jul 2026 13:33:00 -0700 Subject: [PATCH 29/33] Reuse Netty request context queues for keep-alive traffic --- .../netty41/ServerRequestContextTest.java | 27 +++++++++++++++++++ .../netty41/ServerRequestContext.java | 19 ++++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index a532735d086..b73482f4113 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.context.Context; @@ -53,6 +54,32 @@ void reportsOnlyQueuedContextsAsPending() { assertFalse(ServerRequestContext.isPending(attributes, serverContext)); } + @Test + void preservesPipelinedContextWhenCompactingQueue() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + ServerRequestContext first = ServerRequestContext.add(attributes, Context.root(), null); + ServerRequestContext second = ServerRequestContext.add(attributes, Context.root(), null); + + ServerRequestContext.remove(attributes, first); + + assertSame(second, ServerRequestContext.nextResponse(attributes)); + ServerRequestContext.remove(attributes, second); + + ServerRequestContext next = ServerRequestContext.add(attributes, Context.root(), null); + assertSame(next, ServerRequestContext.nextResponse(attributes)); + } + + @Test + void preservesFirstContextWhenCompactingAfterRequestFailure() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + ServerRequestContext first = ServerRequestContext.add(attributes, Context.root(), null); + ServerRequestContext failed = ServerRequestContext.add(attributes, Context.root(), null); + + ServerRequestContext.remove(attributes, failed); + + assertSame(first, ServerRequestContext.nextResponse(attributes)); + } + @Test void tracksBlockedResponseUntilChannelClose() { DefaultAttributeMap attributes = new DefaultAttributeMap(); diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 0f8c8a43869..57c1f54e12e 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -99,22 +99,27 @@ public static void remove( if (isPoisoned(contexts)) { return; } + final boolean removed; if (contexts.peekFirst() == serverContext) { // Response completion consumes the queue head. contexts.pollFirst(); + removed = true; } else { // Request-side failures normally remove the tail. Remove by value to cover later cleanup // after additional pipelined requests were queued. - contexts.remove(serverContext); + removed = contexts.remove(serverContext); } final ServerRequestContext currentContext = contexts.peekLast(); if (currentContext == null) { attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); - // No request-matching state remains. Drop the empty per-channel queue so idle - // keep-alive or upgraded channels do not retain it after the last HTTP request; - // getOrCreate will recreate it lazily if another request arrives. - attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).remove(); } else { + if (removed && contexts.size() == 1) { + // An expanded pipeline must drain through one pending request. Replace its potentially + // large backing array while retaining a compact queue for later keep-alive requests. + final Deque compactContexts = new ArrayDeque<>(1); + compactContexts.addLast(currentContext); + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).set(compactContexts); + } // Keep the context mirror pointed at the current inbound request after removing an older // response context. It may still be copied to a new HTTP/2 stream channel. attributes.attr(CONTEXT_ATTRIBUTE_KEY).set(currentContext.tracingContext()); @@ -155,7 +160,9 @@ private static Deque getOrCreate(final AttributeMap attrib if (contexts == null) { // Netty serializes handler callbacks for a channel on its EventLoop, so this queue does not // need the allocation and atomic overhead of a concurrent collection. - contexts = new ArrayDeque<>(); + // Most keep-alive traffic is sequential, so retain a queue sized for one request until the + // channel closes rather than reallocating it for every request. + contexts = new ArrayDeque<>(1); attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).set(contexts); } return contexts; From b81803e4b67258c2dcce952822b349d349b59c65 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 23 Jul 2026 13:33:00 -0700 Subject: [PATCH 30/33] Reuse Netty request context queues for keep-alive traffic --- .../server/BlockingResponseHandler.java | 4 ++- .../netty41/ServerRequestContextTest.java | 27 +++++++++++++++++++ .../netty41/ServerRequestContext.java | 19 ++++++++----- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java index 1ca8f74b3ae..bb4c926db79 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java @@ -38,7 +38,9 @@ public class BlockingResponseHandler extends ChannelInboundHandlerAdapter { private final String securityResponseId; private final ServerRequestContext serverContext; - private boolean hasBlockedAlready; + // Current callers are event-loop confined; volatile preserves visibility if + // commitBlockingResponse is invoked from another thread. + private volatile boolean hasBlockedAlready; public BlockingResponseHandler( TraceSegment segment, diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index a532735d086..b73482f4113 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.context.Context; @@ -53,6 +54,32 @@ void reportsOnlyQueuedContextsAsPending() { assertFalse(ServerRequestContext.isPending(attributes, serverContext)); } + @Test + void preservesPipelinedContextWhenCompactingQueue() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + ServerRequestContext first = ServerRequestContext.add(attributes, Context.root(), null); + ServerRequestContext second = ServerRequestContext.add(attributes, Context.root(), null); + + ServerRequestContext.remove(attributes, first); + + assertSame(second, ServerRequestContext.nextResponse(attributes)); + ServerRequestContext.remove(attributes, second); + + ServerRequestContext next = ServerRequestContext.add(attributes, Context.root(), null); + assertSame(next, ServerRequestContext.nextResponse(attributes)); + } + + @Test + void preservesFirstContextWhenCompactingAfterRequestFailure() { + DefaultAttributeMap attributes = new DefaultAttributeMap(); + ServerRequestContext first = ServerRequestContext.add(attributes, Context.root(), null); + ServerRequestContext failed = ServerRequestContext.add(attributes, Context.root(), null); + + ServerRequestContext.remove(attributes, failed); + + assertSame(first, ServerRequestContext.nextResponse(attributes)); + } + @Test void tracksBlockedResponseUntilChannelClose() { DefaultAttributeMap attributes = new DefaultAttributeMap(); diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 0f8c8a43869..57c1f54e12e 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -99,22 +99,27 @@ public static void remove( if (isPoisoned(contexts)) { return; } + final boolean removed; if (contexts.peekFirst() == serverContext) { // Response completion consumes the queue head. contexts.pollFirst(); + removed = true; } else { // Request-side failures normally remove the tail. Remove by value to cover later cleanup // after additional pipelined requests were queued. - contexts.remove(serverContext); + removed = contexts.remove(serverContext); } final ServerRequestContext currentContext = contexts.peekLast(); if (currentContext == null) { attributes.attr(CONTEXT_ATTRIBUTE_KEY).remove(); - // No request-matching state remains. Drop the empty per-channel queue so idle - // keep-alive or upgraded channels do not retain it after the last HTTP request; - // getOrCreate will recreate it lazily if another request arrives. - attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).remove(); } else { + if (removed && contexts.size() == 1) { + // An expanded pipeline must drain through one pending request. Replace its potentially + // large backing array while retaining a compact queue for later keep-alive requests. + final Deque compactContexts = new ArrayDeque<>(1); + compactContexts.addLast(currentContext); + attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).set(compactContexts); + } // Keep the context mirror pointed at the current inbound request after removing an older // response context. It may still be copied to a new HTTP/2 stream channel. attributes.attr(CONTEXT_ATTRIBUTE_KEY).set(currentContext.tracingContext()); @@ -155,7 +160,9 @@ private static Deque getOrCreate(final AttributeMap attrib if (contexts == null) { // Netty serializes handler callbacks for a channel on its EventLoop, so this queue does not // need the allocation and atomic overhead of a concurrent collection. - contexts = new ArrayDeque<>(); + // Most keep-alive traffic is sequential, so retain a queue sized for one request until the + // channel closes rather than reallocating it for every request. + contexts = new ArrayDeque<>(1); attributes.attr(SERVER_REQUEST_CONTEXTS_ATTRIBUTE_KEY).set(contexts); } return contexts; From ffb560daac8529ef5f8e43dcceda1ebc68e6c445 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 23 Jul 2026 16:02:28 -0700 Subject: [PATCH 31/33] Align Netty response completion with LastHttpContent Finish server spans and dequeue pipelined request contexts only when Netty writes LastHttpContent. Use the terminal write promise so asynchronous failures retain error reporting and request context. Remove Content-Length, bodyless-response, request-method, and close-delimited completion inference, along with synthetic terminators and the HTTP/2 marker used by that model. Preserve channel-close cleanup for responses that never reach terminal content. Update pipelining, AppSec, HTTP/2, framing, and asynchronous write tests for the terminal-content contract. --- .../NettyChannelPipelineInstrumentation.java | 2 - .../server/BlockingResponseHandler.java | 4 +- .../HttpServerRequestTracingHandler.java | 10 +- .../HttpServerResponseTracingHandler.java | 211 +++--------------- .../netty41/ServerRequestContextTest.java | 9 +- .../HttpServerResponseTracingHandlerTest.java | 57 +++-- .../server/MaybeBlockResponseHandlerTest.java | 3 +- .../server/NettyChunkedResponseSpanTest.java | 125 +++++++++-- .../server/NettyHttp11PipeliningTest.java | 27 ++- .../netty41/AttributeKeys.java | 3 - .../netty41/ServerRequestContext.java | 88 +------- 11 files changed, 212 insertions(+), 327 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java index c48a5d55812..6fc2bc06f10 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/NettyChannelPipelineInstrumentation.java @@ -8,7 +8,6 @@ import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.rootContext; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONNECT_PARENT_CONTINUATION_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.HTTP2_CONNECTION_CODEC_ATTRIBUTE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.HTTP2_STREAM_CODEC_ATTRIBUTE_KEY; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.returns; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -239,7 +238,6 @@ else if (handler instanceof HttpClientCodec) { pipeline, handler, HttpClientResponseTracingHandler.INSTANCE); } else if (NettyHttp2Helper.isHttp2FrameCodec(handler)) { if (NettyHttp2Helper.isServer(handler)) { - pipeline.channel().attr(HTTP2_STREAM_CODEC_ATTRIBUTE_KEY).set(Boolean.TRUE); NettyPipelineHelper.addHandlerAfter( pipeline, handler, diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java index 1ca8f74b3ae..bb4c926db79 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/BlockingResponseHandler.java @@ -38,7 +38,9 @@ public class BlockingResponseHandler extends ChannelInboundHandlerAdapter { private final String securityResponseId; private final ServerRequestContext serverContext; - private boolean hasBlockedAlready; + // Current callers are event-loop confined; volatile preserves visibility if + // commitBlockingResponse is invoked from another thread. + private volatile boolean hasBlockedAlready; public BlockingResponseHandler( TraceSegment segment, diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java index 79b127c5fa9..2d61271ddc0 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerRequestTracingHandler.java @@ -20,8 +20,6 @@ @ChannelHandler.Sharable public class HttpServerRequestTracingHandler extends ChannelInboundHandlerAdapter { public static HttpServerRequestTracingHandler INSTANCE = new HttpServerRequestTracingHandler(); - private static final String INCOMPLETE_RESPONSE_MESSAGE = - "Channel closed before response completed"; @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { @@ -57,7 +55,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { storedParentContext != null ? storedParentContext : DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); final ServerRequestContext serverContext = - ServerRequestContext.add(channel, context, headers, request.method()); + ServerRequestContext.add(channel, context, headers.get("accept")); try (final ContextScope ignored = context.attach()) { final AgentSpan span = AgentSpan.fromContext(context); @@ -122,12 +120,6 @@ private static void finishSpanOnChannelClose(final ServerRequestContext serverCo return; } try (final ContextScope ignored = storedContext.attach()) { - if (!serverContext.isResponseCloseDelimited()) { - // The response declared a body via Content-Length or chunked encoding, but the channel - // closed before that body completed. Close-delimited responses, in contrast, end normally - // when the connection closes. - DECORATE.onError(span, new IllegalStateException(INCOMPLETE_RESPONSE_MESSAGE)); - } if (!serverContext.isBeforeFinishCalled()) { serverContext.markBeforeFinishCalled(); DECORATE.beforeFinish(storedContext); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java index ff83d481a2d..9f330301f28 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandler.java @@ -1,7 +1,6 @@ package datadog.trace.instrumentation.netty41.server; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.HTTP2_STREAM_CODEC_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.WEBSOCKET_SENDER_HANDLER_CONTEXT; import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; @@ -10,20 +9,14 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.websocket.HandlerContext; import datadog.trace.instrumentation.netty41.ServerRequestContext; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufHolder; -import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; -import io.netty.channel.FileRegion; -import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaderValues; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpUtil; import io.netty.handler.codec.http.LastHttpContent; import io.netty.util.concurrent.Future; @@ -33,7 +26,19 @@ public class HttpServerResponseTracingHandler extends ChannelOutboundHandlerAdap @Override public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise prm) { + final boolean isResponse = msg instanceof HttpResponse; + final boolean isLastContent = msg instanceof LastHttpContent; + if (!isResponse && !isLastContent) { + ctx.write(msg, prm); + return; + } + final ServerRequestContext serverContext = ServerRequestContext.nextResponse(ctx.channel()); + if (!isResponse && serverContext != null && !serverContext.isResponseStarted()) { + ctx.write(msg, prm); + return; + } + final Context storedContext = serverContext == null // HTTP/2 multiplex stream channels only inherit the mirrored context attribute from @@ -48,30 +53,18 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann return; } - if (serverContext == null) { - writeMirroredResponse(ctx, msg, prm, storedContext, span); - return; - } - - if (!(msg instanceof HttpResponse) && !serverContext.isResponseStarted()) { - ctx.write(msg, prm); - return; - } - try (final ContextScope ignored = storedContext.attach()) { - final boolean terminalResponse = isTerminalResponse(ctx, span, serverContext, msg); - final boolean knownLengthResponseComplete = - !terminalResponse && serverContext.recordResponseBodyBytes(responseBodyBytes(msg)); - final boolean writeSyntheticLastContent = - !(msg instanceof LastHttpContent) - && (knownLengthResponseComplete - || (msg instanceof HttpResponse - && isHeaderOnly((HttpResponse) msg, serverContext) - && !isWebsocketUpgrade((HttpResponse) msg))); - final boolean finishResponseOnWrite = terminalResponse || knownLengthResponseComplete; + final HttpResponse response = isResponse ? (HttpResponse) msg : null; + final boolean websocketUpgrade = response != null && isWebsocketUpgrade(response); + final boolean informationalResponse = + response != null && isInformationalResponse(response) && !websocketUpgrade; + final boolean finishResponseOnWrite = isLastContent && !informationalResponse; final ChannelPromise writePromise = finishResponseOnWrite && prm.isVoid() ? ctx.newPromise() : prm; try { + if (response != null && !informationalResponse) { + onResponse(ctx, span, serverContext, response, websocketUpgrade); + } if (finishResponseOnWrite) { removeServerContext(ctx, serverContext); writePromise.addListener( @@ -81,9 +74,6 @@ && isHeaderOnly((HttpResponse) msg, serverContext) if (finishResponseOnWrite && (!writePromise.isDone() || writePromise.isSuccess())) { final ServerRequestContext nextResponse = ServerRequestContext.nextResponse(ctx.channel()); - if (writeSyntheticLastContent && nextResponse != null) { - ctx.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER), ctx.voidPromise()); - } BlockingResponseHandler.maybeWriteDeferredBlockResponse(ctx, nextResponse); } } catch (final Throwable throwable) { @@ -100,88 +90,21 @@ && isHeaderOnly((HttpResponse) msg, serverContext) } } - private static void writeMirroredResponse( - final ChannelHandlerContext ctx, - final Object msg, - final ChannelPromise prm, - final Context storedContext, - final AgentSpan span) { - if (!(msg instanceof HttpResponse)) { - ctx.write(msg, prm); - return; - } - - try (final ContextScope ignored = storedContext.attach()) { - final HttpResponse response = (HttpResponse) msg; - final boolean isWebsocketUpgrade = isWebsocketUpgrade(response); - - try { - ctx.write(msg, prm); - } catch (final Throwable throwable) { - DECORATE.onError(span, throwable); - span.setHttpStatusCode(500); - span.finish(); // Finish the span manually since finishSpanOnClose was false - removeServerContext(ctx, null); - throw throwable; - } - if (isInformationalResponse(response) && !isWebsocketUpgrade) { - return; - } - if (isWebsocketUpgrade) { - ctx.channel() - .attr(WEBSOCKET_SENDER_HANDLER_CONTEXT) - .set(new HandlerContext.Sender(span, ctx.channel().id().asShortText())); - } - DECORATE.onResponse(span, response); - DECORATE.beforeFinish(storedContext); - span.finish(); // Finish the span manually since finishSpanOnClose was false - removeServerContext(ctx, null); - } - } - - private static boolean isTerminalResponse( + private static void onResponse( final ChannelHandlerContext ctx, final AgentSpan span, final ServerRequestContext serverContext, - final Object msg) { - if (msg instanceof HttpResponse) { - final HttpResponse response = (HttpResponse) msg; - - final boolean isWebsocketUpgrade = isWebsocketUpgrade(response); - if (isInformationalResponse(response) && !isWebsocketUpgrade) { - return false; - } - - if (isWebsocketUpgrade) { - ctx.channel() - .attr(WEBSOCKET_SENDER_HANDLER_CONTEXT) - .set(new HandlerContext.Sender(span, ctx.channel().id().asShortText())); - } - DECORATE.onResponse(span, response); + final HttpResponse response, + final boolean websocketUpgrade) { + if (websocketUpgrade) { + ctx.channel() + .attr(WEBSOCKET_SENDER_HANDLER_CONTEXT) + .set(new HandlerContext.Sender(span, ctx.channel().id().asShortText())); + } + DECORATE.onResponse(span, response); + if (serverContext != null) { serverContext.markResponseStarted(); - if (isBodylessResponse(serverContext, response) || isWebsocketUpgrade) { - return true; - } - final long contentLength = contentLength(response); - if (contentLength > 0 && !HttpUtil.isTransferEncodingChunked(response)) { - serverContext.setResponseContentLength(contentLength); - } - if (msg instanceof LastHttpContent - && (hasKnownBodyLength(response) || HttpUtil.isContentLengthSet(response))) { - return true; - } - // HTTP/1.x responses with neither a Content-Length nor chunked transfer-encoding are - // delimited by the connection closing. HTTP/2 streams translated through - // Http2StreamFrameToHttpObjectCodec are delimited by END_STREAM/LastHttpContent instead. - if (isCloseDelimitedHttp1Response(ctx, response)) { - serverContext.markResponseCloseDelimited(); - return false; - } - return msg instanceof LastHttpContent; } - return serverContext.isResponseStarted() - && !serverContext.isResponseCloseDelimited() - && msg instanceof LastHttpContent; } private static boolean isInformationalResponse(final HttpResponse response) { @@ -196,53 +119,6 @@ private static boolean isWebsocketUpgrade(final HttpResponse response) { .containsValue(HttpHeaderNames.UPGRADE, HttpHeaderValues.WEBSOCKET, true); } - private static boolean isBodylessResponse( - final ServerRequestContext serverContext, final HttpResponse response) { - final int statusCode = response.status().code(); - return serverContext.isHeadRequest() - || statusCode == 204 - || statusCode == 205 - || statusCode == 304 - || (serverContext.isConnectRequest() && statusCode >= 200 && statusCode < 300) - || (contentLength(response) == 0 && !HttpUtil.isTransferEncodingChunked(response)); - } - - private static boolean hasKnownBodyLength(final HttpResponse response) { - return contentLength(response) >= 0 || HttpUtil.isTransferEncodingChunked(response); - } - - private static boolean isCloseDelimitedHttp1Response( - final ChannelHandlerContext ctx, final HttpResponse response) { - return !hasKnownBodyLength(response) - && !Boolean.TRUE.equals(ctx.channel().attr(HTTP2_STREAM_CODEC_ATTRIBUTE_KEY).get()); - } - - private static long responseBodyBytes(final Object msg) { - if (msg instanceof ByteBuf) { - return ((ByteBuf) msg).readableBytes(); - } - if (msg instanceof ByteBufHolder) { - return ((ByteBufHolder) msg).content().readableBytes(); - } - if (msg instanceof FileRegion) { - return ((FileRegion) msg).count(); - } - return 0; - } - - /** - * Returns the response {@code Content-Length}, or {@code -1} when it is absent or malformed. A - * malformed value is left for Netty's encoder to reject rather than failing the write from the - * tracing handler. - */ - private static long contentLength(final HttpResponse response) { - try { - return HttpUtil.getContentLength(response, -1L); - } catch (final NumberFormatException e) { - return -1L; - } - } - private static void finishSpan( final ServerRequestContext serverContext, final Context storedContext, @@ -265,8 +141,10 @@ private static void finishSpan( private static void beforeFinish( final ServerRequestContext serverContext, final Context storedContext) { - if (!serverContext.isBeforeFinishCalled()) { - serverContext.markBeforeFinishCalled(); + if (serverContext == null || !serverContext.isBeforeFinishCalled()) { + if (serverContext != null) { + serverContext.markBeforeFinishCalled(); + } DECORATE.beforeFinish(storedContext); } } @@ -279,25 +157,4 @@ private static void removeServerContext( ServerRequestContext.remove(ctx.channel(), serverContext); } } - - private static boolean isHeaderOnly( - final HttpResponse response, final ServerRequestContext serverContext) { - final int statusCode = response.status().code(); - return (serverContext != null && serverContext.isHeadRequest()) - || statusCode == HttpResponseStatus.NO_CONTENT.code() - || statusCode == HttpResponseStatus.RESET_CONTENT.code() - || statusCode == HttpResponseStatus.NOT_MODIFIED.code() - || (hasZeroContentLength(response) && !HttpUtil.isTransferEncodingChunked(response)); - } - - private static boolean hasZeroContentLength(final HttpResponse response) { - if (!HttpUtil.isContentLengthSet(response)) { - return false; - } - try { - return HttpUtil.getContentLength(response) == 0; - } catch (final NumberFormatException ignored) { - return false; - } - } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java index 6ecd9896148..b73482f4113 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/ServerRequestContextTest.java @@ -20,13 +20,13 @@ void disablesTrackingWhenPipeliningLimitIsExceeded() { DefaultAttributeMap attributes = new DefaultAttributeMap(); for (int i = 0; i < PIPELINING_LIMIT; i++) { - assertNotNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); + assertNotNull(ServerRequestContext.add(attributes, Context.root(), null)); } assertFalse(ServerRequestContext.canTrackRequest(attributes)); assertNull(ServerRequestContext.nextResponse(attributes)); assertNull(attributes.attr(AttributeKeys.CONTEXT_ATTRIBUTE_KEY).get()); - assertNull(ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders())); + assertNull(ServerRequestContext.add(attributes, Context.root(), null)); } @Test @@ -36,7 +36,7 @@ void capturesAcceptHeaderValue() { headers.set("accept", "text/html"); ServerRequestContext serverContext = - ServerRequestContext.add(attributes, Context.root(), headers); + ServerRequestContext.add(attributes, Context.root(), headers.get("accept")); headers.set("accept", "application/json"); assertEquals("text/html", serverContext.acceptHeader()); @@ -83,8 +83,7 @@ void preservesFirstContextWhenCompactingAfterRequestFailure() { @Test void tracksBlockedResponseUntilChannelClose() { DefaultAttributeMap attributes = new DefaultAttributeMap(); - ServerRequestContext serverContext = - ServerRequestContext.add(attributes, Context.root(), new DefaultHttpHeaders()); + ServerRequestContext serverContext = ServerRequestContext.add(attributes, Context.root(), null); ServerRequestContext.markResponseBlocked(attributes); ServerRequestContext.remove(attributes, serverContext); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java index cfe8179dd3a..d9d90f4d83a 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/HttpServerResponseTracingHandlerTest.java @@ -40,16 +40,22 @@ class HttpServerResponseTracingHandlerTest extends AbstractInstrumentationTest { @Test - void finishesMirroredContextWhenRequestQueueIsAbsent() { + void finishesMirroredContextOnLastContentWhenRequestQueueIsAbsent() { EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); AgentSpan span = startSpan("netty", "mirrored-http2-server"); channel.attr(CONTEXT_ATTRIBUTE_KEY).set(span); - assertTrue(channel.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK))); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + assertTrue(channel.writeOutbound(response)); + + assertSame(span, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); + HttpResponse forwarded = channel.readOutbound(); + assertSame(response, forwarded); + ReferenceCountUtil.release(forwarded); - FullHttpResponse response = channel.readOutbound(); - assertNotNull(response); - response.release(); + assertTrue(channel.writeOutbound(LastHttpContent.EMPTY_LAST_CONTENT)); + + ReferenceCountUtil.release(channel.readOutbound()); assertNull(channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); channel.finishAndReleaseAll(); assertTraces(trace(span().root().operationName("mirrored-http2-server"))); @@ -71,21 +77,21 @@ void forwardsLastContentBeforeFinalResponseWithoutCompletingContext() { } @Test - void headerOnlyResponseCompletesContextAndWritesSyntheticLastContent() { + void headerOnlyResponseWaitsForLastContent() { EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); AgentSpan span = startSpan("netty", "header-only-server"); - ServerRequestContext.add(channel, span, null); + ServerRequestContext serverContext = ServerRequestContext.add(channel, span, null); ServerRequestContext nextServerContext = ServerRequestContext.add(channel, Context.root(), null); HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT); assertTrue(channel.writeOutbound(response)); - assertSame(nextServerContext, ServerRequestContext.nextResponse(channel)); + assertSame(serverContext, ServerRequestContext.nextResponse(channel)); HttpResponse forwarded = channel.readOutbound(); assertSame(response, forwarded); ReferenceCountUtil.release(forwarded); - assertSyntheticLastContent(channel); + assertNull(channel.readOutbound()); assertTrue(channel.writeOutbound(LastHttpContent.EMPTY_LAST_CONTENT)); @@ -97,10 +103,10 @@ void headerOnlyResponseCompletesContextAndWritesSyntheticLastContent() { } @Test - void rawFixedLengthBodyCompletesContextAndWritesSyntheticLastContent() { + void rawFixedLengthBodyWaitsForLastContent() { EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); AgentSpan span = startSpan("netty", "raw-body-server"); - ServerRequestContext.add(channel, span, null); + ServerRequestContext serverContext = ServerRequestContext.add(channel, span, null); ServerRequestContext nextServerContext = ServerRequestContext.add(channel, Context.root(), null); HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); @@ -110,9 +116,9 @@ void rawFixedLengthBodyCompletesContextAndWritesSyntheticLastContent() { ReferenceCountUtil.release(channel.readOutbound()); assertTrue(channel.writeOutbound(Unpooled.wrappedBuffer(new byte[] {1, 2, 3, 4}))); - assertSame(nextServerContext, ServerRequestContext.nextResponse(channel)); + assertSame(serverContext, ServerRequestContext.nextResponse(channel)); ReferenceCountUtil.release(channel.readOutbound()); - assertSyntheticLastContent(channel); + assertNull(channel.readOutbound()); assertTrue(channel.writeOutbound(LastHttpContent.EMPTY_LAST_CONTENT)); @@ -183,9 +189,26 @@ void fullWebsocketUpgradeCompletesContextAndPreservesHandshakeSpan() { assertTraces(trace(span().root().operationName("websocket-handshake-server"))); } - private static void assertSyntheticLastContent(EmbeddedChannel channel) { - Object syntheticLastContent = channel.readOutbound(); - assertTrue(syntheticLastContent instanceof LastHttpContent); - ReferenceCountUtil.release(syntheticLastContent); + @Test + void fullNonWebsocketUpgradeWaitsForFinalResponse() { + EmbeddedChannel channel = new EmbeddedChannel(HttpServerResponseTracingHandler.INSTANCE); + AgentSpan span = startSpan("netty", "h2c-upgrade-server"); + ServerRequestContext serverContext = ServerRequestContext.add(channel, span, null); + FullHttpResponse upgradeResponse = new DefaultFullHttpResponse(HTTP_1_1, SWITCHING_PROTOCOLS); + upgradeResponse.headers().set(UPGRADE, "h2c"); + + assertTrue(channel.writeOutbound(upgradeResponse)); + + assertSame(serverContext, ServerRequestContext.nextResponse(channel)); + assertNull(channel.attr(WEBSOCKET_SENDER_HANDLER_CONTEXT).get()); + ReferenceCountUtil.release(channel.readOutbound()); + + assertTrue(channel.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK))); + + assertNull(ServerRequestContext.nextResponse(channel)); + ReferenceCountUtil.release(channel.readOutbound()); + channel.finishAndReleaseAll(); + + assertTraces(trace(span().root().operationName("h2c-upgrade-server"))); } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java index d700a932560..2f98851a807 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/MaybeBlockResponseHandlerTest.java @@ -118,8 +118,7 @@ void blocksNonWebSocketSwitchingProtocolsResponse() { @Test void dropsWritesAfterBlockedContextHasBeenRemoved() { EmbeddedChannel channel = new EmbeddedChannel(MaybeBlockResponseHandler.INSTANCE); - ServerRequestContext serverContext = - ServerRequestContext.add(channel, Context.root(), new DefaultHttpHeaders()); + ServerRequestContext serverContext = ServerRequestContext.add(channel, Context.root(), null); ServerRequestContext.markResponseBlocked(channel); ServerRequestContext.remove(channel, serverContext); ByteBuf lateResponseChunk = Unpooled.buffer().writeByte(1); diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index c2952606c41..2847a0f64de 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -19,6 +19,8 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.agent.test.assertions.SpanMatcher; @@ -32,6 +34,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.TagContext; +import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.instrumentation.netty41.ServerRequestContext; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; @@ -44,7 +47,6 @@ import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.DefaultHttpContent; -import io.netty.handler.codec.http.DefaultHttpHeaders; import io.netty.handler.codec.http.DefaultHttpResponse; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponseStatus; @@ -120,7 +122,7 @@ void keepsServerSpanOpenUntilLastResponseChunk() throws Exception { } @Test - void keepsServerSpanOpenUntilConnectionDrops() throws Exception { + void finishesServerSpanWithoutFramingErrorWhenConnectionDrops() throws Exception { boolean reportedBeforeConnectionDrop; try (Socket socket = connect()) { socket.getOutputStream().write(request().getBytes(US_ASCII)); @@ -134,7 +136,7 @@ void keepsServerSpanOpenUntilConnectionDrops() throws Exception { assertFalse( reportedBeforeConnectionDrop, "server span should not be reported before connection drop"); - assertTraces(trace(serverSpan(PATH).error())); + assertTraces(trace(serverSpan(PATH))); } @Test @@ -159,7 +161,7 @@ void finishesServerSpanWithoutErrorForCloseDelimitedResponse() throws Exception } @Test - void waitsForChannelCloseForCloseDelimitedFullResponse() throws Exception { + void finishesCloseDelimitedFullResponseOnLastContent() throws Exception { boolean reportedBeforeConnectionClose; try (Socket socket = connect()) { socket @@ -168,29 +170,33 @@ void waitsForChannelCloseForCloseDelimitedFullResponse() throws Exception { socket.getOutputStream().flush(); ChannelHandlerContext responseContext = handler.awaitCloseDelimitedFullResponseWritten(); - reportedBeforeConnectionClose = writer.waitForTracesMax(1, 1); + reportedBeforeConnectionClose = writer.waitForTracesMax(1, 5); closeChannel(responseContext); } - assertFalse( + assertTrue( reportedBeforeConnectionClose, - "server span should not be reported before the connection closes"); + "a FullHttpResponse should finish the server span before the connection closes"); assertTraces(trace(serverSpan(CLOSE_DELIMITED_FULL_RESPONSE_PATH))); } @Test - void finishesServerSpanWithoutErrorForKnownLengthByteBufResponseClosedAfterBody() - throws Exception { + void knownLengthByteBufResponseWaitsForLastContent() throws Exception { + boolean reportedBeforeLastContent; try (Socket socket = connect()) { socket.getOutputStream().write(request(KNOWN_LENGTH_BYTE_BUF_PATH).getBytes(US_ASCII)); socket.getOutputStream().flush(); ChannelHandlerContext responseContext = handler.awaitKnownLengthByteBufWritten(); - closeChannel(responseContext); + reportedBeforeLastContent = writer.waitForTracesMax(1, 1); + writeLastChunk(responseContext); assertEquals("first", readHttpResponseBody(socket.getInputStream())); } + assertFalse( + reportedBeforeLastContent, + "Content-Length body bytes should not finish the span before LastHttpContent"); assertTraces(trace(serverSpan(KNOWN_LENGTH_BYTE_BUF_PATH))); } @@ -300,7 +306,7 @@ public Flow apply(RequestContext context, IGSpanInfo span) { "netty", "netty.request", new TagContext().withRequestContextDataIast(iastRequestData)); - ServerRequestContext.add(channel, span, new DefaultHttpHeaders()); + ServerRequestContext.add(channel, span, null); try { DefaultFullHttpResponse response = @@ -321,23 +327,83 @@ public Flow apply(RequestContext context, IGSpanInfo span) { } @Test - void finishesServerSpanWithoutErrorForKnownLengthFileRegionResponseClosedAfterBody() - throws Exception { + void marksSpanErrorWhenWritePromiseFailsAfterWriteReturns() { + Object iastRequestData = new Object(); + AtomicBoolean requestSpanActive = new AtomicBoolean(); + + Events events = Events.get(); + Subscription requestEnd = + AgentTracer.get() + .getSubscriptionService(RequestContextSlot.IAST) + .registerCallback( + events.requestEnded(), + new BiFunction>() { + @Override + public Flow apply(RequestContext context, IGSpanInfo span) { + if (iastRequestData == context.getData(RequestContextSlot.IAST)) { + AgentSpan activeSpan = AgentTracer.activeSpan(); + requestSpanActive.set( + activeSpan != null && activeSpan.getRequestContext() == context); + } + return Flow.ResultFlow.empty(); + } + }); + + HoldingOutboundHandler holdingHandler = new HoldingOutboundHandler(); + EmbeddedChannel channel = + new EmbeddedChannel(holdingHandler, HttpServerResponseTracingHandler.INSTANCE); + AgentSpan span = + AgentTracer.get() + .startSpan( + "netty", + "netty.request", + new TagContext().withRequestContextDataIast(iastRequestData)); + ServerRequestContext.add(channel, span, null); + IOException writeFailure = new IOException("delayed write failure"); + + try { + DefaultFullHttpResponse response = + new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.copiedBuffer("ok", UTF_8)); + response.headers().set(CONTENT_LENGTH, response.content().readableBytes()); + + channel.writeOutbound(response); + assertFalse(requestSpanActive.get(), "write promise completed synchronously"); + assertNull(ServerRequestContext.nextResponse(channel)); + + holdingHandler.failWrite(writeFailure); + assertThrows(IOException.class, channel::checkException); + + assertTrue(requestSpanActive.get(), "request span was not active during requestEnded"); + assertEquals(500, span.getTag(Tags.HTTP_STATUS)); + assertTraces(trace(span().root().operationName("netty.request").error())); + } finally { + holdingHandler.failWrite(writeFailure); + requestEnd.cancel(); + channel.finishAndReleaseAll(); + } + } + + @Test + void knownLengthFileRegionResponseWaitsForLastContent() throws Exception { + boolean reportedBeforeLastContent; try (Socket socket = connect()) { socket.getOutputStream().write(request(KNOWN_LENGTH_FILE_REGION_PATH).getBytes(US_ASCII)); socket.getOutputStream().flush(); ChannelHandlerContext responseContext = handler.awaitKnownLengthFileRegionWritten(); - closeChannel(responseContext); + reportedBeforeLastContent = writer.waitForTracesMax(1, 1); + writeLastChunk(responseContext); assertEquals("first", readHttpResponseBody(socket.getInputStream())); } + assertFalse( + reportedBeforeLastContent, + "Content-Length body bytes should not finish the span before LastHttpContent"); assertTraces(trace(serverSpan(KNOWN_LENGTH_FILE_REGION_PATH))); } @Test - void marksKnownLengthResponseErrorWhenConnectionClosesBeforeDeclaredBodyComplete() - throws Exception { + void doesNotInferFramingErrorFromContentLengthWhenConnectionCloses() throws Exception { try (Socket socket = connect()) { socket.getOutputStream().write(request(INCOMPLETE_KNOWN_LENGTH_PATH).getBytes(US_ASCII)); socket.getOutputStream().flush(); @@ -346,7 +412,7 @@ void marksKnownLengthResponseErrorWhenConnectionClosesBeforeDeclaredBodyComplete closeChannel(responseContext); } - assertTraces(trace(serverSpan(INCOMPLETE_KNOWN_LENGTH_PATH).error())); + assertTraces(trace(serverSpan(INCOMPLETE_KNOWN_LENGTH_PATH))); } @Test @@ -722,14 +788,17 @@ private void writeIncompleteKnownLengthResponse(ChannelHandlerContext ctx) { private void writeHeaderOnlyResponse( ChannelHandlerContext ctx, String path, HttpResponseStatus status) { - ctx.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, status)) + ctx.write(new DefaultHttpResponse(HTTP_1_1, status)); + ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT) .addListener(future -> headerOnlyWrites.offer(path)); } private void writeContentLengthZeroResponse(ChannelHandlerContext ctx, String path) { DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); response.headers().set(CONTENT_LENGTH, 0); - ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); + ctx.write(response); + ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT) + .addListener(future -> headerOnlyWrites.offer(path)); } private void writeHeaderOnlyWebSocketUpgrade( @@ -737,7 +806,9 @@ private void writeHeaderOnlyWebSocketUpgrade( DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, status); response.headers().set(UPGRADE, "WebSocket"); response.headers().set(CONNECTION, "Upgrade"); - ctx.writeAndFlush(response).addListener(future -> headerOnlyWrites.offer(path)); + ctx.write(response); + ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT) + .addListener(future -> headerOnlyWrites.offer(path)); } private ChannelHandlerContext awaitFirstChunkWritten() throws InterruptedException { @@ -839,11 +910,23 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) } private void completeWrite() { + finishWrite(null); + } + + private void failWrite(Throwable failure) { + finishWrite(failure); + } + + private void finishWrite(Throwable failure) { if (promise == null || promise.isDone()) { return; } try { - promise.setSuccess(); + if (failure == null) { + promise.setSuccess(); + } else { + promise.setFailure(failure); + } } finally { ReferenceCountUtil.release(msg); msg = null; diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java index 34df63ac2eb..a611fbb8cb8 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyHttp11PipeliningTest.java @@ -18,6 +18,7 @@ import static java.util.Collections.emptyMap; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -266,11 +267,17 @@ void requestBlockOnLaterPipelinedRequestFollowsEarlierHeaderOnlyResponse() throw assertTrue( blockedRequestSeen.await(5, SECONDS), "server did not block second pipelined request"); - handler.writeHeaderOnlyResponse(); + handler.writeHeaderOnlyResponseHeaders(); assertTrue( readHeaders(socket.getInputStream()).startsWith("HTTP/1.1 204 "), "first response should be the header-only response"); + assertFalse( + writer.waitForTracesMax(1, 1), + "header-only response should remain pending before LastHttpContent"); + + handler.writeLastContent(); + assertTrue( readHeaders(socket.getInputStream()).startsWith("HTTP/1.1 403 "), "second response should be the deferred blocking response"); @@ -597,7 +604,7 @@ private void writeChunkedResponse() { }); } - private void writeHeaderOnlyResponse() { + private void writeHeaderOnlyResponseHeaders() { ChannelHandlerContext responseContext = context; if (responseContext == null) { throw new IllegalStateException("no request context captured"); @@ -608,12 +615,22 @@ private void writeHeaderOnlyResponse() { () -> { DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT); response.headers().set(CONNECTION, KEEP_ALIVE); - responseContext.write(response); - responseContext.write(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER)); - responseContext.flush(); + responseContext.writeAndFlush(response); }); } + private void writeLastContent() { + ChannelHandlerContext responseContext = context; + if (responseContext == null) { + throw new IllegalStateException("no request context captured"); + } + responseContext + .executor() + .execute( + () -> + responseContext.writeAndFlush(new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER))); + } + private void writeHeadResponse() { ChannelHandlerContext responseContext = context; if (responseContext == null) { diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java index 9eda11c6f6f..addf40b55d6 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/AttributeKeys.java @@ -28,9 +28,6 @@ public final class AttributeKeys { public static final AttributeKey HTTP2_CONNECTION_CODEC_ATTRIBUTE_KEY = attributeKey("datadog.http2.connection.codec"); - public static final AttributeKey HTTP2_STREAM_CODEC_ATTRIBUTE_KEY = - attributeKey("datadog.http2.stream.codec"); - public static final AttributeKey PARENT_CONTEXT_ATTRIBUTE_KEY = attributeKey("datadog.server.parent-context"); diff --git a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java index 9558564f4be..654bf24548f 100644 --- a/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java +++ b/dd-java-agent/instrumentation/netty/netty-common/src/main/java/datadog/trace/instrumentation/netty41/ServerRequestContext.java @@ -4,8 +4,6 @@ import datadog.context.Context; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; import io.netty.util.AttributeKey; import io.netty.util.AttributeMap; import java.util.ArrayDeque; @@ -27,43 +25,12 @@ public static boolean canTrackRequest(final AttributeMap attributes) { /** Adds a request context to the queue tail. */ public static ServerRequestContext add( - final AttributeMap attributes, final Context context, final HttpHeaders requestHeaders) { - return add(attributes, context, requestHeaders, null); - } - - /** Adds a request context to the queue tail. */ - public static ServerRequestContext add( - final AttributeMap attributes, - final Context context, - final String acceptHeader, - final boolean headRequest) { - return add(attributes, context, acceptHeader, headRequest ? HttpMethod.HEAD : null); - } - - /** Adds a request context to the queue tail. */ - public static ServerRequestContext add( - final AttributeMap attributes, - final Context context, - final HttpHeaders requestHeaders, - final HttpMethod requestMethod) { - return add( - attributes, - context, - requestHeaders == null ? null : requestHeaders.get("accept"), - requestMethod); - } - - private static ServerRequestContext add( - final AttributeMap attributes, - final Context context, - final String acceptHeader, - final HttpMethod requestMethod) { + final AttributeMap attributes, final Context context, final String acceptHeader) { final Deque contexts = getOrCreate(attributes); if (!canAdd(attributes, contexts)) { return null; } - final ServerRequestContext serverContext = - new ServerRequestContext(context, acceptHeader, requestMethod); + final ServerRequestContext serverContext = new ServerRequestContext(context, acceptHeader); contexts.addLast(serverContext); // The deque is authoritative for server request/response matching. CONTEXT_ATTRIBUTE_KEY is a // context mirror of the current inbound request used by @@ -162,7 +129,6 @@ public static void remove( /** Closes all pending request contexts on channel close. */ public static void closeAll(final AttributeMap attributes) { - attributes.attr(BLOCKED_RESPONSE_ATTRIBUTE_KEY).remove(); close(removeAll(attributes)); } @@ -253,14 +219,9 @@ private static boolean isPoisoned(final Deque contexts) { private final Context tracingContext; private final String acceptHeader; - private final HttpMethod requestMethod; private boolean responseStarted; - private boolean responseCloseDelimited; private boolean beforeFinishCalled; private boolean responseAnalyzed; - private boolean responseBlocked; - private long responseContentLength = -1; - private long responseBodyBytes; private Object deferredBlockResponse; public Context tracingContext() { @@ -271,14 +232,6 @@ public String acceptHeader() { return acceptHeader; } - public boolean isHeadRequest() { - return HttpMethod.HEAD.equals(requestMethod); - } - - public boolean isConnectRequest() { - return HttpMethod.CONNECT.equals(requestMethod); - } - public boolean isResponseStarted() { return responseStarted; } @@ -287,31 +240,6 @@ public void markResponseStarted() { responseStarted = true; } - public boolean isResponseCloseDelimited() { - return responseCloseDelimited; - } - - public void markResponseCloseDelimited() { - responseCloseDelimited = true; - } - - public void setResponseContentLength(final long contentLength) { - responseContentLength = contentLength; - responseBodyBytes = 0; - } - - public boolean recordResponseBodyBytes(final long bodyBytes) { - if (responseContentLength < 0 || bodyBytes <= 0) { - return false; - } - if (Long.MAX_VALUE - responseBodyBytes < bodyBytes) { - responseBodyBytes = Long.MAX_VALUE; - } else { - responseBodyBytes += bodyBytes; - } - return responseBodyBytes >= responseContentLength; - } - public boolean isBeforeFinishCalled() { return beforeFinishCalled; } @@ -328,14 +256,6 @@ public void markResponseAnalyzed() { responseAnalyzed = true; } - public boolean isResponseBlocked() { - return responseBlocked; - } - - public void markResponseBlocked() { - responseBlocked = true; - } - public Object deferredBlockResponse() { return deferredBlockResponse; } @@ -344,10 +264,8 @@ public void deferBlockResponse(final Object deferredBlockResponse) { this.deferredBlockResponse = deferredBlockResponse; } - private ServerRequestContext( - final Context tracingContext, final String acceptHeader, final HttpMethod requestMethod) { + private ServerRequestContext(final Context tracingContext, final String acceptHeader) { this.tracingContext = tracingContext; this.acceptHeader = acceptHeader; - this.requestMethod = requestMethod; } } From 8a70d5e99ddbb0e7129e5bb2c309d2f4d60d6946 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 24 Jul 2026 11:31:42 -0700 Subject: [PATCH 32/33] Raise agent jar size budget The agent jar exceeds the previous budget by about 1 KiB. Raise the budget to the next 16 KiB boundary to pass the verification check. --- metadata/agent-jar-checks.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata/agent-jar-checks.properties b/metadata/agent-jar-checks.properties index d2565d0f6fd..2538cbe1eed 100644 --- a/metadata/agent-jar-checks.properties +++ b/metadata/agent-jar-checks.properties @@ -1,7 +1,7 @@ # Agent jar structural invariants — edit intentionally, commit with the change that justified it. -# Max agent jar size in bytes. Raise only when the size growth is intentional (~33.02 MiB). -jar.size.budget = 34619392 +# Max agent jar size in bytes. Raise only when the size growth is intentional (~33.03 MiB). +jar.size.budget = 34635776 # Minimum combined class + classdata count in the assembled agent jar. # Set to ~98% of the actual count at the time of the last intentional change. From 670d5c49bc13c681b7a5ff84e790f28549a67ef8 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 24 Jul 2026 12:36:37 -0700 Subject: [PATCH 33/33] Wait for Netty server spans before closing test sockets --- .../netty41/server/NettyChunkedResponseSpanTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java index 2847a0f64de..a68de06d005 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/server/NettyChunkedResponseSpanTest.java @@ -192,6 +192,8 @@ void knownLengthByteBufResponseWaitsForLastContent() throws Exception { reportedBeforeLastContent = writer.waitForTracesMax(1, 1); writeLastChunk(responseContext); assertEquals("first", readHttpResponseBody(socket.getInputStream())); + assertTrue( + writer.waitForTracesMax(1, 5), "server span should be reported after LastHttpContent"); } assertFalse( @@ -394,6 +396,8 @@ void knownLengthFileRegionResponseWaitsForLastContent() throws Exception { reportedBeforeLastContent = writer.waitForTracesMax(1, 1); writeLastChunk(responseContext); assertEquals("first", readHttpResponseBody(socket.getInputStream())); + assertTrue( + writer.waitForTracesMax(1, 5), "server span should be reported after LastHttpContent"); } assertFalse( @@ -640,9 +644,7 @@ private void writeFirstChunk(ChannelHandlerContext responseContext) { } private static void writeLastChunk(ChannelHandlerContext responseContext) { - responseContext - .executor() - .execute(() -> responseContext.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT)); + responseContext.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT).syncUninterruptibly(); } private static void closeChannel(ChannelHandlerContext responseContext) {