netty: add soft Metadata size limit enforcement. (#11603)

Author: ran-su

netty/src/main/java/io/grpc/netty/AbstractNettyHandler.java

@@ -42,7 +42,8 @@ abstract class AbstractNettyHandler extends GrpcHttp2ConnectionHandler {
 
   private final int initialConnectionWindow;
   private final FlowControlPinger flowControlPing;
-
+  protected final int maxHeaderListSize;
+  protected final int softLimitHeaderListSize;
   private boolean autoTuneFlowControlOn;
   private ChannelHandlerContext ctx;
   private boolean initialWindowSent = false;
@@ -58,7 +59,9 @@ abstract class AbstractNettyHandler extends GrpcHttp2ConnectionHandler {
       ChannelLogger negotiationLogger,
       boolean autoFlowControl,
       PingLimiter pingLimiter,
-      Ticker ticker) {
+      Ticker ticker,
+      int maxHeaderListSize,
+      int softLimitHeaderListSize) {
     super(channelUnused, decoder, encoder, initialSettings, negotiationLogger);
 
     // During a graceful shutdown, wait until all streams are closed.
@@ -73,6 +76,8 @@ abstract class AbstractNettyHandler extends GrpcHttp2ConnectionHandler {
     }
     this.flowControlPing = new FlowControlPinger(pingLimiter);
     this.ticker = checkNotNull(ticker, "ticker");
+    this.maxHeaderListSize = maxHeaderListSize;
+    this.softLimitHeaderListSize = softLimitHeaderListSize;
   }
 
   @Override

netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java

@@ -104,6 +104,7 @@ public final class NettyChannelBuilder extends ForwardingChannelBuilder2<NettyCh
   private boolean autoFlowControl = DEFAULT_AUTO_FLOW_CONTROL;
   private int flowControlWindow = DEFAULT_FLOW_CONTROL_WINDOW;
   private int maxHeaderListSize = GrpcUtil.DEFAULT_MAX_HEADER_LIST_SIZE;
+  private int softLimitHeaderListSize = GrpcUtil.DEFAULT_MAX_HEADER_LIST_SIZE;
   private int maxInboundMessageSize = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE;
   private long keepAliveTimeNanos = KEEPALIVE_TIME_NANOS_DISABLED;
   private long keepAliveTimeoutNanos = DEFAULT_KEEPALIVE_TIMEOUT_NANOS;
@@ -452,6 +453,40 @@ public NettyChannelBuilder maxHeaderListSize(int maxHeaderListSize) {
   public NettyChannelBuilder maxInboundMetadataSize(int bytes) {
     checkArgument(bytes > 0, "maxInboundMetadataSize must be > 0");
     this.maxHeaderListSize = bytes;
+    // Clear the soft limit setting, by setting soft limit to maxInboundMetadataSize. The
+    // maxInboundMetadataSize will take precedence be applied before soft limit check.
+    this.softLimitHeaderListSize = bytes;
+    return this;
+  }
+
+  /**
+   * Sets the size of metadata that clients are advised to not exceed. When a metadata with size
+   * larger than the soft limit is encountered there will be a probability the RPC will fail. The
+   * chance of failing increases as the metadata size approaches the hard limit.
+   * {@code Integer.MAX_VALUE} disables the enforcement. The default is implementation-dependent,
+   * but is not generally less than 8 KiB and may be unlimited.
+   *
+   * <p>This is cumulative size of the metadata. The precise calculation is
+   * implementation-dependent, but implementations are encouraged to follow the calculation used
+   * for
+   * <a href="http://httpwg.org/specs/rfc7540.html#rfc.section.6.5.2">HTTP/2's
+   * SETTINGS_MAX_HEADER_LIST_SIZE</a>. It sums the bytes from each entry's key and value, plus 32
+   * bytes of overhead per entry.
+   *
+   * @param soft the soft size limit of received metadata
+   * @param max the hard size limit of received metadata
+   * @return this
+   * @throws IllegalArgumentException if soft and/or max is non-positive, or max smaller than
+   *     soft
+   * @since 1.68.0
+   */
+  @CanIgnoreReturnValue
+  public NettyChannelBuilder maxInboundMetadataSize(int soft, int max) {
+    checkArgument(soft > 0, "softLimitHeaderListSize must be > 0");
+    checkArgument(max > soft,
+        "maxInboundMetadataSize must be greater than softLimitHeaderListSize");
+    this.softLimitHeaderListSize = soft;
+    this.maxHeaderListSize = max;
     return this;
   }
 
@@ -573,10 +608,22 @@ ClientTransportFactory buildTransportFactory() {
 
     ProtocolNegotiator negotiator = protocolNegotiatorFactory.newNegotiator();
     return new NettyTransportFactory(
-        negotiator, channelFactory, channelOptions,
-        eventLoopGroupPool, autoFlowControl, flowControlWindow, maxInboundMessageSize,
-        maxHeaderListSize, keepAliveTimeNanos, keepAliveTimeoutNanos, keepAliveWithoutCalls,
-        transportTracerFactory, localSocketPicker, useGetForSafeMethods, transportSocketType);
+        negotiator,
+        channelFactory,
+        channelOptions,
+        eventLoopGroupPool,
+        autoFlowControl,
+        flowControlWindow,
+        maxInboundMessageSize,
+        maxHeaderListSize,
+        softLimitHeaderListSize,
+        keepAliveTimeNanos,
+        keepAliveTimeoutNanos,
+        keepAliveWithoutCalls,
+        transportTracerFactory,
+        localSocketPicker,
+        useGetForSafeMethods,
+        transportSocketType);
   }
 
   @VisibleForTesting
@@ -710,6 +757,7 @@ private static final class NettyTransportFactory implements ClientTransportFacto
     private final int flowControlWindow;
     private final int maxMessageSize;
     private final int maxHeaderListSize;
+    private final int softLimitHeaderListSize;
     private final long keepAliveTimeNanos;
     private final AtomicBackoff keepAliveBackoff;
     private final long keepAliveTimeoutNanos;
@@ -724,11 +772,20 @@ private static final class NettyTransportFactory implements ClientTransportFacto
     NettyTransportFactory(
         ProtocolNegotiator protocolNegotiator,
         ChannelFactory<? extends Channel> channelFactory,
-        Map<ChannelOption<?>, ?> channelOptions, ObjectPool<? extends EventLoopGroup> groupPool,
-        boolean autoFlowControl, int flowControlWindow, int maxMessageSize, int maxHeaderListSize,
-        long keepAliveTimeNanos, long keepAliveTimeoutNanos, boolean keepAliveWithoutCalls,
-        TransportTracer.Factory transportTracerFactory, LocalSocketPicker localSocketPicker,
-        boolean useGetForSafeMethods, Class<? extends SocketAddress> transportSocketType) {
+        Map<ChannelOption<?>, ?> channelOptions,
+        ObjectPool<? extends EventLoopGroup> groupPool,
+        boolean autoFlowControl,
+        int flowControlWindow,
+        int maxMessageSize,
+        int maxHeaderListSize,
+        int softLimitHeaderListSize,
+        long keepAliveTimeNanos,
+        long keepAliveTimeoutNanos,
+        boolean keepAliveWithoutCalls,
+        TransportTracer.Factory transportTracerFactory,
+        LocalSocketPicker localSocketPicker,
+        boolean useGetForSafeMethods,
+        Class<? extends SocketAddress> transportSocketType) {
       this.protocolNegotiator = checkNotNull(protocolNegotiator, "protocolNegotiator");
       this.channelFactory = channelFactory;
       this.channelOptions = new HashMap<ChannelOption<?>, Object>(channelOptions);
@@ -738,6 +795,7 @@ private static final class NettyTransportFactory implements ClientTransportFacto
       this.flowControlWindow = flowControlWindow;
       this.maxMessageSize = maxMessageSize;
       this.maxHeaderListSize = maxHeaderListSize;
+      this.softLimitHeaderListSize = softLimitHeaderListSize;
       this.keepAliveTimeNanos = keepAliveTimeNanos;
       this.keepAliveBackoff = new AtomicBackoff("keepalive time nanos", keepAliveTimeNanos);
       this.keepAliveTimeoutNanos = keepAliveTimeoutNanos;
@@ -774,13 +832,30 @@ public void run() {
       };
 
       // TODO(carl-mastrangelo): Pass channelLogger in.
-      NettyClientTransport transport = new NettyClientTransport(
-          serverAddress, channelFactory, channelOptions, group,
-          localNegotiator, autoFlowControl, flowControlWindow,
-          maxMessageSize, maxHeaderListSize, keepAliveTimeNanosState.get(), keepAliveTimeoutNanos,
-          keepAliveWithoutCalls, options.getAuthority(), options.getUserAgent(),
-          tooManyPingsRunnable, transportTracerFactory.create(), options.getEagAttributes(),
-          localSocketPicker, channelLogger, useGetForSafeMethods, Ticker.systemTicker());
+      NettyClientTransport transport =
+          new NettyClientTransport(
+              serverAddress,
+              channelFactory,
+              channelOptions,
+              group,
+              localNegotiator,
+              autoFlowControl,
+              flowControlWindow,
+              maxMessageSize,
+              maxHeaderListSize,
+              softLimitHeaderListSize,
+              keepAliveTimeNanosState.get(),
+              keepAliveTimeoutNanos,
+              keepAliveWithoutCalls,
+              options.getAuthority(),
+              options.getUserAgent(),
+              tooManyPingsRunnable,
+              transportTracerFactory.create(),
+              options.getEagAttributes(),
+              localSocketPicker,
+              channelLogger,
+              useGetForSafeMethods,
+              Ticker.systemTicker());
       return transport;
     }
 
@@ -796,11 +871,24 @@ public SwapChannelCredentialsResult swapChannelCredentials(ChannelCredentials ch
       if (result.error != null) {
         return null;
       }
-      ClientTransportFactory factory = new NettyTransportFactory(
-          result.negotiator.newNegotiator(), channelFactory, channelOptions, groupPool,
-          autoFlowControl, flowControlWindow, maxMessageSize, maxHeaderListSize, keepAliveTimeNanos,
-          keepAliveTimeoutNanos, keepAliveWithoutCalls, transportTracerFactory,  localSocketPicker,
-          useGetForSafeMethods, transportSocketType);
+      ClientTransportFactory factory =
+          new NettyTransportFactory(
+              result.negotiator.newNegotiator(),
+              channelFactory,
+              channelOptions,
+              groupPool,
+              autoFlowControl,
+              flowControlWindow,
+              maxMessageSize,
+              maxHeaderListSize,
+              softLimitHeaderListSize,
+              keepAliveTimeNanos,
+              keepAliveTimeoutNanos,
+              keepAliveWithoutCalls,
+              transportTracerFactory,
+              localSocketPicker,
+              useGetForSafeMethods,
+              transportSocketType);
       return new SwapChannelCredentialsResult(factory, result.callCredentials);
     }
 

netty/src/main/java/io/grpc/netty/NettyClientHandler.java

@@ -142,6 +142,7 @@ static NettyClientHandler newHandler(
       boolean autoFlowControl,
       int flowControlWindow,
       int maxHeaderListSize,
+      int softLimitHeaderListSize,
       Supplier<Stopwatch> stopwatchFactory,
       Runnable tooManyPingsRunnable,
       TransportTracer transportTracer,
@@ -171,6 +172,7 @@ static NettyClientHandler newHandler(
         autoFlowControl,
         flowControlWindow,
         maxHeaderListSize,
+        softLimitHeaderListSize,
         stopwatchFactory,
         tooManyPingsRunnable,
         transportTracer,
@@ -190,6 +192,7 @@ static NettyClientHandler newHandler(
       boolean autoFlowControl,
       int flowControlWindow,
       int maxHeaderListSize,
+      int softLimitHeaderListSize,
       Supplier<Stopwatch> stopwatchFactory,
       Runnable tooManyPingsRunnable,
       TransportTracer transportTracer,
@@ -202,6 +205,8 @@ static NettyClientHandler newHandler(
     Preconditions.checkNotNull(lifecycleManager, "lifecycleManager");
     Preconditions.checkArgument(flowControlWindow > 0, "flowControlWindow must be positive");
     Preconditions.checkArgument(maxHeaderListSize > 0, "maxHeaderListSize must be positive");
+    Preconditions.checkArgument(softLimitHeaderListSize > 0,
+        "softLimitHeaderListSize must be positive");
     Preconditions.checkNotNull(stopwatchFactory, "stopwatchFactory");
     Preconditions.checkNotNull(tooManyPingsRunnable, "tooManyPingsRunnable");
     Preconditions.checkNotNull(eagAttributes, "eagAttributes");
@@ -247,7 +252,9 @@ static NettyClientHandler newHandler(
         authority,
         autoFlowControl,
         pingCounter,
-        ticker);
+        ticker,
+        maxHeaderListSize,
+        softLimitHeaderListSize);
   }
 
   private NettyClientHandler(
@@ -264,9 +271,20 @@ private NettyClientHandler(
       String authority,
       boolean autoFlowControl,
       PingLimiter pingLimiter,
-      Ticker ticker) {
-    super(/* channelUnused= */ null, decoder, encoder, settings,
-        negotiationLogger, autoFlowControl, pingLimiter, ticker);
+      Ticker ticker,
+      int maxHeaderListSize,
+      int softLimitHeaderListSize) {
+    super(
+        /* channelUnused= */ null,
+        decoder,
+        encoder,
+        settings,
+        negotiationLogger,
+        autoFlowControl,
+        pingLimiter,
+        ticker,
+        maxHeaderListSize,
+        softLimitHeaderListSize);
     this.lifecycleManager = lifecycleManager;
     this.keepAliveManager = keepAliveManager;
     this.stopwatchFactory = stopwatchFactory;
@@ -380,6 +398,28 @@ private void onHeadersRead(int streamId, Http2Headers headers, boolean endStream
     if (streamId != Http2CodecUtil.HTTP_UPGRADE_STREAM_ID) {
       NettyClientStream.TransportState stream = clientStream(requireHttp2Stream(streamId));
       PerfMark.event("NettyClientHandler.onHeadersRead", stream.tag());
+      // check metadata size vs soft limit
+      int h2HeadersSize = Utils.getH2HeadersSize(headers);
+      boolean shouldFail =
+          Utils.shouldRejectOnMetadataSizeSoftLimitExceeded(
+              h2HeadersSize, softLimitHeaderListSize, maxHeaderListSize);
+      if (shouldFail && endStream) {
+        stream.transportReportStatus(Status.RESOURCE_EXHAUSTED
+            .withDescription(
+                String.format(
+                    "Server Status + Trailers of size %d exceeded Metadata size soft limit: %d",
+                    h2HeadersSize,
+                    softLimitHeaderListSize)), true, new Metadata());
+        return;
+      } else if (shouldFail) {
+        stream.transportReportStatus(Status.RESOURCE_EXHAUSTED
+            .withDescription(
+                String.format(
+                    "Server Headers of size %d exceeded Metadata size soft limit: %d",
+                    h2HeadersSize,
+                    softLimitHeaderListSize)), true, new Metadata());
+        return;
+      }
       stream.transportHeadersReceived(headers, endStream);
     }
 

netty/src/main/java/io/grpc/netty/NettyClientTransport.java

@@ -83,6 +83,7 @@ class NettyClientTransport implements ConnectionClientTransport {
   private final int flowControlWindow;
   private final int maxMessageSize;
   private final int maxHeaderListSize;
+  private final int softLimitHeaderListSize;
   private KeepAliveManager keepAliveManager;
   private final long keepAliveTimeNanos;
   private final long keepAliveTimeoutNanos;
@@ -106,15 +107,28 @@ class NettyClientTransport implements ConnectionClientTransport {
   private final Ticker ticker;
 
   NettyClientTransport(
-      SocketAddress address, ChannelFactory<? extends Channel> channelFactory,
-      Map<ChannelOption<?>, ?> channelOptions, EventLoopGroup group,
-      ProtocolNegotiator negotiator, boolean autoFlowControl, int flowControlWindow,
-      int maxMessageSize, int maxHeaderListSize,
-      long keepAliveTimeNanos, long keepAliveTimeoutNanos,
-      boolean keepAliveWithoutCalls, String authority, @Nullable String userAgent,
-      Runnable tooManyPingsRunnable, TransportTracer transportTracer, Attributes eagAttributes,
-      LocalSocketPicker localSocketPicker, ChannelLogger channelLogger,
-      boolean useGetForSafeMethods, Ticker ticker) {
+      SocketAddress address,
+      ChannelFactory<? extends Channel> channelFactory,
+      Map<ChannelOption<?>, ?> channelOptions,
+      EventLoopGroup group,
+      ProtocolNegotiator negotiator,
+      boolean autoFlowControl,
+      int flowControlWindow,
+      int maxMessageSize,
+      int maxHeaderListSize,
+      int softLimitHeaderListSize,
+      long keepAliveTimeNanos,
+      long keepAliveTimeoutNanos,
+      boolean keepAliveWithoutCalls,
+      String authority,
+      @Nullable String userAgent,
+      Runnable tooManyPingsRunnable,
+      TransportTracer transportTracer,
+      Attributes eagAttributes,
+      LocalSocketPicker localSocketPicker,
+      ChannelLogger channelLogger,
+      boolean useGetForSafeMethods,
+      Ticker ticker) {
 
     this.negotiator = Preconditions.checkNotNull(negotiator, "negotiator");
     this.negotiationScheme = this.negotiator.scheme();
@@ -126,6 +140,7 @@ class NettyClientTransport implements ConnectionClientTransport {
     this.flowControlWindow = flowControlWindow;
     this.maxMessageSize = maxMessageSize;
     this.maxHeaderListSize = maxHeaderListSize;
+    this.softLimitHeaderListSize = softLimitHeaderListSize;
     this.keepAliveTimeNanos = keepAliveTimeNanos;
     this.keepAliveTimeoutNanos = keepAliveTimeoutNanos;
     this.keepAliveWithoutCalls = keepAliveWithoutCalls;
@@ -220,18 +235,19 @@ public Runnable start(Listener transportListener) {
     }
 
     handler = NettyClientHandler.newHandler(
-        lifecycleManager,
-        keepAliveManager,
-        autoFlowControl,
-        flowControlWindow,
-        maxHeaderListSize,
-        GrpcUtil.STOPWATCH_SUPPLIER,
-        tooManyPingsRunnable,
-        transportTracer,
-        eagAttributes,
-        authorityString,
-        channelLogger,
-        ticker);
+            lifecycleManager,
+            keepAliveManager,
+            autoFlowControl,
+            flowControlWindow,
+            maxHeaderListSize,
+            softLimitHeaderListSize,
+            GrpcUtil.STOPWATCH_SUPPLIER,
+            tooManyPingsRunnable,
+            transportTracer,
+            eagAttributes,
+            authorityString,
+            channelLogger,
+            ticker);
 
     ChannelHandler negotiationHandler = negotiator.newHandler(handler);