Working

Author: robotdan

src/main/java/io/fusionauth/http/io/ReadLimitedInputStream.java

@@ -168,6 +168,15 @@ default T withLoggerFactory(LoggerFactory loggerFactory) {
     return (T) this;
   }
 
+  /**
+   * The maximum size of the HTTP request body in bytes when the Content-Type is application/x-www-form-urlencoded. Defaults to 10
+   * Megabytes.
+   * <p>
+   * Set this to -1 to disable this limitation.
+   *
+   * @param maxFormDataSize the maximum size of the HTTP request body in bytes.
+   * @return This.
+   */
   default T withMaxFormDataSize(int maxFormDataSize) {
     configuration().withMaxFormDataSize(maxFormDataSize);
     return (T) this;
@@ -203,7 +212,9 @@ default T withMaxRequestBodySize(int maxRequestBodySize) {
   }
 
   /**
-   * Sets the maximum size of the HTTP request header. If this limit is exceeded, the connection will be closed. Defaults to 128 Kilobytes.
+   * Sets the maximum size of the HTTP request header. The request header includes the HTTP request line, and all HTTP request headers,
+   * essentially everything except the request body. If this maximum limit is exceeded, the connection will be closed. Defaults to 128
+   * Kilobytes.
    * <p>
    * Set this to -1 to disable this limitation.
    *
@@ -228,7 +239,8 @@ default T withMaxRequestsPerConnection(int maxRequestsPerConnection) {
   }
 
   /**
-   * This configures the maximum size of a chunk in the response when the server is using chunked response encoding. Defaults to 16 Kilobytes.
+   * This configures the maximum size of a chunk in the response when the server is using chunked response encoding. Defaults to 16
+   * Kilobytes.
    *
    * @param size The size in bytes.
    * @return This.
@@ -342,8 +354,10 @@ default T withRequestBufferSize(int requestBufferSize) {
    * Sets the size of the buffer that is used to store the HTTP response before any bytes are written back to the client. This is useful
    * when the server is generating the response but encounters an error. In this case, the server will throw out the response and change to
    * a 500 error response. This defaults to 64 Kilobytes. Negative values disable the response buffer.
+   * <p>
+   * Set to -1 do disable buffering completely.
    *
-   * @param responseBufferSize The size of the buffer. Set to -1 to disable buffering completely.
+   * @param responseBufferSize The size of the buffer.
    * @return This.
    */
   default T withResponseBufferSize(int responseBufferSize) {

src/main/java/io/fusionauth/http/server/Configurable.java

@@ -226,6 +226,7 @@ public int getMaxRequestBodySize() {
    * @return the maximum size of the HTTP request header in bytes. This configuration does not affect the HTTP response header. Defaults to
    *     128 Kilobytes.
    */
+  // TODO : Naming : Preamble
   public int getMaxRequestHeaderSize() {
     // TODO : Notes:
     //          https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html
@@ -478,7 +479,10 @@ public HTTPServerConfiguration withLoggerFactory(LoggerFactory loggerFactory) {
    */
   @Override
   public HTTPServerConfiguration withMaxFormDataSize(int maxFormDataSize) {
-    // 0 turns this off
+    if (maxFormDataSize != -1 && maxFormDataSize <= 0) {
+      throw new IllegalArgumentException("The maximum form data size must be greater than 0. Set to -1 to disable this limitation.");
+    }
+
     this.maxFormDataSize = maxFormDataSize;
     return this;
   }
@@ -541,7 +545,7 @@ public HTTPServerConfiguration withMaxResponseChunkSize(int size) {
    */
   @Override
   public HTTPServerConfiguration withMaximumBytesToDrain(int maxBytesToDrain) {
-    if (maxBytesToDrain < 1024 || maxBytesToDrain >= 256 * 1024 * 1024) {
+    if (maxBytesToDrain < 1024 || maxBytesToDrain > 256 * 1024 * 1024) {
       throw new IllegalArgumentException("The maximum bytes to drain must be greater than or equal to 1024 and less than or equal to 268,435,456 (256 Megabytes)");
     }
 

src/main/java/io/fusionauth/http/server/HTTPServerConfiguration.java

@@ -22,18 +22,15 @@
 import java.net.SocketTimeoutException;
 import java.nio.file.Files;
 
-import io.fusionauth.http.ContentTooLargeException;
 import io.fusionauth.http.HTTPProcessingException;
 import io.fusionauth.http.HTTPValues;
 import io.fusionauth.http.HTTPValues.Connections;
 import io.fusionauth.http.HTTPValues.ContentTypes;
 import io.fusionauth.http.HTTPValues.Headers;
 import io.fusionauth.http.HTTPValues.Protocols;
 import io.fusionauth.http.ParseException;
-import io.fusionauth.http.RequestHeadersTooLargeException;
 import io.fusionauth.http.io.MultipartConfiguration;
 import io.fusionauth.http.io.PushbackInputStream;
-import io.fusionauth.http.io.ReadLimitedInputStream;
 import io.fusionauth.http.log.Logger;
 import io.fusionauth.http.server.ExceptionHandlerContext;
 import io.fusionauth.http.server.HTTPHandler;
@@ -69,8 +66,6 @@ public class HTTPWorker implements Runnable {
 
   private final Logger logger;
 
-  private final ReadLimitedInputStream readLimitedInputStream;
-
   private final Socket socket;
 
   private final long startInstant;
@@ -90,8 +85,7 @@ public HTTPWorker(Socket socket, HTTPServerConfiguration configuration, Instrume
     this.throughput = throughput;
     this.buffers = new HTTPBuffers(configuration);
     this.logger = configuration.getLoggerFactory().getLogger(HTTPWorker.class);
-    this.readLimitedInputStream = new ReadLimitedInputStream(new ThroughputInputStream(socket.getInputStream(), throughput));
-    this.inputStream = new PushbackInputStream(readLimitedInputStream, instrumenter);
+    this.inputStream = new PushbackInputStream(new ThroughputInputStream(socket.getInputStream(), throughput), instrumenter);
     this.state = State.Read;
     this.startInstant = System.currentTimeMillis();
     logger.trace("[{}] Starting HTTP worker.", Thread.currentThread().threadId());
@@ -134,14 +128,10 @@ public void run() {
         HTTPOutputStream outputStream = new HTTPOutputStream(configuration, request.getAcceptEncodings(), response, throughputOutputStream, buffers, () -> state = State.Write);
         response.setOutputStream(outputStream);
 
-        // Limit the maximum header size
-        readLimitedInputStream.setMaximumBytesToRead(null, configuration.getMaxRequestHeaderSize(), maxSize ->
-            new RequestHeadersTooLargeException(maxSize, "The maximum size of the request header has been exceeded. The maximum size is [" + maxSize + "] bytes."));
-
         // Not this line of code will block
         // - When a client is using Keep-Alive - we will loop and block here while we wait for the client to send us bytes.
         byte[] requestBuffer = buffers.requestBuffer();
-        HTTPTools.parseRequestPreamble(inputStream, request, requestBuffer, () -> state = State.Read);
+        HTTPTools.parseRequestPreamble(inputStream, configuration.getMaxRequestHeaderSize(), request, requestBuffer, () -> state = State.Read);
         if (logger.isTraceEnabled()) {
           int availableBufferedBytes = inputStream.getAvailableBufferedBytesRemaining();
           if (availableBufferedBytes != 0) {
@@ -155,14 +145,9 @@ public void run() {
           instrumenter.acceptedRequest();
         }
 
-        // Limit the maximum body size
-        var maximumContentLength = ContentTypes.Form.equalsIgnoreCase(request.getContentType())
-            ? configuration.getMaxFormDataSize()
-            : configuration.getMaxRequestBodySize();
-        readLimitedInputStream.setMaximumBytesToRead(request.getContentLength(), maximumContentLength, maxSize ->
-            new ContentTooLargeException(maxSize, "The maximum size of the request body has been exceeded. The maximum size is [" + maxSize + "] bytes."));
-
-        httpInputStream = new HTTPInputStream(configuration, request, inputStream);
+        // Configure maximum content length
+        int maximumContentLength = getMaximumContentLength(request);
+        httpInputStream = new HTTPInputStream(configuration, request, inputStream, maximumContentLength);
         request.setInputStream(httpInputStream);
 
         // Set the Connection response header as soon as possible
@@ -251,8 +236,7 @@ public void run() {
       logger.trace("[{}] Closing socket. Client closed the connection. Reason [{}].", Thread.currentThread().threadId(), e.getMessage());
       closeSocketOnly(CloseSocketReason.Expected);
     } catch (HTTPProcessingException e) {
-      // Note that I am only tracing this. This is sort of expected - in that it is possible that the request handler will catch this exception and handle it. If the request handler
-      // does not handle this exception, it is totally fine to handle it here.
+      // Note that I am only tracing this, because this exception is mostly expected. Use closeSocketOnError so we can attempt to write a response.
       logger.trace("[{}] Closing socket with status [{}]. An unhandled [{}] exception was taken. Reason [{}].", Thread.currentThread().threadId(), e.getStatus(), e.getClass().getSimpleName(), e.getMessage());
       closeSocketOnError(response, e.getStatus());
     } catch (TooManyBytesToDrainException e) {
@@ -327,9 +311,20 @@ private void closeSocketOnError(HTTPResponse response, int status) {
         response.setHeader(Headers.Connection, Connections.Close);
         response.setStatus(status);
         response.setContentLength(0L);
+//        System.out.println("return ["  + status + "]");
+
+        // Here
+        // Close this sucker out!
+//        socket.setSoLinger(true, 0);
+
+//        socket.shutdownInput();
+//        socket.getInputStream().close();
         response.close();
       }
     } catch (IOException e) {
+      System.out.println("\n\n\nHere!");
+      System.out.println(e.getClass().getSimpleName());
+      System.out.println(e.getMessage());
       logger.debug(String.format("[%s] Could not close the HTTP response.", Thread.currentThread().threadId()), e);
     } finally {
       // It is plausible that calling response.close() could throw an exception. We must ensure we close the socket.
@@ -349,6 +344,19 @@ private void closeSocketOnly(CloseSocketReason reason) {
     }
   }
 
+  private int getMaximumContentLength(HTTPRequest request) {
+    var maximumContentLength = -1;
+    if (ContentTypes.Form.equalsIgnoreCase(request.getContentType())) {
+      maximumContentLength = configuration.getMaxFormDataSize();
+    }
+
+    if (maximumContentLength == -1) {
+      maximumContentLength = configuration.getMaxRequestBodySize();
+    }
+
+    return maximumContentLength;
+  }
+
   private boolean handleExpectContinue(HTTPRequest request) throws IOException {
     var expectResponse = new HTTPResponse();
     configuration.getExpectValidator().validate(request, expectResponse);

src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java

@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 
+import io.fusionauth.http.ContentTooLargeException;
 import io.fusionauth.http.io.ChunkedInputStream;
 import io.fusionauth.http.io.PushbackInputStream;
 import io.fusionauth.http.log.Logger;
@@ -43,10 +44,14 @@ public class HTTPInputStream extends InputStream {
 
   private final int maximumBytesToDrain;
 
+  private final int maximumContentLength;
+
   private final PushbackInputStream pushbackInputStream;
 
   private final HTTPRequest request;
 
+  private int bytesRead;
+
   private long bytesRemaining;
 
   private boolean closed;
@@ -57,14 +62,16 @@ public class HTTPInputStream extends InputStream {
 
   private boolean drained;
 
-  public HTTPInputStream(HTTPServerConfiguration configuration, HTTPRequest request, PushbackInputStream pushbackInputStream) {
+  public HTTPInputStream(HTTPServerConfiguration configuration, HTTPRequest request, PushbackInputStream pushbackInputStream,
+                         int maximumContentLength) {
     this.logger = configuration.getLoggerFactory().getLogger(HTTPInputStream.class);
     this.instrumenter = configuration.getInstrumenter();
     this.request = request;
     this.delegate = pushbackInputStream;
     this.pushbackInputStream = pushbackInputStream;
     this.chunkedBufferSize = configuration.getChunkedBufferSize();
     this.maximumBytesToDrain = configuration.getMaxBytesToDrain();
+    this.maximumContentLength = maximumContentLength;
 
     // Start the countdown
     if (request.getContentLength() != null) {
@@ -144,7 +151,9 @@ public int read(byte[] b, int off, int len) throws IOException {
 
     // When we have a fixed length request, read beyond the remainingBytes if possible.
     // - If we have read past the end of the current request, push those bytes back onto the InputStream.
-    int read = delegate.read(b, off, len);
+    int maxLen = maximumContentLength == -1 ? len : Math.min(len, maximumContentLength - bytesRead + 1);
+    int read = delegate.read(b, off, maxLen);
+
     int reportBytesRead = read;
     if (fixedLength && read > 0) {
       int extraBytes = (int) (read - bytesRemaining);
@@ -161,14 +170,32 @@ public int read(byte[] b, int off, int len) throws IOException {
     }
 
     // TODO : Daniel : Review : If we push back n bytes, don't we need to return read - n? This was previously read which ignored bytes pushed back.
+    // TODO : Daniel : Write a test to prove this, send a content-length of 100, buffer size 80 (as an example), ensure this returns 80, and then
+    //                 the next call returns 20?
+
+
+    bytesRead += reportBytesRead;
+
+    // This won't cause us to fail as fast as we could, but it keeps the code a bit simpler.
+    // - This means we will have read past the maximum by n where n is > 0 && < len. This seems like an acceptable over-read, in practice the buffers will be
+    if (maximumContentLength != -1) {
+      if (bytesRead > maximumContentLength) {
+        String detailedMessage = "The maximum request size has been exceeded.The maximum request size is [" + maximumContentLength + "] bytes.";
+        throw new ContentTooLargeException(maximumContentLength, detailedMessage);
+      }
+    }
+
     return reportBytesRead;
   }
 
   private void commit() {
     committed = true;
 
+    // TODO : Handle : Content-Encoding
+
     // Note that isChunked() should take precedence over the fact that we have a Content-Length.
     // - The client should not send both, but in the case they are both present we ignore Content-Length
+    //   In practice, we will remove the Content-Length header when sent in addition to Transfer-Encoding. See HTTPWorker.validatePreamble.
     Long contentLength = request.getContentLength();
     boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked();
     if (!hasBody) {
@@ -184,5 +211,15 @@ private void commit() {
     } else {
       logger.trace("Client indicated it was NOT sending an entity-body in the request");
     }
+
+    // If we have a maximumContentLength, and this is a fixed content length request, before we read any bytes, fail early.
+    // For good measure do this last so if anyone downstream wants to read from the InputStream they could in theory because
+    // we will have set up the InputStream.
+    if (contentLength != null && maximumContentLength != -1) {
+      if (contentLength > maximumContentLength) {
+        String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes.";
+        throw new ContentTooLargeException(maximumContentLength, detailedMessage);
+      }
+    }
   }
 }