BodyProcessor.java

package com.renomad.minum.web;

import com.renomad.minum.logging.ILogger;
import com.renomad.minum.security.ForbiddenUseException;
import com.renomad.minum.state.Constants;
import com.renomad.minum.state.Context;
import com.renomad.minum.utils.StringUtils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * This code is responsible for extracting the {@link Body} from
 * an HTTP request.
 */
final class BodyProcessor implements IBodyProcessor {

    private final ILogger logger;
    private final IInputStreamUtils inputStreamUtils;
    private final Constants constants;

    BodyProcessor(Context context) {
        this.constants = context.getConstants();
        this.logger = context.getLogger();
        this.inputStreamUtils = new InputStreamUtils(constants.maxReadLineSizeBytes);
    }

    @Override
    public Body extractData(InputStream is, Headers h) {
        final var contentType = h.contentType();

        if (h.contentLength() >= 0L) {
            if (h.contentLength() >= (long) constants.maxReadSizeBytes) {
                throw new ForbiddenUseException("It is disallowed to process a body with a length more than " + constants.maxReadSizeBytes + " bytes");
            }
        } else {
            // we don't process chunked transfer encodings.  just bail.
            List<String> transferEncodingHeaders = h.valueByKey("transfer-encoding");
            if (List.of("chunked").equals(transferEncodingHeaders)) {
                throw new BadRequestException("client sent chunked transfer-encoding.  Minum does not automatically read bodies of this type.");
            }
            return Body.EMPTY;
        }

        return extractBodyFromInputStream(h.contentLength(), contentType, is);
    }

    /**
     * Handles the parsing of the body data for either form-urlencoded or
     * multipart/form-data
     *
     * @param contentType a mime value which must be either application/x-www-form-urlencoded
     *                    or multipart/form-data.  Anything else will cause a new Body to
     *                    be created with the body bytes, unparsed.  There are a number of
     *                    cases where this makes sense - if the user is sending us plain text,
     *                    html, json, or css, we want to simply accept the data and not try to parse it.
     */
    Body extractBodyFromInputStream(long contentLength, String contentType, InputStream is) {
        // if the body is zero bytes long, just return
        if (contentLength == 0L) {
            logger.logDebug(() -> "the length of the body was 0, returning an empty Body");
            return Body.EMPTY;
        }

        if (contentType.contains("application/x-www-form-urlencoded")) {
            return parseUrlEncodedForm(is, contentLength);
        } else if (contentType.contains("multipart/form-data")) {
            String boundaryValue = determineBoundaryValue(contentType);
            return parseMultipartForm(contentLength, boundaryValue, is);
        } else {
            logger.logDebug(() -> "did not recognize a key-value pattern content-type, returning the raw bytes for the body.  Content-Type was: " + contentType);
            // we can return the whole byte array here because we never read from it
            try {
                return new Body(Map.of(), inputStreamUtils.read((int) contentLength, is), List.of(), BodyType.UNRECOGNIZED);
            } catch (IOException e) {
                throw new WebServerException("Error in BodyProcessor.extractBodyFromInputStream", e);
            }
        }
    }

    /**
     * Parse multipart/form protocol.
     * @param contentLength the length of incoming data, found in the "content-length" header
     * @param boundaryValue the randomly-generated boundary value between the partitions.  Research
     *                      multipart/form data protocol for further information.
     * @param inputStream A stream of bytes coming from the socket.
     */
    private Body parseMultipartForm(long contentLength, String boundaryValue, InputStream inputStream) {
        List<Partition> partitions = new ArrayList<>();

        try {
            int countOfPartitions = 0;
            for (StreamingMultipartPartition p : getMultiPartIterable(inputStream, boundaryValue, contentLength)) {
                countOfPartitions += 1;
                if (countOfPartitions >= MAX_BODY_KEYS_URL_ENCODED) {
                    throw new WebServerException("Error: body had excessive number of partitions (" + countOfPartitions + ").  Maximum allowed: " + MAX_BODY_KEYS_URL_ENCODED);
                }
                partitions.add(new Partition(p.getHeaders(), p.readAllBytes(), p.getContentDisposition()));
            }


        } catch (Exception ex) {
            throw new BadRequestException("Unable to parse the body as a multipart/form-data content-type", ex);
        }

        if (partitions.isEmpty()) {
            return new Body(Map.of(), new byte[0], List.of(), BodyType.UNRECOGNIZED);
        } else {
            return new Body(Map.of(), new byte[0], partitions, BodyType.MULTIPART);
        }
    }

    /**
     * Given the "content-type" header, determine the boundary value.  A typical
     * multipart content-type header might look like this: <pre>Content-Type: multipart/form-data; boundary=i_am_a_boundary</pre>
     */
    static String determineBoundaryValue(String contentType) {
        String boundaryKey = "boundary=";
        String boundaryValue;
        int indexOfBoundaryKey = contentType.indexOf(boundaryKey);
        if (indexOfBoundaryKey >= 0) {
            // grab all the text after the key to obtain the boundary value
            boundaryValue = contentType.substring(indexOfBoundaryKey + boundaryKey.length());
            // the index after the end of the boundary value, used to trim the string if necessary
            int indexEndOfBoundaryValue = 0;
            for (int i = 0; i < boundaryValue.length(); i++) {
                char c = boundaryValue.charAt(i);
                if (c == ' ') break;
                if (c == ';') break;
                indexEndOfBoundaryValue += 1;
            }
            return boundaryValue.substring(0, indexEndOfBoundaryValue);
        }
        throw new BadRequestException("Did not find a valid boundary value for the multipart input. Header was: " + contentType);
    }


    /**
     * Parse data formatted by application/x-www-form-urlencoded
     * See <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST">...</a>
     * <p>
     * See here for the encoding: <a href="https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding">...</a>
     * <p>
     * for example, {@code valuea=3&valueb=this+is+something}
     */
    Body parseUrlEncodedForm(InputStream is, long contentLength) {
        if (contentLength == 0L) {
            return Body.EMPTY;
        }
        final var postedPairs = new HashMap<String, byte[]>();

        try {
            int countOfPartitions = 0;
            for (final var keyValue : getUrlEncodedDataIterable(is, contentLength)) {
                countOfPartitions += 1;
                if (countOfPartitions >= MAX_BODY_KEYS_URL_ENCODED) {
                    throw new WebServerException("Error: body had excessive number of partitions ("+countOfPartitions+").  Maximum allowed: " + MAX_BODY_KEYS_URL_ENCODED);
                }
                String value = new String(keyValue.getUedg().readAllBytes(), StandardCharsets.US_ASCII);
                String key = keyValue.getKey();
                final var decodedValue = StringUtils.decode(value);
                final var convertedValue = decodedValue == null ? "".getBytes(StandardCharsets.UTF_8) : decodedValue.getBytes(StandardCharsets.UTF_8);

                final var result = postedPairs.put(key, convertedValue);

                if (result != null) {
                    throw new BadRequestException("Unexpected: key (" +key + ") was duplicated in the post body - previous value was " + new String(result, StandardCharsets.US_ASCII) + " and was overwritten by " + decodedValue);
                }
            }
        } catch (Exception ex) {
            throw new BadRequestException("Unable to parse the request body as a URL-encoded content type", ex);
        }
        // we return nothing for the raw bytes because the code for parsing the streaming data
        // doesn't begin with a fully-read byte array - it pulls data off the stream one byte
        // at a time.
        return new Body(postedPairs, new byte[0], List.of(), BodyType.FORM_URL_ENCODED);
    }

    /**
     * A regex used to extract the name value from the headers in multipart/form
     * For example, in the following code, you can see that the name is "image_uploads"
     * <pre>
     * {@code
     * --i_am_a_boundary
     * Content-Type: text/plain
     * Content-Disposition: form-data; name="text1"
     *
     * I am a value that is text
     * --i_am_a_boundary
     * Content-Type: application/octet-stream
     * Content-Disposition: form-data; name="image_uploads"; filename="photo_preview.jpg"
     * }
     * </pre>
     */
    private static final Pattern multiformNameRegex = Pattern.compile("\\bname\\b=\"(?<namevalue>.*?)\"");
    private static final Pattern multiformFilenameRegex = Pattern.compile("\\bfilename\\b=\"(?<namevalue>.*?)\"");

    @Override
    public Iterable<UrlEncodedKeyValue> getUrlEncodedDataIterable(InputStream inputStream, long contentLength) {
        return () -> new Iterator<>() {

            final CountBytesRead countBytesRead = new CountBytesRead();

            @Override
            public boolean hasNext() {
                return countBytesRead.getCount() < contentLength;
            }

            @Override
            public UrlEncodedKeyValue next() {
                if (!hasNext()) {
                    throw new NoSuchElementException();
                }
                String key = "";
                ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
                while(true) {
                    int result;
                    try {
                        result = inputStream.read();
                        countBytesRead.increment();
                    } catch (IOException e) {
                        throw new WebServerException("Error in getUrlEncodedDataIterable.next", e);
                    }
                    // if this is true, the inputstream is closed
                    if (result == -1) break;
                    byte myByte = (byte) result;
                    // if this is true, we're done with the key
                    if (myByte == '=') {
                        // URL encoding is in ASCII only.
                        key = byteArrayOutputStream.toString(StandardCharsets.US_ASCII);
                        break;
                    } else {
                        if (byteArrayOutputStream.size() >= MAX_KEY_SIZE_BYTES) {
                            throw new WebServerException("Maximum size for name attribute is " + MAX_KEY_SIZE_BYTES + " ascii characters");
                        }
                        byteArrayOutputStream.write(myByte);
                    }
                }

                if (key.isBlank()) {
                    throw new WebServerException("Unable to parse this body. no key found during parsing");
                } else if (countBytesRead.getCount() == contentLength) {
                    // if the only thing sent was the key and there's no further data, return the key with a null input stream
                    // that will immediately return
                    return new UrlEncodedKeyValue(key, new UrlEncodedDataGetter(InputStream.nullInputStream(), countBytesRead, contentLength));
                } else {
                    return new UrlEncodedKeyValue(key, new UrlEncodedDataGetter(inputStream, countBytesRead, contentLength));
                }
            }
        };
    }


    @Override
    public Iterable<StreamingMultipartPartition> getMultiPartIterable(InputStream inputStream, String boundaryValue, long contentLength) {
        return () -> new Iterator<>() {

            final CountBytesRead countBytesRead = new CountBytesRead();
            boolean hasReadFirstPartition = false;

            @Override
            public boolean hasNext() {
                // determining if we have more to read is a little tricky because we have a buffer
                // filled by reading ahead, looking for the boundary value
                return (contentLength - countBytesRead.getCount()) > (long) boundaryValue.length();
            }

            @Override
            public StreamingMultipartPartition next() {
                try {
                    if (!hasNext()) {
                        throw new NoSuchElementException();
                    }
                    // confirm that the boundary value is as expected, as a sanity check,
                    // and avoid including the boundary value in the first set of headers
                    if (!hasReadFirstPartition) {
                        String s;
                        s = inputStreamUtils.readLine(inputStream);
                        if (s == null) {
                            throw new BadRequestException("Unexpectedly encountered end of stream while reading in BodyProcessor.next()");
                        }
                        countBytesRead.incrementBy(s.getBytes(StandardCharsets.UTF_8).length + 2);
                        hasReadFirstPartition = true;

                        if (!s.contains(boundaryValue)) {
                            throw new BadRequestException("Error: First line must contain the expected boundary value. Expected to find: " + boundaryValue + " in: " + s);
                        }
                    }
                    List<String> allHeaders;
                    allHeaders = Headers.getAllHeaders(inputStream, inputStreamUtils);
                    int lengthOfHeaders = allHeaders.stream().map(x -> x.getBytes(StandardCharsets.UTF_8).length).reduce(0, Integer::sum);
                    // each line has a CR + LF (that's two bytes) and the headers end with a second pair of CR+LF.
                    int extraCrLfs = (2 * allHeaders.size()) + 2;
                    countBytesRead.incrementBy(lengthOfHeaders + extraCrLfs);

                    Headers headers = new Headers(allHeaders);

                    List<String> cds = headers.valueByKey("Content-Disposition");
                    if (cds == null) {
                        throw new WebServerException("Error: no Content-Disposition header on partition in Multipart/form data");
                    }
                    String contentDisposition = String.join(";", cds);

                    Matcher nameMatcher = multiformNameRegex.matcher(contentDisposition);
                    Matcher filenameMatcher = multiformFilenameRegex.matcher(contentDisposition);

                    String name;
                    if (nameMatcher.find()) {
                        name = nameMatcher.group("namevalue");
                    } else {
                        throw new WebServerException("Error: No name value set on multipart partition");
                    }
                    String filename = "";
                    if (filenameMatcher.find()) {
                        filename = filenameMatcher.group("namevalue");
                    }

                    // at this point our inputstream pointer is at the beginning of the
                    // body data.  From here until the end it's pure data.

                    return new StreamingMultipartPartition(headers, inputStream, new ContentDisposition(name, filename), boundaryValue, countBytesRead, contentLength);
                } catch (IOException ex) {
                    throw new WebServerException("Error in BodyProcessor.getMultiPartIterable.next", ex);
                }
            }


        };
    }

}



























































Active mutators

CONDITIONALS_BOUNDARY
EMPTY_RETURNS
FALSE_RETURNS
INCREMENTS
INVERT_NEGS
MATH
NEGATE_CONDITIONALS
NULL_RETURNS
PRIMITIVE_RETURNS
TRUE_RETURNS
VOID_METHOD_CALLS

Tests examined

com.renomad.minum.web.BoundaryBugTest (31 ms)
com.renomad.minum.web.BodyProcessorTests (133 ms)
com.renomad.minum.web.RequestTests (189 ms)
com.renomad.minum.web.FindingsWebTests (529 ms)
com.renomad.minum.web.WebTests (637 ms)
com.renomad.minum.FunctionalTests.testEndToEnd_Functional(com.renomad.minum.FunctionalTests) (2350 ms)
com.renomad.minum.FunctionalTests.test_EdgeCase_Response_MultiCookies(com.renomad.minum.FunctionalTests) (620 ms)
com.renomad.minum.FunctionalTests.test_PathFunction_Response_Range(com.renomad.minum.FunctionalTests) (636 ms)
com.renomad.minum.FunctionalTests.test_PathFunction_Response(com.renomad.minum.FunctionalTests) (638 ms)
com.renomad.minum.FunctionalTests.test_EdgeCase_PostHandler_IgnoreBody(com.renomad.minum.FunctionalTests) (652 ms)

Report generated by PIT 1.17.0

1		package com.renomad.minum.web;
2
3		import com.renomad.minum.logging.ILogger;
4		import com.renomad.minum.security.ForbiddenUseException;
5		import com.renomad.minum.state.Constants;
6		import com.renomad.minum.state.Context;
7		import com.renomad.minum.utils.StringUtils;
8
9		import java.io.ByteArrayOutputStream;
10		import java.io.IOException;
11		import java.io.InputStream;
12		import java.nio.charset.StandardCharsets;
13		import java.util.*;
14		import java.util.regex.Matcher;
15		import java.util.regex.Pattern;
16
17		/**
18		* This code is responsible for extracting the {@link Body} from
19		* an HTTP request.
20		*/
21		final class BodyProcessor implements IBodyProcessor {
22
23		private final ILogger logger;
24		private final IInputStreamUtils inputStreamUtils;
25		private final Constants constants;
26
27		BodyProcessor(Context context) {
28		this.constants = context.getConstants();
29		this.logger = context.getLogger();
30		this.inputStreamUtils = new InputStreamUtils(constants.maxReadLineSizeBytes);
31		}
32
33		@Override
34		public Body extractData(InputStream is, Headers h) {
35		final var contentType = h.contentType();
36
37	2 1. extractData : changed conditional boundary → TIMED_OUT 2. extractData : negated conditional → KILLED	if (h.contentLength() >= 0L) {
38	2 1. extractData : negated conditional → TIMED_OUT 2. extractData : changed conditional boundary → TIMED_OUT	if (h.contentLength() >= (long) constants.maxReadSizeBytes) {
39		throw new ForbiddenUseException("It is disallowed to process a body with a length more than " + constants.maxReadSizeBytes + " bytes");
40		}

BodyProcessor.java

Mutations

Active mutators

Tests examined