fix: binary-safe HTTP body buffering for multi-segment POST/PUT

Makefile

@@ -348,6 +348,7 @@ devices-test: build/v86-debug.wasm
 	./tests/devices/virtio_console.js
 	./tests/devices/fetch_network.js
 	USE_VIRTIO=1 ./tests/devices/fetch_network.js
+	./tests/devices/fetch_network_post.js
 	./tests/devices/wisp_network.js
 	./tests/devices/virtio_balloon.js
 

src/browser/fetch_network.js

@@ -13,6 +13,22 @@ import {
 // For Types Only
 import { BusConnector } from "../bus.js";
 
+
+const CR = 0x0D;
+const LF = 0x0A;
+
+function find_crlfcrlf(haystack)
+{
+    for(let i = 0; i + 3 < haystack.length; i++)
+    {
+        if(haystack[i] === CR && haystack[i + 1] === LF && haystack[i + 2] === CR && haystack[i + 3] === LF)
+        {
+            return i;
+        }
+    }
+    return -1;
+}
+
 /**
  * @constructor
  *
@@ -69,116 +85,213 @@ FetchNetworkAdapter.prototype.tcp_probe = function(port)
 };
 
 /**
+ * HTTP data handler for port-80 TCP connections.
+ *
+ * Incoming TCP segments are buffered as raw bytes and searched for the
+ * \r\n\r\n header/body separator.  Once found the header portion alone is
+ * text-decoded and parsed; the body stays as a binary Uint8Array.
+ *
+ * When a POST/PUT body spans multiple TCP segments the partial body is
+ * stored in this.pendingBody and subsequent segments are accumulated there until
+ * Content-Length is satisfied, at which point the deferred fetch is fired.
+ *
+ * NOTE: Transfer-Encoding: chunked is not supported.  Requests using it
+ * will not be dispatched.
+ *
  * @this {TCPConnection}
  * @param {!ArrayBuffer} data
  */
 async function on_data_http(data)
 {
-    this.read = this.read || "";
-    this.read += new TextDecoder().decode(data);
-    if(this.read && this.read.indexOf("\r\n\r\n") !== -1) {
-        let offset = this.read.indexOf("\r\n\r\n");
-        let headers = this.read.substring(0, offset).split(/\r\n/);
-        let data = this.read.substring(offset + 4);
-        this.read = "";
-
-        let first_line = headers[0].split(" ");
-        let target;
-        if(/^https?:/.test(first_line[1])) {
-            // HTTP proxy
-            target = new URL(first_line[1]);
-        }
-        else {
-            target = new URL("http://host" + first_line[1]);
+    // If we're buffering a partial request body, accumulate chunks until
+    // Content-Length is satisfied, then fire the deferred fetch.
+    if(this.pendingBody)
+    {
+        const chunk = data instanceof Uint8Array ? data : new Uint8Array(data);
+        const combined = new Uint8Array(this.pendingBody.buf.length + chunk.length);
+        combined.set(this.pendingBody.buf);
+        combined.set(chunk, this.pendingBody.buf.length);
+        this.pendingBody.buf = combined;
+        if(this.pendingBody.buf.length >= this.pendingBody.cl)
+        {
+            const body = this.pendingBody.buf;
+            const done = this.pendingBody.done;
+            this.pendingBody = null;
+            done(body);
         }
-        if(typeof window !== "undefined" && target.protocol === "http:" && window.location.protocol === "https:") {
-            // fix "Mixed Content" errors
-            target.protocol = "https:";
+        return;
+    }
+
+    // Accumulate raw bytes (not text) so binary body data is preserved.
+    const chunk = data instanceof Uint8Array ? data : new Uint8Array(data);
+    if(this.rawBuffer)
+    {
+        const combined = new Uint8Array(this.rawBuffer.length + chunk.length);
+        combined.set(this.rawBuffer);
+        combined.set(chunk, this.rawBuffer.length);
+        this.rawBuffer = combined;
+    }
+    else
+    {
+        this.rawBuffer = chunk;
+    }
+
+    const sep_index = find_crlfcrlf(this.rawBuffer);
+    if(sep_index === -1) return;
+
+    // Split into header (text) and body (binary).
+    const header_bytes = this.rawBuffer.slice(0, sep_index);
+    const body_bytes = this.rawBuffer.slice(sep_index + 4);
+    this.rawBuffer = null;
+
+    const header_text = new TextDecoder().decode(header_bytes);
+    const header_lines = header_text.split(/\r\n/);
+
+    const first_line = header_lines[0].split(" ");
+    let target;
+    if(/^https?:/.test(first_line[1]))
+    {
+        // HTTP proxy
+        target = new URL(first_line[1]);
+    }
+    else
+    {
+        target = new URL("http://host" + first_line[1]);
+    }
+    if(typeof window !== "undefined" && target.protocol === "http:" && window.location.protocol === "https:")
+    {
+        // fix "Mixed Content" errors
+        target.protocol = "https:";
+    }
+
+    const req_headers = new Headers();
+    for(let i = 1; i < header_lines.length; ++i)
+    {
+        const header = this.net.parse_http_header(header_lines[i]);
+        if(!header)
+        {
+            console.warn('The request contains an invalid header: "%s"', header_lines[i]);
+            this.net.respond_text_and_close(this, 400, "Bad Request", `Invalid header in request: ${header_lines[i]}`);
+            return;
         }
+        if(header.key.toLowerCase() === "host") target.host = header.value;
+        else req_headers.append(header.key, header.value);
+    }
 
-        let req_headers = new Headers();
-        for(let i = 1; i < headers.length; ++i) {
-            const header = this.net.parse_http_header(headers[i]);
-            if(!header) {
-                console.warn('The request contains an invalid header: "%s"', headers[i]);
-                this.net.respond_text_and_close(this, 400, "Bad Request", `Invalid header in request: ${headers[i]}`);
-                return;
-            }
-            if( header.key.toLowerCase() === "host" ) target.host = header.value;
-            else req_headers.append(header.key, header.value);
+    if(!this.net.cors_proxy && /^\d+\.external$/.test(target.hostname))
+    {
+        dbg_log("Request to localhost: " + target.href, LOG_FETCH);
+        const localport = parseInt(target.hostname.split(".")[0], 10);
+        if(!isNaN(localport) && localport > 0 && localport < 65536)
+        {
+            target.protocol = "http:";
+            target.hostname = "localhost";
+            target.port = localport.toString(10);
         }
+        else
+        {
+            console.warn('Unknown port for localhost: "%s"', target.href);
+            this.net.respond_text_and_close(this, 400, "Bad Request", `Unknown port for localhost: ${target.href}`);
+            return;
+        }
+    }
+
+    dbg_log("HTTP Dispatch: " + target.href, LOG_FETCH);
+    this.name = target.href;
+
+    const opts = {
+        method: first_line[0],
+        headers: req_headers,
+    };
 
-        if(!this.net.cors_proxy && /^\d+\.external$/.test(target.hostname)) {
-            dbg_log("Request to localhost: " + target.href, LOG_FETCH);
-            const localport = parseInt(target.hostname.split(".")[0], 10);
-            if(!isNaN(localport) && localport > 0 && localport < 65536) {
-                target.protocol = "http:";
-                target.hostname = "localhost";
-                target.port = localport.toString(10);
-            } else {
-                console.warn('Unknown port for localhost: "%s"', target.href);
-                this.net.respond_text_and_close(this, 400, "Bad Request", `Unknown port for localhost: ${target.href}`);
-                return;
-            }
+    const fetch_url = this.net.cors_proxy
+        ? this.net.cors_proxy + encodeURIComponent(target.href)
+        : target.href;
+
+    if(["put", "post"].indexOf(opts.method.toLowerCase()) !== -1)
+    {
+        // The body may span multiple TCP segments.
+        // If Content-Length is present and larger than what we have so far,
+        // buffer the partial body and wait for remaining chunks.
+        const content_length = parseInt(req_headers.get("content-length") || "0", 10);
+        if(content_length > 0 && body_bytes.length < content_length)
+        {
+            this.pendingBody = {
+                buf: body_bytes,
+                cl: content_length,
+                done: (body) => {
+                    opts.body = body;
+                    dispatch_fetch(this, fetch_url, opts);
+                },
+            };
+            return;
         }
+        opts.body = body_bytes;
+    }
 
-        dbg_log("HTTP Dispatch: " + target.href, LOG_FETCH);
-        this.name = target.href;
-        let opts = {
-            method: first_line[0],
-            headers: req_headers,
-        };
-        if(["put", "post"].indexOf(opts.method.toLowerCase()) !== -1) {
-            opts.body = data;
+    dispatch_fetch(this, fetch_url, opts);
+}
+
+/**
+ * Execute the HTTP fetch and pipe the response back to the guest.
+ *
+ * @param {TCPConnection} conn
+ * @param {string} fetch_url
+ * @param {!Object} opts
+ */
+function dispatch_fetch(conn, fetch_url, opts)
+{
+    let response_started = false;
+    let handler = (resp) => {
+        let resp_headers = new Headers(resp.headers);
+        resp_headers.delete("content-encoding");
+        resp_headers.delete("keep-alive");
+        resp_headers.delete("content-length");
+        resp_headers.delete("transfer-encoding");
+        resp_headers.set("x-was-fetch-redirected", `${!!resp.redirected}`);
+        resp_headers.set("x-fetch-resp-url", resp.url);
+        resp_headers.set("connection", "close");
+
+        conn.write(conn.net.form_response_head(resp.status, resp.statusText, resp_headers));
+        response_started = true;
+
+        if(resp.body && resp.body.getReader)
+        {
+            const resp_reader = resp.body.getReader();
+            const pump = ({ value, done }) => {
+                if(value)
+                {
+                    conn.write(value);
+                }
+                if(done)
+                {
+                    conn.close();
+                }
+                else
+                {
+                    return resp_reader.read().then(pump);
+                }
+            };
+            resp_reader.read().then(pump);
+        }
+        else
+        {
+            resp.arrayBuffer().then(buffer => {
+                conn.write(new Uint8Array(buffer));
+                conn.close();
+            });
         }
+    };
 
-        const fetch_url = this.net.cors_proxy ? this.net.cors_proxy + encodeURIComponent(target.href) : target.href;
-        const encoder = new TextEncoder();
-        let response_started = false;
-        let handler = (resp) => {
-            let resp_headers = new Headers(resp.headers);
-            resp_headers.delete("content-encoding");
-            resp_headers.delete("keep-alive");
-            resp_headers.delete("content-length");
-            resp_headers.delete("transfer-encoding");
-            resp_headers.set("x-was-fetch-redirected", `${!!resp.redirected}`);
-            resp_headers.set("x-fetch-resp-url", resp.url);
-            resp_headers.set("connection", "close");
-
-            this.write(this.net.form_response_head(resp.status, resp.statusText, resp_headers));
-            response_started = true;
-
-            if(resp.body && resp.body.getReader) {
-                const resp_reader = resp.body.getReader();
-                const pump = ({ value, done }) => {
-                    if(value) {
-                        this.write(value);
-                    }
-                    if(done) {
-                        this.close();
-                    }
-                    else {
-                        return resp_reader.read().then(pump);
-                    }
-                };
-                resp_reader.read().then(pump);
-            } else {
-                resp.arrayBuffer().then(buffer => {
-                    this.write(new Uint8Array(buffer));
-                    this.close();
-                });
-            }
-        };
-
-        this.net.fetch(fetch_url, opts).then(handler)
-        .catch((e) => {
-            console.warn("Fetch Failed: " + fetch_url + "\n" + e);
-            if(!response_started) {
-                this.net.respond_text_and_close(this, 502, "Fetch Error", `Fetch ${fetch_url} failed:\n\n${e.stack || e.message}`);
-            }
-            this.close();
-        });
-    }
+    conn.net.fetch(fetch_url, opts).then(handler)
+    .catch((e) => {
+        console.warn("Fetch Failed: " + fetch_url + "\n" + e);
+        if(!response_started)
+        {
+            conn.net.respond_text_and_close(conn, 502, "Fetch Error", `Fetch ${fetch_url} failed:\n\n${e.stack || e.message}`);
+        }
+        conn.close();
+    });
 }
 
 FetchNetworkAdapter.prototype.fetch = async function(url, options)
@@ -211,7 +324,8 @@ FetchNetworkAdapter.prototype.form_response_head = function(status_code, status_
         `HTTP/1.1 ${status_code} ${status_text}`
     ];
 
-    for(const [key, value] of headers.entries()) {
+    for(const [key, value] of headers.entries())
+    {
         lines.push(`${key}: ${value}`);
     }
 
@@ -232,7 +346,8 @@ FetchNetworkAdapter.prototype.respond_text_and_close = function(conn, status_cod
 FetchNetworkAdapter.prototype.parse_http_header = function(header)
 {
     const parts = header.match(/^([^:]*):(.*)$/);
-    if(!parts) {
+    if(!parts)
+    {
         dbg_log("Unable to parse HTTP header", LOG_FETCH);
         return;
     }