add support for chunked encoding

src/puter-js/src/modules/networking/requests.js

@@ -61,7 +61,7 @@ export function pFetch(...args) {
             headers.set("user-agent", navigator.userAgent);
         }
 
-        let reqHead = `${reqObj.method} ${parsedURL.pathname} HTTP/1.1\r\nHost: ${parsedURL.host}\r\nConnection: close\r\n`;
+        let reqHead = `${reqObj.method} ${parsedURL.pathname}${parsedURL.search} HTTP/1.1\r\nHost: ${parsedURL.host}\r\nConnection: close\r\n`;
         for (const [key, value] of headers) {
             reqHead += `${key}: ${value}\r\n`;
         }
@@ -77,18 +77,93 @@ export function pFetch(...args) {
         let responseReturned = false;
         let contentLength = -1;
         let ingestedContent = 0;
+        let chunkedTransfer = false;
+        let currentChunkLeft = -1;
+        let buffer = new Uint8Array(0);
 
         const outStream = new ReadableStream({
             start(controller) {
+                // This is annoyingly long
+                function parseIncomingChunk(data) {
+                    // append new data to our rolling buffer
+                    const tmp = new Uint8Array(buffer.length + data.length);
+                    tmp.set(buffer, 0);
+                    tmp.set(data, buffer.length);
+                    buffer = tmp;
+
+                    // pull out as many complete chunks (or headers) as we can
+                    while (true) {
+                        if (currentChunkLeft > 0) {
+                            // we’re in the middle of reading a chunk body
+                            // need size + 2 bytes (for trailing \r\n)
+                            if (buffer.length >= currentChunkLeft + 2) {
+                                // full body + CRLF available
+                                const chunk = buffer.slice(0, currentChunkLeft);
+                                controller.enqueue(chunk);
+
+                                // strip body + CRLF and reset for next header
+                                buffer = buffer.slice(currentChunkLeft + 2);
+                                currentChunkLeft = 0;
+                            } else {
+                                // only a partial body available
+                                controller.enqueue(buffer);
+                                currentChunkLeft -= buffer.length;
+                                buffer = new Uint8Array(0);
+                                break; // wait for more data
+                            }
+                        } else {
+                            // we need to parse the next size line
+                            // find the first "\r\n"
+                            let idx = -1;
+                            for (let i = 0; i + 1 < buffer.length; i++) {
+                                if (
+                                    buffer[i] === 0x0d &&
+                                    buffer[i + 1] === 0x0a
+                                ) {
+                                    idx = i;
+                                    break;
+                                }
+                            }
+                            if (idx < 0) {
+                                // we don’t yet have a full size line
+                                break;
+                            }
+
+                            // decode just the size line as ASCII hex
+                            const sizeText = decoder
+                                .decode(buffer.slice(0, idx))
+                                .trim();
+                            currentChunkLeft = parseInt(sizeText, 16);
+                            if (isNaN(currentChunkLeft)) { 
+                                controller.error("Invalid chunk length from server")
+                            }
+                            // strip off the size line + CRLF
+                            buffer = buffer.slice(idx + 2);
+
+                            // zero-length => end of stream
+                            if (currentChunkLeft === 0) {
+                                responseReturned = true;
+                                controller.close();
+                                return;
+                            }
+                        }
+                    }
+                }
                 socket.on("data", (data) => {
                     // Dataoffset is set to another value once head is returned, its safe to assume all remaining data is body
-                    if (dataOffset !== -1) {
+                    if (dataOffset !== -1 && !chunkedTransfer) {
                         controller.enqueue(data);
                         ingestedContent += data.length;
                     }
 
-                    fullDataParts.push(data);
-                    responseHead += decoder.decode(data, { stream: true });
+                    // We dont have the full responseHead yet
+                    if (dataOffset === -1) {
+                        fullDataParts.push(data);
+                        responseHead += decoder.decode(data, { stream: true });
+                    }
+                    if (chunkedTransfer) {
+                        parseIncomingChunk(data);
+                    }
 
                     // See if we have the HEAD of an HTTP/1.1 yet
                     if (responseHead.indexOf("\r\n\r\n") !== -1) {
@@ -98,20 +173,28 @@ export function pFetch(...args) {
                         contentLength = Number(
                             parsedHead.headers.get("content-length"),
                         );
+                        chunkedTransfer =
+                            parsedHead.headers.get("transfer-encoding") ===
+                            "chunked";
                         // Return initial response object
                         res(new Response(outStream, parsedHead));
 
-                        // Add any content we have but isn't part of the head into the body stream
                         const residualBody = mergeUint8Arrays(
                             ...fullDataParts,
                         ).slice(dataOffset + 4);
-                        ingestedContent += residualBody.length;
-                        controller.enqueue(residualBody);
+                        if (!chunkedTransfer) {
+                            // Add any content we have but isn't part of the head into the body stream
+                            ingestedContent += residualBody.length;
+                            controller.enqueue(residualBody);
+                        } else {
+                            parseIncomingChunk(residualBody);
+                        }
                     }
 
                     if (
                         contentLength !== -1 &&
-                        ingestedContent === contentLength
+                        ingestedContent === contentLength &&
+                        !chunkedTransfer
                     ) {
                         // Work around for the close bug for compliant HTTP/1.1 servers
                         if (!responseReturned) {