Unexpected websocket buffer data w/ masked fragmented messages: As data grows, server starts receiving what looks random data, not matching frame spec

huangapple
go评论73阅读模式
英文:

Unexpected websocket buffer data w/ masked fragmented messages: As data grows, server starts receiving what looks random data, not matching frame spec

问题

我正在尝试在NodeJS中从头开始实现WebSocket服务器，遵循RFC 6455，以下是我目前可以运行和测试的NodeJS代码（仍然具有一些用于测试的控制台日志）：

// 代码部分...

尝试使用本地主机上的任何客户端连接到它[尚未测试是否在远程服务器上更改]。它应该能够正常工作，直到一定数量的数据，但随着数据变得更大（仍然小于2的64次方），它开始接收看起来像随机数据的内容，与帧规范不匹配。

例如，如果我运行以下代码：

var something = new WebSocket(
    "ws://localhost:8081"
)

something.onopen = () => {
    console.log("opened")
    something.send("BH".repeat(1000000))
}

something.onmessage = e=>{
    console.log(e.data)
}

您可以看到消息的长度仍然小于2^64，因此应该能够正常工作。在服务器端，发送的第一帧看起来正常，FIN位设置为0，表示继续，但接下来的“帧”似乎没有与第一个帧具有相同的格式，甚至有时会占用前3位“保留”位的一些内容，并且“掩码”位并不总是设置为0。这是一些输出的示例[在一点后截断]：

// 输出部分...

因此，输出的大部分内容是不可预测的，一些操作码甚至没有值，大多数内容甚至没有被掩码等等。

RFC中没有关于“掩码”分段消息的示例，这里提供了一些示例：

// 示例部分...

也许这是读取帧顺序的问题？

英文:

I'm trying to implement a websocket server from scratch in NodeJS following the RFC 6455, below is my NodeJS code (still has some console-logs for testing) that can be run and tested currently:

//B"H
var http = require("http")
var crypto = require("crypto")

var awdawneem = [

];

var server  = http.createServer(function(
    request,
    response
) {
    awdawneem.forEach(a => {
        koysayv("Happy hey teves! " + Date.now(), a)
    })
    response.end("Boruch Hashem!")
}).listen(8081, () => {
    server.on("upgrade", (request, socket) => {
        var ki = request.headers[
            "sec-websocket-key"
        ]

        if(!ki) {
            return;
        }

        awdawneem.push(socket);

        var added = ki + 
            "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
        
        var base64string = crypto
        .createHash("sha1")
        .update(
            added
        ).digest("base64")

        var responseHeaders = [
            "HTTP/1.1 101 Awtsmoos",
            "upgrade: websocket",
            "connection: upgrade",
            "sec-websocket-accept: "
                + base64string
        ]

        socket.write(
            responseHeaders.join(
                "\r\n"
            ) + "\r\n\r\n"
        )

        
        koysayv("Shalom", socket)
        shoymayuh(socket, payloads => {
            console.log("Got pay!",payloads)
        })
    })
})

function koysayv(str,socket) {
    var encoded = encodeDayuh(str)
    if(encoded) {
        try {
            socket.write(encoded)
        } catch(e) {
            console.log("something",e)
        }
    }
    else {
        console.log("Nothing")
    }
}

function shoymayuh/*listen*/(socket,cb) {
    var payloads = [];
    var isInMiddleOfGetting = false;
    var currentFrameIndex = 0;
    var isBeginning = false;

    var callback = typeof(cb) == "function"
        ? cb : (() => {})
    socket.on("data", buff => {
        if(!buff || buff.length < 1) {
            payloads = [];
            return callback({
                error: "No length!",
                buff
            })
        }
        var FIN = (
            buff[0] >> 7 
            & 1
        )
        

        var opcode = (
            buff[0] &
            0b00001111
        )

        if(!FIN) {
            isInMiddleOfGetting = true;
            currentFrameIndex = payloads.length;
            if(opcode) {
                isBeginning = true;
            } else {
                isBeginning = false;
            }
        }

        var isMasked = (
            buff[1] >> 7 & 1
        )

        if(!isMasked) {
            return callback({
                error: "Not masked",
                isMasked,buff
            })
        }

        var lengthCode = (
            buff[1] &
            0b01111111
        )

        var whereToReadMask = 2/*3rd byte*/
        var lengthOfPayload;
        if(lengthCode < 126) {
            lengthOfPayload = lengthCode;
        } else if(lengthCode < Math.pow(2,16)) {
            lengthOfPayload = buff.readUInt16BE(2)
            whereToReadMask = 4
        } else if(lengthCode < Math.pow(2,64)) {
            lengthOfPayload = buff.readBigUInt64BE(2)
            whereToReadMask = 10
        } else {
            return callback("Too big")
        }

        var maskData = buff.slice(
            whereToReadMask,
            whereToReadMask + 4
        );
        var whereToReadPayload = whereToReadMask + 4

        
        if(
            whereToReadPayload + 
            lengthOfPayload > buff.length 
        ) {
            return callback({
                error: "Length problem",
                info: {
                    lengthOfPayload,
                    maskedPayload,
                    maskData,
                    isMasked,
                    FIN,
                    isMasked,
                    opcode
                }
            })
        }

        var maskedPayload = buff.slice(
            whereToReadPayload,
            whereToReadPayload + 
            lengthOfPayload
        )

        

        var unmaskedPayload = Buffer.alloc(maskedPayload.length)
        var i;
        var maskIndex;
        for(
            i = 0;
            i < maskedPayload.length;
            i++
        ) {
            maskIndex = maskData[i % 4]
            unmaskedPayload[i] = 
            maskedPayload[i] ^ maskIndex
        }

        console.log(
            "Pushing payload: ",
            unmaskedPayload,
            unmaskedPayload.length,{
                FIN,
                opcode,
                whereToReadPayload,
                whereToReadMask,
                buffLength:buff.length,
                maskData,
                isMasked,
                buff
            }
        )
        payloads.push(unmaskedPayload)

        if(FIN) {
            callback({
                success: {
                    strings: payloads.map(q=>q.toString()),
                    original: payloads
                },
                info: {
                    FIN,
                    opcode,
                    whereToReadMask,
                    whereToReadPayload,
                    maskData,
                    isMasked,
                    lengthOfPayload,
                    maskedLength:maskedPayload.length,
                    bufferLength: buff.length,
                    lengthOfAllPayloads:payloads.length

                }
            })
            payloads = [];
        }
    })
}

function encodeDayuh(mightBeBuffer) {
    if(!mightBeBuffer) return;

    var str = mightBeBuffer.toString();

    var lengthOfPayload = Buffer.byteLength(str)
    var howManyBytesToAdd;
    if(lengthOfPayload < 126) {
        howManyBytesToAdd = 0
    } else if(lengthOfPayload < Math.pow(2,16)) {
        howManyBytesToAdd = 2
    } else if(lengthOfPayload < Math.pow(2,64)) {
        howManyBytesToAdd = 8
    } else return console.log("TOO BIG");

    var bufferToSend = Buffer.alloc(
        2 
        + howManyBytesToAdd
        + lengthOfPayload
    );

    var offset = 0;
    bufferToSend.writeUInt8(
        0b10000001,
        offset
    )

    offset = 1;
    if(lengthOfPayload < 126) {
        bufferToSend.writeUInt8(
            lengthOfPayload,
            offset
        )
        offset = 2
    } else if(lengthOfPayload < Math.pow(2, 16)) {
        bufferToSend.writeUInt8(
            0b01111110/*126*/,
            offset
        )
        offset = 2;
        bufferToSend.writeUInt16BE(
            lengthOfPayload,
            offset
        )

        
        offset = 4;
    } else if(lengthOfPayload < Math.pow(2, 64)) {
        bufferToSend.writeUInt8(
            0b01111111,
            offset
        )

        offset = 2;
        bufferToSend.writeBigUInt64BE(
            lengthOfPayload,
            offset
        )
        offset = 10
    }

    bufferToSend.write(
        str,
        offset
    )
    

    return bufferToSend;

}

Try connecting to it with any client on localhost [haven't tested if it changes when on a remote server]. It should work up until a certain amount, but as the data gets bigger (still less than 2 to the power of 64), it starts receiving what appears to be random data which doesn't match the frame specification.

For example if I run the following:

var something = new WebSocket(
    "ws://localhost:8081"
)

something.onopen = () => {
    console.log("opened")
    something.send("BH".repeat(1000000))
}

something.onmessage = e=>{
    console.log(e.data)
}
"BH".repeat(1000000).length < Math.pow(2,64);

You can see that the length of the message is still less than 2^64, so it should work. On the server side the first frame that's sent looks normal, with the FIN bit set to 0 indicating a continuation, but then the next "frame" doesn't appear to have the same format as the first one, even [some of] the first 3 "reserved" bits are sometimes taken, and the "masking" bit isn't always set to 0, here's an example of some of the output [cut off after a little bit]:

Pushing payload:  <Buffer > 0 {
  FIN: 0,
  opcode: 1,
  whereToReadPayload: 8,
  whereToReadMask: 4,
  buffLength: 32768,
  maskData: <Buffer 00 00 00 01>,
  isMasked: 1,
  buff: <Buffer 01 ff 00 00 00 00 00 01 ff b8 32 85 76 d3 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b ... 32718 more bytes>
}
Pushing payload:  <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00> 27 {
  FIN: 0,
  opcode: 4,
  whereToReadPayload: 6,
  whereToReadMask: 2,
  buffLength: 65536,
  maskData: <Buffer 70 cd 34 9b>,
  isMasked: 1,
  buff: <Buffer 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b ... 65486 more bytes>
}
Pushing payload:  <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00> 27 {
  FIN: 0,
  opcode: 4,
  whereToReadPayload: 6,
  whereToReadMask: 2,
  buffLength: 65536,
  maskData: <Buffer 70 cd 34 9b>,
  isMasked: 1,
  buff: <Buffer 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b 70 cd 34 9b ... 65486 more bytes>
}
Got pay! {
  error: 'Not masked',
  isMasked: 0,
  buff: <Buffer 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a ... 65486 more bytes>
}
Got pay! {
  error: 'Not masked',
  isMasked: 0,
  buff: <Buffer 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a 96 98 33 1a ... 65486 more bytes>
}

So a lot of the output is simply unpredictable, some of the opcodes aren't even value, most of it isn't even masked etc.

There's no example in the RFC for masked fragmented messages, here at the examples given:

5.7.  Examples

   o  A single-frame unmasked text message

      *  0x81 0x05 0x48 0x65 0x6c 0x6c 0x6f (contains "Hello")

   o  A single-frame masked text message

      *  0x81 0x85 0x37 0xfa 0x21 0x3d 0x7f 0x9f 0x4d 0x51 0x58
         (contains "Hello")

   o  A fragmented unmasked text message

      *  0x01 0x03 0x48 0x65 0x6c (contains "Hel")

      *  0x80 0x02 0x6c 0x6f (contains "lo")







Fette & Melnikov             Standards Track                   [Page 38]

RFC 6455                 The WebSocket Protocol            December 2011


   o  Unmasked Ping request and masked Ping response

      *  0x89 0x05 0x48 0x65 0x6c 0x6c 0x6f (contains a body of "Hello",
         but the contents of the body are arbitrary)

      *  0x8a 0x85 0x37 0xfa 0x21 0x3d 0x7f 0x9f 0x4d 0x51 0x58
         (contains a body of "Hello", matching the body of the ping)

   o  256 bytes binary message in a single unmasked frame

      *  0x82 0x7E 0x0100 [256 bytes of binary data]

   o  64KiB binary message in a single unmasked frame

      *  0x82 0x7F 0x0000000000010000 [65536 bytes of binary data]

Maybe it's a problem with reading the order of the frames?

答案1

得分: 3

以下是翻译好的部分：

你错误地假设整个帧都在你的buff块中。你需要将这些块连接成一个单一的缓冲区，然后尝试解析出帧，如果缓冲区中有足够的数据。

你还错误地读取了负载长度。你应该检查lengthCode的值是否为126和127，而不是它们是否小于2的16次方或2的64次方。一个7位整数甚至不会达到2的7次方。

当你正确读取帧时，你会注意到第一帧的负载长度（131000）超过了buff块的长度（32768）。

还要注意你的“第二帧”只是重复了第一帧中的相同4个字节，这表明它仍然只是负载数据。

我修复了你的代码并添加了一些调试输出，以便在接收到块和实际解析帧时清楚地看到。

const OPCODE = {
    CONTINUATION: 0,
    TEXT: 1,
    BINARY: 2,
    CLOSE: 8,
    PING: 9,
    PONG: 10,
};

function shoymayuh(socket, callback) {
    let fragmented_message_opcode;
    let fragmented_message_payload;
    let buff = Buffer.alloc(0);
    let frame_count = 0;
    let chunk_count = 0;

    socket.on("data", chunk => {
        if (!chunk || chunk.length == 0) {
            return;
        }
        // 将块添加到缓冲区
        buff = Buffer.concat([buff, chunk]);
        console.log(`收到块 #${++chunk_count}，大小：${chunk.length} 字节（缓冲区：${buff.length} 字节）`);

        // 缓冲区中可能包含多个帧，尝试从缓冲区中解析，直到无法继续
        for (;;) {
            // 如果没有足够的数据来读取头部的前两个字节，跳过并等待更多数据
            if (buff.length < 2) {
                return;
            }
            const FIN = Boolean(buff[0] & 0b10000000);

            let opcode = buff[0] & 0b00001111;

            const isMasked = Boolean(buff[1] & 0b10000000);

            const lengthCode = buff[1] & 0b01111111;

            // 读取负载长度
            let offset = 2;
            let lengthOfPayload;
            if (lengthCode <= 125) {
                lengthOfPayload = lengthCode;
            } else if (lengthCode == 126) {
                // 如果没有足够的数据来读取负载长度，跳过并等待更多数据
                if (buff.length < offset + 2) {
                    return;
                }
                lengthOfPayload = buff.readUInt16BE(offset);
                offset += 2;
            } else if (lengthCode == 127) {
                // 如果没有足够的数据来读取负载长度，跳过并等待更多数据
                if (buff.length < offset + 8) {
                    return;
                }
                // readBigUInt64BE 返回一个 BigInt，将其转换为数字
                lengthOfPayload = Number(buff.readBigUInt64BE(offset));
                offset += 8;
            }

            // 如果没有足够的数据来读取负载，跳过并等待更多数据
            if (offset + lengthOfPayload > buff.length) {
                return;
            }

            // 提取负载
            let payload = buff.slice(offset, offset + lengthOfPayload);

            // 如果有掩码，则解除掩码
            if (isMasked) {
                payload = payload.map((b, i) => b ^ maskData[i % 4]);
            }

            // 从缓冲区中删除已解析的帧
            buff = buff.slice(offset + lengthOfPayload);

            // 如果是分段消息，将负载连接起来
            if (opcode == OPCODE.CONTINUATION) {
                fragmented_message_payload = Buffer.concat([fragmented_message_payload, payload]);
            }

            console.log("已解析帧", `#${++frame_count}`, {
                FIN,
                opcode,
                isMasked,
                lengthCode,
                lengthOfPayload,
                maskData,
                payload,
            });
            console.log(buff.length, "字节在缓冲区中剩余")

            if (FIN) {
                if (opcode == OPCODE.CONTINUATION) {
                    opcode = fragmented_message_opcode;
                    payload = fragmented_message_payload;
                }

                if (opcode == OPCODE.PING) {
                    callback({
                        ping: payload,
                    });
                }
                else if (opcode == OPCODE.PONG) {
                    callback({
                        pong: payload,
                    });
                }
                else if (opcode == OPCODE.CLOSE) {
                    let code = 1005;
                    if (payload.length >= 2) {
                        code = payload.readUInt16BE();
                        payload = payload.slice(2);
                    }
                    callback({
                        close: code,
                        payload: payload,
                    });
                }
                else {
                    // 如果是文本帧，将其转换为字符串
                    if (opcode == OPCODE.TEXT) {
                        payload = payload.toString("utf8");
                    }
                    callback({
                        message: payload,
                    });
                }
                fragmented_message_opcode = undefined;
                fragmented_message_payload = undefined;
            }
            else {
                if (opcode == OPCODE.TEXT || opcode == OPCODE.BINARY) {
                    fragmented_message_opcode = opcode;
                    fragmented_message_payload = payload;
                }
                else if (opcode != OPCODE.CONTINUATION) {
                    throw new Exception("分段控制帧");
                }
            }
        }
    });
}

希望这可以帮助你理解代码的逻辑。如果有任何问题，请随时提出。

英文:

You are falsely assuming that the whole frame will be in your buff chunk. You need to concatenate those chunks into a single buffer and try to parse out the frame, if you have enough data in the buffer.

You are also reading the payload length wrong. You are supposed to check lengthCode for value of 126 and 127, not if they are smaller than 2<sup>16</sup> or 2<sup>64</sup>. A 7 bit integer won't even reach 2<sup>7</sup>.

Whey you properly read the frame you will notice that the payload length (131000) of the first frame exceeds the length of the buff chunk (32768).

Also notice that your "second frame" just repeats the same 4 bytes like your payload in the first, which indicates it's still just payload data.

I fixed your code and added some debugging output put to make it clear when a chunk is received and when is the frame actually parsed.

const OPCODE = {
    CONTUNIATION: 0,
    TEXT: 1,
    BINARY: 2,
    CLOSE: 8,
    PING: 9,
    PONG: 10,
};

function shoymayuh(socket, callback) {
    let fragmented_message_opcode;
    let fragmented_message_payload;
    let buff = Buffer.alloc(0);
    let frame_count = 0;
    let chunk_count = 0;

    socket.on(&quot;data&quot;, chunk =&gt; {
        if(!chunk || chunk.length == 0) {
            return;
        }
        // add chunk to the buffer
        buff = Buffer.concat([buff, chunk]);
        console.log(`got chunk #${++chunk_count} of size: ${chunk.length} bytes (buffer: ${buff.length} bytes)`);
        
        // there might be multiple frames in the buffer, try to parse from the buffer until we can&#39;t anymore
        for (;;) {
            // if not enough data to read the first 2 bytes of the header, skip and wait for more
            if (buff.length &lt; 2) {
                return;
            }
            const FIN = Boolean(buff[0] &amp; 0b10000000);
            
            let opcode = buff[0] &amp; 0b00001111;

            const isMasked = Boolean(buff[1] &amp; 0b10000000);

            const lengthCode = buff[1] &amp; 0b01111111;

            // read the payload length
            let offset = 2;
            let lengthOfPayload;
            if(lengthCode &lt;= 125) {
                lengthOfPayload = lengthCode;
            } else if(lengthCode == 126) {
                // if not enough data to read the payload length, skip and wait for more
                if (buff.length &lt; offset + 2) {
                    return;
                }
                lengthOfPayload = buff.readUInt16BE(offset);
                offset += 2
            } else if(lengthCode == 127) {
                // if not enough data to read the payload length, skip and wait for more
                if (buff.length &lt; offset + 8) {
                    return;
                }
                // readBigUInt64BE returns a BigInt, convert it to Number
                lengthOfPayload = Number(buff.readBigUInt64BE(offset));
                offset += 8
            }

            // read the mask if masked
            let maskData;
            if (isMasked) {
                // if not enough data to read the mask, skip and wait for more
                if (buff.length &lt; offset + 4) {
                    return;
                }
                maskData = buff.slice(offset, offset + 4);
                offset += 4;
            }

            // if not enough data to read the payload, skip and wait for more
            if(offset + lengthOfPayload &gt; buff.length) {
                return;
            }

            // extract the payload
            let payload = buff.slice(offset, offset + lengthOfPayload);

            // unmask the payload if masked
            if (isMasked) {
                payload = payload.map((b, i) =&gt; b ^ maskData[i%4]);
            }

            // remove the parsed frame from the buffer
            buff = buff.slice(offset + lengthOfPayload);

            // concatanate payload if fragmented message
            if (opcode == OPCODE.CONTUNIATION) {
                fragmented_message_payload = Buffer.concat([fragmented_message_payload, payload]);
            }

            console.log(&quot;parsed frame&quot;, &quot;#&quot;+(++frame_count), {
                FIN,
                opcode,
                isMasked,
                lengthCode,
                lengthOfPayload,
                maskData,
                payload,
            });
            console.log(buff.length, &quot;bytes left in buffer&quot;)

            if(FIN) {
                if (opcode == OPCODE.CONTUNIATION) {
                    opcode = fragmented_message_opcode;
                    payload = fragmented_message_payload;
                }

                if (opcode == OPCODE.PING) {
                    callback({
                        ping: payload,
                    });
                }
                else if (opcode == OPCODE.PONG) {
                    callback({
                        pong: payload,
                    });
                }
                else if (opcode == OPCODE.CLOSE) {
                    let code = 1005;
                    if (payload.length &gt;= 2) {
                        code = payload.readUInt16BE();
                        payload = payload.slice(2);
                    }
                    callback({
                        close: code,
                        payload: payload,
                    });
                }
                else {
                    // if it was a text frame, convert it to string
                    if (opcode == OPCODE.TEXT) {
                        payload = payload.toString(&quot;utf8&quot;);
                    }
                    callback({
                        message: payload,
                    });
                }
                fragmented_message_opcode = undefined;
                fragmented_message_payload = undefined;
            }
            else {
                if (opcode == OPCODE.TEXT || opcode == OPCODE.BINARY) {
                    fragmented_message_opcode = opcode;
                    fragmented_message_payload = payload;
                }
                else if (opcode != OPCODE.CONTUNIATION) {
                    throw new Exception(&quot;Fragmented control frame&quot;);
                }
            }
        }
    });
}

huangapple
  • 本文由 发表于 2023年1月6日 14:16:40
  • 转载请务必保留本文链接：https://go.coder-hub.com/75027584.html
  • javascript
  • node.js
  • websocket
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定