For my new project I wanted to use RSA public/private key encryption for the WebSocket API to ensure security. I use node.js v16 with the ws package (to handle connections) and node-forge (to generate the keypairs, encrypt, decrypt, etc).

When I generate keypairs, convert to pem, encrypt with the public then decrypt with the private key, everything works. However when I try to send the encrypted message over WebSocket to the server (as the client, chrome), the server (which tries to decrypt it) gives me Error: Encrypted message length is invalid. This is my trace log:

/home/user/directory/node_modules/node-forge/lib/rsa.js:544
    throw error;
    ^

Error: Encrypted message length is invalid.
    at Object.pki.rsa.decrypt (/home/user/directory/node_modules/node-forge/lib/rsa.js:541:17)
    at Object.key.decrypt (/home/user/directory/node_modules/node-forge/lib/rsa.js:1183:21)
    at WebSocket.<anonymous> (/home/user/directory/src/websocket.js:28:58)
    at WebSocket.emit (node:events:402:35)
    at Receiver.receiverOnMessage (/home/user/directory/node_modules/ws/lib/websocket.js:1098:20)
    at Receiver.emit (node:events:390:28)
    at Receiver.dataMessage (/home/user/directory/node_modules/ws/lib/receiver.js:528:14)
    at Receiver.getData (/home/user/directory/node_modules/ws/lib/receiver.js:446:17)
    at Receiver.startLoop (/home/user/directory/node_modules/ws/lib/receiver.js:148:22)
    at Receiver._write (/home/user/directory/node_modules/ws/lib/receiver.js:83:10) {
  length: 138,
  expected: 64
}

I've countlessly tried using forge.util.encode64/decode64 and forge.util.encodeUtf8/decodeUtf8 function to fix the issue, but the fact that WebSocket messes up the encoding due which the message length increases stays.

This is the code I use:

const forge = require("node-forge");

module.exports = (socket) => {
    socket.on("message", (message_buffer) => {
        const message = message_buffer.toString();
        if (message === "REGI") {
            forge.pki.rsa.generateKeyPair({
                "bits": 2048,
                "workers": -1,
                "workerScript": "https://cdnjs.cloudflare.com/ajax/libs/forge/0.10.0/prime.worker.min.js"
            }, (err, keypair) => {
                if (err) {
                    console.error(err);
                    socket.send("An error occurred while generating the keypair.");
                    socket.close();
                    return;
                }
                socket.send((forge.pki.publicKeyToPem(keypair.publicKey)).replace(/(\r\n|\n|\r)/gm, ""));
                socket.on("message", (message_buffer) => {
                    const message = message_buffer.toString();
                    const decrypted = keypair.privateKey.decrypt(forge.util.encodeUtf8(message));
                    socket.send(decrypted);
                });
            });
        }
    });
};

Could anyone help me? Thanks in advance!