Is it possible to download a file and upload it simultaneously to another source without storage?

I am looking for a nodeJS boiler plate, if technically possible, to download a given file and upload it simultaneously as it downloads.

It implies that the whole file isn’t downloaded nor stored entirely before being uploaded, so memory usage shouldn’t be relevant here.

I understand that streams may be the solution, but I am unsure what does it imply in terms of source and destination requirements (multipart support for instance) ?

You can do this with the build in http module and streams.

const http = require("http");


const request = (options = {}) => {

    options = Object.assign({
        method: "GET"
    }, options);

    return http.request(options);

};


let download = request({
    url: "http://127.0.0.1:80",
    path: "/bytes/100"
});

// handle get/download response
download.on("response", (res) => {

    console.log("Response from download", res.headers);

    let upload = request({
        url: "http://127.0.0.1:80",
        path: "/anything/foo",
        method: "POST"
    });


    // handle post/upload response
    upload.on("response", (res) => {

        let chunks = [];

        res.on("data", (chunk) => {
            chunks.push(chunk)
        });

        res.on("end", () => {
            console.log("Body", Buffer.concat(chunks).toString())
        });

    });

    // pipe download to upload
    res.pipe(upload);

});

download.end();

I used the httpbin container from "kennethreitz".
docker run -p 80:80 kennethreitz/httpbin: https://httpbin.org/

The example code above "downloads" 100 bytes, and pipe them to "/anything/foo", which response with debug information about the made request.

Example output:

Response from download {
server: 'gunicorn/19.9.0',
date: 'Thu, 20 Apr 2023 07:43:08 GMT',
connection: 'close',
'content-type': 'application/octet-stream',
'content-length': '100',
'access-control-allow-origin': '*',
'access-control-allow-credentials': 'true'
}
Response from upload {
server: 'gunicorn/19.9.0',
date: 'Thu, 20 Apr 2023 07:43:08 GMT',
connection: 'close',
'content-type': 'application/json',
'content-length': '456',
'access-control-allow-origin': '*',
'access-control-allow-credentials': 'true'
}
Body {
"args": {}, 
"data": "data:application/octet-stream;base64,KnyWTt134iwvCP8AHAx7eXfPrxpjZUuZMiqUI3y/PAemFqBmAGDZNI7IlP5oQ+pUjKYaKPXH3CjI0HeaSrGefPtztVsJh+R+BR8UaCQAzGCpyCS/fR34k26AnG4b+jK8D1A6vA==", 
"files": {}, 
"form": {}, 
"headers": {
"Connection": "close", 
"Host": "localhost", 
"Transfer-Encoding": "chunked"
}, 
"json": null, 
"method": "POST", 
"origin": "192.168.16.1", 
"url": "http://localhost/anything/foo"
}

In the "upload response" you can see that the uploaded data is encoded as base64 for debugging purpose, but its the exact same data you received from "/bytes/100"

Example code for comparing download with upload buffer:

<!-- begin snippet: js hide: true console: false babel: false -->

<!-- language: lang-js -->

download.on("response", (res) => {
console.log("Response from download", res.headers);
let recvBuffer = Buffer.alloc(0);
let sendBuffer = Buffer.alloc(0);
let upload = request({
url: "http://127.0.0.1:80",
path: "/anything/foo",
method: "POST"
});
let chunks = [];
res.on("data", (chunk) => {
chunks.push(chunk);
});
res.on("end", () => {
recvBuffer = Buffer.concat(chunks);
});
// handle post/upload response
upload.on("response", (res) => {
console.log("Response from upload", res.headers)
let chunks = [];
res.on("data", (chunk) => {
chunks.push(chunk)
});
res.on("end", () => {
// pase response as json & extract received/send data
let json = JSON.parse(Buffer.concat(chunks).toString());
sendBuffer = Buffer.from(json.data.split(",")[1], "base64");
console.log("Download = Upload:", Buffer.compare(recvBuffer, sendBuffer) === 0);
});
});
// pipe download to upload
res.pipe(upload);
});

<!-- end snippet -->

Since it uses streams, the memory footprint is pretty low.

Note that the exact solution is depending on your targets that provide the download/uploads endpoints. But since you wanted a boilerplate this should be a good start for you.