Upload multiple files to BackBlaze according to the original filename

I'm trying to upload images to BackBlaze using backblaze-b2 npm module from a folder that has multiple images and for each image successfully uploading I push the fileId to an array all the files in the folder is named from 0001.jpg to how many files there are in this was if there are 12 images in the folder the last image will be named 0012.jpg but after all images are uploaded and the ids are pushed into the array it'll be like 2nd image's id will place placed on 5th or 6th in the array the last image will be placed at the first

Here is the code I'm using

export const uploadToBackBlaze = async (req, res, next) => {
// Defining backblaze masterkey and application key
const b2 = new B2({
applicationKeyId: process.env.BACKBLAZE_MASTER_APPLICATION_KEY_ID,
applicationKey: process.env.BACKBLAZE_MASTER_APPLICATION_ID,
});
const __dirname = path.resolve(); // Defining directory
let tempDir = path.join(__dirname, "chapterTemp"); // the temp directory
const imageIds = []; // Empty array where the file IDs will be pushed
try {
await b2.authorize().then(async () => {
const bucketId = process.env.BACKBLAZE_BUCKET_ID;
fs.readdir(tempDir, async function (err, files) {
if (err) {
console.error(err);
res.sendStatus(500).json({ message: err.message });
return;
}
// Unique id
let uid = uuidv4();
// Reading file from temp Dir and upload to backblaze
const uploadPromises = files.map(async (file) => {
const fileData = fs.readFileSync(path.join(tempDir, file));
const uploadFileName = path.join(uid, file); // changing file name to uniques
const uploadUrl = await b2.getUploadUrl(bucketId); // Getting upload URL and auth token
const response = await b2.uploadFile({
uploadUrl: uploadUrl.data.uploadUrl,
uploadAuthToken: uploadUrl.data.authorizationToken,
filename: uploadFileName,
data: fileData,
mime: "image/png" || "image/jpg" || "image/jpeg" || "image/webp", // replace with the appropriate MIME type for your files
});
imageIds.push(response.data.fileId);
});
await Promise.all(uploadPromises);
req.imageIds = imageIds;
next();
});
});
} catch (error) {
return res.status(500).json({ message: error.message });
}
};

I want the array to have the file ids in order like 1.jpg's file id should one on the imageIds[0] position and 2.jpg should be on the imageIds[1] position and so on!

This problem actually has nothing to do with Backblaze B2. It's a problem related to asynchronous programming. You have the code const uploadPromises = files.map(...) (where ... is the code that maps the files to promises). This code creates a promise for each file. This is good! It's what you want. You want some code to execute for each file. You even take advantage of Node.js support for concurrency by having each promise begin its work immediately.

Then you have the code await Promise.all(uploadPromises); which blocks the program until all of the promises have finished executing. This is also good. You probably don't want your program to continue executing (returning an HTTP response to the client) until all of the files have been dealt with.

But the processing for each file will complete in a nondeterministic order. Each time you run the program, the order will be different. This is because of factors like network IO. There's no way to ensure the network request for 0001.jpg will finish before the network request for 0002.jpg, and so on.

You can see this in action with a simplified version of what your code is doing.

async function startTasks() {
    let results = [];

    let tasks = ['1', '2', '3'].map((str) => {
        return new Promise(async (resolve) => {
            let delay = Math.floor(Math.random() * 3000) + 1;
            await new Promise(r => setTimeout(r, delay));
            results.push(str);
            resolve();
        });
    });

    await Promise.all(tasks);
    console.log(results);
}

startTasks();

Here, numbers between 1 and 3, as strings, are mapped to promises. The promises here just wait a random number of milliseconds between 1 and 3000 inclusively, designed to simulate work like using the Backblaze library like your code does.

If you run the code, you'll see that it completes the tasks in a different order each time.

$ node index.js 
[ '1', '3', '2' ]
$ node index.js 
[ '1', '2', '3' ]

To retain the order of tasks, you could return the result (string identifier) from each promise instead of pushing it into the results array inside the promise. Then, you can directly assign the results of Promise.all to the results array.

async function startTasks() {
    let tasks = ['1', '2', '3'].map((str) => {
        return new Promise(async (resolve) => {
            let delay = Math.floor(Math.random() * 3000) + 1;
            await new Promise(r => setTimeout(r, delay));
            resolve(str);
        });
    });

    let results = await Promise.all(tasks);
    console.log(results);
}

startTasks();

This time, when you print the results, the order matches the order they were started in.

$ node index.js 
[ '1', '2', '3' ]
$ node index.js 
[ '1', '2', '3' ]

Note that this doesn't account for error handling. If you adapted this example for your use case, you should add error handling. Here's an expanded version of the above example that accounts for errors.

async function startTasks() {
    let tasks = ['1', '2', '3'].map((str) => {
        return new Promise(async (resolve, reject) => {
            try {
                let delay = Math.floor(Math.random() * 3000) + 1;
                await new Promise(r => setTimeout(r, delay));

                // 50% chance to throw an error
                if (Math.random() < 0.5) {
                    throw new Error(`Error encountered in task ${str}`);
                }

                resolve(str);
            } catch (error) {
                reject(error);
            }
        });
    });

    try {
        let results = await Promise.all(tasks);
        console.log(results);
    } catch (error) {
        console.error('One or more tasks failed: ', error);
    }
}

startTasks();

Here, there's a 50% chance of an error being thrown. When you run this code, it might log errors, and it might not.

$ node index.js 
One or more tasks failed:  Error: Error encountered in task 2
at ...
$ node index.js 
One or more tasks failed:  Error: Error encountered in task 3
at ...

In the real world, for your use case, this error would be thrown by the Backblaze library you're using. Note that because there's more than one task being executed that could throw an error, you have to collect the errors the same way you would collect results from successful tasks, and then deal with them.

Or, you could change your code to fail as soon as one of the tasks failed (and discard any successful results collected thus far from tasks that hadn't failed). Or, you could change your code to execute the tasks sequentially, saving successful results as they occur, and returning an error for the first task that failed (without even starting the rest).

Doing this fixed the issue and It's been working fine after I tried it multiple times

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-js -->

import fs from "fs";
import B2 from "backblaze-b2";
import path from "path";
import dotenv from "dotenv";
import { v4 as uuidv4 } from "uuid";
dotenv.config();
export const uploadToBackBlaze = async (req, res, next) => {
// Defining backblaze masterkey and application key
const b2 = new B2({
applicationKeyId: process.env.BACKBLAZE_MASTER_APPLICATION_KEY_ID,
applicationKey: process.env.BACKBLAZE_MASTER_APPLICATION_ID,
});
const __dirname = path.resolve(); // Defining directory
let tempDir = path.join(__dirname, "processed"); // the temp directory
try {
await b2.authorize().then(async () => {
const bucketId = process.env.BACKBLAZE_BUCKET_ID;
fs.readdir(tempDir, async function (err, files) {
if (err) {
console.error(err);
res.sendStatus(500).json({ message: err.message });
return;
}
// Unique id
let uid = uuidv4();
// Reading file from temp Dir and upload to backblaze
const uploadPromises = files.map(async (file) => {
const fileData = fs.readFileSync(path.join(tempDir, file));
const uploadFileName = path.join(uid, file); // changing file name to uniques
const uploadUrl = await b2.getUploadUrl(bucketId); // Getting upload URL and auth token
const response = await b2.uploadFile({
uploadUrl: uploadUrl.data.uploadUrl,
uploadAuthToken: uploadUrl.data.authorizationToken,
filename: uploadFileName,
data: fileData,
mime: "image/png" || "image/jpg" || "image/jpeg" || "image/webp", // replace with the appropriate MIME type for your files
});
return response.data.fileId;
});
req.imageIds = await Promise.all(uploadPromises);
fs.rmdirSync(tempDir, { recursive: true }); // Removing Files From Temp Dir
next();
});
});
} catch (error) {
fs.rmdirSync(tempDir, { recursive: true }); // Removing Files From Temp Dir
return res.status(500).json({ message: error.message });
}
};

<!-- end snippet -->

Instead of pushing the Image Ids into an array if I just send the Promise.all(uploadPromises) as req.imageIds the image Ids are resolved into an array in the some order of their original File Name