I am new to concept of “promises” and seeking an enterprise grade or best practice to process a huge text file (1GB max) in NodeJS and then use each row to fire off an API request. The goal is to collect some part of API response and write to a file as well as capture which API requests had an error.
Below is a trimmed down version of my code and it works to the point that I am able to print all API responses (success or failed) to console just fine. However I am seeking a better way to finally able to collect everything in an array (like gt_partner_rows below) and then use it to generate an output file.
Appreciate your thoughts on this.
import * as https from "https";
import * as fs from "fs";
import * as csv_parse from "csv-parse";
import { escape } from "querystring";
let gr_http_request_options = {};
let gt_partner_rows = [];
let gv_rows_in_file = 0;
fs.createReadStream(lv_csv_absolute_filepath)
.pipe(csv_parse.parse({ columns: true, bom: true })) .on("error", (error) => console.error(error))
.on("data", (row) => {
gv_rows_in_file ++;
console.log(`row data = ${row}`);
gr_http_request_options = {};
gr_http_request_options = prepare_API_request(row);
trigger_API_request(gr_http_request_options, row);
})
.on("end", () => {
console.log(`Parsed ${gv_rows_in_file} rows`)
});
//Let's print the final array of all API responses.
gt_partner_rows.forEach((item, index) => {
console.log(item);
function prepare_API_request(row) {
//prepare request according to file data
return http_request_options;
}
function trigger_API_request(ir_http_request_options, row) {
let data = "";
const req = https.request(ir_http_request_options, (res) => {
res.on("data", (chunk) => {
data += chunk;
});
res.on("end", () => {
// Process the data received in the response
//console.log(data);
let lr_response = JSON.parse(data);
try {
//Parse JSON response and collect in array
gt_partner_rows.push(lr_response);
}
);
} catch (error) {
console.error(error);
}
});
});
req.on("error", (error) => {
console.error(error);
});
req.end();
}
Already tried an approach to read a file in stream mode and fire off the API request. I am at the last step now to find a cleaner strategy to collect all API responses and export to a file.