I have a Socket server which accept about 250000 connections as the same time. So I need to create a Log function to save status and errors of each connections.

I used Winston to save logs and the worker_thread module to offload the logging process from the main thread.

I used a loop to write a million log lines in the worker_thread to verify if the main thread continues to process normally.

Unfortunately, the main thread becomes unresponsive during the logging process.

Does anyone know how to resolve the issue mentioned above?

const logQueue: Array<LogParams | (ErrorLogParams & { log_type: string })> = [];
const batchSize = 1000;
let processing = false;
let test = true;

/** */
function processLogQueue() {
    if (processing || logQueue.length === 0) return;
    processing = true;

    const batch = logQueue.splice(0, batchSize);
    batch.forEach((data) => {
        if (Libs.isBlank(data.log_type) || Libs.isBlank(data.message)) return;
        switch (data.log_type) {
        case 'info':
            FLLogger.info(data as LogParams);
            break;
        case 'warn':
            FLLogger.warn(data as LogParams);
            break;
        case 'error':
            FLLogger.error(data as ErrorLogParams);
            break;
        case 'critical':
            FLLogger.critical(data as ErrorLogParams);
            break;
        }
    });

    processing = false;

    // Process the next batch
    if (logQueue.length > 0) {
        setImmediate(processLogQueue);
    }
}

if (parentPort) {
    parentPort.on('message', (data: LogParams | (ErrorLogParams & { log_type: string })) => {
        logQueue.push(data);
        if (!processing) {
            setImmediate(processLogQueue);
        }
    });

    if (test) {
        test = false;
        for (let i=0; i< 10000000; i++) {
            logQueue.push({
                fileName: 'LogWorker',
                methodName: 'Stress test',
                message: `Test message ${i} - is main thread: ${isMainThread }`,
                deviceType: Constants.DEVICE_TYPE.SERVER,
                log_type: Constants.LOG_TYPE.WARN,
            });
        }
    }

    setInterval(() => {
        if (!processing && logQueue.length > 0) {
            setImmediate(processLogQueue);
        }
    }, 1000);
}