I have an app where I use firebase as a backend. I have two gen1 firebase function, the first downloads a file and save it to firebase storage as a json file. My second function is a HTTP endpoint what downloads this file from storage and response with the json file.
It worked perfectly until I had too much user with too much request. In google cloud log I can see only this message, what is not so informative:
insertId: "16qoz24f6z01ka"
execution_id: "gm3e668c9lxj"
runtime_version: "nodejs20_20240421_20_12_1_RC00"
logName: "projects/currency-converter-1d6e8/logs/cloudfunctions.googleapis.com%2Fcloud-functions"
receiveTimestamp: "2024-05-03T08:37:34.877763494Z"
function_name: "premiumapi"
project_id: "currency-converter-1d6e8"
textPayload: "Function execution took 5 ms, finished with status code: 429"
timestamp: "2024-05-03T08:37:34.569502107Z"
trace: "projects/currency-converter-1d6e8/traces/a541c12b1432ac4067ec5cdd1ad10a90"
<code>{
insertId: "16qoz24f6z01ka"
labels: {
execution_id: "gm3e668c9lxj"
runtime_version: "nodejs20_20240421_20_12_1_RC00"
}
logName: "projects/currency-converter-1d6e8/logs/cloudfunctions.googleapis.com%2Fcloud-functions"
receiveTimestamp: "2024-05-03T08:37:34.877763494Z"
resource: {
labels: {
function_name: "premiumapi"
project_id: "currency-converter-1d6e8"
region: "us-central1"
}
type: "cloud_function"
}
severity: "DEBUG"
textPayload: "Function execution took 5 ms, finished with status code: 429"
timestamp: "2024-05-03T08:37:34.569502107Z"
trace: "projects/currency-converter-1d6e8/traces/a541c12b1432ac4067ec5cdd1ad10a90"
</code>
{
insertId: "16qoz24f6z01ka"
labels: {
execution_id: "gm3e668c9lxj"
runtime_version: "nodejs20_20240421_20_12_1_RC00"
}
logName: "projects/currency-converter-1d6e8/logs/cloudfunctions.googleapis.com%2Fcloud-functions"
receiveTimestamp: "2024-05-03T08:37:34.877763494Z"
resource: {
labels: {
function_name: "premiumapi"
project_id: "currency-converter-1d6e8"
region: "us-central1"
}
type: "cloud_function"
}
severity: "DEBUG"
textPayload: "Function execution took 5 ms, finished with status code: 429"
timestamp: "2024-05-03T08:37:34.569502107Z"
trace: "projects/currency-converter-1d6e8/traces/a541c12b1432ac4067ec5cdd1ad10a90"
This is my code:
<code>export const premiumapi = functions
.https.onRequest(premiumGetLatestPricesFromStorage);
const limiter = rateLimit({
windowMs: 60 * 1000, // 1 minute
max: 5, // limit each IP to 10 requests per windowMs
export const premiumGetLatestPricesFromStorage = express()
.get(`/${apiVersion}/rates`, limiter, (request: express.Request, response: express.Response) => {
downloadFromStorageAndDecodeJsonData(path.latestPriceStoragePath)
if (Result.success === false) {
// Send (500) Internal Server Error
functions.logger.error(`❌ [ERROR HTTP GET REQUEST] Couldn't get latest prices from Storage, error: ${Result.log}`);
response.status(500).send("Rates not available!");
latestPrices = Result.data;
response.set("Cache-Control", "no-cache, no-store");
response.status(200).json(latestPrices);
functions.logger.log(`✅ [SUCCESSFULLY GET HTTP REQUEST] ${Result.log}`);
functions.logger.error(`❌ [ERROR HTTP GET REQUEST] error: ${error}`);
response.status(500).send("Rates not available!");
export async function downloadFromStorageAndDecodeJsonData(storagePath: string): Promise<Result> {
const file = bucket.file(storagePath);
const fileBuffer = await file.download();
const fileContents = fileBuffer[0].toString();
const jsonData = JSON.parse(fileContents);
return {success: true, data: jsonData, log: `✅ [SUCCESSFULLY DOWNLOADED AND PARSED DATA FROM STORAGE] path: ${storagePath}`};
functions.logger.error(`❌ [ERROR DOWNLOADING DATA FROM STORAGE] path: ${storagePath}, error: ${error}`);
return {success: false, log: `❌ [ERROR DOWNLOADING DATA FROM STORAGE] path: ${storagePath}, error: ${error}`};
export async function encodeJsonDataAndUploadToStorage(storagePath: string, downloadedData: any): Promise<Result> {
const file = bucket.file(storagePath);
const jsonEncodedData = JSON.stringify(downloadedData, null, 4);
await file.save(jsonEncodedData, {
contentType: "application/json",
cacheControl: "public, max-age=10",
return {success: true, log: `✅ [SUCCESSFULLY UPLOADED TO STORAGE] path: ${storagePath}`};
functions.logger.error(`❌ [ERROR UPLOADING TO STORAGE] path: ${storagePath}, error: ${error}`);
return {success: false, log: `❌ [ERROR UPLOADING TO STORAGE] path: ${storagePath}, error: ${error}`};
<code>export const premiumapi = functions
.runWith({
timeoutSeconds: 60,
})
.https.onRequest(premiumGetLatestPricesFromStorage);
const limiter = rateLimit({
windowMs: 60 * 1000, // 1 minute
max: 5, // limit each IP to 10 requests per windowMs
});
export const premiumGetLatestPricesFromStorage = express()
.get(`/${apiVersion}/rates`, limiter, (request: express.Request, response: express.Response) => {
downloadFromStorageAndDecodeJsonData(path.latestPriceStoragePath)
.then((Result) => {
if (Result.success === false) {
// Send (500) Internal Server Error
functions.logger.error(`❌ [ERROR HTTP GET REQUEST] Couldn't get latest prices from Storage, error: ${Result.log}`);
response.status(500).send("Rates not available!");
} else {
// Send the JSON data
latestPrices = Result.data;
response.set("Cache-Control", "no-cache, no-store");
response.status(200).json(latestPrices);
functions.logger.log(`✅ [SUCCESSFULLY GET HTTP REQUEST] ${Result.log}`);
}
})
.catch((error) => {
functions.logger.error(`❌ [ERROR HTTP GET REQUEST] error: ${error}`);
response.status(500).send("Rates not available!");
});
});
export async function downloadFromStorageAndDecodeJsonData(storagePath: string): Promise<Result> {
try {
const file = bucket.file(storagePath);
const fileBuffer = await file.download();
const fileContents = fileBuffer[0].toString();
const jsonData = JSON.parse(fileContents);
return {success: true, data: jsonData, log: `✅ [SUCCESSFULLY DOWNLOADED AND PARSED DATA FROM STORAGE] path: ${storagePath}`};
} catch (error) {
functions.logger.error(`❌ [ERROR DOWNLOADING DATA FROM STORAGE] path: ${storagePath}, error: ${error}`);
return {success: false, log: `❌ [ERROR DOWNLOADING DATA FROM STORAGE] path: ${storagePath}, error: ${error}`};
}
}
export async function encodeJsonDataAndUploadToStorage(storagePath: string, downloadedData: any): Promise<Result> {
try {
const file = bucket.file(storagePath);
const jsonEncodedData = JSON.stringify(downloadedData, null, 4);
await file.save(jsonEncodedData, {
metadata: {
contentType: "application/json",
cacheControl: "public, max-age=10",
},
});
return {success: true, log: `✅ [SUCCESSFULLY UPLOADED TO STORAGE] path: ${storagePath}`};
} catch (error) {
functions.logger.error(`❌ [ERROR UPLOADING TO STORAGE] path: ${storagePath}, error: ${error}`);
return {success: false, log: `❌ [ERROR UPLOADING TO STORAGE] path: ${storagePath}, error: ${error}`};
}
}
</code>
export const premiumapi = functions
.runWith({
timeoutSeconds: 60,
})
.https.onRequest(premiumGetLatestPricesFromStorage);
const limiter = rateLimit({
windowMs: 60 * 1000, // 1 minute
max: 5, // limit each IP to 10 requests per windowMs
});
export const premiumGetLatestPricesFromStorage = express()
.get(`/${apiVersion}/rates`, limiter, (request: express.Request, response: express.Response) => {
downloadFromStorageAndDecodeJsonData(path.latestPriceStoragePath)
.then((Result) => {
if (Result.success === false) {
// Send (500) Internal Server Error
functions.logger.error(`❌ [ERROR HTTP GET REQUEST] Couldn't get latest prices from Storage, error: ${Result.log}`);
response.status(500).send("Rates not available!");
} else {
// Send the JSON data
latestPrices = Result.data;
response.set("Cache-Control", "no-cache, no-store");
response.status(200).json(latestPrices);
functions.logger.log(`✅ [SUCCESSFULLY GET HTTP REQUEST] ${Result.log}`);
}
})
.catch((error) => {
functions.logger.error(`❌ [ERROR HTTP GET REQUEST] error: ${error}`);
response.status(500).send("Rates not available!");
});
});
export async function downloadFromStorageAndDecodeJsonData(storagePath: string): Promise<Result> {
try {
const file = bucket.file(storagePath);
const fileBuffer = await file.download();
const fileContents = fileBuffer[0].toString();
const jsonData = JSON.parse(fileContents);
return {success: true, data: jsonData, log: `✅ [SUCCESSFULLY DOWNLOADED AND PARSED DATA FROM STORAGE] path: ${storagePath}`};
} catch (error) {
functions.logger.error(`❌ [ERROR DOWNLOADING DATA FROM STORAGE] path: ${storagePath}, error: ${error}`);
return {success: false, log: `❌ [ERROR DOWNLOADING DATA FROM STORAGE] path: ${storagePath}, error: ${error}`};
}
}
export async function encodeJsonDataAndUploadToStorage(storagePath: string, downloadedData: any): Promise<Result> {
try {
const file = bucket.file(storagePath);
const jsonEncodedData = JSON.stringify(downloadedData, null, 4);
await file.save(jsonEncodedData, {
metadata: {
contentType: "application/json",
cacheControl: "public, max-age=10",
},
});
return {success: true, log: `✅ [SUCCESSFULLY UPLOADED TO STORAGE] path: ${storagePath}`};
} catch (error) {
functions.logger.error(`❌ [ERROR UPLOADING TO STORAGE] path: ${storagePath}, error: ${error}`);
return {success: false, log: `❌ [ERROR UPLOADING TO STORAGE] path: ${storagePath}, error: ${error}`};
}
}
When the request successfully returns with 200, I can see my log message in google cloud log.
Anyone has any suggestion how to decide if it is a storage limitation problem or http request limitation on what else?
How could I debug a little bit more detailed?
I tried to set cacheControl of the file in the storage to “public, max-age=10”, but no change.