So I am trying to scrape a page on twitter to get the tweets:
I want to get the elements; text, image, video separately but I keep getting an empty Array
//Scraper.js
const puppeteer = require('puppeteer');
const fs = require('fs');
async function scrapeTwitter() {
try {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://twitter.com/coindesk');
await page.waitForLoadState('networkidle2');
const html = await page.content();
const $ = cheerio.load(html);
const tweets = $('[data-testid="tweet"]');
const posts = [];
tweets.each(function () {
const text = $(this).find('.tweet-text').text().trim();
const image = $(this).find('.tweet-image').attr('src');
const video = $(this).find('.tweet-video').attr('src');
posts.push({ text, image, video });
});
await browser.close();
return posts;
} catch (error) {
console.error('Error scraping Twitter:', error);
return [];
}
}
module.exports = scrapeTwitter;