I’m trying to make a Fantasy Football Optimizer. Since ESPN doesn’t have an API and requires a login to access the draft room, I’m using Selenium. My goal is to have a website with a launch button that, when clicked, creates a driver and opens a new window. Once the user navigates to their draft room, they hit a connect button, which should scrape the required information in that window and display it on the site. The problem is that when the user clicks connect, the correct information appears on my site for a split second but then disappears. Even the console log resets. However, if I save the dataframe created by the scraper and upload it directly to the backend, it works perfectly. What can cause this and how do I fix it?

@app.route('/launch-ESPN', methods=['POST'])
def launch_ESPN():
    global manager
    data = request.json
    browser = data.get('browser')
    manager = scraper.ESPNManager()
    return manager.launch_ESPN(browser)
    
@app.route('/scrape-ESPN', methods=['POST'])
def scrape_ESPN():
    global manager
    if manager:
        return manager.scrape_ESPN()
    else:
        return jsonify({"error": "Manager not initialized"}), 400

class ESPNManager:
    def __init__(self):
        self.driver = None

    def launch_ESPN(self, browser):
        if browser == 'chrome':
            self.driver = webdriver.Chrome()
        elif browser == 'firefox':
            self.driver = webdriver.Firefox()
        else:
            return 'Please select a valid browser'
        
        self.driver.get('https://www.espn.com/fantasy/football/')
        return 'ESPN launched successfully'

    def scrape_ESPN(self):
        dropdown_element = self.driver.find_element(By.XPATH, '/html/body/div[1]/div[1]/section/div/div[2]/main/div/div/div[3]/div[1]/div[1]/div[2]/div[1]/div/select')

        # Create a Select object
        dropdown = Select(dropdown_element)
        teams = dropdown.options
        table = self.driver.find_element(By.XPATH, "/html/body/div[1]/div[1]/section/div/div[2]/main/div/div/div[3]/div[1]/div[1]/div[2]/div[2]/div/div/div/div/div[2]/table")
        roster_size = len(table.find_elements(By.TAG_NAME, "tr")) - 1
        # roster is df where column names are team names and rows are roster requirements
        positions = []
        uniq_num = 0
        roster = pd.DataFrame()
        for i in range(len(teams)):
            dropdown.select_by_index(i)
            team_df = pd.DataFrame(columns=["Player", "Position"])
            for j in range(1, roster_size + 1):
                team_ele = self.driver.find_element(By.XPATH, f"/html/body/div[1]/div[1]/section/div/div[2]/main/div/div/div[3]/div[1]/div[1]/div[2]/div[2]/div/div/div/div/div[2]/table/tbody/tr[{j}]")
                # get the second time it says ("tag name","td")
                try:
                    element = self.driver.find_elements(By.CSS_SELECTOR, ".jsx-2810852873.table--cell.player-column")[j]
                    player = element.get_attribute("title")
                    if player is None or player == "":
                        player = " "
                except:
                    player = " "
                position_ele = team_ele.find_element(By.TAG_NAME, "td")
                team_df.loc[j] = [player, position_ele.text]
                if i == 0:
                    positions.append(position_ele.text + str(uniq_num))
                    uniq_num += 1
            roster[teams[i].text] = team_df["Player"]
        roster.index = positions
        roster.to_csv('roster.csv')
        return roster.to_json()

As shown above, I’m saving the dataframe to roster.csv. If the function just uploads that csv and returns it, it works.