There is a website that I am trying to send post requests to. I am trying to scrape the data that loads after I type a keyword in the search bar of that page. In order to get to the search bars, there are several steps. I have sent requests that work until that point, but the last request which includes sending the request for the keyword, does not work. Even though it gives a 200 status code, the response is still not expected.
this is what the framework of my code looks like:
import requests
import re
url = "..."
# Create a session
session = requests.Session()
# Get initial response to retrieve session cookies and CSRF token
initial_response = session.get(url)
response_text = initial_response.text
# Extract session ID and CSRF token
match2 = re.search(r'"session_id":"([^"]+)"', response_text)
session_id = match2.group(1)
match3 = re.search(r'"page_id":"(d+)"', response_text)
page_id = match3.group(1)
match = re.search(r'"csrf_token":"([^"]+)"', response_text)
csrf_token = match.group(1)
cookie_header = '; '.join([f"{cookie.name}={cookie.value}" for cookie in session.cookies])
header1 = {
{
payload1 = {
}
response1 =session.post(url, headers=headers1, json=payload1)
I do this until the last request that takes me to the search bar, and the response I get is what I expect so far.
this is what my last header and payload looks like:
headers10 = {
"Accept": "application/json, text/plain, */*",
"Accept-Encoding": "gzip, deflate, br, zstd",
"Accept-Language": "en-US,en;q=0.9",
"Adv-Action-Code": "search",
"Adv-Action-Type": "searchAction",
"Adv-Conversation-Id": "0971763290845014221",
"Adv-Page-Id": page_id,
"Adv-Request-Id": "9",
"Adv-Window-Id": "44a041d0-fb9d-471e-b1a8-dcd791bf224d",
"Connection": "keep-alive",
"Content-Length": "1351",
"Content-Type": "application/json;charset=UTF-8",
"Cookie": cookie_header,
"Host": "prd.co.cgiadvantage.com",
"Origin": "https://prd.co.cgiadvantage.com",
"Referer": "https://prd.co.cgiadvantage.com/PRDVSS1X1/Advantage4",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"sec-ch-ua": ""Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": ""Windows""
}
payload10 = {
"action": {
"appActionCode": None,
"customActionName": None,
"applicationAction": "search",
"shouldIgnoreSysFeedback": False,
"backgroundAction": "userInitiated",
"isShiftKey": False,
"hideActionButton": False,
"key": "vss.page.VVSSX10019.gridView1.group1.cardSearch.searchActions.search",
"actionType": "searchAction",
"actionCode": "search",
"dsNameList": "T1SO_SRCH_QRY",
"dataSource": "T1SO_SRCH_QRY",
"viewName": "gridView1",
"hotkey": "SHIFT+E",
"name": "search"
},
"checksum": {
"DS_DATA": {
"T1SO_SRCH_QRY": "-1"
},
"DATASOURCE": {
"T1SO_SRCH_QRY": 40331019
},
"VIEW": {
"gridView1": 1338231594
}
},
"viewState": {
"vss.page.VVSSX10019.gridView1.group1.cardSearch.search1": {
"editable": True
},
"vss.page.VVSSX10019.gridView1.group1.cardSearch": {
"editable": True
},
"vss.page.VVSSX10019": {
"editable": False,
"hidden": False,
"closed": False,
"required": False,
"protected": False
},
"TOP_LEVEL_KV_PAIRS_MAP": {
"vss.page.VAXXX03153.carouselView.carousel.newVendor": "27614832911314142937",
"CURR_LINK_KEY": "vss.page.VAXXX03153.carouselView.carousel.solicitations",
"CURR_LINK_INDEX": "4",
"vss.page.VAXXX03153.carouselView.carousel.solicitations": "87603590628600383103"
}
},
"data": {
"page_data": {},
"ds_query_data": {
"T1SO_SRCH_QRY": {
"SHOW_TXT": "3",
"QRY_SRCH_STRING": "*RESEARCH*"
}
}
},
"session_info": {
"session_id": session_id,
"page_id": page_id,
"csrf_token": csrf_token
}
}
response10 = session.post(url, headers=headers10, json=payload10)
the response I am getting looks like this:
{
"pageChange": false,
"action": {
"name": "search",
"key": "vss.page.VVSSX10019.gridView1.group1.cardSearch.searchActions.search",
"params": {
"targetLocation": "viewManager"
},
"actionType": "searchAction",
"actionCode": "search",
"viewName": "gridView1",
"targetQualifiedName": "vss.page.VAXXX03150",
"dsNameList": "T1SO_SRCH_QRY",
"dataSource": "T1SO_SRCH_QRY",
"allowAcceptData": false,
"targetLocation": "viewManager",
"isShiftKey": false,
"shouldIgnoreSysFeedback": false,
"backgroundAction": "userInitiated",
"customActionName": null,
"applicationAction": "search",
"hotkey": "SHIFT+E",
"hideActionButton": false
},
"page_metadata": {
"key": "vss.page.VAXXX03150",
"type": "page",
"name": "ViewManager",
"title": "viewManager",
"order": 1,
"leafElemsMap": {},
"containerElemsMap": {
"viewManager": {
"key": "vss.page.VAXXX03150.viewManager",
"type": "view",
"name": "viewManager",
"title": "viewManager",
"template": "stdNoNav_Hdr3_Main101",
"order": 1,
"leafElemsMap": {
"advfavicon": {
"key": "vss.page.VAXXX03150.viewManager.advfavicon",
"type": "logo",
"name": "advfavicon",
"title": "",
"template": "advfavicon",
"order": 1,
"image": "https://prd.co.cgiadvantage.com:443/PRDVSS1X1/Advantage4/../advantage/ImageProvider?image_name=11cd1429-5569-4401-87b4-26d73c84b003"
}
},
"containerElemsMap": {
"viewManagerActionGroup": {
"key": "vss.page.VAXXX03150.viewManager.viewManagerActionGroup",
"type": "actiongroup",
"name": "viewManagerActionGroup",
"order": 1,
"leafElemsMap": {
"loginAction": {
"key": "vss.page.VAXXX03150.viewManager.viewManagerActionGroup.loginAction",
"type": "nav",
"name": "loginAction",
"title": "Login",
"bypassTxnCatalog": false,
"order": 1,
"actionType": "navAction",
"dsNameList": "",
"protected": false,
"applicationUrl": "https://prd.co.cgiadvantage.com:443/PRDVSS1X1/Advantage4",
"targetComponentType": "LoginPage",
"targetLocation": "header_right",
"targetQualifiedName": "vss.page.VAXXX03152",
"group": 1
},
"footerAction": {
"key": "vss.page.VAXXX03150.viewManager.viewManagerActionGroup.footerAction",
"type": "nav",
"name": "footerAction",
"title": "Footer",
"bypassTxnCatalog": false,
"order": 1,
"actionType": "navAction",
"dsNameList": "",
"protected": false,
"applicationUrl": "https://prd.co.cgiadvantage.com:443/PRDVSS1X1/Advantage4",
"targetComponentType": "CgiFooterPage",
"targetLocation": "footer",
"targetQualifiedName": "vss.page.VAXXX03154",
"group": 1
},
"logoAction": {
"key": "vss.page.VAXXX03150.viewManager.viewManagerActionGroup.logoAction",
"type": "nav",
"name": "logoAction",
"title": "Logo",
"bypassTxnCatalog": false,
"order": 1,
"actionType": "navAction",
"dsNameList": "",
"viewName": "logoViewWithOutHamburgerMenu",
"protected": false,
"applicationUrl": "https://prd.co.cgiadvantage.com:443/PRDVSS1X1/Advantage4",
"targetComponentType": "LogoPage",
"targetLocation": "header_left",
"targetQualifiedName": "vss.page.AAXXX03151",
"group": 1
},
"emptyAction": {
"key": "vss.page.VAXXX03150.viewManager.viewManagerActionGroup.emptyAction",
"type": "nav",
"name": "emptyAction",
"title": "Empty",
"bypassTxnCatalog": false,
"order": 1,
"actionType": "navAction",
"dsNameList": "",
"protected": false,
"applicationUrl": "https://prd.co.cgiadvantage.com:443/PRDVSS1X1/Advantage4",
"targetComponentType": "EmptyPage",
"targetLocation": "header_center",
"targetQualifiedName": "vss.page.AAXXX71633",
"group": 1
},
"carousalAction": {
"key": "vss.page.VAXXX03150.viewManager.viewManagerActionGroup.carousalAction",
"type": "nav",
"name": "carousalAction",
"title": "What would you like to do?",
"bypassTxnCatalog": false,
"order": 1,
"actionType": "navAction",
"dsNameList": "",
"protected": false,
"applicationUrl": "https://prd.co.cgiadvantage.com:443/PRDVSS1X1/Advantage4",
"targetComponentType": "CustomCarouselPage",
"targetLocation": "display",
"targetQualifiedName": "vss.page.VAXXX03153",
"group": 1
}
},
"containerElemsMap": {},
"group": 1
}
},
"dsNameList": "",
"requiredIndicator": false
}
}
},
"currentView": "viewManager",
"defaultView": "viewManager",
"datasources": {},
"pagelets": [],
"referencedItemsMap": {}
},
"data": {
"page_data": {
"global_params": {
"column_title_wrap_count": "10",
"spinner_lag_time": "3000",
"server_time_zone": "US/Mountain",
"GUEST_SESSION": "true",
"spinner_max_time": "300000",
"laser_barcode_scanner_wait_after_scan": "2000",
"min_inputfield_char_display": "10",
"feature_params": {
"enableCVLFilter": false,
"enableGridPinning": false,
"enableFlow": false,
"enableHtmlSnippet": false,
"enableSortByDimensionFeature": true,
"timeinout": false,
"enableBusinessCardPopUp": false,
"loadingBarMobile": true,
"quickentry": false,
"gridPinningPageKeys": [
"BF0201",
"BF0200",
"BF0400",
"BF0186",
"BF0086",
"BF0067",
"BF0050"
],
"redis": false,
"enableUserInfoTracking": false,
"googleAnalytics": false,
"onlineBatchProcesses": false,
"viewAllRecordsOnGrid": false,
"enableAttachmentOptions": false,
"flushCacheOnConfigFormulaUpdatesEnable": false,
"enableAppRolesSecurity": false,
"transactionSummaryTab": true,
"columnTitleWrap": false,
"enableFirstColumnWrap": false,
"transactionWorkflowActions": false
},
"enable_improve_aggrid_scroll_performance": "false",
"enable_laser_barcode_scanner": "false",
"enable_500_pagination_option": "false",
"base_app": "vss",
"session_timeout": "900000",
"session_timeout_warning": "60000",
"view_all_max_row_count": "1000",
"internationalization_params": {
"currency": "USD",
"locale": "en-US"
}
}
},
"ds_data": {},
"ds_query_data": {
"T1SO_SRCH_QRY": {
"QRY_SRCH_STRING": "*RESEARCH*",
"SHOW_TXT": "3"
}
},
"visualization_data": {}
},
"systemFeedback": {
"feedbackCode": 1,
"showSuccessIndicator": true
},
"session_info": {
"session_id": "4RFui035Do6qSzRe6iW0rTDWxyyBBY2sc1su1sAL",
"page_id": "680493036276572912585",
"csrf_token": "AgHSYtLr6gAGoBFkhVfT"
},
"global_state_data": {
"enableAutoTab": "false",
"enableMinimizeInfoTab": "false",
"enableMinimizeGridToolbar": "false",
"enableFieldDbHelp": "false"
},
"next_directives": [],
"viewState": {
"TOP_LEVEL_KV_PAIRS_MAP": {
"vss.page.VAXXX03153.carouselView.carousel.newVendor": "27614832911314142937",
"CURR_LINK_KEY": "vss.page.VAXXX03153.carouselView.carousel.solicitations",
"CURR_LINK_INDEX": "4",
"vss.page.VAXXX03153.carouselView.carousel.solicitations": "87603590628600383103"
},
"vss.page.VVSSX10019.gridView1.group1.cardSearch.search1": {
"editable": true
},
"vss.page.VVSSX10019.gridView1.group1.cardSearch": {
"editable": true
},
"vss.page.VVSSX10019": {
"closed": false,
"hidden": false,
"editable": false,
"protected": false,
"required": false
}
},
"checksum": {
"VIEW": {
"gridView1": 1338231594,
"viewManager": 3446746938
},
"DATASOURCE": {
"T1SO_SRCH_QRY": 40331019
},
"DS_DATA": {
"T1SO_SRCH_QRY": -1
}
}
}
and this is the response I should get:
{
"pageChange": false,
"action": {
"name": "search",
"key": "vss.page.VVSSX10019.gridView1.group1.cardSearch.searchActions.search",
"params": {
"targetLocation": "noDisplay",
"targetComponentType": "SystemInquiryPage"
},
"actionType": "searchAction",
"actionCode": "search",
"viewName": "gridView1",
"targetQualifiedName": "vss.page.VVSSX10019",
"dsNameList": "T1SO_SRCH_QRY",
"dataSource": "T1SO_SRCH_QRY",
"allowAcceptData": false,
"targetLocation": "noDisplay",
"isShiftKey": false,
"shouldIgnoreSysFeedback": false,
"backgroundAction": "userInitiated",
"customActionName": null,
"applicationAction": "search",
"hotkey": "SHIFT+E",
"hideActionButton": false
},
"page_metadata": {
"key": "vss.page.VVSSX10019",
"type": "page",
"name": "ViewPubSoCarousel",
"title": "View Published Solicitation",
"order": 1,
"leafElemsMap": {},
"containerElemsMap": {},
"currentView": "gridView1",
"defaultView": "gridView1",
"datasources": {},
"pagelets": [],
"referencedItemsMap": {
"vss.page.VVSSX10019.gridView1.group1.cardGrid.grid1.solNumTypCat.DOC_REF.DOC_REF_Detail": {
"key": "vss.page.VVSSX10019.gridView1.group1.cardGrid.grid1.solNumTypCat.DOC_REF.DOC_REF_Detail",
"type": "action",
"name": "DOC_REF_Detail",
"bypassTxnCatalog": false,
"order": 1,
"referenceKey": "vss.page.VVSSX10019.gridView1.group1.cardGrid.grid1.solNumTypCat.DOC_REF",
"actionType": "transitionAction",
"actionCode": "docTransition",
"dsNameList": "T1SO_SRCH_QRY",
"applicationAction": "docTransition",
"protected": false,
"applicationUrl": "https://prd.co.cgiadvantage.com:443/PRDVSS1X1/Advantage4",
"targetLocation": "display",
"layoutName": "stdNoNav_Hdr3_Main101",
"group": 1
}
}
},
"data": {
"page_data": {},
"ds_data": {
"T1SO_SRCH_QRY": {
"name": "T1SO_SRCH_QRY",
"metaname": "SO_SRCH_QRY",
"rows_sent": 1,
"rows_total": 1,
"row_data": [
{
"ADV_ROW_ID": "LgOowLyfAEmXwv9t",
"ADV_ROW_SEL": false,
"SHOW_TXT": "",
"QRY_SRCH_STRING": "",
"SO_CAT_CD": "18",
"SO_STA": "M",
"DOC_CD": "RFP",
"DOC_DSCR": "SB21-181 Health Equity Research and Report Writing",
"DEPT_NM": "CDPHE - Department of Public Health and Environment",
"BUYR_NM": "Allan Smith",
"DOC_REF": "[RFP,FAAA,2025000002,2][RFP-FAAA-2025000002-2]",
"DOC_CD_CONCAT": "Request for Proposals (RFP)",
"SO_CLSNG_DT_TM": 1723579200000,
"SO_TIME_LEFT": "580280000",
"PUB_DT": 1721973600000,
"AMND_DT": 1721973600000,
"INTENT_POSTED_DT": "",
"PUB_BID_OP_DT": "",
"BUYR_EMAIL_AD": "[email protected]",
"BUYR_PH_NO": "",
"BUYR_FAX_NO": ""
}
],
"row_data_mod": [
{
"ADV_ROW_ID": "LgOowLyfAEmXwv9t",
"ADV_ROW_EDITABLE": true,
"UNREAD": false,
"SO_CAT_CD": {
"EDITABLE": "true"
},
"BUYR_PH_NO": {
"EDITABLE": "true"
},
"PUB_DT": {
"EDITABLE": "false"
},
"SO_TIME_LEFT": {
"EDITABLE": "false"
},
"BUYR_NM": {
"EDITABLE": "true"
},
"AMND_DT": {
"EDITABLE": "false"
},
"PUB_BID_OP_DT": {
"EDITABLE": "true"
},
"SHOW_TXT": {
"EDITABLE": "false"
},
"DOC_CD": {
"EDITABLE": "false"
},
"DEPT_NM": {
"EDITABLE": "true"
},
"BUYR_EMAIL_AD": {
"EDITABLE": "true"
},
"DOC_DSCR": {
"EDITABLE": "true"
},
"BUYR_FAX_NO": {
"EDITABLE": "true"
},
"INTENT_POSTED_DT": {
"EDITABLE": "false"
},
"QRY_SRCH_STRING": {
"EDITABLE": "true"
},
"SO_STA": {
"EDITABLE": "false"
},
"DOC_CD_CONCAT": {
"EDITABLE": "false"
},
"SO_CLSNG_DT_TM": {
"EDITABLE": "false"
},
"DOC_REF": {
"EDITABLE": "false"
},
"protectElements": {
"ELEMENTS": "RespondGuest"
}
}
],
"order_by": "",
"rows_per_page": 20,
"data_dirty": false,
"start_data_window": 1,
"end_data_window": 1,
"start_page_window": 1,
"end_page_window": 1,
"total_count_suffix": "",
"single_select_model": true,
"readonly": false,
"current_row_id": "LgOowLyfAEmXwv9t",
"selected_rows_index": ""
}
},
"ds_query_data": {
"T1SO_SRCH_QRY": {
"QRY_SRCH_STRING": "*RESEARCH*",
"SHOW_TXT": "3"
}
},
"visualization_data": {}
},
"systemFeedback": {
"feedbackCode": 1,
"showSuccessIndicator": false
},
"session_info": {
"session_id": "uuLD35bA_c0zMlbi6-mRqjzYaSOGJNfL7UlKlSsC",
"page_id": "351191671710415212533",
"csrf_token": "JkgN7fkl2REDbFxEhQYm"
},
"global_state_data": {
"enableAutoTab": "false",
"enableMinimizeInfoTab": "false",
"enableMinimizeGridToolbar": "false",
"enableFieldDbHelp": "false"
},
"next_directives": [],
"viewState": {
"TOP_LEVEL_KV_PAIRS_MAP": {
"vss.page.VAXXX03153.carouselView.carousel.newVendor": "083786901363776212502",
"CURR_LINK_KEY": "vss.page.VAXXX03153.carouselView.carousel.solicitations",
"CURR_LINK_INDEX": "4",
"vss.page.VAXXX03153.carouselView.carousel.solicitations": "351191671710415212533"
},
"vss.page.VVSSX10019.gridView1.group1.cardSearch.search1": {
"editable": true
},
"vss.page.VVSSX10019.gridView1.group1.cardSearch": {
"editable": true
},
"vss.page.VVSSX10019": {
"closed": false,
"hidden": false,
"editable": false,
"protected": false,
"required": false
}
},
"checksum": {
"VIEW": {
"gridView1": 784260451
},
"DATASOURCE": {
"T1SO_SRCH_QRY": 1828225798
},
"DS_DATA": {
"T1SO_SRCH_QRY": 2465394914
}
}
}
why does the targetLocation in my response viewManager and that of the expected response noDisplay?