I’m trying to create a script using the requests module and the BeautifulSoup library from this website that will do the following:
Select the Strata plan number button, input 11 in the input box, and then hit the search button. Finally, scrape the address from the result.
After running the script, When I verify the result, I don’t see the address within it.
import re
import requests
from bs4 import BeautifulSoup
link = 'https://www.nsw.gov.au/housing-and-construction/strata/strata-search'
url = 'https://www.stratahub.nsw.gov.au/prweb/PRAuth/app/ssr_4380/6nxCgYjOTS_fVOVfeekVPA*/!SchemeSearch?pzTransactionId=cc5ddc1ecec1c095231675db14450f87&pzFromFrame=&pzPrimaryPageName=pyDisplayHarness&AJAXTrackID=22'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'accept': '*/*',
'accept-encoding': 'gzip, deflate, br, zstd',
'accept-language': 'en-US,en;q=0.9',
'X-Requested-With': 'XMLHttpRequest',
'origin': 'https://www.stratahub.nsw.gov.au',
}
payload = {
"$PSchemeSearch$pSearchBy": "Strata Plan Number",
"$PSchemeSearch$pSchemePlanNumber": 11,
"pzuiactionzzz": "",
"PreActivitiesList": "",
"sectionParam": "",
"ActivityParams": "=",
"$ODesktopWrapperInclude": "",
"$ODeterminePortalTop": "",
"$ODynamicLayout": "",
"$ODynamicLayoutCell": "",
"$OEvalDOMScripts_Include": "",
"$OForm": "",
"$OHarness": "",
"$OHarnessStaticJSEnd": "",
"$OHarnessStaticJSStart": "",
"$OHarnessStaticScriptsClientValidation": "",
"$OPMCHarnessStaticScripts": "",
"$OSessionUser": "",
"$OSurveyStaticScripts": "",
"$OWorkformStyles": "",
"$OpxAutoComplete": "",
"$OpxButton": "",
"$OpxDisplayText": "",
"$OpxHarnessContent": "",
"$OpxLayoutContainer": "",
"$OpxNonTemplate": "",
"$OpxRadioButtons": "",
"$OpxSection": "",
"$OpxVisible": "",
"$OpxWorkArea": "",
"$OpxWorkAreaContent": "",
"$OpyDirtyCheckConfirm": "",
"$OpyWorkFormStandardEnd": "",
"$OpyWorkFormStandardStart": "",
"$OpzAutoCompleteAGIncludes": "",
"$OpzHarnessInlineScriptsEnd": "",
"$OpzHarnessInlineScriptsStart": "",
"$OpzPortalFavIcon": "",
"$OpzPortalIcon": "",
"$Opzpega_ui_harnesscontext": "",
"$Opzpega_web_mashup": "",
"$OpxTextInput": "",
"$OpzDecimalInclude": "",
"pyEncodedParameters": True,
"pzKeepPageMessages": False,
"strPHarnessClass": "Data-Portal",
"strPHarnessPurpose": "SearchStrataScheme",
"UITemplatingStatus": "Y",
"StreamName": "SchemeSearch",
"BaseReference": "SchemeSearch",
"bClientValidation": True,
"FormError": "NONE",
"pyCustomError": "DisplayErrors",
"UsingPage": True,
"HeaderButtonSectionName": "-1",
"PagesToRemove": "",
"pzHarnessID": "HID387D2E2FCEE4EC200B5BAEA8C6A5D859",
"inStandardsMode": True
}
with requests.Session() as s:
s.headers.update(headers)
res = s.get(link)
soup = BeautifulSoup(res.text,"lxml")
code_url = soup.select_one("iframe[title='Strata Search Production']")['data-src']
s.headers['referer'] = code_url
payload['pzuiactionzzz'] = code_url.split("?")[-1]
r = s.post(url,data=payload)
print(r.status_code)
print(r.text)
How can I generate the result containing the address I’m after?