#
Example implementation
First of all: Regardless of the programming language used, it is crucial to ensure the use of an appropriate HTTP client that can accurately mimic a browser’s behavior and handle TLS as browsers do. This ensures that communication with servers closely resembles actual browser traffic, which is essential for bypassing certain website security mechanisms. In Python, this can be achieved using the tls_client library, for example, by creating a session with from tls_client import Session. For curl, you can use curl-impersonate, a modified version of curl designed to better emulate web browsers.
import sys
import re
import time
import json
from tls_client import Session
PROXY = None
URL = "https://www.ticketmaster.es"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
RAPIDAPI_HOST = "https://imperva-incapsula-reese84.p.rapidapi.com"
RAPIDAPI_KEY = "FILL_YOUR_API_KEY_HERE"
def save_response_to_file(response, step):
"""Save both response content and headers to separate log files"""
filename = f"{sanitized_url}_{timestamp}_step{step}_text.log"
with open(filename, 'w', encoding='utf-8') as f:
f.write(response.text)
print(f"STEP{step}: response saved to file: {filename}")
filename = f"{sanitized_url}_{timestamp}_step{step}_headers.log"
with open(filename, 'w', encoding='utf-8') as f:
f.write(f"Status Code: {response.status_code}\n\n")
f.write("Headers:\n")
for header, value in response.headers.items():
f.write(f"{header}: {value}\n")
print(f"STEP{step}: headers headers saved to file: {filename}")
if __name__ == "__main__":
sanitized_url = re.sub(r'[^\w\-_\. ]', '_', URL)
timestamp = int(time.time())
session = Session(client_identifier="chrome_111")
print('Opening URL ', URL)
# -------------------------------
# STEP 1 - Try to fetch page
# -------------------------------
headers = {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-GB,en;q=0.9',
'cache-control': 'max-age=0',
'priority': 'u=0, i',
"sec-ch-ua": '"Brave";v="129", "Not=A?Brand";v="8", "Chromium";v="129"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "none",
"sec-fetch-user": "?1",
"sec-gpc": "1",
"upgrade-insecure-requests": "1",
}
response = session.get(URL, headers=headers, proxy=PROXY)
# -------------------------------
# Save response to file
# -------------------------------
save_response_to_file(response, 1)
# -------------------------------
# Check if we received challenge page
# -------------------------------
if not (('Request unsuccessful. Incapsula incident ID: ' in response.text or \
"Let's Get Your Identity Verified" in response.text) \
and response.status_code == 403):
print("Page loaded successfully, no challenge found")
sys.exit(0)
# incapsula script on page is /epsf/eps-d (look at the dev tools network)
script_url = f'{URL}/epsf/eps-d'
print('Incapsula script is ', script_url)
# -------------------------------
# STEP2 - solve Incapsula challenge
# -------------------------------
print('Solving Incapsula challenge...')
response = session.post(
f"{RAPIDAPI_HOST}/solve",
headers={
"X-RapidApi-Key": RAPIDAPI_KEY,
"User-Agent": UA,
},
json={
"scriptUrl": script_url,
}
)
save_response_to_file(response, 2)
if response.status_code != 200:
print("Failed to solve challenge, http status code: ", response.status_code)
print("body: ", response.text)
exit(0)
payload = response.json()
# -------------------------------
# STEP3 - post solution to Incapsula
# -------------------------------
print("Posting solution... to ", script_url)
challenge_payload = json.dumps(payload['payload'])
response = session.post(
script_url,
headers={
"accept": "application/json; charset=utf-8",
"accept-language": "en-GB,en;q=0.9",
"content-type": "text/plain; charset=utf-8",
"origin": "null",
"priority": "u=0, i",
"sec-ch-ua": '"Brave";v="129", "Not=A?Brand";v="8", "Chromium";v="129"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"macOS\"",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "cross-site",
"user-agent": UA
},
data=challenge_payload,
proxy=PROXY,
params={"d": URL[8:]} # only host
)
if response.status_code != 200:
print("Incapsula rejected our payload, http status code: ", response.status_code)
exit(0)
print('Incapsula response after receiving solution: ', response.status_code)
save_response_to_file(response, 3)
response = response.json()
# -------------------------------
# Use provided cookie
# -------------------------------
if cookie := response['token']:
session.cookies.set("reese84", cookie)
print(f"Got reese84 cookie => {cookie[:30]}...")
# -------------------------------
# STEP4 - try to load page again
# -------------------------------
headers = {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-GB,en;q=0.9',
'cache-control': 'max-age=0',
'priority': 'u=0, i',
'referer': URL,
"priority": "u=0, i",
"sec-ch-ua": '"Brave";v="129", "Not=A?Brand";v="8", "Chromium";v="129"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"macOS\"",
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'same-origin',
"user-agent": UA
}
response = session.get(URL, headers=headers, proxy=PROXY)
save_response_to_file(response, 4)
if (('Request unsuccessful. Incapsula incident ID: ' in response.text or \
"Let's Get Your Identity Verified" in response.text) \
and response.status_code == 403):
print("Passing Incapsula failed, challenge found")
sys.exit(0)
print('Page loaded successfully, no challenge found')