Pre Cookie Setup – Screenshots and Pages
Overview
Captures pre-implementation records by taking visual screenshots of existing site setups and archiving full HTML website exports prior to cookie privacy deployment.
Implementation Guide
Create a folder called “[Client Code] HTML”.
Create the two following python files inside:
- dom_downloader.py
- screenshot_downloader.py
This Python script is an automated XML sitemap web scraper and HTML downloader. It uses curl_cffi to impersonate a Chrome browser (chrome120) to bypass common anti-bot security blocks (like Cloudflare 403 errors).
Here is the step-by-step breakdown of what it does:
-
Fetches Sitemaps: It reads through the target XML sitemaps defined in the
SITEMAPSarray. -
Extracts Web Page URLs: It parses the XML to find all
<loc>tags, filtering out direct image files (.png,.jpg, etc.) and duplicate links to compile a list of clean web page URLs. -
Downloads Raw HTML: It iterates through each discovered URL, sends a GET request (with a 1-second delay between requests to avoid rate limits), and downloads the raw HTML page content.
-
Saves Clean Files Locally: It sanitizes each URL into a safe filename (e.g.,
[https://domain.com/about-us](https://domain.com/about-us)becomesabout-us.html) and saves the files into adownloaded_pages/folder, adding numbers to prevent overwriting duplicate filenames.
Note: Be sure to update the domain and sitemap slugs to reflect the current website.
import os
import re
import time
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from curl_cffi import requests
# --- CONFIGURATION ---
SITEMAPS = [
"https://merrymed.com/page-sitemap.xml",
"https://merrymed.com/product-sitemap.xml"
]
OUTPUT_FOLDER = "downloaded_pages"
DELAY = 1.0 # Time in seconds to wait between requests
BROWSER_PROFILE = "chrome120" # Impersonates Chrome to pass security checks
# ---------------------
visited_urls = set()
urls_to_visit = []
if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER)
def safe_filename(url):
"""Converts a URL into a safe, clean filename, stripping trailing slashes."""
parsed_url = urlparse(url)
path = parsed_url.path.strip("/")
if not path:
return "index.html"
path_segments = path.split("/")
last_segment = path_segments[-1]
if "." in last_segment:
last_segment = last_segment.rsplit(".", 1)[0]
filename = re.sub(r'[^\w\-_\. ]', '_', last_segment)
return f"{filename}.html"
print("š Extracting page and post URLs from sitemaps...")
for sitemap_url in SITEMAPS:
try:
# Using curl_cffi with browser impersonation to bypass 403 blocks
response = requests.get(sitemap_url, impersonate=BROWSER_PROFILE, timeout=15)
if response.status_code == 200:
soup = BeautifulSoup(response.content, "xml")
links = []
# Find all loc tags regardless of structural namespace
for element in soup.find_all("loc"):
url_text = element.text.strip()
# Filter out image extensions and attachment pages
if not url_text.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg')):
links.append(url_text)
valid_links = [l for l in links if l.startswith("http")]
urls_to_visit.extend(valid_links)
print(f" š Found {len(valid_links)} page/post URLs in: {sitemap_url}")
else:
print(f" ā Failed to fetch sitemap (Status {response.status_code}): {sitemap_url}")
except Exception as e:
print(f" ā Error reading sitemap {sitemap_url}: {e}")
urls_to_visit = list(set(urls_to_visit))
print(f"\nš Total unique web pages to download: {len(urls_to_visit)}")
for current_url in urls_to_visit:
print(f"š„ Downloading HTML for: {current_url}")
try:
response = requests.get(current_url, impersonate=BROWSER_PROFILE, timeout=15)
if response.status_code == 200:
html_content = response.text
filename = safe_filename(current_url)
file_path = os.path.join(OUTPUT_FOLDER, filename)
counter = 1
original_path = file_path
while os.path.exists(file_path):
file_path = original_path.replace(".html", f"_{counter}.html")
counter += 1
with open(file_path, "w", encoding="utf-8") as f:
f.write(html_content)
else:
print(f" ā ļø Skipped (Status {response.status_code})")
time.sleep(DELAY)
except Exception as e:
print(f" ā Error downloading {current_url}: {e}")
print(f"\nā
Finished! All actual page HTML files saved to: '{OUTPUT_FOLDER}/'")
This Python script is an automated full-page website screenshot generator. It combines curl_cffi for fast, anti-bot sitemap parsing with Playwright’s headless browser to render pages and capture complete visual screenshots.
Here is the step-by-step breakdown of what it does:
-
Extracts Page URLs: Reads target XML sitemaps using
curl_cffi(impersonating Chrome to bypass WAF/Cloudflare blocks) and compiles a clean, deduplicated list of page URLs. -
Launches Headless Browser: Starts a Playwright-controlled Chromium browser configured with anti-bot arguments and a standard desktop User-Agent.
-
Loads & Prepares Pages: Navigates to each URL and automatically scrolls to the bottom of the page to trigger lazy-loaded images, dynamic scripts, and layouts.
-
Captures Full-Page Screenshots: Takes a full-length
.pngscreenshot of the rendered page, converts the URL into a safe filename, and saves it into adownloaded_screenshots/folder without overwriting existing files.
Note: Be sure to update the domain and sitemap slugs to reflect the current website.
import os
import re
import time
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from curl_cffi import requests # Bypasses 403 blocks on sitemap fetching
from playwright.sync_api import sync_playwright
# --- CONFIGURATION ---
SITEMAPS = [
"https://merrymed.com/page-sitemap.xml",
"https://merrymed.com/product-sitemap.xml" # Fixed typo (.xmll -> .xml)
]
OUTPUT_FOLDER = "downloaded_screenshots"
BROWSER_PROFILE = "chrome120"
# ---------------------
urls_to_visit = []
if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER)
def safe_filename(url):
"""Converts a URL into a safe, clean filename."""
parsed_url = urlparse(url)
path = parsed_url.path.strip("/")
if not path:
return "index.png"
path_segments = path.split("/")
last_segment = path_segments[-1]
if "." in last_segment:
last_segment = last_segment.rsplit(".", 1)[0]
filename = re.sub(r'[^\w\-_\. ]', '_', last_segment)
return f"{filename}.png"
print("š Extracting page and post URLs from sitemaps...")
# Gather the URLs using curl_cffi to prevent 403 Forbidden errors
for sitemap_url in SITEMAPS:
try:
response = requests.get(sitemap_url, impersonate=BROWSER_PROFILE, timeout=15)
if response.status_code == 200:
soup = BeautifulSoup(response.content, "xml")
links = []
for element in soup.find_all("loc"):
url_text = element.text.strip()
if not url_text.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg')):
links.append(url_text)
valid_links = [l for l in links if l.startswith("http")]
urls_to_visit.extend(valid_links)
print(f" š Found {len(valid_links)} URLs in: {sitemap_url}")
else:
print(f" ā Failed to fetch sitemap (Status {response.status_code}): {sitemap_url}")
except Exception as e:
print(f" ā Error reading sitemap {sitemap_url}: {e}")
urls_to_visit = list(set(urls_to_visit))
print(f"\nš Found {len(urls_to_visit)} unique pages to capture.")
if not urls_to_visit:
print("ā ļø No URLs found. Exiting.")
exit()
# Start the automated browser engine
print("\nš¬ Launching automated browser...")
with sync_playwright() as p:
# Launch browser with real user args to minimize bot detection
browser = p.chromium.launch(
headless=True,
args=["--disable-blink-features=AutomationControlled"]
)
# Create browser context with full user headers
context = browser.new_context(
viewport={"width": 1280, "height": 800},
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
page = context.new_page()
for idx, current_url in enumerate(urls_to_visit, 1):
print(f"šø [{idx}/{len(urls_to_visit)}] Capturing: {current_url}")
filename = safe_filename(current_url)
file_path = os.path.join(OUTPUT_FOLDER, filename)
counter = 1
original_path = file_path
while os.path.exists(file_path):
file_path = original_path.replace(".png", f"_{counter}.png")
counter += 1
try:
# Navigate to the page
page.goto(current_url, wait_until="domcontentloaded", timeout=30000)
# Scroll down slowly to trigger lazy-loaded images/animations
page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(1.5) # Short pause for layout/images to settle
# Take full-page screenshot
page.screenshot(path=file_path, full_page=True)
except Exception as e:
print(f" ā Failed to screenshot {current_url}: {e}")
browser.close()
print(f"\nā
Finished! All screenshots saved to folder: '{OUTPUT_FOLDER}/'")
Commands in terminal to run:
cd ~/Desktop/”[Client Code] HTML”
python3 dom_downloader.py
In a separate terminal:
cd ~/Desktop/”[Client Code] HTML”
python3 screenshot_downloader.py
Note: If python3 is not installed on your device, you will need to do that before running the installs.