Manual SEO audits do not scale. When you are dealing with thousands of URLs, dozens of properties, or daily monitoring requirements, clicking through dashboards is too slow and too unreliable. This module teaches you to replace repetitive manual work with code: Python and JavaScript scripts that call APIs, parse data, detect problems, and surface insights automatically. By the end you will have a personal automation toolkit that saves hours every week and catches regressions before clients or your boss do.
Before writing a single line of code, ask three questions about any task:
High-value automation targets in technical SEO:
Python is the dominant language for SEO automation. Its ecosystem covers HTTP requests, HTML parsing, spreadsheet generation, API clients, and data analysis in a way no other language matches for accessibility.
# Create and activate a virtual environment first
python -m venv seo-env
source seo-env/bin/activate # Windows: seo-env\Scripts\activate
pip install \
requests \ # HTTP requests and API calls
beautifulsoup4 \ # HTML parsing
lxml \ # Fast HTML/XML parser (used by BS4)
pandas \ # Tabular data, analysis, CSV export
google-auth \ # Google OAuth2 for API access
google-auth-httplib2 \
google-api-python-client \ # GSC, PageSpeed, etc.
advertools \ # High-level SEO utilities (sitemaps, log files, robots)
python-dotenv # Load secrets from .env files safely
Always store API keys and OAuth credentials in a .env file. Never commit
credentials to version control.
# .env (add .env to .gitignore)
GSC_CREDENTIALS_FILE=credentials.json
PSI_API_KEY=your_key_here
The GSC UI shows you 1,000 rows and 16 months of data. The API removes the row cap and lets you automate date-range comparisons, segment by device or country, and store data in your own database for long-term trend analysis.
The GSC API requires OAuth2, not a simple API key, because it accesses private property data.
credentials.json file.from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
import pickle, os
SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly']
def authenticate_gsc():
creds = None
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as f:
creds = pickle.load(f)
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
with open('token.pickle', 'wb') as f:
pickle.dump(creds, f)
return build('searchconsole', 'v1', credentials=creds)
service = authenticate_gsc()
The core endpoint is searchanalytics.query. You specify the property, a date
range, and which dimensions to group by (query, page, device, country, searchAppearance).
import pandas as pd
def fetch_gsc_data(service, site_url, start_date, end_date,
dimensions=['query', 'page'], row_limit=25000):
"""Fetch search analytics rows from GSC API."""
body = {
'startDate': start_date,
'endDate': end_date,
'dimensions': dimensions,
'rowLimit': row_limit,
'startRow': 0
}
response = service.searchanalytics().query(
siteUrl=site_url, body=body).execute()
rows = response.get('rows', [])
records = []
for row in rows:
record = {dim: row['keys'][i]
for i, dim in enumerate(dimensions)}
record['clicks'] = row.get('clicks', 0)
record['impressions'] = row.get('impressions', 0)
record['ctr'] = row.get('ctr', 0)
record['position'] = row.get('position', 0)
records.append(record)
return pd.DataFrame(records)
df = fetch_gsc_data(
service,
site_url='sc-domain:example.com',
start_date='2024-01-01',
end_date='2024-03-31'
)
df.to_csv('gsc_q1_2024.csv', index=False)
print(df.head())
One of the highest-value automations is a weekly or monthly change report that flags URLs whose clicks or position have dropped significantly.
def compare_periods(service, site_url,
current_start, current_end,
previous_start, previous_end):
current = fetch_gsc_data(service, site_url,
current_start, current_end,
dimensions=['page'])
previous = fetch_gsc_data(service, site_url,
previous_start, previous_end,
dimensions=['page'])
merged = current.merge(previous, on='page',
suffixes=('_cur', '_prev'))
merged['click_delta'] = merged['clicks_cur'] - merged['clicks_prev']
merged['pos_delta'] = merged['position_cur'] - merged['position_prev']
# Pages with the worst click loss
losers = merged.nsmallest(20, 'click_delta')[
['page', 'clicks_cur', 'clicks_prev', 'click_delta',
'position_cur', 'position_prev', 'pos_delta']
]
return losers
losers = compare_periods(
service, 'sc-domain:example.com',
'2024-04-01', '2024-04-30',
'2024-03-01', '2024-03-31'
)
losers.to_csv('traffic_losers_april.csv', index=False)
The PSI API returns both field data (CrUX, real users) and lab data (Lighthouse). You can use it to monitor Core Web Vitals across your URL list on a schedule and alert when a score regresses.
import requests, os
from dotenv import load_dotenv
load_dotenv()
PSI_KEY = os.getenv('PSI_API_KEY')
def fetch_psi(url, strategy='mobile'):
endpoint = 'https://www.googleapis.com/pagespeedonline/v5/runPagespeed'
params = {
'url': url,
'key': PSI_KEY,
'strategy': strategy,
'category': ['performance', 'accessibility', 'best-practices', 'seo']
}
resp = requests.get(endpoint, params=params, timeout=60)
resp.raise_for_status()
return resp.json()
data = fetch_psi('https://example.com/')
# Lighthouse score (0–1, multiply by 100)
perf_score = data['lighthouseResult']['categories']['performance']['score']
print(f"Performance score: {perf_score * 100:.0f}")
# Core Web Vitals from Lighthouse audits
audits = data['lighthouseResult']['audits']
lcp = audits['largest-contentful-paint']['displayValue']
cls = audits['cumulative-layout-shift']['displayValue']
inp = audits.get('interaction-to-next-paint', {}).get('displayValue', 'N/A')
print(f"LCP: {lcp} | CLS: {cls} | INP: {inp}")
The PSI API has a free quota of 25,000 requests per day and enforces rate limits. Use
time.sleep between requests to stay within quota without being throttled.
import time, pandas as pd
def bulk_psi_audit(url_list, strategy='mobile', delay=1.5):
results = []
for i, url in enumerate(url_list, 1):
print(f" [{i}/{len(url_list)}] {url}")
try:
data = fetch_psi(url, strategy)
audits = data['lighthouseResult']['audits']
cats = data['lighthouseResult']['categories']
# Pull field data (CrUX) if available
crux = data.get('loadingExperience', {})
crux_lcp = crux.get('metrics', {}).get(
'LARGEST_CONTENTFUL_PAINT_MS', {}).get('category', 'N/A')
crux_cls = crux.get('metrics', {}).get(
'CUMULATIVE_LAYOUT_SHIFT_SCORE', {}).get('category', 'N/A')
crux_inp = crux.get('metrics', {}).get(
'INTERACTION_TO_NEXT_PAINT', {}).get('category', 'N/A')
results.append({
'url': url,
'perf_score': round(cats['performance']['score'] * 100),
'seo_score': round(cats['seo']['score'] * 100),
'lcp_lab': audits['largest-contentful-paint']['displayValue'],
'cls_lab': audits['cumulative-layout-shift']['displayValue'],
'inp_lab': audits.get(
'interaction-to-next-paint', {}).get('displayValue', 'N/A'),
'crux_lcp_rating': crux_lcp,
'crux_cls_rating': crux_cls,
'crux_inp_rating': crux_inp,
})
except Exception as e:
results.append({'url': url, 'error': str(e)})
time.sleep(delay)
return pd.DataFrame(results)
urls = [
'https://example.com/',
'https://example.com/about/',
'https://example.com/products/',
]
df = bulk_psi_audit(urls)
df.to_csv('psi_audit.csv', index=False)
Compare today's scores against a stored baseline and print (or email) a warning whenever performance drops below a threshold.
PERFORMANCE_THRESHOLD = 70 # alert if score falls below this
def check_regressions(current_df, baseline_csv='baseline_psi.csv'):
try:
baseline = pd.read_csv(baseline_csv)
except FileNotFoundError:
print("No baseline found — saving current run as baseline.")
current_df.to_csv(baseline_csv, index=False)
return
merged = current_df.merge(baseline, on='url', suffixes=('_now', '_base'))
merged['score_delta'] = merged['perf_score_now'] - merged['perf_score_base']
regressions = merged[
(merged['perf_score_now'] < PERFORMANCE_THRESHOLD) |
(merged['score_delta'] < -10) # dropped more than 10 points
]
if not regressions.empty:
print("\n⚠️ PERFORMANCE REGRESSIONS DETECTED:")
print(regressions[['url', 'perf_score_now', 'perf_score_base',
'score_delta']].to_string(index=False))
else:
print("✅ No regressions detected.")
check_regressions(df)
Screaming Frog exports and log files are the raw material of technical audits. Python lets you go far beyond what the tool's built-in filters can surface.
Export All Inlinks, Internal HTML, and Response Codes as CSVs from Screaming Frog, then load them into pandas.
import pandas as pd
# Load internal HTML pages export
pages = pd.read_csv('internal_html.csv', low_memory=False)
# Flag pages missing a meta description
no_desc = pages[pages['Meta Description 1'].isna() |
(pages['Meta Description 1'].str.strip() == '')]
print(f"Pages missing meta description: {len(no_desc)}")
no_desc[['Address', 'Title 1', 'Meta Description 1']].to_csv(
'missing_meta_desc.csv', index=False)
# Flag duplicate title tags
dupes = pages[pages.duplicated(subset='Title 1', keep=False)]
print(f"Pages with duplicate titles: {len(dupes)}")
dupes[['Address', 'Title 1']].sort_values('Title 1').to_csv(
'duplicate_titles.csv', index=False)
# Flag thin content (word count under threshold)
WORD_THRESHOLD = 300
if 'Word Count' in pages.columns:
thin = pages[pages['Word Count'].fillna(0) < WORD_THRESHOLD]
print(f"Thin content pages (< {WORD_THRESHOLD} words): {len(thin)}")
thin[['Address', 'Word Count']].to_csv('thin_content.csv', index=False)
Log file analysis is the only way to see what Googlebot actually crawled, as opposed to what
you think it crawled. Use advertools for fast parsing, or write a custom parser
for custom log formats.
import advertools as adv
import pandas as pd
# Parse an Apache/Nginx combined log file
logs = adv.logs_to_df(
log_file='access.log',
fields=['host', 'user_agent', 'request', 'status', 'size',
'referer', 'time_local'],
log_format='combined',
encoding='utf-8-sig'
)
# Filter to Googlebot only
googlebot = logs[logs['user_agent'].str.contains(
'Googlebot', case=False, na=False)]
# Crawl frequency by URL path
crawl_counts = (googlebot.groupby('request')
.size()
.reset_index(name='crawl_count')
.sort_values('crawl_count', ascending=False))
print("Top 20 most-crawled URLs:")
print(crawl_counts.head(20).to_string(index=False))
# Status code distribution for Googlebot
print("\nStatus code distribution:")
print(googlebot['status'].value_counts())
# Pages crawled but returning 404 — wasted crawl budget
wasted = googlebot[googlebot['status'] == '404']
wasted[['request', 'time_local']].to_csv('googlebot_404s.csv', index=False)
print(f"\nGooglebot hitting 404s: {len(wasted)} requests")
Compare two sitemap snapshots (before and after a deploy) to detect unintended URL removals or additions. This is a critical check for migrations and large site deployments.
import advertools as adv
before = adv.sitemap_to_df('https://example.com/sitemap_before.xml')
after = adv.sitemap_to_df('https://example.com/sitemap.xml')
before_urls = set(before['loc'])
after_urls = set(after['loc'])
removed = before_urls - after_urls
added = after_urls - before_urls
print(f"URLs removed from sitemap: {len(removed)}")
print(f"URLs added to sitemap: {len(added)}")
pd.DataFrame(sorted(removed), columns=['removed_url']).to_csv(
'sitemap_removed.csv', index=False)
pd.DataFrame(sorted(added), columns=['added_url']).to_csv(
'sitemap_added.csv', index=False)
The Rich Results Test is a one-URL tool. For large sites, you need to validate JSON-LD across hundreds of URLs automatically. The Schema.org validator exposes an HTTP API.
import requests, json
from bs4 import BeautifulSoup
def extract_json_ld(url):
"""Fetch a page and return all JSON-LD blocks as parsed dicts."""
resp = requests.get(url, timeout=15,
headers={'User-Agent': 'SEO-Audit-Bot/1.0'})
soup = BeautifulSoup(resp.text, 'lxml')
blocks = []
for tag in soup.find_all('script', type='application/ld+json'):
try:
blocks.append(json.loads(tag.string))
except json.JSONDecodeError as e:
blocks.append({'_parse_error': str(e), '_raw': tag.string})
return blocks
def validate_json_ld(schema_dict):
"""Send a JSON-LD block to the Schema.org validator API."""
endpoint = 'https://validator.schema.org/validate'
resp = requests.post(endpoint,
data={'code': json.dumps(schema_dict)},
timeout=15)
return resp.json()
urls_to_check = [
'https://example.com/products/widget-pro/',
'https://example.com/blog/how-to-guide/',
]
for url in urls_to_check:
blocks = extract_json_ld(url)
print(f"\n{url} — {len(blocks)} JSON-LD block(s)")
for i, block in enumerate(blocks, 1):
if '_parse_error' in block:
print(f" Block {i}: ❌ JSON parse error: {block['_parse_error']}")
else:
result = validate_json_ld(block)
errors = result.get('errors', [])
if errors:
print(f" Block {i} ({block.get('@type', 'unknown')}): "
f"❌ {len(errors)} error(s)")
for err in errors:
print(f" - {err.get('message', err)}")
else:
print(f" Block {i} ({block.get('@type', 'unknown')}): ✅ Valid")
Screaming Frog has a command-line interface that lets you trigger crawls from a script, CI pipeline, or scheduled cron job — without opening the GUI.
# Basic CLI crawl — saves results to /output directory
screamingfrogseospider \
--crawl https://example.com \
--headless \
--save-crawl \
--export-format csv \
--output-folder /output/crawls/example \
--overwrite \
--export-tabs "Internal:HTML,Response Codes,Canonicals,Meta Robots"
# Crawl with a custom config file (pre-configured rules, exclusions, etc.)
screamingfrogseospider \
--crawl https://example.com \
--headless \
--config /configs/production.seospiderconfig \
--save-crawl \
--export-format csv \
--output-folder /output/crawls/example \
--overwrite
Wrap the CLI call in a Python subprocess and then immediately process the output CSVs with your audit scripts:
import subprocess, pathlib, pandas as pd
from datetime import date
OUTPUT_DIR = pathlib.Path(f'/output/crawls/{date.today()}')
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
result = subprocess.run([
'screamingfrogseospider',
'--crawl', 'https://example.com',
'--headless',
'--export-format', 'csv',
'--output-folder', str(OUTPUT_DIR),
'--overwrite',
'--export-tabs', 'Internal:HTML,Response Codes'
], capture_output=True, text=True)
if result.returncode != 0:
print("Crawl failed:", result.stderr)
else:
print("Crawl complete. Processing output...")
html_csv = OUTPUT_DIR / 'internal_html.csv'
if html_csv.exists():
pages = pd.read_csv(html_csv, low_memory=False)
print(f"Total pages crawled: {len(pages)}")
A script that runs once is a one-off fix. A script that runs on a schedule is infrastructure. There are three main options depending on your environment.
# Edit crontab
crontab -e
# Run the PSI monitor every day at 07:00
0 7 * * * /path/to/seo-env/bin/python /scripts/psi_monitor.py >> /logs/psi.log 2>&1
# Run the GSC weekly report every Monday at 08:00
0 8 * * 1 /path/to/seo-env/bin/python /scripts/gsc_weekly.py >> /logs/gsc.log 2>&1
# .github/workflows/seo-monitor.yml
name: Weekly SEO Monitor
on:
schedule:
- cron: '0 8 * * 1' # Every Monday at 08:00 UTC
workflow_dispatch: # Allow manual trigger
jobs:
monitor:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install dependencies
run: pip install -r requirements.txt
- name: Run PSI monitor
env:
PSI_API_KEY: ${{ secrets.PSI_API_KEY }}
run: python scripts/psi_monitor.py
- name: Upload results
uses: actions/upload-artifact@v4
with:
name: psi-results
path: psi_audit.csv