// seo-audit.mjs — crawl a website and report common on-page and technical SEO problems. // // node seo-audit.mjs [--max=200] [--json] // node seo-audit.mjs http://127.0.0.1:8080/ // // Needs Node.js 18 or later (built-in fetch). No packages to install. // It follows links within the same site, reads robots.txt and the sitemap, // and checks each page the way the lessons in Code Craft Academy's SEO // Fundamentals course explain. It only reads pages; it never changes anything. // Exit code: 1 if any errors were found, 0 otherwise. const args = process.argv.slice(2); const start = args.find((a) => !a.startsWith("--")); const MAX = Number((args.find((a) => a.startsWith("--max=")) ?? "--max=200").slice(6)); const JSON_OUT = args.includes("--json"); if (!start) { console.error("Usage: node seo-audit.mjs [--max=200] [--json]"); process.exit(2); } const origin = new URL(start).origin; const issues = []; // { level: "error" | "warning" | "note", rule, url, detail } const add = (level, rule, url, detail = "") => issues.push({ level, rule, url, detail }); // ── Tiny HTML helpers (enough for well-formed pages; no packages needed) ── const attr = (tag, name) => { const m = new RegExp(`\\s${name}\\s*=\\s*("([^"]*)"|'([^']*)'|([^\\s>]+))`, "i").exec(tag); return m ? (m[2] ?? m[3] ?? m[4] ?? "") : null; }; const tags = (html, name) => html.match(new RegExp(`<${name}\\b[^>]*>`, "gi")) ?? []; const text = (s) => s.replace(/<[^>]+>/g, " ").replace(/ /g, " ").replace(/&/g, "&").replace(/\s+/g, " ").trim(); const between = (html, name) => [...html.matchAll(new RegExp(`<${name}\\b[^>]*>([\\s\\S]*?)`, "gi"))].map((m) => m[1]); const meta = (html, name) => { for (const t of tags(html, "meta")) if ((attr(t, "name") ?? "").toLowerCase() === name) return attr(t, "content") ?? ""; return null; }; const clean = (u) => { const x = new URL(u); x.hash = ""; return x.href; }; // ── robots.txt: the group for "*", longest matching rule wins, allow wins ties ── async function readRobots() { const res = await fetch(`${origin}/robots.txt`, { redirect: "follow" }).catch(() => null); if (!res || res.status !== 200) { add("note", "No robots.txt", `${origin}/robots.txt`, `status ${res ? res.status : "unreachable"}: crawlers may crawl everything`); return { rules: [], sitemaps: [] }; } const rules = [], sitemaps = []; let inStar = false, lastWasAgent = false; for (const raw of (await res.text()).split(/\r?\n/)) { const line = raw.replace(/#.*/, "").trim(); const m = /^([a-z-]+)\s*:\s*(.*)$/i.exec(line); if (!m) continue; const [key, value] = [m[1].toLowerCase(), m[2].trim()]; if (key === "sitemap") { sitemaps.push(value); continue; } if (key === "user-agent") { inStar = (lastWasAgent && inStar) || value === "*"; lastWasAgent = true; continue; } lastWasAgent = false; if (inStar && (key === "allow" || key === "disallow") && value) rules.push({ allow: key === "allow", path: value }); } return { rules, sitemaps }; } function blockedBy(rules, url) { const p = new URL(url).pathname + new URL(url).search; let best = null; for (const r of rules) { const re = new RegExp("^" + r.path.replace(/[.+?^{}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\\\$$/, "$")); if (re.test(p) && (!best || r.path.length > best.path.length || (r.path.length === best.path.length && r.allow))) best = r; } return best && !best.allow ? best.path : null; } // ── Fetch without following redirects, so we can report them ── async function get(url) { let res = await fetch(url, { redirect: "manual" }).catch(() => null); if (!res) return { status: 0, url }; if (res.status >= 300 && res.status < 400) { const to = new URL(res.headers.get("location") ?? "", url).href; return { status: res.status, url, redirectTo: to }; } const type = res.headers.get("content-type") ?? ""; return { status: res.status, url, html: type.includes("text/html") ? await res.text() : null, robotsHeader: res.headers.get("x-robots-tag") ?? "" }; } // ── Crawl ── const robots = await readRobots(); const pages = new Map(); // url -> page info const linksTo = new Map(); // url -> Set(source urls) const queue = [clean(start)]; const seen = new Set(queue); const GENERIC = /^(click here|here|read more|more|link|this page)$/i; while (queue.length && pages.size < MAX) { const url = queue.shift(); const blocked = blockedBy(robots.rules, url); if (blocked) add("warning", "Blocked by robots.txt", url, `rule "Disallow: ${blocked}". Google can't read this page; it may still be indexed without its content`); const r = await get(url); if (r.redirectTo) { add("warning", "Internal link to a redirect", url, `${r.status} → ${r.redirectTo}. Link straight to the final URL`); const to = clean(r.redirectTo); if (to.startsWith(origin) && !seen.has(to)) { seen.add(to); queue.push(to); } pages.set(url, { status: r.status }); continue; } pages.set(url, { status: r.status }); if (r.status !== 200) { add("error", "Broken page", url, `status ${r.status || "unreachable"}`); continue; } if (!r.html) continue; const html = r.html; const head = between(html, "head")[0] ?? ""; const info = pages.get(url); info.title = text(between(head, "title")[0] ?? ""); info.description = meta(head, "description"); const robotsMeta = `${meta(head, "robots") ?? ""} ${r.robotsHeader}`.toLowerCase(); info.noindex = robotsMeta.includes("noindex"); const canonicalTag = tags(head, "link").find((t) => (attr(t, "rel") ?? "").toLowerCase() === "canonical"); info.canonical = canonicalTag ? attr(canonicalTag, "href") : null; if (!info.title) add("error", "Missing ", url); else if (info.title.length > 60) add("note", "Long title", url, `${info.title.length} characters; Google shortens title links to fit the screen, so put the important words first`); if (info.description === null || info.description.trim() === "") add("warning", "Missing meta description", url, "Google may write its own snippet from the page"); if (!/<html\b[^>]*\slang\s*=/i.test(html)) add("warning", "Missing lang attribute", url, 'add <html lang="en">'); if (meta(head, "viewport") === null) add("warning", "Missing viewport meta tag", url, "the page won't fit phone screens"); if (info.noindex) add("note", "noindex", url, "this page asks search engines not to index it"); const h1s = between(html, "h1").length; if (h1s !== 1) add("warning", h1s === 0 ? "No <h1>" : "More than one <h1>", url, `found ${h1s}`); for (const img of tags(html, "img")) { if (attr(img, "alt") === null) add("error", "Image without alt attribute", url, attr(img, "src") ?? ""); if (attr(img, "width") === null || attr(img, "height") === null) add("warning", "Image without width and height", url, `${attr(img, "src")}: the layout jumps when it loads`); } if (!info.canonical) add("note", "No canonical link", url, "add a self-referencing rel=canonical"); else if (!/^https?:\/\//i.test(info.canonical)) add("warning", "Relative canonical URL", url, `"${info.canonical}": Google recommends absolute URLs`); else if (clean(info.canonical) !== url) add("note", "Canonical points elsewhere", url, info.canonical); for (const block of between(html, "script").filter((_, i) => /type\s*=\s*["']application\/ld\+json/i.test(tags(html, "script")[i] ?? ""))) { let data; try { data = JSON.parse(block); } catch (e) { add("error", "Invalid JSON-LD", url, e.message); continue; } for (const item of [data, ...(data["@graph"] ?? [])].flat()) { const types = [item?.["@type"]].flat().filter(Boolean); const local = types.some((t) => /LocalBusiness|Bakery|Restaurant|Store|Cafe|CafeOrCoffeeShop|FoodEstablishment|Plumber|Electrician|Dentist/i.test(t)); if (local) { if (!item.name) add("error", "LocalBusiness missing name", url, "name is required"); if (!item.address) add("error", "LocalBusiness missing address", url, "address is required"); const visible = text(html); if (item.telephone && !visible.replace(/\s/g, "").includes(String(item.telephone).replace(/\s/g, ""))) add("warning", "Phone in structured data not shown on page", url, String(item.telephone)); } } } for (const a of html.matchAll(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi)) { const href = attr(`<a ${a[1]}>`, "href"); if (!href || /^(mailto:|tel:|javascript:|#)/i.test(href)) continue; const anchor = text(a[2]) || (attr(a[2], "alt") ?? ""); let target; try { target = clean(new URL(href, url).href); } catch { add("error", "Malformed link", url, href); continue; } if (!target.startsWith(origin)) continue; if (!anchor) add("warning", "Link with no anchor text", url, href); else if (GENERIC.test(anchor)) add("warning", "Vague anchor text", url, `"${anchor}" → ${href}. Say what the page is about`); if (!linksTo.has(target)) linksTo.set(target, new Set()); if (target !== url) linksTo.get(target).add(url); if (!seen.has(target)) { seen.add(target); queue.push(target); } } } // ── Site-wide checks ── const ok = [...pages].filter(([, p]) => p.status === 200 && p.title !== undefined); for (const [field, rule] of [["title", "Duplicate title"], ["description", "Duplicate meta description"]]) { const by = new Map(); for (const [u, p] of ok) if (p[field]) by.set(p[field], [...(by.get(p[field]) ?? []), u]); for (const [value, urls] of by) if (urls.length > 1) add("warning", rule, urls.join(", "), `"${value}"`); } for (const [u, from] of linksTo) { const p = pages.get(u); if (p && p.status !== 200 && p.status !== 301 && p.status !== 308) add("error", "Broken internal link", u, `linked from ${[...from].join(", ")}`); } // Sitemap: listed URLs must work, and pages should be reachable by links. const sitemapUrls = robots.sitemaps.length ? robots.sitemaps : [`${origin}/sitemap.xml`]; let listed = []; for (const sm of sitemapUrls) { const res = await fetch(sm).catch(() => null); if (!res || res.status !== 200) { add(robots.sitemaps.length ? "error" : "note", "No sitemap", sm, `status ${res ? res.status : "unreachable"}`); continue; } listed.push(...[...(await res.text()).matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((m) => m[1])); } for (const u of listed) { if (!/^https?:\/\//.test(u)) { add("error", "Relative URL in sitemap", u, "sitemaps need full, absolute URLs"); continue; } const r = pages.get(clean(u)) ?? (await get(u)); if (r.redirectTo || (r.status >= 300 && r.status < 400)) add("warning", "Sitemap lists a redirect", u, "list the final URL instead"); else if (r.status !== 200) add("error", "Sitemap lists a broken URL", u, `status ${r.status}`); else if (pages.get(clean(u))?.noindex) add("warning", "Sitemap lists a noindex page", u); if (clean(u) !== clean(start) && !linksTo.get(clean(u))?.size) add("warning", "Orphan page", u, "in the sitemap, but no internal link points to it"); } if (listed.length) for (const [u] of ok) if (!listed.map(clean).includes(u) && !pages.get(u).noindex) add("note", "Page not in sitemap", u); // ── Report ── const order = { error: 0, warning: 1, note: 2 }; issues.sort((a, b) => order[a.level] - order[b.level] || a.rule.localeCompare(b.rule)); const count = (l) => issues.filter((i) => i.level === l).length; if (JSON_OUT) { console.log(JSON.stringify({ origin, pagesCrawled: pages.size, errors: count("error"), warnings: count("warning"), notes: count("note"), issues }, null, 2)); } else { console.log(`SEO audit of ${origin}: ${pages.size} URLs crawled`); console.log(`${count("error")} errors, ${count("warning")} warnings, ${count("note")} notes\n`); for (const i of issues) console.log(`[${i.level.toUpperCase()}] ${i.rule}: ${i.url}${i.detail ? `\n ${i.detail}` : ""}`); } process.exitCode = count("error") ? 1 : 0;