XML 사이트맵 검사기

// Node 18 or later.  npm i @xmldom/xmldom
import { DOMParser } from '@xmldom/xmldom';

const SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9';
const MAX_URLS = 50_000;
const MAX_BYTES = 52_428_800;

const response = await fetch('https://example.com/sitemap.xml');
const xml = await response.text();
const bytes = Buffer.byteLength(xml, 'utf8');

// xmldom does not resolve external entities, so there is no XXE surface here.
const doc = new DOMParser().parseFromString(xml, 'text/xml');
const root = doc.documentElement;

if (root.namespaceURI !== SITEMAP_NS) {
  throw new Error('Namespace is "' + root.namespaceURI + '", expected "' + SITEMAP_NS + '"');
}

const locs = doc.getElementsByTagNameNS(SITEMAP_NS, 'loc');
console.log(locs.length + ' URLs, ' + bytes + ' bytes uncompressed');
if (locs.length > MAX_URLS) throw new Error('Over the 50,000 URL limit');
if (bytes > MAX_BYTES) throw new Error('Over the 50 MB limit');

const lastmods = doc.getElementsByTagNameNS(SITEMAP_NS, 'lastmod');
for (let i = 0; i < lastmods.length; i++) {
  const v = lastmods.item(i).textContent.trim();
  // W3C Datetime: a bare date, or a timestamp that carries a zone.
  const ok = /^\d{4}-\d{2}-\d{2}$/.test(v)
    || (/^\d{4}-\d{2}-\d{2}T/.test(v) && /(Z|[+-]\d{2}:\d{2})$/.test(v));
  if (!ok) console.error('Invalid lastmod: ' + v);
}

# pip install lxml requests
import re
import sys
import requests
from lxml import etree

SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9'
MAX_URLS, MAX_BYTES = 50_000, 52_428_800

raw = requests.get('https://example.com/sitemap.xml', timeout=30).content

# The three flags are the safe form. lxml will otherwise expand entities and
# fetch a DTD the document references.
parser = etree.XMLParser(resolve_entities=False, no_network=True, load_dtd=False)
root = etree.fromstring(raw, parser)

if etree.QName(root).namespace != SITEMAP_NS:
    sys.exit('Wrong namespace: %s' % etree.QName(root).namespace)

locs = root.findall('{%s}url/{%s}loc' % (SITEMAP_NS, SITEMAP_NS))
print('%d URLs, %d bytes uncompressed' % (len(locs), len(raw)))
if len(locs) > MAX_URLS:
    sys.exit('Over the 50,000 URL limit')
if len(raw) > MAX_BYTES:
    sys.exit('Over the 50 MB limit')

W3C = re.compile(r'^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2}))?$')
for lm in root.iter('{%s}lastmod' % SITEMAP_NS):
    if not W3C.match((lm.text or '').strip()):
        print('Invalid lastmod:', lm.text)

# The protocol publishes an XSD, which lxml can check structure against:
#   schema = etree.XMLSchema(etree.parse('sitemap.xsd'))
#   schema.assertValid(root.getroottree())

import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.ByteArrayInputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;

final String NS = "http://www.sitemaps.org/schemas/sitemap/0.9";

byte[] bytes = HttpClient.newHttpClient()
    .send(HttpRequest.newBuilder(URI.create("https://example.com/sitemap.xml")).build(),
          HttpResponse.BodyHandlers.ofByteArray())
    .body();

DocumentBuilderFactory f = DocumentBuilderFactory.newInstance();
// Without setNamespaceAware(true) every namespace lookup below returns nothing.
// This is the single most common reason sitemap parsing code silently finds zero URLs.
f.setNamespaceAware(true);
f.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
f.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
f.setXIncludeAware(false);

Document doc = f.newDocumentBuilder().parse(new ByteArrayInputStream(bytes));
NodeList locs = doc.getElementsByTagNameNS(NS, "loc");

System.out.printf("%d URLs, %d bytes uncompressed%n", locs.getLength(), bytes.length);
if (locs.getLength() > 50_000) System.err.println("Over the 50,000 URL limit");
if (bytes.length > 52_428_800) System.err.println("Over the 50 MB limit");

NodeList lastmods = doc.getElementsByTagNameNS(NS, "lastmod");
for (int i = 0; i < lastmods.getLength(); i++) {
    String v = lastmods.item(i).getTextContent().trim();
    try {
        if (v.length() == 10) LocalDate.parse(v);
        else OffsetDateTime.parse(v);   // rejects a timestamp with no zone
    } catch (DateTimeParseException e) {
        System.err.println("Invalid lastmod: " + v);
    }
}

using System.Globalization;
using System.Xml;

const string Ns = "http://www.sitemaps.org/schemas/sitemap/0.9";

// XmlReader streams, so a 50 MB sitemap never becomes a 400 MB DOM.
var settings = new XmlReaderSettings
{
    DtdProcessing = DtdProcessing.Prohibit,  // external entities are never fetched
    XmlResolver = null,
    IgnoreWhitespace = true,
};

string[] w3c =
{
    "yyyy-MM-dd",
    "yyyy-MM-ddTHH:mmK",
    "yyyy-MM-ddTHH:mm:ssK",
    "yyyy-MM-ddTHH:mm:ss.FFFFFFFK",
};

byte[] bytes = await new HttpClient().GetByteArrayAsync("https://example.com/sitemap.xml");

int urls = 0;
using var stream = new MemoryStream(bytes);
using var reader = XmlReader.Create(stream, settings);
while (reader.Read())
{
    if (reader.NodeType != XmlNodeType.Element || reader.NamespaceURI != Ns) continue;

    if (reader.LocalName == "loc")
    {
        urls++;
    }
    else if (reader.LocalName == "lastmod")
    {
        var v = reader.ReadElementContentAsString().Trim();
        // TryParse would accept 03/14/2026. TryParseExact against the four
        // permitted layouts is what the protocol actually asks for.
        if (!DateTimeOffset.TryParseExact(v, w3c, CultureInfo.InvariantCulture,
                                          DateTimeStyles.None, out _))
        {
            Console.Error.WriteLine("Invalid lastmod: " + v);
        }
    }
}

Console.WriteLine(urls + " URLs, " + bytes.Length + " bytes uncompressed");
if (urls > 50_000) Console.Error.WriteLine("Over the 50,000 URL limit");
if (bytes.Length > 52_428_800) Console.Error.WriteLine("Over the 50 MB limit");

# The protocol publishes an XSD, so xmllint can check the structure offline.
curl -sL https://example.com/sitemap.xml -o sitemap.xml
curl -sO https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd

xmllint --noout --nonet --schema sitemap.xsd sitemap.xml

# The two hard limits:
xmllint --nonet --xpath 'count(//*[local-name()="loc"])' sitemap.xml   # cap 50,000
wc -c < sitemap.xml                                                    # cap 52,428,800

# A gzipped sitemap must be decompressed before either check:
curl -sL https://example.com/sitemap.xml.gz | gunzip > sitemap.xml

# Pull every URL out, one per line, for a crawler or a spreadsheet:
xmllint --nonet --xpath '//*[local-name()="loc"]/text()' sitemap.xml | tr -s '\n'