{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":[]},"type":"markdown"},"seo":{"title":"Audit & Provenance — kwery Docs","siteUrl":"https://docs.kwery.co","description":"Every result's crawl trail — enable provenance, read the meta.crawls[] array, and audit cached vs. live data."},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"audit--provenance","__idx":0},"children":["Audit & provenance"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For every result, kwery can hand you the complete trail of ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["where the data came from"]},":"," ","which URLs were fetched, when, from which country, how long each fetch took, and what HTTP"," ","status the marketplace returned. This is the audit backbone for MAP enforcement, compliance"," ","work, and any situation where \"we saw this price\" needs to be ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["provable"]},", not just stated."]},{"$$mdtype":"Tag","name":"blockquote","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Rolling out."]}," The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta"]}," provenance block is being enabled account by account"," ","and may not yet be attached to your results. Treat ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta"]}," as ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["optional"]}," in your pipeline"," ","for now, and ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://kwery.co/contact"},"children":["contact us"]}," if your workflows need the crawl trail"," ","today — we'll prioritize your account. The shape documented below is final."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"what","level":2,"__idx":1},"children":["What provenance answers "]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["\"Prove this price existed.\""]}," — the exact source URL plus an observation timestamp, per"," ","fetch."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["\"When exactly was this observed?\""]}," — ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["observed_at"]},", ISO 8601, UTC, per fetch."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["\"Was this seen from the right market?\""]}," — ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["geo"]},", the country the request was made from"," ","(a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":[".de"]}," price observed from ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["de"]},", not through some other locale)."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["\"Is this data point disputable?\""]}," — every fetch has a stable ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["crawl_id"]},"; quote it to"," ","support and we can trace that individual fetch end to end."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"enabling","level":2,"__idx":2},"children":["Enabling it "]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Provenance is ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["opt-in per request"]},", so you only carry the extra payload when you need it:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Batch"]}," (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["POST /job"]},") — set ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta: true"]}," on the request."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Stream"]}," (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["POST /stream"]},") — set ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta: true"]},"; the provenance block is attached"," ","to each result in every webhook delivery."]}]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"curl -s https://api.kwery.co/job \\\n  -H \"Authorization: Bearer $KWERY_API_KEY\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\n    \"source\": \"idealo\",\n    \"country\": \"de\",\n    \"key\": \"gtin\",\n    \"values\": [\"4006381333962\"],\n    \"include_meta\": true\n  }'\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["There is no extra charge — provenance rides along with the result you already paid for."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"crawls","level":2,"__idx":3},"children":["The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta.crawls[]"]}," trail "]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta"]}," enabled, each result carries ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta.crawls[]"]}," — ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["one entry per fetch"," ","that contributed to the result"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"\"meta\": {\n  \"crawls\": [\n    {\n      \"crawl_id\": \"9f1c2d3e4b5a6c7d8e9f0a1b\",\n      \"url\": \"https://www.idealo.de/preisvergleich/OffersOfProduct/...\",\n      \"http_status\": 200,\n      \"duration_ms\": 2341,\n      \"observed_at\": \"2026-07-01T12:00:00Z\",\n      \"geo\": \"de\",\n      \"job_type\": \"idealo.search\"\n    }\n  ]\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Field"},"children":["Field"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"What it tells you"},"children":["What it tells you"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["crawl_id"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Stable ID for this individual fetch — quote it to support when asking about a specific data point."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["url"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The public page the fetch corresponds to — the storefront page (without query string) when the data came from one, otherwise the storefront's address, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["null"]}," when there is no public equivalent. Google search URLs keep only the search term."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["http_status"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The HTTP status the target returned (e.g. ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["200"]},")."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["duration_ms"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["How long the fetch took, in milliseconds."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["observed_at"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["When the data was observed (ISO 8601, UTC)."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["geo"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The country the request was made from."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["job_type"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The kind of crawl step, `{source}.{search"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"multiple-crawls","level":3,"__idx":4},"children":["Why one result can list several crawls "]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Most marketplaces need more than one fetch to assemble a complete answer, and every fetch"," ","appears as its own entry:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["A GTIN lookup"]}," typically runs a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["search"]}," fetch (resolve the GTIN to the marketplace's"," ","product) plus a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["product/offer"]}," fetch."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["search_and_offers"]}]}," adds a seller-list fetch on sources where offers live on a"," ","separate endpoint (Amazon, Kaufland)."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Paginated offer lists"]}," (Idealo) add one entry per extra page fetched."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["A ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["cache-served"]}," result carries the trail of the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["original"]}," crawls — ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["observed_at"]}," ","shows you exactly how old the observation is (the same age the"," ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/trust#components"},"children":["trust ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["freshness"]}," component"]}," scores)."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Read together, the entries are the full story of how your answer was produced."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"served-from","level":3,"__idx":5},"children":["Cached or live? "]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You don't need ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta"]}," to know whether a result was fetched for you or re-served"," ","from cache — every result carries two fields regardless:"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Field"},"children":["Field"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Meaning"},"children":["Meaning"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["served_from"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["live"]}," — at least one fetch was made for this job. ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["cache"]}," — re-served from a cached observation, no fetch."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["observed_at"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["When the data was observed (ISO 8601, UTC): the last fetch for ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["live"]},", the cached observation's date for ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["cache"]}," (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["null"]}," when nothing was observed)."]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["observed_at"]}," equals the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["observed_at"]}," of the last entry in"," ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta.crawls[]"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"workflows","level":2,"__idx":6},"children":["Audit workflows "]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["MAP / pricing disputes"]}," — store ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["url"]}," + ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["observed_at"]}," + ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["crawl_id"]}," alongside the"," ","price. A violation report that says ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["\"€189.00 on idealo.de, observed 2026-07-01"," ","12:00 UTC from Germany, crawl 9f1c…\""]}," is evidence; a bare price is an anecdote."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Compliance & retention"]}," — results (with their provenance) are downloadable for"," ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["72 hours"]},"; persist them on your side as part of your evidence chain. Enterprise plans"," ","add managed ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["audit-trail export"]}," — the complete provenance stream delivered to your"," ","storage — see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/pricing#enterprise"},"children":["Pricing → Enterprise"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Debugging with support"]}," — anything odd about a data point, send us the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["crawl_id"]},"."," ","It pins the conversation to one specific fetch instead of \"sometime yesterday\"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"vs-trust","level":2,"__idx":7},"children":["Relation to Trust "]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/trust"},"children":["Confidence score"]}," and provenance answer different questions about the same result:"," ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["trust"]}," scores ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["how much to rely on it"]},"; ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["provenance"]}," proves ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["where it came from"]},"."," ","High-stakes pipelines typically use both: route on ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["trust.score"]},", archive ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta.crawls"]}," ","for whatever you act on."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"faq","level":2,"__idx":8},"children":["Questions we get "]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Does it cost extra?"]}," No — no extra credits. It's off by default only to keep payloads"," ","small."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["How much bigger are payloads?"]}," One crawl entry is ~200 bytes; a typical result carries"," ","one to three entries (plus one per extra Idealo offer page)."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Can I get provenance retroactively for a past result?"]}," Within the 72-hour retention"," ","window, re-download the job — provenance is part of the stored result if the job was"," ","submitted with ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta: true"]},". It cannot be reconstructed for jobs submitted"," ","without it, so watch lists that may ever need evidence should always set it."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Is the raw HTML available?"]}," Not via the API. For enterprise evidence-chain"," ","requirements (stored page snapshots), ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://kwery.co/contact"},"children":["talk to us"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"next-steps","__idx":9},"children":["Next steps"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/trust"},"children":["Confidence score"]}," — the confidence score carried by every result."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/change-events"},"children":["Change events"]}," — notifications when watched products change."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/webhooks"},"children":["Webhooks"]}," — stream deliveries, where provenance is attached per result."]}]}]},"headings":[{"value":"Audit & provenance","id":"audit--provenance","depth":1},{"value":"What provenance answers","id":"what","depth":2},{"value":"Enabling it","id":"enabling","depth":2},{"value":"The meta.crawls[] trail","id":"crawls","depth":2},{"value":"Why one result can list several crawls","id":"multiple-crawls","depth":3},{"value":"Cached or live?","id":"served-from","depth":3},{"value":"Audit workflows","id":"workflows","depth":2},{"value":"Relation to Trust","id":"vs-trust","depth":2},{"value":"Questions we get","id":"faq","depth":2},{"value":"Next steps","id":"next-steps","depth":2}],"frontmatter":{"seo":{"title":"Audit & Provenance — kwery Docs","description":"Every result's crawl trail — enable provenance, read the meta.crawls[] array, and audit cached vs. live data."}},"lastModified":"2026-10-14T13:44:44.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/guides/audit-and-provenance","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}