{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":[]},"type":"markdown"},"seo":{"title":"Webhooks (Stream API)","description":"Real-time price and offer data across major marketplaces."},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"webhooks-stream-api","__idx":0},"children":["Webhooks (Stream API)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Stream API is built for large jobs — up to ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["100 000 values"]},". Instead of polling, kwery"," ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["pushes results to your endpoint"]}," as they complete, in signed, batched deliveries."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"1-submit-a-stream-job","__idx":1},"children":["1. Submit a stream job"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["POST /stream"]}," takes the same fields as ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["POST /job"]},", plus a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["callback_url"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"curl -s https://api.kwery.co/stream \\\n  -H \"Authorization: Bearer $KWERY_API_KEY\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\n    \"source\": \"idealo\",\n    \"country\": \"de\",\n    \"key\": \"gtin\",\n    \"values\": [\"4006381333962\", \"4719512101148\"],\n    \"callback_url\": \"https://your-app.example.com/kwery/webhook\",\n    \"client_ref\": \"batch-2026-04-03\"\n  }'\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Response:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{\n  \"error\": false,\n  \"job_id\": \"67e1234abc...\",\n  \"webhook_secret\": \"a3f9...(64 hex chars)...\"\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"blockquote","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Store ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["webhook_secret"]}," immediately."]}," It is returned ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["once"]},", only at submission, and is"," ","required to verify delivery signatures. It cannot be retrieved later."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"optional-fields","__idx":2},"children":["Optional fields"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Field"},"children":["Field"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Default"},"children":["Default"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Description"},"children":["Description"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["client_ref"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["—"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Echoed back on every delivery. Use it to correlate deliveries to your own batch."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["delivery_batch_size"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["100"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["How many results to group per delivery."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["false"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["When ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["true"]},", each result includes a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta"]}," provenance block (see below)."]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"2-receive-deliveries","__idx":3},"children":["2. Receive deliveries"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["kwery sends ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["POST"]}," requests to your ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["callback_url"]}," as results accumulate, and once more at the"," ","end (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["is_final: true"]},"). The payload embeds the results — you do ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["not"]}," need to poll:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{\n  \"job_id\": \"67e1234abc...\",\n  \"source\": \"idealo\",\n  \"country\": \"de\",\n  \"batch_sequence\": 3,\n  \"is_final\": false,\n  \"values_total\": 1000,\n  \"values_done\": 75,\n  \"values_errors\": 1,\n  \"client_ref\": \"batch-2026-04-03\",\n  \"results\": [\n    {\n      \"key\": \"gtin:4006381333962\",\n      \"index\": 0,\n      \"completed_at\": \"2026-04-03T10:01:12.000Z\",\n      \"result\": {\n        \"query\": { \"source\": \"idealo\", \"country\": \"de\", \"key\": \"gtin\", \"value\": \"4006381333962\" },\n        \"content\": { \"...\": \"parsed product + offers\" },\n        \"success\": true,\n        \"reason\": null,\n        \"updated_at\": \"2026-04-03T10:01:12.000Z\"\n      }\n    }\n  ]\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["batch_sequence"]}," increases by one per delivery, so you can detect gaps and ordering. When"," ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["is_final"]}," is ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["true"]},", the job is complete."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"provenance--include_meta-true-","__idx":4},"children":["Provenance (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta: true"]},")"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Submit with ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["include_meta: true"]}," to attach a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta"]}," block to each result, describing how the"," ","data was gathered (provenance is ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/audit-and-provenance"},"children":["rolling out"]}," — treat ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["meta"]}," as"," ","optional in your receiver for now):"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"\"meta\": {\n  \"crawls\": [\n    {\n      \"crawl_id\": \"9f1c...\",\n      \"url\": \"https://www.idealo.de/preisvergleich/OffersOfProduct/...\",\n      \"http_status\": 200,\n      \"duration_ms\": 1840,\n      \"observed_at\": \"2026-04-03T10:01:11.000Z\",\n      \"geo\": \"de\"\n    }\n  ],\n  \"partial\": false\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["partial"]}," (Google only) indicates the marketplace reports more offers than were gathered in"," ","this snapshot."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"3-verify-the-signature","__idx":5},"children":["3. Verify the signature"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Every delivery carries three headers:"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Header"},"children":["Header"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Description"},"children":["Description"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["X-Kwery-Signature"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["v1="]}," + HMAC-SHA256 of ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["\"{timestamp}.{raw_body}\""]},", keyed with your ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["webhook_secret"]},"."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["X-Kwery-Timestamp"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Unix seconds when the delivery was signed."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["X-Kwery-Delivery-Id"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Unique ID for this delivery (useful for idempotency and support)."]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To verify:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Read the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["raw request body"]}," before any JSON parsing."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Read ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["X-Kwery-Timestamp"]},"; reject if it is more than ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["5 minutes"]}," old (replay protection)."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Recompute ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["v1=HMAC_SHA256(webhook_secret, \"{timestamp}.{raw_body}\")"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Compare against ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["X-Kwery-Signature"]}," using a constant-time comparison."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Reject with ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["401"]}," on mismatch."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A complete, runnable receiver is in ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/code-samples"},"children":["Code samples"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"4-acknowledge-quickly","__idx":6},"children":["4. Acknowledge quickly"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Return a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["2xx"]}," status as soon as you have received and verified the delivery, then process"," ","asynchronously. kwery treats the response status as follows:"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Your response"},"children":["Your response"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"kwery's action"},"children":["kwery's action"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["2xx"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Marked delivered."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["4xx"]}," (except ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["429"]},")"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Permanent failure — ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["not retried"]},"."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["3xx"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["5xx"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["429"]},", network error"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Retried with backoff."]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"id":"retries","level":3,"__idx":7},"children":["Retry schedule "]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Failed deliveries are retried up to ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["5 times"]}," with this backoff (±20% jitter):"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Attempt"},"children":["Attempt"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Delay"},"children":["Delay"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["immediate"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["2"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["+30 s"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["+2 min"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["4"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["+10 min"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["+30 min"]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After the 5th attempt a delivery is marked ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["dead-lettered"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"5-replay-and-polling-fallback","__idx":8},"children":["5. Replay and polling fallback"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your endpoint was down, you can replay deliveries:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["POST /stream/{id}/replay-all"]}," — re-enqueue all dead-lettered deliveries for a job."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["POST /stream/deliveries/{delivery_id}/replay"]}," — replay a single delivery."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can also poll instead of (or in addition to) webhooks:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["GET /stream/{id}"]}," — job status and counters."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["GET /stream/{id}/results?offset=0&limit=100"]}," — results sorted by ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["index"]}," (max ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["limit"]}," 1000)."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["GET /stream/{id}/deliveries"]}," — delivery history and statuses."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"next-steps","__idx":9},"children":["Next steps"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/code-samples"},"children":["Code samples"]}," — a full signature-verifying receiver in Node.js and Python."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/guides/errors-and-limits"},"children":["Errors & limits"]}," — limits, retention, and the response envelope."]}]}]},"headings":[{"value":"Webhooks (Stream API)","id":"webhooks-stream-api","depth":1},{"value":"1. Submit a stream job","id":"1-submit-a-stream-job","depth":2},{"value":"Optional fields","id":"optional-fields","depth":3},{"value":"2. Receive deliveries","id":"2-receive-deliveries","depth":2},{"value":"Provenance ( include_meta: true )","id":"provenance--include_meta-true-","depth":3},{"value":"3. Verify the signature","id":"3-verify-the-signature","depth":2},{"value":"4. Acknowledge quickly","id":"4-acknowledge-quickly","depth":2},{"value":"Retry schedule","id":"retries","depth":3},{"value":"5. Replay and polling fallback","id":"5-replay-and-polling-fallback","depth":2},{"value":"Next steps","id":"next-steps","depth":2}],"frontmatter":{"seo":{"title":"Webhooks (Stream API)"}},"lastModified":"2026-08-25T12:32:20.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/guides/webhooks","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}