{
  "title": "searchSDK benchmark protocol",
  "protocolVersion": "0.2.0-draft",
  "resultSchemaVersion": "1.0.0",
  "status": "planned-boat-run",
  "resultsPublished": false,
  "executionVenue": { "planned": "Boat", "liveRunCompleted": false, "docs": "https://docs.boat.dev/llms.txt" },
  "tracks": [
    { "id": "open-web-search", "status": "harness-prepared", "providers": ["context", "tinyfish", "exa", "tavily", "brave", "perplexity", "parallel", "linkup", "firecrawl", "you", "serper", "serpapi", "valyu", "searxng"], "task": "Normalized ranked web results; SDK adapter defaults, result ceiling 10, no generated answers, ingestion, or implicit pagination." },
    { "id": "corpus-retrieval", "status": "separate-protocol-required", "providers": ["wondersearch"], "nonComparableReason": "WonderSearch retrieves from a configured corpus. A frozen public or permissioned corpus, access rules, ingestion state, and corpus-specific judgments are required. It cannot share an open-web leaderboard. No corpus ingestion or native mutation is performed." }
  ],
  "dataset": { "id": "public-web-v1", "version": "1.0.0", "license": "CC0-1.0", "queryCount": 12, "provenance": "Original public-domain prompts about public documentation, science, and reference topics. No personal data, production queries, or account identifiers.", "limitation": "Smoke dataset only; too small and narrow for general provider rankings. Relevance judgments are not yet collected." },
  "defaults": { "resultDepth": 10, "warmupsPerProvider": 1, "runs": 3, "concurrency": 1, "timeoutMs": 15000, "minIntervalMs": 1000, "maxRequests": 600, "maxDurationMs": 1800000, "maxSpendUsd": 0, "retry": "off", "cache": "off", "deduplicate": false, "fallback": "off", "seed": 1729 },
  "safety": { "defaultMode": "plan", "fixtureMode": "injected synthetic responses; no provider calls or credentials", "liveOptIn": ["--mode live", "--allow-live", "SEARCHSDK_BENCHMARK_ALLOW_LIVE=1", "validated runtime, provider plan, pricing date, per-request reservation and total budget"], "secrets": "Environment only. Never serialized: environment values, headers, full URLs, result content, raw exceptions, account identifiers, or private hostnames.", "budget": "Warmups, errors and timeouts consume the same request and conservative cost-reservation budgets. No retries. Reservations are operator-verified bounds, not actual charges or a guarantee of remote billing. Reconcile provider usage before publication." },
  "metrics": [
    { "id": "relevance", "measure": "nDCG@10 after independent judgments", "status": "not-scored", "requiredEvidence": ["lawfully reusable evidence", "provider-blinded rubric scored 0-3", "two independent judges", "agreement and adjudication", "per-query judgments", "query-cluster bootstrap 95% confidence intervals with published seed and resamples"] },
    { "id": "latency", "measure": "nearest-rank p50 and p95 milliseconds for schema-valid measured responses", "requiredEvidence": ["monotonic end-to-end durations", "all error and timeout durations retained separately", "warmups excluded from headline percentiles", "sample count and deadline", "fixed region and runtime", "provider-side cache and network limitations"] },
    { "id": "cost", "measure": "reconciled USD per 1000 measured attempts", "status": "not-priced", "requiredEvidence": ["named plan", "pricing verification date", "provider-reported units preserved separately", "billing reconciliation", "unknown charges remain null", "warmup charges disclosed separately", "credits and funding disclosures"] },
    { "id": "reliability", "measure": "schema-valid responses / all started measured attempts", "requiredEvidence": ["success and empty counts separately", "rate limits, auth, provider errors and timeouts separately", "unstarted requests excluded from denominator and disclosed", "warmups separate", "partial runs labelled"] }
  ],
  "fairness": ["Seeded query and provider order, interleaved per query and repeated in three rounds", "Same public prompts, result ceiling, timeouts and serial dispatch policy", "Publish provider API revision, exact adapter profile, plan, source/build hash and exceptions", "Provider defaults may differ in retrieval depth, content work, geography and index; no assertion of identical native work", "SearXNG depends on an operator-selected instance and upstream engines; use a public alias and a separately approved sanitized engine manifest", "Do not equate process warmup with controlled cold or warm provider caches", "Never compare provider-native relevance scores directly", "Publish failures, exclusions, sample sizes, conflicts of interest and limitations"],
  "requiredRunMetadata": ["runId", "protocolVersion", "protocolSha256", "dataset version and sha256", "configSha256", "sdkCommit or archive content hashes", "workingTreeDirty", "sdkSourceSha256", "sdkBuildSha256", "harnessSha256", "packageLockSha256", "Node version", "OS and architecture", "machine class", "verified execution region", "provider API version and adapter profile", "provider plan", "pricing date", "seed", "timeouts", "warmup and measured schedules", "request and reservation budgets", "startedAt", "finishedAt", "stopReason", "funding and credits"],
  "artifacts": { "manifest": "run.json", "attempts": "attempts.jsonl (benchmark-attempt schema v1.0.0)", "results": "results.json with summaries, exclusions, null relevance and unreconciled cost", "publication": "All outputs are unreviewed, publicationReady=false. Public fixtures must stay labelled synthetic. No automatic website leaderboard or upload." },
  "publicationPolicy": ["No invented scores or fixture timings presented as provider performance", "No ranking before representative dataset, independent judgments and reproducible reviewed run artifacts", "Publish lawful sanitized evidence only; keep secrets and private instance details out", "Record exclusions and incomplete runs; never drop failed providers silently", "Recheck current Boat and provider documentation before any future live execution"],
  "runs": []
}