{ "schema": "varexis-index-free-publication-preview/1", "catalogue_version": "ice-08-tranche-67-v1", "route_id": "agentic_reviewed_langsmith_evaluator_cap_inflight_overshoot", "record_id": "VXI-EXT-ICE08-LANGSMITH-EVALUATOR-CAP-INFLIGHT-OVERSHOOT-01-v1", "question": "Is an evaluator weekly cap an exact ceiling on every in-flight charge?", "answer": "LangSmith records spend after each evaluator run finishes. Concurrent in-flight runs can therefore push the final amount above the effective cap before new evaluations are paused.", "records": [ { "record_id": "VXI-EXT-ICE08-LANGSMITH-EVALUATOR-CAP-INFLIGHT-OVERSHOOT-01-v1", "url": "/varexis-index/records/langsmith-evaluator-cap-inflight-overshoot.json", "content_sha256": "fe98ac7f48534a9e9501f0e4ad8331b57507b2a2fb41f8cef2e5fb379bb29dcb", "evidence": { "schema": "varexis-public-evidence-record/0.3", "generated_at_utc": "2026-10-30T12:29:00.575278+00:00", "record_id": "VXI-EXT-ICE08-LANGSMITH-EVALUATOR-CAP-INFLIGHT-OVERSHOOT-01-v1", "record_kind": "external_reference", "title": "Is an evaluator weekly cap an exact ceiling on every in-flight charge", "system": "LangChain", "evidence_family": "vendor_evidence_review", "index_status": "PUBLIC_SAFE_SOURCE_REVIEW", "evidence_state": "PRIMARY_DOCUMENTATION_REVIEW_ONLY", "canonical_question": "Is an evaluator weekly cap an exact ceiling on every in-flight charge?", "near_questions": [ "Can concurrent evaluations exceed a configured cap?", "When does LangSmith add an evaluator run to spend?", "Does reaching a cap cancel already-running evaluations?", "What explains a small evaluator-budget overshoot?" ], "supported_claims": [ { "claim": "LangSmith records spend after each evaluator run finishes. Concurrent in-flight runs can therefore push the final amount above the effective cap before new evaluations are paused.", "reliance": "DOCUMENTED_CAPABILITY_ONLY" } ], "not_demonstrated_claims": [ "Retain in-flight completion and cost evidence before asserting an exact maximum.", "Buyer implementation, live runtime enforcement, settled payment, verified deletion or certification are not established.", "No expanded capabilities or licences for existing INDEX executables." ], "public_summary": "LangSmith records spend after each evaluator run finishes. Concurrent in-flight runs can therefore push the final amount above the effective cap before new evaluations are paused.", "related_record_ids": [], "source_review": { "source_type": "official_external_documentation", "source_title": "Track and limit evaluator spend", "source_url": "https://docs.langchain.com/langsmith/evaluator-spend.md", "checked_on": "2026-10-30", "candidate_record_id": "VXI-EXT-ICE08-LANGSMITH-EVALUATOR-CAP-INFLIGHT-OVERSHOOT-01-v1", "candidate_record_sha256": "df7493665e082e225482fc459a733aebe87acc0c2e3a46a920815f7a94bb20a8", "method": "Primary documentation reviewed for the published bounded statement.", "deployment_tested_by_varexis": false, "revalidation": "The delivered snapshot preserves the review date shown. Recheck current primary documentation and deployment context before relying on present applicability; payment does not trigger fresh research." }, "answer_policy": { "classification": "DOCUMENTED_WITH_DEPLOYMENT_BOUNDARY", "bounded_answer": "LangSmith records spend after each evaluator run finishes. Concurrent in-flight runs can therefore push the final amount above the effective cap before new evaluations are paused.", "required_buyer_context": "effective cap; running evaluations; completion times; recorded cost. Buyer deployment not inspected.", "missing_evidence": "A broader claim needs evidence matching the intended use. Payment does not upgrade the evidence state." }, "private_boundary": "Raw logs, thresholds, private paths, source code, protected mechanisms and reconstruction-level details are excluded from public and paid delivery.", "decision_profile": { "schema": "varexis-vendor-evidence-review/1", "review_status": "documentation_review_only", "checked_on": "2026-10-30", "claim_to_check": "Is an evaluator weekly cap an exact ceiling on every in-flight charge?", "documented_statement": { "status": "documented", "statement": "LangSmith records spend after each evaluator run finishes. Concurrent in-flight runs can therefore push the final amount above the effective cap before new evaluations are paused.", "source_ids": [ "langsmith-evaluator-spend" ] }, "review_inference": { "status": "review_inference", "statement": "Is an evaluator weekly cap an exact ceiling on every in-flight charge? Use the stated boundary to identify missing scope, source or outcome evidence before closing the review. Evaluator post-run accounting differs from API-provider limit enforcement. This is an operational review inference, not an observed deployment outcome.", "source_ids": [ "langsmith-evaluator-spend" ] }, "buyer_evidence_required": [ "effective cap", "running evaluations", "completion times", "recorded cost" ], "review_condition": { "status": "varexis_review_policy", "statement": "Retain in-flight completion and cost evidence before asserting an exact maximum." }, "unknowns": "The buyer implementation, effective authority, source-data completeness and actual business outcomes were not inspected.", "unknown_finding": { "status": "unknown", "statement": "No buyer deployment or transaction was observed." }, "evidence_sources": [ { "source_id": "langsmith-evaluator-spend", "url": "https://docs.langchain.com/langsmith/evaluator-spend.md", "final_url": "https://docs.langchain.com/langsmith/evaluator-spend.md", "captured_at_utc": "2026-10-30T12:18:09.149088+00:00", "status": 200, "content_type": "text/markdown; charset=utf-8", "bytes": 10815, "sha256": "836b56c6ac8597506c3d35f5899e72ee44bbf7499c72fe9bfaff1aef32833aa9", "title": "Track and limit evaluator spend", "publisher": "LangChain", "checked_on": "2026-10-30", "capture_status": 200, "claim_review_completed": true, "hash_basis": "SHA-256 of captured HTTP response-body bytes; not a signature or factual guarantee.", "version_context": "Official SDK/service documentation captured 2026-10-30. Rolling documentation, not a pinned implementation release; recheck current version and configuration before reliance.", "rights_disposition": "Original bounded paraphrase with source attribution. Captured source body remains private; no source-body redistribution. Vendor documentation rights, code licences and marks are not transferred.", "reviewed_sections": [ "How enforcement works", "When a limit is reached", "Override the default for an attached project or dataset", "Troubleshooting" ] } ], "observed_deployment": false, "commercial_reliance_boundary": "Documentation supports only the stated boundary. Existing licensed checkers analyse supported supplied declarations locally; they do not inspect provider accounts, verify live behaviour, enforce decisions or certify compliance.", "paid_value": { "decision_context": "Is an evaluator weekly cap an exact ceiling on every in-flight charge? Use the stated boundary to identify missing scope, source or outcome evidence before closing the review.", "free": "Complete source review, questions, provenance and limitations; free to read and save locally.", "single_request": "Unavailable for this publication-only record; historical saved retrieval is unchanged.", "evidence_receipt": "Unavailable for this publication-only record; no paid retrieval entitlement is added.", "excluded": [ "Fresh source research", "Native provider integration or schema support", "Live inspection or runtime enforcement", "Payment settlement or verified deletion", "Certification or expanded licences" ], "integrity": "Hashes identify content versions; they do not prove factual truth or provide a signature." }, "revalidation": { "trigger": "Recheck current primary documentation and the applicable API, service, account and configuration before reliance. No ongoing monitoring is included.", "specific_watch": "How enforcement works" }, "buyer_relevance": "Is an evaluator weekly cap an exact ceiling on every in-flight charge? Use the stated boundary to identify missing scope, source or outcome evidence before closing the review.", "refusal_conditions": [ "Retain in-flight completion and cost evidence before asserting an exact maximum.", "Do not infer deployment conformance, authority or successful outcomes from source documentation." ], "decision_context": { "operator": "Agent platform operator or cost reviewer", "scenario": "Is an evaluator weekly cap an exact ceiling on every in-flight charge?", "consequence": "Use the stated boundary to identify missing scope, source or outcome evidence before closing the review.", "confidence": "High for captured documentation; applicability and deployed behaviour unverified." }, "novelty_review": { "candidate_id": "ICE08-031", "disposition": "EXTENSION", "nearest_record_id": "VXI-EXT-PROCUREMENT-OPENAI-SPEND-ALERT-HARD-LIMIT-RATE-BOUNDARY-01-v1", "nearest_record_url": "/varexis-index/records/openai-spend-alert-hard-limit-rate-boundary.json", "nearest_record_sha256": "3c926fb131306cfd2364fa074555556f9abb50b11b39b312f855404efdfd4e91", "material_distinction": "Evaluator post-run accounting differs from API-provider limit enforcement.", "review_status": "PASS: implementing-agent source and duplication review" }, "optional_selection": { "guide_id": "agent-run-cost-trace-evidence", "guide_url": "/index-guides/agent-run-cost-trace-evidence", "guide_manifest": "/varexis-index/decision-guides.json", "checker": { "product_id": "agent-audit-coverage", "price_gbp_excluding_vat": 79, "purchase_url": "/index-access?route=agentic_pack_audit_coverage_v1", "licence_scope": "Existing separate internal-use licence; subscriptions do not include the executable.", "benefit": "Optional local analysis of supported supplied declarations only; no native provider integration or verification is added." }, "boundary": "Optional Audit Coverage only reconciles supported supplied event-chain declarations. It does not connect to these providers, calculate model charges, verify live permissions, export traces or enforce budgets." }, "exact_access": { "endpoint": "/mcp/catalogue", "tool": "get_publication_record", "arguments": { "record_id": "VXI-EXT-ICE08-LANGSMITH-EVALUATOR-CAP-INFLIGHT-OVERSHOOT-01-v1" }, "record_url": "/varexis-index/records/langsmith-evaluator-cap-inflight-overshoot.json", "original_mcp_search": false, "paid_saved_retrieval": false } }, "niche": { "niche_id": "ice08-audit", "label": "Audit, Provenance & Evidence Accountability", "classification_id": "audit", "classification_label": "Audit, Provenance & Evidence Accountability", "topic_id": "audit", "buyer_intent": "Is an evaluator weekly cap an exact ceiling on every in-flight charge? Use the stated boundary to identify missing scope, source or outcome evidence before closing the review.", "comparison_role": "supplementary_discovery" }, "paid_access_note": "The full published answer and boundaries are free. Paid access saves the selected route and source versions, with an unsigned receipt when purchased. Preview coverage before paying. Fresh research, bespoke assessment and protected evidence are excluded.", "service_route": { "request_guide": "/varexis-index/request.html", "request_workflow": "/varexis-index/request-workflow.json", "service_id": "single_request", "service_routes_url": "/varexis-index/service-routes.json", "payment_required": true, "availability_endpoint": "/api/index/capabilities", "operational_status": "AUTOMATIC_SERVICE_CAPABILITIES_GATED", "checkout_url": "/index-access" }, "authority_boundary": "VAREXIS-INDEX informs evidence reliance. It does not provide advice, certification, deployment authority, security assurance, production approval, or guarantee.", "human_page": "/varexis-index/catalog.html#langsmith-evaluator-cap-inflight-overshoot", "machine_readable_record": "/varexis-index/records/langsmith-evaluator-cap-inflight-overshoot.json" } } ], "access": { "mode": "exact_http_only", "payment_required": false, "purchase_url": null, "mcp_preview_available": false, "paid_saved_retrieval_available": false, "natural_language_search_qualified": false }, "boundary": "Catalogue additions are available free through the exact HTTP JSON links below. Enhanced natural-language discovery is not yet qualified for these records. They are outside MCP search, MCP preview and paid saved retrieval. No search qualification, executable capability or entitlement is added.", "content_sha256": "d85340cfa4a0e60ad29e8ee8bd46af4b669823c8b1b3c51dc5341592b1872b2a" }