{"components":[{"slug":"toloka","name":"Toloka","summary":"Human-in-the-loop training data, RL environments, and evaluation for AI agents and LLMs.","kind":"resource","homepage":"https://toloka.ai","publisher":"Toloka","hasMcp":false,"hasApi":true,"costRange":"enterprise","composabilityRationale":"Toloka is a data and evaluation vendor, not an agent runtime; it feeds training and eval loops rather than composing as an in-agent tool.","functions":["evaluate","classify","generate","observe"],"slices":[{"functionId":"evaluate","label":"RL environments and model evaluation","note":"Complex multi-step RL environments used to test and improve agent behavior."},{"functionId":"classify","label":"Human annotation and labeling","note":"Expert labeling and annotation for training and safety datasets."},{"functionId":"generate","label":"Training data generation","note":"Human-plus-tech pipelines that produce agentic, coding, and safety data."},{"functionId":"observe","label":"Human expertise feedback","note":"Expert review that surfaces model failure modes and improvement signals."}],"tweets":[],"sources":[{"kind":"web","label":"web","url":"https://toloka.ai","excerpt":"Apr 15, 2026 — The definitive list: 20 production-ready MCP servers across developer tools, productivity, data, search, design, SEO, and human expertise ...","at":"2026-09-11T16:37:36.852Z"},{"kind":"crawl","label":"TinyFish fetch","url":"https://toloka.ai","excerpt":"# Training data for AI agents and LLMs\n\nFrom agentic skills to coding and AI safety — we build data solutions integrating human expertise and technology to accelerate AI development.\n\nTalk to an expert\n\nTrusted by Leading AI Teams\n\n# What o","at":"2026-09-11T16:37:36.853Z"}],"origin":"intake","notes":[{"authorKind":"agent","body":"Intake via web.","at":"2026-09-11T16:37:36.853Z"}],"costNotes":"Enterprise data-collection engagements; no public self-serve MCP or list pricing on the homepage evidence.","score":12,"explain":{"score":12,"formula":"100 × sum(weight × value) over observed factors only. Unobserved factors are omitted, not zeroed.","factors":[{"id":"github_health","label":"GitHub health","weight":0.18,"why":"insufficient evidence","sources":[],"evidence":"insufficient"},{"id":"agent_use","label":"Agent use","weight":0.16,"why":"insufficient evidence","sources":[],"evidence":"insufficient"},{"id":"human_notes","label":"Human notes","weight":0.14,"value":0.0867528411160754,"contribution":0.012145397756250557,"why":"1 stored human notes (component-record).","sources":[],"evidence":"observed"},{"id":"composability","label":"Composability","weight":0.16,"why":"insufficient evidence","sources":[{"kind":"crawl","label":"TinyFish fetch","url":"https://toloka.ai","excerpt":"# Training data for AI agents and LLMs\n\nFrom agentic skills to coding and AI safety — we build data solutions integrating human expertise and technology to accelerate AI development.\n\nTalk to an expert\n\nTrusted by Leading AI Teams\n\n# What o","at":"2026-09-11T16:37:36.853Z"}],"evidence":"insufficient"},{"id":"checklist","label":"Checklist","weight":0.1,"value":0.6666666666666666,"contribution":0.06666666666666667,"why":"No MCP surface is declared. An API is declared. Cost is enterprise. 2 of 3 checklist fields are complete.","sources":[{"kind":"crawl","label":"TinyFish fetch","url":"https://toloka.ai","excerpt":"# Training data for AI agents and LLMs\n\nFrom agentic skills to coding and AI safety — we build data solutions integrating human expertise and technology to accelerate AI development.\n\nTalk to an expert\n\nTrusted by Leading AI Teams\n\n# What o","at":"2026-09-11T16:37:36.853Z"}],"evidence":"observed"},{"id":"evidence_quality","label":"Evidence quality","weight":0.14,"value":0.3284952023094184,"contribution":0.04598932832331858,"why":"2 sources cover 2 evidence types. The latest dated evidence is 2026-09-11. Freshness is omitted unless observed as a signal.","sources":[{"kind":"web","label":"web","url":"https://toloka.ai","excerpt":"Apr 15, 2026 — The definitive list: 20 production-ready MCP servers across developer tools, productivity, data, search, design, SEO, and human expertise ...","at":"2026-09-11T16:37:36.852Z"},{"kind":"crawl","label":"TinyFish fetch","url":"https://toloka.ai","excerpt":"# Training data for AI agents and LLMs\n\nFrom agentic skills to coding and AI safety — we build data solutions integrating human expertise and technology to accelerate AI development.\n\nTalk to an expert\n\nTrusted by Leading AI Teams\n\n# What o","at":"2026-09-11T16:37:36.853Z"}],"evidence":"observed"},{"id":"benchmark","label":"Benchmarks","weight":0.12,"why":"insufficient evidence","sources":[],"evidence":"insufficient"}],"missing":["GitHub health: insufficient evidence","Agent use: insufficient evidence","Composability: insufficient evidence","Benchmarks: insufficient evidence"]},"createdAt":"2026-09-11T16:37:36.854Z","updatedAt":"2026-09-13T08:56:02.862Z","tags":["agents"],"recordId":"cmp_toloka","claims":{"name":"Toloka","summary":"Human-in-the-loop training data, RL environments, and evaluation for AI agents and LLMs.","kind":"resource","publisher":"Toloka","homepage":"https://toloka.ai","costRange":"enterprise","costNotes":"Enterprise data-collection engagements; no public self-serve MCP or list pricing on the homepage evidence.","functions":["evaluate","classify","generate","observe"],"slices":[{"functionId":"evaluate","label":"RL environments and model evaluation","note":"Complex multi-step RL environments used to test and improve agent behavior."},{"functionId":"classify","label":"Human annotation and labeling","note":"Expert labeling and annotation for training and safety datasets."},{"functionId":"generate","label":"Training data generation","note":"Human-plus-tech pipelines that produce agentic, coding, and safety data."},{"functionId":"observe","label":"Human expertise feedback","note":"Expert review that surfaces model failure modes and improvement signals."}],"interfaces":[{"kind":"http"}],"composabilityRationale":"Toloka is a data and evaluation vendor, not an agent runtime; it feeds training and eval loops rather than composing as an in-agent tool."},"interfaces":[{"kind":"http"}],"relations":[{"from":"toloka","to":"langsmith","kind":"fills-same-slot-as","note":"Shared functions evaluate, observe. Can fill the same function slot."},{"from":"toloka","to":"openrouter","kind":"fills-same-slot-as","note":"Shared functions generate, observe. Can fill the same function slot."},{"from":"toloka","to":"typesafe-jev","kind":"fills-same-slot-as","note":"Shared functions classify, evaluate. Can fill the same function slot."},{"from":"toloka","to":"41-open-source-llms-evaluated-on-19-benchmarks","kind":"fills-same-slot-as","note":"Shared functions evaluate, generate. Can fill the same function slot."}],"signals":[{"key":"human_note_count","value":1,"source":"component-record","method":"stored-notes","observedAt":"2026-09-13T08:56:02.862Z"}],"functionProbabilities":[],"claimVerification":[]},{"slug":"typesafe-jev","name":"TypeSafe Jev","publisher":"TypeSafe","kind":"model","summary":"Returns typed judgments and probabilities from software state and narrowly defined questions.","homepage":"https://typesafe.ai/","docsUrl":"https://docs.typesafe.ai/llms.txt","functions":["classify","evaluate","reason"],"costRange":"unknown","recordId":"cmp_typesafe-jev","hasMcp":false,"hasApi":true,"costNotes":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","composabilityRationale":"Use its declared http surface for the listed capability slices. Source review does not establish integration success or observed usage.","interfaces":[{"kind":"http","specUrl":"https://docs.typesafe.ai/llms.txt"}],"origin":"intake","tweets":[],"notes":[],"sources":[{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/llms.txt","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/images","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/primitives/box","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"}],"slices":[{"functionId":"classify","label":"Classify","note":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","cost":"unknown"},{"functionId":"evaluate","label":"Evaluate","note":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","cost":"unknown"},{"functionId":"reason","label":"Reason","note":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","cost":"unknown"}],"claims":{"name":"TypeSafe Jev","summary":"Returns typed judgments and probabilities from software state and narrowly defined questions.","kind":"model","publisher":"TypeSafe","homepage":"https://typesafe.ai/","docsUrl":"https://docs.typesafe.ai/llms.txt","costRange":"unknown","costNotes":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","functions":["classify","evaluate","reason"],"slices":[{"functionId":"classify","label":"Classify","note":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","cost":"unknown"},{"functionId":"evaluate","label":"Evaluate","note":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","cost":"unknown"},{"functionId":"reason","label":"Reason","note":"Images and Box localization are experimental, off by default and require the documented danger-zone header; request format and accuracy may change.","cost":"unknown"}],"interfaces":[{"kind":"http","specUrl":"https://docs.typesafe.ai/llms.txt"}],"composabilityRationale":"Use its declared http surface for the listed capability slices. Source review does not establish integration success or observed usage."},"relations":[{"from":"typesafe-jev","to":"toloka","kind":"fills-same-slot-as","note":"Shared functions classify, evaluate. Can fill the same function slot."},{"from":"typesafe-jev","to":"41-open-source-llms-evaluated-on-19-benchmarks","kind":"fills-same-slot-as","note":"Shared functions evaluate, reason. Can fill the same function slot."}],"signals":[],"functionProbabilities":[],"claimVerification":[],"explain":{"score":7,"formula":"100 × sum(weight × value) over observed factors only. Unobserved factors are omitted, not zeroed.","factors":[{"id":"github_health","label":"GitHub health","weight":0.18,"why":"insufficient evidence","sources":[],"evidence":"insufficient"},{"id":"agent_use","label":"Agent use","weight":0.16,"why":"insufficient evidence","sources":[],"evidence":"insufficient"},{"id":"human_notes","label":"Human notes","weight":0.14,"why":"insufficient evidence","sources":[],"evidence":"insufficient"},{"id":"composability","label":"Composability","weight":0.16,"why":"insufficient evidence","sources":[{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/llms.txt","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/images","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/primitives/box","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"}],"evidence":"insufficient"},{"id":"checklist","label":"Checklist","weight":0.1,"value":0.3333333333333333,"contribution":0.03333333333333333,"why":"No MCP surface is declared. An API is declared. Cost is unknown. 1 of 3 checklist fields are complete.","sources":[{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/llms.txt","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/images","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/primitives/box","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"}],"evidence":"observed"},{"id":"evidence_quality","label":"Evidence quality","weight":0.14,"value":0.26214289265639185,"contribution":0.036700004971894865,"why":"3 sources cover 1 evidence types. The latest dated evidence is undated. Freshness is omitted unless observed as a signal.","sources":[{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/llms.txt","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/images","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"},{"kind":"docs","label":"TypeSafe Jev official documentation","url":"https://docs.typesafe.ai/primitives/box","checkedAt":"2026-11-07T13:46:44.976Z","basis":"vendor-claim"}],"evidence":"observed"},{"id":"benchmark","label":"Benchmarks","weight":0.12,"why":"insufficient evidence","sources":[],"evidence":"insufficient"}],"missing":["GitHub health: insufficient evidence","Agent use: insufficient evidence","Human notes: insufficient evidence","Composability: insufficient evidence","Benchmarks: insufficient evidence"]},"score":7}]}