Skip to main content
POST
/
v1
/
eval
Launch an eval
curl --request POST \
  --url https://api.braintrust.dev/v1/eval \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "project_id": "<string>",
  "data": {
    "dataset_id": "<string>",
    "dataset_version": "<string>",
    "dataset_environment": "<string>",
    "_internal_btql": {}
  },
  "task": {
    "function_id": "<string>",
    "version": "<string>"
  },
  "scores": [
    {
      "function_id": "<string>",
      "version": "<string>"
    }
  ],
  "name": "<string>",
  "parameters": {},
  "experiment_name": "<string>",
  "metadata": {},
  "parent": {
    "object_id": "<string>",
    "row_ids": {
      "id": "<string>",
      "span_id": "<string>",
      "root_span_id": "<string>"
    },
    "propagated_event": {}
  },
  "stream": true,
  "trial_count": 123,
  "is_public": true,
  "timeout": 123,
  "max_concurrency": 10,
  "base_experiment_name": "<string>",
  "base_experiment_id": "<string>",
  "git_metadata_settings": {
    "fields": []
  },
  "repo_info": {
    "commit": "<string>",
    "branch": "<string>",
    "tag": "<string>",
    "dirty": true,
    "author_name": "<string>",
    "author_email": "<string>",
    "commit_message": "<string>",
    "commit_time": "<string>",
    "git_diff": "<string>"
  },
  "strict": true,
  "stop_token": "<string>",
  "extra_messages": "<string>",
  "tags": [
    "<string>"
  ],
  "mcp_auth": {}
}
'
import requests

url = "https://api.braintrust.dev/v1/eval"

payload = {
    "project_id": "<string>",
    "data": {
        "dataset_id": "<string>",
        "dataset_version": "<string>",
        "dataset_environment": "<string>",
        "_internal_btql": {}
    },
    "task": {
        "function_id": "<string>",
        "version": "<string>"
    },
    "scores": [
        {
            "function_id": "<string>",
            "version": "<string>"
        }
    ],
    "name": "<string>",
    "parameters": {},
    "experiment_name": "<string>",
    "metadata": {},
    "parent": {
        "object_id": "<string>",
        "row_ids": {
            "id": "<string>",
            "span_id": "<string>",
            "root_span_id": "<string>"
        },
        "propagated_event": {}
    },
    "stream": True,
    "trial_count": 123,
    "is_public": True,
    "timeout": 123,
    "max_concurrency": 10,
    "base_experiment_name": "<string>",
    "base_experiment_id": "<string>",
    "git_metadata_settings": { "fields": [] },
    "repo_info": {
        "commit": "<string>",
        "branch": "<string>",
        "tag": "<string>",
        "dirty": True,
        "author_name": "<string>",
        "author_email": "<string>",
        "commit_message": "<string>",
        "commit_time": "<string>",
        "git_diff": "<string>"
    },
    "strict": True,
    "stop_token": "<string>",
    "extra_messages": "<string>",
    "tags": ["<string>"],
    "mcp_auth": {}
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)
const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    project_id: '<string>',
    data: {
      dataset_id: '<string>',
      dataset_version: '<string>',
      dataset_environment: '<string>',
      _internal_btql: {}
    },
    task: {function_id: '<string>', version: '<string>'},
    scores: [{function_id: '<string>', version: '<string>'}],
    name: '<string>',
    parameters: {},
    experiment_name: '<string>',
    metadata: {},
    parent: {
      object_id: '<string>',
      row_ids: {id: '<string>', span_id: '<string>', root_span_id: '<string>'},
      propagated_event: {}
    },
    stream: true,
    trial_count: 123,
    is_public: true,
    timeout: 123,
    max_concurrency: 10,
    base_experiment_name: '<string>',
    base_experiment_id: '<string>',
    git_metadata_settings: {fields: []},
    repo_info: {
      commit: '<string>',
      branch: '<string>',
      tag: '<string>',
      dirty: true,
      author_name: '<string>',
      author_email: '<string>',
      commit_message: '<string>',
      commit_time: '<string>',
      git_diff: '<string>'
    },
    strict: true,
    stop_token: '<string>',
    extra_messages: '<string>',
    tags: ['<string>'],
    mcp_auth: {}
  })
};

fetch('https://api.braintrust.dev/v1/eval', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));
<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.braintrust.dev/v1/eval",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'project_id' => '<string>',
    'data' => [
        'dataset_id' => '<string>',
        'dataset_version' => '<string>',
        'dataset_environment' => '<string>',
        '_internal_btql' => [
                
        ]
    ],
    'task' => [
        'function_id' => '<string>',
        'version' => '<string>'
    ],
    'scores' => [
        [
                'function_id' => '<string>',
                'version' => '<string>'
        ]
    ],
    'name' => '<string>',
    'parameters' => [
        
    ],
    'experiment_name' => '<string>',
    'metadata' => [
        
    ],
    'parent' => [
        'object_id' => '<string>',
        'row_ids' => [
                'id' => '<string>',
                'span_id' => '<string>',
                'root_span_id' => '<string>'
        ],
        'propagated_event' => [
                
        ]
    ],
    'stream' => true,
    'trial_count' => 123,
    'is_public' => true,
    'timeout' => 123,
    'max_concurrency' => 10,
    'base_experiment_name' => '<string>',
    'base_experiment_id' => '<string>',
    'git_metadata_settings' => [
        'fields' => [
                
        ]
    ],
    'repo_info' => [
        'commit' => '<string>',
        'branch' => '<string>',
        'tag' => '<string>',
        'dirty' => true,
        'author_name' => '<string>',
        'author_email' => '<string>',
        'commit_message' => '<string>',
        'commit_time' => '<string>',
        'git_diff' => '<string>'
    ],
    'strict' => true,
    'stop_token' => '<string>',
    'extra_messages' => '<string>',
    'tags' => [
        '<string>'
    ],
    'mcp_auth' => [
        
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}
package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.braintrust.dev/v1/eval"

	payload := strings.NewReader("{\n  \"project_id\": \"<string>\",\n  \"data\": {\n    \"dataset_id\": \"<string>\",\n    \"dataset_version\": \"<string>\",\n    \"dataset_environment\": \"<string>\",\n    \"_internal_btql\": {}\n  },\n  \"task\": {\n    \"function_id\": \"<string>\",\n    \"version\": \"<string>\"\n  },\n  \"scores\": [\n    {\n      \"function_id\": \"<string>\",\n      \"version\": \"<string>\"\n    }\n  ],\n  \"name\": \"<string>\",\n  \"parameters\": {},\n  \"experiment_name\": \"<string>\",\n  \"metadata\": {},\n  \"parent\": {\n    \"object_id\": \"<string>\",\n    \"row_ids\": {\n      \"id\": \"<string>\",\n      \"span_id\": \"<string>\",\n      \"root_span_id\": \"<string>\"\n    },\n    \"propagated_event\": {}\n  },\n  \"stream\": true,\n  \"trial_count\": 123,\n  \"is_public\": true,\n  \"timeout\": 123,\n  \"max_concurrency\": 10,\n  \"base_experiment_name\": \"<string>\",\n  \"base_experiment_id\": \"<string>\",\n  \"git_metadata_settings\": {\n    \"fields\": []\n  },\n  \"repo_info\": {\n    \"commit\": \"<string>\",\n    \"branch\": \"<string>\",\n    \"tag\": \"<string>\",\n    \"dirty\": true,\n    \"author_name\": \"<string>\",\n    \"author_email\": \"<string>\",\n    \"commit_message\": \"<string>\",\n    \"commit_time\": \"<string>\",\n    \"git_diff\": \"<string>\"\n  },\n  \"strict\": true,\n  \"stop_token\": \"<string>\",\n  \"extra_messages\": \"<string>\",\n  \"tags\": [\n    \"<string>\"\n  ],\n  \"mcp_auth\": {}\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}
HttpResponse<String> response = Unirest.post("https://api.braintrust.dev/v1/eval")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"project_id\": \"<string>\",\n  \"data\": {\n    \"dataset_id\": \"<string>\",\n    \"dataset_version\": \"<string>\",\n    \"dataset_environment\": \"<string>\",\n    \"_internal_btql\": {}\n  },\n  \"task\": {\n    \"function_id\": \"<string>\",\n    \"version\": \"<string>\"\n  },\n  \"scores\": [\n    {\n      \"function_id\": \"<string>\",\n      \"version\": \"<string>\"\n    }\n  ],\n  \"name\": \"<string>\",\n  \"parameters\": {},\n  \"experiment_name\": \"<string>\",\n  \"metadata\": {},\n  \"parent\": {\n    \"object_id\": \"<string>\",\n    \"row_ids\": {\n      \"id\": \"<string>\",\n      \"span_id\": \"<string>\",\n      \"root_span_id\": \"<string>\"\n    },\n    \"propagated_event\": {}\n  },\n  \"stream\": true,\n  \"trial_count\": 123,\n  \"is_public\": true,\n  \"timeout\": 123,\n  \"max_concurrency\": 10,\n  \"base_experiment_name\": \"<string>\",\n  \"base_experiment_id\": \"<string>\",\n  \"git_metadata_settings\": {\n    \"fields\": []\n  },\n  \"repo_info\": {\n    \"commit\": \"<string>\",\n    \"branch\": \"<string>\",\n    \"tag\": \"<string>\",\n    \"dirty\": true,\n    \"author_name\": \"<string>\",\n    \"author_email\": \"<string>\",\n    \"commit_message\": \"<string>\",\n    \"commit_time\": \"<string>\",\n    \"git_diff\": \"<string>\"\n  },\n  \"strict\": true,\n  \"stop_token\": \"<string>\",\n  \"extra_messages\": \"<string>\",\n  \"tags\": [\n    \"<string>\"\n  ],\n  \"mcp_auth\": {}\n}")
  .asString();
require 'uri'
require 'net/http'

url = URI("https://api.braintrust.dev/v1/eval")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"project_id\": \"<string>\",\n  \"data\": {\n    \"dataset_id\": \"<string>\",\n    \"dataset_version\": \"<string>\",\n    \"dataset_environment\": \"<string>\",\n    \"_internal_btql\": {}\n  },\n  \"task\": {\n    \"function_id\": \"<string>\",\n    \"version\": \"<string>\"\n  },\n  \"scores\": [\n    {\n      \"function_id\": \"<string>\",\n      \"version\": \"<string>\"\n    }\n  ],\n  \"name\": \"<string>\",\n  \"parameters\": {},\n  \"experiment_name\": \"<string>\",\n  \"metadata\": {},\n  \"parent\": {\n    \"object_id\": \"<string>\",\n    \"row_ids\": {\n      \"id\": \"<string>\",\n      \"span_id\": \"<string>\",\n      \"root_span_id\": \"<string>\"\n    },\n    \"propagated_event\": {}\n  },\n  \"stream\": true,\n  \"trial_count\": 123,\n  \"is_public\": true,\n  \"timeout\": 123,\n  \"max_concurrency\": 10,\n  \"base_experiment_name\": \"<string>\",\n  \"base_experiment_id\": \"<string>\",\n  \"git_metadata_settings\": {\n    \"fields\": []\n  },\n  \"repo_info\": {\n    \"commit\": \"<string>\",\n    \"branch\": \"<string>\",\n    \"tag\": \"<string>\",\n    \"dirty\": true,\n    \"author_name\": \"<string>\",\n    \"author_email\": \"<string>\",\n    \"commit_message\": \"<string>\",\n    \"commit_time\": \"<string>\",\n    \"git_diff\": \"<string>\"\n  },\n  \"strict\": true,\n  \"stop_token\": \"<string>\",\n  \"extra_messages\": \"<string>\",\n  \"tags\": [\n    \"<string>\"\n  ],\n  \"mcp_auth\": {}\n}"

response = http.request(request)
puts response.read_body
{
  "project_name": "<string>",
  "experiment_name": "<string>",
  "project_url": "<string>",
  "experiment_url": "<string>",
  "comparison_experiment_name": "<string>",
  "scores": {},
  "metrics": {}
}

Authorizations

​
Authorization
string
header
required

Most Braintrust endpoints are authenticated by providing your API key as a header Authorization: Bearer [api_key] to your HTTP request. You can create an API key in the Braintrust organization settings page.

Body

application/json

Eval launch parameters

​
project_id
string
required

Unique identifier for the project to run the eval in

​
data
dataset_id · object
required

The dataset to use

​
task
function_id · object
required

The function to evaluate

​
scores
(function_id · object | project_slug · object | global_function · object | prompt_session_id · object | inline_code · object | inline_function · object | inline_prompt · object)[]
required

The functions to score the eval on

Options for identifying a function

​
name
string

The name of the eval to run when multiple evals available

​
parameters
object

Values for any parameters used in the eval

​
experiment_name
string

An optional name for the experiment created by this eval. If it conflicts with an existing experiment, it will be suffixed with a unique identifier.

​
metadata
object

Optional experiment-level metadata to store about the evaluation. You can later use this to slice & dice across experiments.

​
parent

Options for tracing the evaluation

​
stream
boolean

Whether to stream the results of the eval. If true, the request will return two events: one to indicate the experiment has started, and another upon completion. If false, the request will return the evaluation's summary upon completion.

​
trial_count
number | null

The number of times to run the evaluator per input. This is useful for evaluating applications that have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the variance in the results.

​
is_public
boolean | null

Whether the experiment should be public. Defaults to false.

​
timeout
number | null

The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout.

​
max_concurrency
number | null
default:10

The maximum number of tasks/scorers that will be run concurrently. Defaults to 10. If null is provided, no max concurrency will be used.

​
base_experiment_name
string | null

An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this experiment.

​
base_experiment_id
string | null

An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this experiment.

​
git_metadata_settings
object | null

Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in.

​
repo_info
object | null

Optionally explicitly specify the git metadata for this experiment. This takes precedence over gitMetadataSettings if specified.

​
strict
boolean | null

If true, throw an error if one of the variables in the prompt is not present in the input

​
stop_token
string | null

The token to stop the run

​
extra_messages
string

A template path of extra messages to append to the conversion. These messages will be appended to the end of the conversation, after the last message.

​
tags
string[]

Optional tags that will be added to the experiment.

​
mcp_auth
object

Response

200 - application/json

Eval launch response

Summary of an experiment

​
project_name
string
required

Name of the project that the experiment belongs to

​
experiment_name
string
required

Name of the experiment

​
project_url
string<uri>
required

URL to the project's page in the Braintrust app

​
experiment_url
string<uri>
required

URL to the experiment's page in the Braintrust app

​
comparison_experiment_name
string | null

The experiment which scores are baselined against

​
scores
object | null

Summary of the experiment's scores

​
metrics
object | null

Summary of the experiment's metrics