curl --request POST \
--url https://api.deepinfra.com/v1/chat/completions \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"model": "meta-llama/Llama-2-70b-chat-hf",
"messages": [
{
"content": "<string>",
"tool_call_id": "<string>",
"cache_control": {},
"role": "tool"
}
],
"fail_fast": false,
"models": [
"<string>"
],
"stream": false,
"temperature": 1,
"top_p": 1,
"min_p": 0,
"top_k": 0,
"max_tokens": 5000000,
"stop": "<string>",
"stop_token_ids": [
123
],
"n": 1,
"presence_penalty": 0,
"frequency_penalty": 0,
"tools": [
{
"function": {
"name": "<string>",
"description": "<string>",
"parameters": {}
},
"cache_control": {},
"type": "function"
}
],
"response_format": {
"type": "text"
},
"repetition_penalty": 1,
"user": "<string>",
"seed": 4611686018427388000,
"logprobs": true,
"stream_options": {
"include_usage": true,
"continuous_usage_stats": false
},
"reasoning": {
"enabled": true
},
"prompt_cache_key": "<string>",
"prompt_cache_options": {},
"chat_template_kwargs": {},
"continue_final_message": true,
"ignore_eos": true
}
'import requests
url = "https://api.deepinfra.com/v1/chat/completions"
payload = {
"model": "meta-llama/Llama-2-70b-chat-hf",
"messages": [
{
"content": "<string>",
"tool_call_id": "<string>",
"cache_control": {},
"role": "tool"
}
],
"fail_fast": False,
"models": ["<string>"],
"stream": False,
"temperature": 1,
"top_p": 1,
"min_p": 0,
"top_k": 0,
"max_tokens": 5000000,
"stop": "<string>",
"stop_token_ids": [123],
"n": 1,
"presence_penalty": 0,
"frequency_penalty": 0,
"tools": [
{
"function": {
"name": "<string>",
"description": "<string>",
"parameters": {}
},
"cache_control": {},
"type": "function"
}
],
"response_format": { "type": "text" },
"repetition_penalty": 1,
"user": "<string>",
"seed": 4611686018427388000,
"logprobs": True,
"stream_options": {
"include_usage": True,
"continuous_usage_stats": False
},
"reasoning": { "enabled": True },
"prompt_cache_key": "<string>",
"prompt_cache_options": {},
"chat_template_kwargs": {},
"continue_final_message": True,
"ignore_eos": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
model: 'meta-llama/Llama-2-70b-chat-hf',
messages: [
{content: '<string>', tool_call_id: '<string>', cache_control: {}, role: 'tool'}
],
fail_fast: false,
models: ['<string>'],
stream: false,
temperature: 1,
top_p: 1,
min_p: 0,
top_k: 0,
max_tokens: 5000000,
stop: '<string>',
stop_token_ids: [123],
n: 1,
presence_penalty: 0,
frequency_penalty: 0,
tools: [
{
function: {name: '<string>', description: '<string>', parameters: {}},
cache_control: {},
type: 'function'
}
],
response_format: {type: 'text'},
repetition_penalty: 1,
user: '<string>',
seed: 4611686018427388000,
logprobs: true,
stream_options: {include_usage: true, continuous_usage_stats: false},
reasoning: {enabled: true},
prompt_cache_key: '<string>',
prompt_cache_options: {},
chat_template_kwargs: {},
continue_final_message: true,
ignore_eos: true
})
};
fetch('https://api.deepinfra.com/v1/chat/completions', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.deepinfra.com/v1/chat/completions",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'model' => 'meta-llama/Llama-2-70b-chat-hf',
'messages' => [
[
'content' => '<string>',
'tool_call_id' => '<string>',
'cache_control' => [
],
'role' => 'tool'
]
],
'fail_fast' => false,
'models' => [
'<string>'
],
'stream' => false,
'temperature' => 1,
'top_p' => 1,
'min_p' => 0,
'top_k' => 0,
'max_tokens' => 5000000,
'stop' => '<string>',
'stop_token_ids' => [
123
],
'n' => 1,
'presence_penalty' => 0,
'frequency_penalty' => 0,
'tools' => [
[
'function' => [
'name' => '<string>',
'description' => '<string>',
'parameters' => [
]
],
'cache_control' => [
],
'type' => 'function'
]
],
'response_format' => [
'type' => 'text'
],
'repetition_penalty' => 1,
'user' => '<string>',
'seed' => 4611686018427388000,
'logprobs' => true,
'stream_options' => [
'include_usage' => true,
'continuous_usage_stats' => false
],
'reasoning' => [
'enabled' => true
],
'prompt_cache_key' => '<string>',
'prompt_cache_options' => [
],
'chat_template_kwargs' => [
],
'continue_final_message' => true,
'ignore_eos' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.deepinfra.com/v1/chat/completions"
payload := strings.NewReader("{\n \"model\": \"meta-llama/Llama-2-70b-chat-hf\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_call_id\": \"<string>\",\n \"cache_control\": {},\n \"role\": \"tool\"\n }\n ],\n \"fail_fast\": false,\n \"models\": [\n \"<string>\"\n ],\n \"stream\": false,\n \"temperature\": 1,\n \"top_p\": 1,\n \"min_p\": 0,\n \"top_k\": 0,\n \"max_tokens\": 5000000,\n \"stop\": \"<string>\",\n \"stop_token_ids\": [\n 123\n ],\n \"n\": 1,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 0,\n \"tools\": [\n {\n \"function\": {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"parameters\": {}\n },\n \"cache_control\": {},\n \"type\": \"function\"\n }\n ],\n \"response_format\": {\n \"type\": \"text\"\n },\n \"repetition_penalty\": 1,\n \"user\": \"<string>\",\n \"seed\": 4611686018427388000,\n \"logprobs\": true,\n \"stream_options\": {\n \"include_usage\": true,\n \"continuous_usage_stats\": false\n },\n \"reasoning\": {\n \"enabled\": true\n },\n \"prompt_cache_key\": \"<string>\",\n \"prompt_cache_options\": {},\n \"chat_template_kwargs\": {},\n \"continue_final_message\": true,\n \"ignore_eos\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.deepinfra.com/v1/chat/completions")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"model\": \"meta-llama/Llama-2-70b-chat-hf\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_call_id\": \"<string>\",\n \"cache_control\": {},\n \"role\": \"tool\"\n }\n ],\n \"fail_fast\": false,\n \"models\": [\n \"<string>\"\n ],\n \"stream\": false,\n \"temperature\": 1,\n \"top_p\": 1,\n \"min_p\": 0,\n \"top_k\": 0,\n \"max_tokens\": 5000000,\n \"stop\": \"<string>\",\n \"stop_token_ids\": [\n 123\n ],\n \"n\": 1,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 0,\n \"tools\": [\n {\n \"function\": {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"parameters\": {}\n },\n \"cache_control\": {},\n \"type\": \"function\"\n }\n ],\n \"response_format\": {\n \"type\": \"text\"\n },\n \"repetition_penalty\": 1,\n \"user\": \"<string>\",\n \"seed\": 4611686018427388000,\n \"logprobs\": true,\n \"stream_options\": {\n \"include_usage\": true,\n \"continuous_usage_stats\": false\n },\n \"reasoning\": {\n \"enabled\": true\n },\n \"prompt_cache_key\": \"<string>\",\n \"prompt_cache_options\": {},\n \"chat_template_kwargs\": {},\n \"continue_final_message\": true,\n \"ignore_eos\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.deepinfra.com/v1/chat/completions")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"model\": \"meta-llama/Llama-2-70b-chat-hf\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_call_id\": \"<string>\",\n \"cache_control\": {},\n \"role\": \"tool\"\n }\n ],\n \"fail_fast\": false,\n \"models\": [\n \"<string>\"\n ],\n \"stream\": false,\n \"temperature\": 1,\n \"top_p\": 1,\n \"min_p\": 0,\n \"top_k\": 0,\n \"max_tokens\": 5000000,\n \"stop\": \"<string>\",\n \"stop_token_ids\": [\n 123\n ],\n \"n\": 1,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 0,\n \"tools\": [\n {\n \"function\": {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"parameters\": {}\n },\n \"cache_control\": {},\n \"type\": \"function\"\n }\n ],\n \"response_format\": {\n \"type\": \"text\"\n },\n \"repetition_penalty\": 1,\n \"user\": \"<string>\",\n \"seed\": 4611686018427388000,\n \"logprobs\": true,\n \"stream_options\": {\n \"include_usage\": true,\n \"continuous_usage_stats\": false\n },\n \"reasoning\": {\n \"enabled\": true\n },\n \"prompt_cache_key\": \"<string>\",\n \"prompt_cache_options\": {},\n \"chat_template_kwargs\": {},\n \"continue_final_message\": true,\n \"ignore_eos\": true\n}"
response = http.request(request)
puts response.read_body{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Openai Chat Completions
curl --request POST \
--url https://api.deepinfra.com/v1/chat/completions \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"model": "meta-llama/Llama-2-70b-chat-hf",
"messages": [
{
"content": "<string>",
"tool_call_id": "<string>",
"cache_control": {},
"role": "tool"
}
],
"fail_fast": false,
"models": [
"<string>"
],
"stream": false,
"temperature": 1,
"top_p": 1,
"min_p": 0,
"top_k": 0,
"max_tokens": 5000000,
"stop": "<string>",
"stop_token_ids": [
123
],
"n": 1,
"presence_penalty": 0,
"frequency_penalty": 0,
"tools": [
{
"function": {
"name": "<string>",
"description": "<string>",
"parameters": {}
},
"cache_control": {},
"type": "function"
}
],
"response_format": {
"type": "text"
},
"repetition_penalty": 1,
"user": "<string>",
"seed": 4611686018427388000,
"logprobs": true,
"stream_options": {
"include_usage": true,
"continuous_usage_stats": false
},
"reasoning": {
"enabled": true
},
"prompt_cache_key": "<string>",
"prompt_cache_options": {},
"chat_template_kwargs": {},
"continue_final_message": true,
"ignore_eos": true
}
'import requests
url = "https://api.deepinfra.com/v1/chat/completions"
payload = {
"model": "meta-llama/Llama-2-70b-chat-hf",
"messages": [
{
"content": "<string>",
"tool_call_id": "<string>",
"cache_control": {},
"role": "tool"
}
],
"fail_fast": False,
"models": ["<string>"],
"stream": False,
"temperature": 1,
"top_p": 1,
"min_p": 0,
"top_k": 0,
"max_tokens": 5000000,
"stop": "<string>",
"stop_token_ids": [123],
"n": 1,
"presence_penalty": 0,
"frequency_penalty": 0,
"tools": [
{
"function": {
"name": "<string>",
"description": "<string>",
"parameters": {}
},
"cache_control": {},
"type": "function"
}
],
"response_format": { "type": "text" },
"repetition_penalty": 1,
"user": "<string>",
"seed": 4611686018427388000,
"logprobs": True,
"stream_options": {
"include_usage": True,
"continuous_usage_stats": False
},
"reasoning": { "enabled": True },
"prompt_cache_key": "<string>",
"prompt_cache_options": {},
"chat_template_kwargs": {},
"continue_final_message": True,
"ignore_eos": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
model: 'meta-llama/Llama-2-70b-chat-hf',
messages: [
{content: '<string>', tool_call_id: '<string>', cache_control: {}, role: 'tool'}
],
fail_fast: false,
models: ['<string>'],
stream: false,
temperature: 1,
top_p: 1,
min_p: 0,
top_k: 0,
max_tokens: 5000000,
stop: '<string>',
stop_token_ids: [123],
n: 1,
presence_penalty: 0,
frequency_penalty: 0,
tools: [
{
function: {name: '<string>', description: '<string>', parameters: {}},
cache_control: {},
type: 'function'
}
],
response_format: {type: 'text'},
repetition_penalty: 1,
user: '<string>',
seed: 4611686018427388000,
logprobs: true,
stream_options: {include_usage: true, continuous_usage_stats: false},
reasoning: {enabled: true},
prompt_cache_key: '<string>',
prompt_cache_options: {},
chat_template_kwargs: {},
continue_final_message: true,
ignore_eos: true
})
};
fetch('https://api.deepinfra.com/v1/chat/completions', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.deepinfra.com/v1/chat/completions",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'model' => 'meta-llama/Llama-2-70b-chat-hf',
'messages' => [
[
'content' => '<string>',
'tool_call_id' => '<string>',
'cache_control' => [
],
'role' => 'tool'
]
],
'fail_fast' => false,
'models' => [
'<string>'
],
'stream' => false,
'temperature' => 1,
'top_p' => 1,
'min_p' => 0,
'top_k' => 0,
'max_tokens' => 5000000,
'stop' => '<string>',
'stop_token_ids' => [
123
],
'n' => 1,
'presence_penalty' => 0,
'frequency_penalty' => 0,
'tools' => [
[
'function' => [
'name' => '<string>',
'description' => '<string>',
'parameters' => [
]
],
'cache_control' => [
],
'type' => 'function'
]
],
'response_format' => [
'type' => 'text'
],
'repetition_penalty' => 1,
'user' => '<string>',
'seed' => 4611686018427388000,
'logprobs' => true,
'stream_options' => [
'include_usage' => true,
'continuous_usage_stats' => false
],
'reasoning' => [
'enabled' => true
],
'prompt_cache_key' => '<string>',
'prompt_cache_options' => [
],
'chat_template_kwargs' => [
],
'continue_final_message' => true,
'ignore_eos' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.deepinfra.com/v1/chat/completions"
payload := strings.NewReader("{\n \"model\": \"meta-llama/Llama-2-70b-chat-hf\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_call_id\": \"<string>\",\n \"cache_control\": {},\n \"role\": \"tool\"\n }\n ],\n \"fail_fast\": false,\n \"models\": [\n \"<string>\"\n ],\n \"stream\": false,\n \"temperature\": 1,\n \"top_p\": 1,\n \"min_p\": 0,\n \"top_k\": 0,\n \"max_tokens\": 5000000,\n \"stop\": \"<string>\",\n \"stop_token_ids\": [\n 123\n ],\n \"n\": 1,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 0,\n \"tools\": [\n {\n \"function\": {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"parameters\": {}\n },\n \"cache_control\": {},\n \"type\": \"function\"\n }\n ],\n \"response_format\": {\n \"type\": \"text\"\n },\n \"repetition_penalty\": 1,\n \"user\": \"<string>\",\n \"seed\": 4611686018427388000,\n \"logprobs\": true,\n \"stream_options\": {\n \"include_usage\": true,\n \"continuous_usage_stats\": false\n },\n \"reasoning\": {\n \"enabled\": true\n },\n \"prompt_cache_key\": \"<string>\",\n \"prompt_cache_options\": {},\n \"chat_template_kwargs\": {},\n \"continue_final_message\": true,\n \"ignore_eos\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.deepinfra.com/v1/chat/completions")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"model\": \"meta-llama/Llama-2-70b-chat-hf\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_call_id\": \"<string>\",\n \"cache_control\": {},\n \"role\": \"tool\"\n }\n ],\n \"fail_fast\": false,\n \"models\": [\n \"<string>\"\n ],\n \"stream\": false,\n \"temperature\": 1,\n \"top_p\": 1,\n \"min_p\": 0,\n \"top_k\": 0,\n \"max_tokens\": 5000000,\n \"stop\": \"<string>\",\n \"stop_token_ids\": [\n 123\n ],\n \"n\": 1,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 0,\n \"tools\": [\n {\n \"function\": {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"parameters\": {}\n },\n \"cache_control\": {},\n \"type\": \"function\"\n }\n ],\n \"response_format\": {\n \"type\": \"text\"\n },\n \"repetition_penalty\": 1,\n \"user\": \"<string>\",\n \"seed\": 4611686018427388000,\n \"logprobs\": true,\n \"stream_options\": {\n \"include_usage\": true,\n \"continuous_usage_stats\": false\n },\n \"reasoning\": {\n \"enabled\": true\n },\n \"prompt_cache_key\": \"<string>\",\n \"prompt_cache_options\": {},\n \"chat_template_kwargs\": {},\n \"continue_final_message\": true,\n \"ignore_eos\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.deepinfra.com/v1/chat/completions")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"model\": \"meta-llama/Llama-2-70b-chat-hf\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_call_id\": \"<string>\",\n \"cache_control\": {},\n \"role\": \"tool\"\n }\n ],\n \"fail_fast\": false,\n \"models\": [\n \"<string>\"\n ],\n \"stream\": false,\n \"temperature\": 1,\n \"top_p\": 1,\n \"min_p\": 0,\n \"top_k\": 0,\n \"max_tokens\": 5000000,\n \"stop\": \"<string>\",\n \"stop_token_ids\": [\n 123\n ],\n \"n\": 1,\n \"presence_penalty\": 0,\n \"frequency_penalty\": 0,\n \"tools\": [\n {\n \"function\": {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"parameters\": {}\n },\n \"cache_control\": {},\n \"type\": \"function\"\n }\n ],\n \"response_format\": {\n \"type\": \"text\"\n },\n \"repetition_penalty\": 1,\n \"user\": \"<string>\",\n \"seed\": 4611686018427388000,\n \"logprobs\": true,\n \"stream_options\": {\n \"include_usage\": true,\n \"continuous_usage_stats\": false\n },\n \"reasoning\": {\n \"enabled\": true\n },\n \"prompt_cache_key\": \"<string>\",\n \"prompt_cache_options\": {},\n \"chat_template_kwargs\": {},\n \"continue_final_message\": true,\n \"ignore_eos\": true\n}"
response = http.request(request)
puts response.read_body{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
model name
"meta-llama/Llama-2-70b-chat-hf"
conversation messages: (user,assistant,tool)*,user including one system message anywhere
- ChatCompletionToolMessage
- ChatCompletionAssistantMessage
- ChatCompletionUserMessage
- ChatCompletionSystemMessage
Show child attributes
Show child attributes
The service tier used for processing the request. 'priority' processes the request with higher priority (premium rate); 'flex' processes it at lower priority for a discount, served only when spare capacity exists and may be retried/timed out under load. Both apply only to models that support the respective tier. For compatibility, 'auto' is treated as 'priority' and 'standard_only' as 'default'.
default, priority, flex If true, the request is rejected immediately with HTTP 429 when the model has no spare capacity, instead of waiting in the queue. Opt-in; the default (false) keeps standard queueing behavior.
Ordered list of up to 4 fallback models. The request is attempted on each model in order: when a model rejects it for lack of capacity (HTTP 429 model-busy / flex no-capacity), the next model is tried server-side. The first model that accepts serves the request; the response's model field and billing reflect that model, at that model's pricing. Models before the last are attempted without queueing (as if fail_fast were set); the last model honors the request's own fail_fast value. When models is set, the model field is ignored. Entries must be plain model names (no deploy_id:, custom_hostport, or :revision specifiers); duplicate entries are ignored, keeping the first occurrence.
1 - 4 elementswhether to stream the output via SSE or return the full response
What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic
0 <= x <= 2An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.
x <= 1Float that represents the minimum probability for a token to be considered, relative to the probability of the most likely token. Must be in [0, 1]. Set to 0 to disable this.
0 <= x <= 1Sample from the best k (number of) tokens. 0 means off
x >= 0The maximum number of tokens to generate in the chat completion.
The total length of input tokens and generated tokens is limited by the model's context length. If explicitly set to None it will be the model's max context length minus input length or 65536, whichever is smaller.
0 < x <= 10000000up to 16 sequences where the API will stop generating further tokens
Up to 16 token IDs where the API will stop generating further tokens. Merged with the model's built-in stop tokens. Intended for private deployments.
number of sequences to return
1 <= x <= 4Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
-2 <= x <= 2Positive values penalize new tokens based on how many times they appear in the text so far, increasing the model's likelihood to talk about new topics.
-2 <= x <= 2A list of tools the model may call. Currently, only functions are supported as a tool.
Show child attributes
Show child attributes
Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. required means the model must call a function. defined tool means the model must call that specific tool. none is the default when no functions are present. auto is the default if functions are present.
none, auto, required The format of the response. Currently, only json is supported.
- TextResponseFormat
- JsonObjectResponseFormat
- JsonSchemaResponseFormat
- RegexResponseFormat
Show child attributes
Show child attributes
Alternative penalty for repetition, but multiplicative instead of additive (> 1 penalize, < 1 encourage)
0.01 <= x <= 5A unique identifier representing your end-user, which can help monitor and detect abuse. Avoid sending us any identifying information. We recommend hashing user identifiers.
Seed for random number generator. If not provided, a random seed is used. Determinism is not guaranteed.
-9223372036854776000 <= x < 18446744073709552000Whether to return log probabilities of the output tokens or not.If true, returns the log probabilities of each output token returned in the content of message.
streaming options
Show child attributes
Show child attributes
Constrains effort on reasoning for reasoning models. Currently supported values are none, minimal, low, medium, high, xhigh, and max. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response. Setting to none disables reasoning entirely if the model supports.
none, minimal, low, medium, high, xhigh, max Reasoning configuration.
Show child attributes
Show child attributes
A key to identify prompt cache for reuse across requests. If provided, the prompt will be cached and can be reused in subsequent requests with the same key.
Prompt cache options for this request's prefix, e.g. {"ttl": "1h"}.
Show child attributes
Show child attributes
Chat template kwargs.
If set, the final assistant message is used as a prefix for the model to continue generating from, rather than starting a new turn. Only applicable when the last message in the conversation is an assistant message.
Keep generating until max_tokens instead of stopping at the end-of-sequence token. Only honoured on models tagged with the allow_ignore_eos feature flag; ignored otherwise. Intended for benchmarking, where a fixed output length is needed.
Response
Successful Response