Using extra_param to configure a model at runtime.
During model generation, you can send extra configuration to tweak your model behavior at API-call time. These parameters will be used to amend or change the existing model configuration existing within a Deployment.
This is a powerful tool to access parameters not directly exposed by the Orq.ai panel, or to modify preexisting setting depending on a particular scenario.
Unsupported Parameters
Not all parameters offered by model providers are natively supported by Orq.ai when using Invoke.
Our API offers a way to provide parameters that are not supported, using the extra_param
field.
Example:
Here we are injecting the
presence_penalty
parameter on our model generation. This parameter is available with our provider but not natively exposed through the orq API.
curl --request POST \
--url https://my.orq.ai/v2/deployments/invoke \
--header 'accept: application/json' \
--header 'authorization: Bearer <orq-api-key>' \
--header 'content-type: application/json' \
--data '
{
"key": "my-deployment",
"context": {
"environment": "production"
},
"extra_params": {
"presence_penalty": 1.0
}
}
'
generation = client.deployments.invoke(
key="my-deployment",
context={"environments": "production"},
extra_params={"presence_penalty": 1.0}
)
print(generation.choices[0].message.content)
const deployment = await client.deployments.invoke({
key: 'my-deployment',
context: { environments: 'production' },
extraParams: { presence_penalty: 1.0 },
});
console.log(deployment?.choices[0].message.content);
Overwriting Existing Parameters
Overwriting existing parameter can impact your model configuration, use this feature with caution.
Using the extra_params
field can also be used to overwrite the Model Configuration defined within the Deployment.
At runtime, you can dynamically override previously defined parameters within Orq.ai.
Example: Overwritingtemperature
Here we are using extra_params to override the
temperature
parameter that can be also defined within your Prompt Configuration
curl --request POST \
--url https://my.dev.orq.ai/v2/deployments/invoke \
--header 'accept: application/json' \
--header 'content-type: application/json' \
--data '
{
"key": "my-deployment",
"context": {
"environment": "production"
},
"extra_params": {
"temperature": 0.4
}
}
'
generation = client.deployments.invoke(
key="my-deployment",
context={"environments": "production"},
extra_params={"temperature": 0.4}
)
print(generation.choices[0].message.content)
const deployment = await client.deployments.invoke({
key: 'my-deployment',
context: { environments: 'production' },
extraParams: { temperature : 0.4 },
});
console.log(deployment?.choices[0].message.content);
Example: Overwritingresponse_format
All parameters can be overwritten including complex ones, in this example, we're overwriting
response_format
to dynamically set the response format for the generation to a predefined JSON object.
curl --request POST \
--url https://my.dev.orq.ai/v2/deployments/invoke \
--header 'accept: application/json' \
--header 'content-type: application/json' \
--data '
{
"key": "my-deployment",
"context": {
"environment": "production"
},
"extra_params": {
"response_format": {
json_schema: <schema>,
type: "json_schema"
}
}
}
'
# Here <schema> is a valid JSON Object containing the definition of fields to return
# Example:
# {
# "name": "object_name",
# "strict": true,
# "schema": {
# "type": "object",
# "properties": {
# "field1": {
# "type": "integer",
# "description": "First integer field"
# },
# "field2": {
# "type": "integer",
# "description": "Second integer field"
# }
# },
# "additionalProperties": false,
# "required": [
# "field1",
# "field2"
# ]
# }
# }
generation = client.deployments.invoke(
key="my-deployment",
context={"environments": "production"},
extra_params={
"response_format": {
"json_schema": <schema>,
"type": "json_schema"
}
}}
)
# Here <schema> is a valid JSON Object containing the definition of fields to return
# Example:
# {
# "name": "object_name",
# "strict": true,
# "schema": {
# "type": "object",
# "properties": {
# "field1": {
# "type": "integer",
# "description": "First integer field"
# },
# "field2": {
# "type": "integer",
# "description": "Second integer field"
# }
# },
# "additionalProperties": false,
# "required": [
# "field1",
# "field2"
# ]
# }
# }
print(generation.choices[0].message.content)
const deployment = await client.deployments.invoke({
key: 'my-deployment',
context: { environments: 'production' },
extraParams: { response_format : {
json_schema: <schema>,
type: "json_schema"
}
}},
});
/*
Here <schema> is a valid JSON Object containing the definition of fields to return
Example:
{
name: "object_name",
strict: true,
schema: {
type: "object",
properties: {
field1: {
type: "integer",
description: "First integer field"
},
field2: {
type: "integer",
description: "Second integer field"
}
},
additionalProperties: false,
required: [
"field1",
"field2"
]
}
}
*/
console.log(deployment?.choices[0].message.content);