feat(api): add evals api to config

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
-configured_endpoints: 35
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/togetherai%2Ftogetherai-563dc194d092117305e98b4b79965e7b4cafc03b7d610ffa21ea1f4ba5c119e9.yml
+configured_endpoints: 41
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/togetherai%2Ftogetherai-2d787a1d9fe261bee11bff3e707fcb9c957f759e397032b64241ed9703b98cae.yml
 openapi_spec_hash: 0675cf7a85dee80cbb0818d54af3fe33
-config_hash: 2887483246d24a361689cbd1258db7cf
+config_hash: e2d1be538fd1fb65bfc566a2a168cc16

MIGRATION.md

@@ -69,6 +69,7 @@ This affects the following methods:
 
 - `client.endpoints.list()`
 - `client.hardware.list()`
+- `client.evaluations.list()`
 
 ### Removed `httpAgent` in favor of `fetchOptions`
 

README.md

@@ -28,7 +28,7 @@ const client = new Together({
 
 const chatCompletion = await client.chat.completions.create({
   messages: [{ role: 'user', content: 'Say this is a test!' }],
-  model: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+  model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',
 });
 
 console.log(chatCompletion.choices);
@@ -45,7 +45,7 @@ const client = new Together();
 
 const stream = await client.chat.completions.create({
   messages: [{ role: 'user', content: 'Say this is a test' }],
-  model: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+  model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',
   stream: true,
 });
 for await (const chatCompletionChunk of stream) {
@@ -70,7 +70,7 @@ const client = new Together({
 
 const params: Together.Chat.CompletionCreateParams = {
   messages: [{ role: 'user', content: 'Say this is a test' }],
-  model: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+  model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',
 };
 const chatCompletion: Together.Chat.ChatCompletion = await client.chat.completions.create(params);
 ```
@@ -137,7 +137,7 @@ a subclass of `APIError` will be thrown:
 const chatCompletion = await client.chat.completions
   .create({
     messages: [{ role: 'user', content: 'Say this is a test' }],
-    model: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+    model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',
   })
   .catch(async (err) => {
     if (err instanceof Together.APIError) {
@@ -179,7 +179,7 @@ const client = new Together({
 });
 
 // Or, configure per-request:
-await client.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], model: 'mistralai/Mixtral-8x7B-Instruct-v0.1' }, {
+await client.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' }, {
   maxRetries: 5,
 });
 ```
@@ -196,7 +196,7 @@ const client = new Together({
 });
 
 // Override per-request:
-await client.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], model: 'mistralai/Mixtral-8x7B-Instruct-v0.1' }, {
+await client.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' }, {
   timeout: 5 * 1000,
 });
 ```
@@ -222,7 +222,7 @@ const client = new Together();
 const response = await client.chat.completions
   .create({
     messages: [{ role: 'user', content: 'Say this is a test' }],
-    model: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+    model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',
   })
   .asResponse();
 console.log(response.headers.get('X-My-Header'));
@@ -231,7 +231,7 @@ console.log(response.statusText); // access the underlying Response object
 const { data: chatCompletion, response: raw } = await client.chat.completions
   .create({
     messages: [{ role: 'user', content: 'Say this is a test' }],
-    model: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+    model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',
   })
   .withResponse();
 console.log(raw.headers.get('X-My-Header'));

api.md

@@ -235,3 +235,33 @@ Methods:
 - <code title="post /batches">client.batches.<a href="./src/resources/batches.ts">create</a>({ ...params }) -> BatchCreateResponse</code>
 - <code title="get /batches/{id}">client.batches.<a href="./src/resources/batches.ts">retrieve</a>(id) -> BatchRetrieveResponse</code>
 - <code title="get /batches">client.batches.<a href="./src/resources/batches.ts">list</a>() -> BatchListResponse</code>
+
+# Evaluation
+
+Types:
+
+- <code><a href="./src/resources/evaluation.ts">EvaluationJudgeModelConfig</a></code>
+- <code><a href="./src/resources/evaluation.ts">EvaluationModelRequest</a></code>
+- <code><a href="./src/resources/evaluation.ts">EvaluationCreateResponse</a></code>
+- <code><a href="./src/resources/evaluation.ts">EvaluationRetrieveResponse</a></code>
+- <code><a href="./src/resources/evaluation.ts">EvaluationGetStatusResponse</a></code>
+- <code><a href="./src/resources/evaluation.ts">EvaluationUpdateStatusResponse</a></code>
+
+Methods:
+
+- <code title="post /evaluation">client.evaluation.<a href="./src/resources/evaluation.ts">create</a>({ ...params }) -> EvaluationCreateResponse</code>
+- <code title="get /evaluation/{id}">client.evaluation.<a href="./src/resources/evaluation.ts">retrieve</a>(id) -> EvaluationRetrieveResponse</code>
+- <code title="get /evaluation/{id}/status">client.evaluation.<a href="./src/resources/evaluation.ts">getStatus</a>(id) -> EvaluationGetStatusResponse</code>
+- <code title="post /evaluation/{id}/update">client.evaluation.<a href="./src/resources/evaluation.ts">updateStatus</a>(id, { ...params }) -> EvaluationUpdateStatusResponse</code>
+
+# Evaluations
+
+Types:
+
+- <code><a href="./src/resources/evaluations.ts">EvaluationListResponse</a></code>
+- <code><a href="./src/resources/evaluations.ts">EvaluationGetAllowedModelsResponse</a></code>
+
+Methods:
+
+- <code title="get /evaluations">client.evaluations.<a href="./src/resources/evaluations.ts">list</a>({ ...params }) -> EvaluationListResponse</code>
+- <code title="get /evaluations/model-list">client.evaluations.<a href="./src/resources/evaluations.ts">getAllowedModels</a>() -> EvaluationGetAllowedModelsResponse</code>

src/client.ts

@@ -48,6 +48,23 @@ import {
   EndpointUpdateResponse,
   Endpoints,
 } from './resources/endpoints';
+import {
+  Evaluation,
+  EvaluationCreateParams,
+  EvaluationCreateResponse,
+  EvaluationGetStatusResponse,
+  EvaluationJudgeModelConfig,
+  EvaluationModelRequest,
+  EvaluationRetrieveResponse,
+  EvaluationUpdateStatusParams,
+  EvaluationUpdateStatusResponse,
+} from './resources/evaluation';
+import {
+  EvaluationGetAllowedModelsResponse,
+  EvaluationListParams,
+  EvaluationListResponse,
+  Evaluations,
+} from './resources/evaluations';
 import {
   FileDeleteResponse,
   FileListResponse,
@@ -828,6 +845,8 @@ export class Together {
   endpoints: API.Endpoints = new API.Endpoints(this);
   hardware: API.Hardware = new API.Hardware(this);
   batches: API.Batches = new API.Batches(this);
+  evaluation: API.Evaluation = new API.Evaluation(this);
+  evaluations: API.Evaluations = new API.Evaluations(this);
 }
 
 Together.Chat = Chat;
@@ -843,6 +862,8 @@ Together.Jobs = Jobs;
 Together.Endpoints = Endpoints;
 Together.Hardware = Hardware;
 Together.Batches = Batches;
+Together.Evaluation = Evaluation;
+Together.Evaluations = Evaluations;
 
 export declare namespace Together {
   export type RequestOptions = Opts.RequestOptions;
@@ -963,4 +984,23 @@ export declare namespace Together {
     type BatchListResponse as BatchListResponse,
     type BatchCreateParams as BatchCreateParams,
   };
+
+  export {
+    Evaluation as Evaluation,
+    type EvaluationJudgeModelConfig as EvaluationJudgeModelConfig,
+    type EvaluationModelRequest as EvaluationModelRequest,
+    type EvaluationCreateResponse as EvaluationCreateResponse,
+    type EvaluationRetrieveResponse as EvaluationRetrieveResponse,
+    type EvaluationGetStatusResponse as EvaluationGetStatusResponse,
+    type EvaluationUpdateStatusResponse as EvaluationUpdateStatusResponse,
+    type EvaluationCreateParams as EvaluationCreateParams,
+    type EvaluationUpdateStatusParams as EvaluationUpdateStatusParams,
+  };
+
+  export {
+    Evaluations as Evaluations,
+    type EvaluationListResponse as EvaluationListResponse,
+    type EvaluationGetAllowedModelsResponse as EvaluationGetAllowedModelsResponse,
+    type EvaluationListParams as EvaluationListParams,
+  };
 }