Merge branch 'main' of github.com:firebase/firebase-js-sdk into markduckworth/ppl-node-backport

Author: MarkDuckworth

.changeset/lazy-donuts-agree.md

@@ -0,0 +1,9 @@
+---
+'firebase': minor
+'@firebase/ai': minor
+---
+
+Added a `sendFunctionResponses` method to `LiveSession`, allowing function responses to be sent during realtime sessions.
+Fixed an issue where function responses during audio conversations caused the WebSocket connection to close. See [GitHub Issue #9264](https://github.com/firebase/firebase-js-sdk/issues/9264).
+ - **Breaking Change**: Changed the `functionCallingHandler` property in `StartAudioConversationOptions` so that it now must return a `Promise<FunctionResponse>`.
+   This breaking change is allowed in a minor release since the Live API is in Public Preview.

.changeset/poor-cobras-dream.md

@@ -0,0 +1,5 @@
+---
+'@firebase/ai': patch
+---
+
+Updated SDK to handle empty parts when streaming.

.changeset/poor-rings-admire.md

@@ -0,0 +1,6 @@
+---
+'firebase': minor
+'@firebase/ai': minor
+---
+
+Added support for the URL context tool, which allows the model to access content from provided public web URLs to inform and enhance its responses.

.changeset/tender-meals-clap.md

@@ -0,0 +1,5 @@
+---
+'@firebase/ai': patch
+---
+
+Tag code execution with beta tag (public preview).

common/api-review/ai.api.md

@@ -175,13 +175,13 @@ export interface CitationMetadata {
     citations: Citation[];
 }
 
-// @public
+// @beta
 export interface CodeExecutionResult {
     outcome?: Outcome;
     output?: string;
 }
 
-// @public
+// @beta
 export interface CodeExecutionResultPart {
     // (undocumented)
     codeExecutionResult?: CodeExecutionResult;
@@ -203,7 +203,7 @@ export interface CodeExecutionResultPart {
     thoughtSignature?: never;
 }
 
-// @public
+// @beta
 export interface CodeExecutionTool {
     codeExecution: {};
 }
@@ -271,13 +271,13 @@ export interface ErrorDetails {
     reason?: string;
 }
 
-// @public
+// @beta
 export interface ExecutableCode {
     code?: string;
     language?: Language;
 }
 
-// @public
+// @beta
 export interface ExecutableCodePart {
     // (undocumented)
     codeExecutionResult?: never;
@@ -449,6 +449,10 @@ export interface GenerateContentCandidate {
     index: number;
     // (undocumented)
     safetyRatings?: SafetyRating[];
+    // Warning: (ae-incompatible-release-tags) The symbol "urlContextMetadata" is marked as @public, but its signature references "URLContextMetadata" which is marked as @beta
+    //
+    // (undocumented)
+    urlContextMetadata?: URLContextMetadata;
 }
 
 // @public
@@ -600,6 +604,8 @@ export interface GoogleAIGenerateContentCandidate {
     index: number;
     // (undocumented)
     safetyRatings?: SafetyRating[];
+    // (undocumented)
+    urlContextMetadata?: URLContextMetadata;
 }
 
 // Warning: (ae-internal-missing-underscore) The name "GoogleAIGenerateContentResponse" should be prefixed with an underscore because the declaration is marked as @internal
@@ -836,13 +842,13 @@ export class IntegerSchema extends Schema {
     constructor(schemaParams?: SchemaParams);
 }
 
-// @public
+// @beta
 export const Language: {
     UNSPECIFIED: string;
     PYTHON: string;
 };
 
-// @public
+// @beta
 export type Language = (typeof Language)[keyof typeof Language];
 
 // @beta
@@ -988,6 +994,7 @@ export class LiveSession {
     isClosed: boolean;
     receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
     send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
+    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
     sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
     sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
     }
@@ -1058,17 +1065,20 @@ export interface OnDeviceParams {
     promptOptions?: LanguageModelPromptOptions;
 }
 
-// @public
+// @beta
 export const Outcome: {
     UNSPECIFIED: string;
     OK: string;
     FAILED: string;
     DEADLINE_EXCEEDED: string;
 };
 
-// @public
+// @beta
 export type Outcome = (typeof Outcome)[keyof typeof Outcome];
 
+// Warning: (ae-incompatible-release-tags) The symbol "Part" is marked as @public, but its signature references "ExecutableCodePart" which is marked as @beta
+// Warning: (ae-incompatible-release-tags) The symbol "Part" is marked as @public, but its signature references "CodeExecutionResultPart" which is marked as @beta
+//
 // @public
 export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart | ExecutableCodePart | CodeExecutionResultPart;
 
@@ -1254,7 +1264,7 @@ export function startAudioConversation(liveSession: LiveSession, options?: Start
 
 // @beta
 export interface StartAudioConversationOptions {
-    functionCallingHandler?: (functionCalls: LiveServerToolCall['functionCalls']) => Promise<Part>;
+    functionCallingHandler?: (functionCalls: FunctionCall[]) => Promise<FunctionResponse>;
 }
 
 // @public
@@ -1304,8 +1314,11 @@ export interface ThinkingConfig {
     thinkingBudget?: number;
 }
 
+// Warning: (ae-incompatible-release-tags) The symbol "Tool" is marked as @public, but its signature references "CodeExecutionTool" which is marked as @beta
+// Warning: (ae-incompatible-release-tags) The symbol "Tool" is marked as @public, but its signature references "URLContextTool" which is marked as @beta
+//
 // @public
-export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool;
+export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool;
 
 // @public
 export interface ToolConfig {
@@ -1316,6 +1329,38 @@ export interface ToolConfig {
 // @public
 export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema;
 
+// @beta
+export interface URLContext {
+}
+
+// @beta
+export interface URLContextMetadata {
+    urlMetadata: URLMetadata[];
+}
+
+// @beta
+export interface URLContextTool {
+    urlContext: URLContext;
+}
+
+// @beta
+export interface URLMetadata {
+    retrievedUrl?: string;
+    urlRetrievalStatus?: URLRetrievalStatus;
+}
+
+// @beta
+export const URLRetrievalStatus: {
+    URL_RETRIEVAL_STATUS_UNSPECIFIED: string;
+    URL_RETRIEVAL_STATUS_SUCCESS: string;
+    URL_RETRIEVAL_STATUS_ERROR: string;
+    URL_RETRIEVAL_STATUS_PAYWALL: string;
+    URL_RETRIEVAL_STATUS_UNSAFE: string;
+};
+
+// @beta
+export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];
+
 // @public
 export interface UsageMetadata {
     // (undocumented)
@@ -1327,6 +1372,8 @@ export interface UsageMetadata {
     // (undocumented)
     promptTokensDetails?: ModalityTokenCount[];
     thoughtsTokenCount?: number;
+    toolUsePromptTokenCount?: number;
+    toolUsePromptTokensDetails?: ModalityTokenCount[];
     // (undocumented)
     totalTokenCount: number;
 }

docs-devsite/_toc.yaml

@@ -202,6 +202,14 @@ toc:
     path: /docs/reference/js/ai.thinkingconfig.md
   - title: ToolConfig
     path: /docs/reference/js/ai.toolconfig.md
+  - title: URLContext
+    path: /docs/reference/js/ai.urlcontext.md
+  - title: URLContextMetadata
+    path: /docs/reference/js/ai.urlcontextmetadata.md
+  - title: URLContextTool
+    path: /docs/reference/js/ai.urlcontexttool.md
+  - title: URLMetadata
+    path: /docs/reference/js/ai.urlmetadata.md
   - title: UsageMetadata
     path: /docs/reference/js/ai.usagemetadata.md
   - title: VertexAIBackend

docs-devsite/ai.codeexecutionresult.md

@@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk
 {% endcomment %}
 
 # CodeExecutionResult interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 The results of code execution run by the model.
 
 <b>Signature:</b>
@@ -22,11 +25,14 @@ export interface CodeExecutionResult
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [outcome](./ai.codeexecutionresult.md#codeexecutionresultoutcome) | [Outcome](./ai.md#outcome) | The result of the code execution. |
-|  [output](./ai.codeexecutionresult.md#codeexecutionresultoutput) | string | The output from the code execution, or an error message if it failed. |
+|  [outcome](./ai.codeexecutionresult.md#codeexecutionresultoutcome) | [Outcome](./ai.md#outcome) | <b><i>(Public Preview)</i></b> The result of the code execution. |
+|  [output](./ai.codeexecutionresult.md#codeexecutionresultoutput) | string | <b><i>(Public Preview)</i></b> The output from the code execution, or an error message if it failed. |
 
 ## CodeExecutionResult.outcome
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 The result of the code execution.
 
 <b>Signature:</b>
@@ -37,6 +43,9 @@ outcome?: Outcome;
 
 ## CodeExecutionResult.output
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 The output from the code execution, or an error message if it failed.
 
 <b>Signature:</b>

docs-devsite/ai.codeexecutionresultpart.md

@@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk
 {% endcomment %}
 
 # CodeExecutionResultPart interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 Represents the code execution result from the model.
 
 <b>Signature:</b>
@@ -22,17 +25,20 @@ export interface CodeExecutionResultPart
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [codeExecutionResult](./ai.codeexecutionresultpart.md#codeexecutionresultpartcodeexecutionresult) | [CodeExecutionResult](./ai.codeexecutionresult.md#codeexecutionresult_interface) |  |
-|  [executableCode](./ai.codeexecutionresultpart.md#codeexecutionresultpartexecutablecode) | never |  |
-|  [fileData](./ai.codeexecutionresultpart.md#codeexecutionresultpartfiledata) | never |  |
-|  [functionCall](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctioncall) | never |  |
-|  [functionResponse](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctionresponse) | never |  |
-|  [inlineData](./ai.codeexecutionresultpart.md#codeexecutionresultpartinlinedata) | never |  |
-|  [text](./ai.codeexecutionresultpart.md#codeexecutionresultparttext) | never |  |
-|  [thought](./ai.codeexecutionresultpart.md#codeexecutionresultpartthought) | never |  |
+|  [codeExecutionResult](./ai.codeexecutionresultpart.md#codeexecutionresultpartcodeexecutionresult) | [CodeExecutionResult](./ai.codeexecutionresult.md#codeexecutionresult_interface) | <b><i>(Public Preview)</i></b> |
+|  [executableCode](./ai.codeexecutionresultpart.md#codeexecutionresultpartexecutablecode) | never | <b><i>(Public Preview)</i></b> |
+|  [fileData](./ai.codeexecutionresultpart.md#codeexecutionresultpartfiledata) | never | <b><i>(Public Preview)</i></b> |
+|  [functionCall](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctioncall) | never | <b><i>(Public Preview)</i></b> |
+|  [functionResponse](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctionresponse) | never | <b><i>(Public Preview)</i></b> |
+|  [inlineData](./ai.codeexecutionresultpart.md#codeexecutionresultpartinlinedata) | never | <b><i>(Public Preview)</i></b> |
+|  [text](./ai.codeexecutionresultpart.md#codeexecutionresultparttext) | never | <b><i>(Public Preview)</i></b> |
+|  [thought](./ai.codeexecutionresultpart.md#codeexecutionresultpartthought) | never | <b><i>(Public Preview)</i></b> |
 
 ## CodeExecutionResultPart.codeExecutionResult
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript
@@ -41,6 +47,9 @@ codeExecutionResult?: CodeExecutionResult;
 
 ## CodeExecutionResultPart.executableCode
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript
@@ -49,6 +58,9 @@ executableCode?: never;
 
 ## CodeExecutionResultPart.fileData
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript
@@ -57,6 +69,9 @@ fileData: never;
 
 ## CodeExecutionResultPart.functionCall
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript
@@ -65,6 +80,9 @@ functionCall?: never;
 
 ## CodeExecutionResultPart.functionResponse
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript
@@ -73,6 +91,9 @@ functionResponse?: never;
 
 ## CodeExecutionResultPart.inlineData
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript
@@ -81,6 +102,9 @@ inlineData?: never;
 
 ## CodeExecutionResultPart.text
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript
@@ -89,6 +113,9 @@ text?: never;
 
 ## CodeExecutionResultPart.thought
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 <b>Signature:</b>
 
 ```typescript

docs-devsite/ai.codeexecutiontool.md

@@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk
 {% endcomment %}
 
 # CodeExecutionTool interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 A tool that enables the model to use code execution.
 
 <b>Signature:</b>
@@ -22,10 +25,13 @@ export interface CodeExecutionTool
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [codeExecution](./ai.codeexecutiontool.md#codeexecutiontoolcodeexecution) | {} | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. |
+|  [codeExecution](./ai.codeexecutiontool.md#codeexecutiontoolcodeexecution) | {} | <b><i>(Public Preview)</i></b> Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. |
 
 ## CodeExecutionTool.codeExecution
 
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options.
 
 <b>Signature:</b>