Agent derived states (#840)

pblazej · web-flow · commit 4aa74ada9d1e · 2025-11-13T09:24:53.000+01:00
- Adds more computed `is_` props to the (private for now) agent state - JS livekit/components-js#1231 - `buffering` state was already handled inside the `.connecting` enum case (associated value is less error-prone) - Adds disconnected == failed concept - JS livekit/components-js#1228 - Checks actual dispatch response when waiting for the agent - JS livekit/components-js#1226
diff --git a/Sources/LiveKit/Agent/Agent.swift b/Sources/LiveKit/Agent/Agent.swift
@@ -35,11 +35,14 @@ public struct Agent: Loggable {
 
     public enum Error: LocalizedError {
         case timeout
+        case left
 
         public var errorDescription: String? {
             switch self {
             case .timeout:
-                "Agent did not connect"
+                "Agent did not connect to the room"
+            case .left:
+                "Agent left the room unexpectedly"
             }
         }
     }
@@ -90,50 +93,120 @@ public struct Agent: Loggable {
             log("Invalid transition from \(state) to connected", .warning)
         }
     }
+}
+
+// MARK: - Derived State
+
+public extension Agent {
+    /// A boolean value indicating whether the agent is connected to the client.
+    ///
+    /// Returns `true` when the agent is actively connected and in a conversational state
+    /// (listening, thinking, or speaking).
+    var isConnected: Bool {
+        switch state {
+        case let .connected(agentState, _, _):
+            switch agentState {
+            case .listening, .thinking, .speaking:
+                true
+            default:
+                false
+            }
+        default:
+            false
+        }
+    }
+
+    /// A boolean value indicating whether the client could be listening for user speech.
+    ///
+    /// Returns `true` when the agent is in a state where it can receive user input,
+    /// either through pre-connect buffering or active conversation states.
+    ///
+    /// - Note: This may not mean that the agent is actually connected. The audio pre-connect
+    ///   buffer could be active and recording user input before the agent actually connects.
+    var canListen: Bool {
+        switch state {
+        case let .connecting(buffering):
+            buffering
+        case let .connected(agentState, _, _):
+            switch agentState {
+            case .listening, .thinking, .speaking:
+                true
+            default:
+                false
+            }
+        default:
+            false
+        }
+    }
 
-    // MARK: - Public
+    /// A boolean value indicating whether the agent is currently connecting or setting itself up.
+    ///
+    /// Returns `true` during the connection phase (before pre-connect buffering begins) or
+    /// when the agent is initializing after connection.
+    var isPending: Bool {
+        switch state {
+        case let .connecting(buffering):
+            !buffering
+        case let .connected(agentState, _, _):
+            switch agentState {
+            case .initializing, .idle:
+                true
+            default:
+                false
+            }
+        default:
+            false
+        }
+    }
 
-    /// A boolean value indicating whether the agent is connected.
-    public var isConnected: Bool {
+    /// A boolean value indicating whether the client has disconnected from the agent.
+    ///
+    /// Returns `true` when the agent session has ended, either for an expected or unexpected reason
+    /// (including failures).
+    var isFinished: Bool {
         switch state {
-        case .connected: true
-        default: false
+        case .disconnected, .failed:
+            true
+        default:
+            false
         }
     }
 
     /// The current conversational state of the agent.
-    public var agentState: AgentState? {
+    var agentState: AgentState? {
         switch state {
         case let .connected(agentState, _, _): agentState
         default: nil
         }
     }
 
     /// The agent's audio track.
-    public var audioTrack: (any AudioTrack)? {
+    var audioTrack: (any AudioTrack)? {
         switch state {
         case let .connected(_, audioTrack, _): audioTrack
         default: nil
         }
     }
 
     /// The agent's avatar video track.
-    public var avatarVideoTrack: (any VideoTrack)? {
+    var avatarVideoTrack: (any VideoTrack)? {
         switch state {
         case let .connected(_, _, avatarVideoTrack): avatarVideoTrack
         default: nil
         }
     }
 
     /// The last error that occurred.
-    public var error: Error? {
+    var error: Error? {
         switch state {
         case let .failed(error): error
         default: nil
         }
     }
 }
 
+// MARK: - Extension
+
 private extension Participant {
     var agentAudioTrack: (any AudioTrack)? {
         audioTracks.first(where: { $0.source == .microphone })?.track as? AudioTrack
diff --git a/Sources/LiveKit/Agent/Session.swift b/Sources/LiveKit/Agent/Session.swift
@@ -213,6 +213,8 @@ open class Session: ObservableObject {
             agent.disconnected()
         } else if let firstAgent = room.agentParticipants.values.first {
             agent.connected(participant: firstAgent)
+        } else if agent.isConnected {
+            agent.failed(error: .left)
         } else {
             agent.connecting(buffering: options.preConnectAudio)
         }
@@ -248,30 +250,34 @@ open class Session: ObservableObject {
             let response = try await self.tokenSourceConfiguration.fetch()
             try await self.room.connect(url: response.serverURL.absoluteString,
                                         token: response.participantToken)
+            return response.dispatchesAgent()
         }
 
         do {
+            let dispatchesAgent: Bool
             if options.preConnectAudio {
-                try await room.withPreConnectAudio(timeout: timeout) {
+                dispatchesAgent = try await room.withPreConnectAudio(timeout: timeout) {
                     await MainActor.run {
                         self.connectionState = .connecting
                         self.agent.connecting(buffering: true)
                     }
-                    try await connect()
+                    return try await connect()
                 }
             } else {
                 connectionState = .connecting
                 agent.connecting(buffering: false)
-                try await connect()
+                dispatchesAgent = try await connect()
                 try await room.localParticipant.setMicrophone(enabled: true)
             }
 
-            waitForAgentTask = Task { [weak self] in
-                try await Task.sleep(nanoseconds: UInt64(timeout * Double(NSEC_PER_SEC)))
-                try Task.checkCancellation()
-                guard let self else { return }
-                if isConnected, !agent.isConnected {
-                    agent.failed(error: .timeout)
+            if dispatchesAgent {
+                waitForAgentTask = Task { [weak self] in
+                    try await Task.sleep(nanoseconds: UInt64(timeout * Double(NSEC_PER_SEC)))
+                    try Task.checkCancellation()
+                    guard let self else { return }
+                    if isConnected, !agent.isConnected {
+                        agent.failed(error: .timeout)
+                    }
                 }
             }
         } catch {
diff --git a/Sources/LiveKit/Token/CachingTokenSource.swift b/Sources/LiveKit/Token/CachingTokenSource.swift
@@ -157,11 +157,4 @@ public extension TokenSourceResponse {
 
         return true
     }
-
-    /// Extracts the JWT payload from the participant token.
-    ///
-    /// - Returns: The JWT payload if successfully parsed, nil otherwise
-    internal func jwt() -> LiveKitJWTPayload? {
-        LiveKitJWTPayload.fromUnverified(token: participantToken)
-    }
 }
diff --git a/Sources/LiveKit/Token/JWT.swift b/Sources/LiveKit/Token/JWT.swift
@@ -19,6 +19,17 @@ internal import JWTKit
 
 /// JWT payload structure for LiveKit authentication tokens.
 struct LiveKitJWTPayload: JWTPayload, Codable, Equatable {
+    /// Room configuration embedded in the JWT token.
+    struct RoomConfiguration: Codable, Equatable {
+        /// Agent dispatch configuration.
+        struct AgentDispatch: Codable, Equatable {
+            let agentName: String?
+            let metadata: String?
+        }
+
+        let agents: [AgentDispatch]?
+    }
+
     /// Video-specific permissions and room access grants for the participant.
     struct VideoGrant: Codable, Equatable {
         /// Name of the room. Required for admin or join permissions.
@@ -90,6 +101,14 @@ struct LiveKitJWTPayload: JWTPayload, Codable, Equatable {
     let metadata: String?
     /// Video-specific permissions and room access grants.
     let video: VideoGrant?
+    /// Room configuration, including agent dispatch information.
+    let roomConfiguration: RoomConfiguration?
+
+    enum CodingKeys: String, CodingKey {
+        case exp, iss, nbf, sub
+        case name, metadata, video
+        case roomConfiguration = "roomConfig"
+    }
 
     /// Verifies the JWT token's validity by checking expiration and not-before claims.
     func verify(using _: JWTSigner) throws {
diff --git a/Sources/LiveKit/Token/TokenSource.swift b/Sources/LiveKit/Token/TokenSource.swift
@@ -139,3 +139,23 @@ public struct TokenSourceResponse: Decodable, Sendable {
         self.roomName = roomName
     }
 }
+
+public extension TokenSourceResponse {
+    /// Extracts the JWT payload from the participant token.
+    ///
+    /// - Returns: The JWT payload if successfully parsed, nil otherwise
+    internal func jwt() -> LiveKitJWTPayload? {
+        LiveKitJWTPayload.fromUnverified(token: participantToken)
+    }
+
+    /// Checks if the JWT token contains agent dispatch configuration.
+    ///
+    /// - Returns: `true` if the token is configured to dispatch one or more agents, `false` otherwise
+    func dispatchesAgent() -> Bool {
+        guard let jwt = jwt(), let agents = jwt.roomConfiguration?.agents else {
+            return false
+        }
+
+        return !agents.isEmpty
+    }
+}
diff --git a/Tests/LiveKitTestSupport/TokenGenerator.swift b/Tests/LiveKitTestSupport/TokenGenerator.swift
@@ -30,6 +30,7 @@ public class TokenGenerator {
     public var name: String?
     public var metadata: String?
     public var videoGrant: LiveKitJWTPayload.VideoGrant?
+    public var roomConfiguration: LiveKitJWTPayload.RoomConfiguration?
 
     // MARK: - Private
 
@@ -58,7 +59,8 @@ public class TokenGenerator {
                                   sub: .init(stringLiteral: identity),
                                   name: name,
                                   metadata: metadata,
-                                  video: videoGrant)
+                                  video: videoGrant,
+                                  roomConfiguration: roomConfiguration)
 
         return try signers.sign(p)
     }

Original file line number	Diff line number	Diff line change
`@@ -157,11 +157,4 @@ public extension TokenSourceResponse {`
`157`	`157`
`158`	`158`	`return true`
`159`	`159`	`}`
`160`		`-`
`161`		`- /// Extracts the JWT payload from the participant token.`
`162`		`- ///`
`163`		`- /// - Returns: The JWT payload if successfully parsed, nil otherwise`
`164`		`- internal func jwt() -> LiveKitJWTPayload? {`
`165`		`- LiveKitJWTPayload.fromUnverified(token: participantToken)`
`166`		`- }`
`167`	`160`	`}`