Add getting LLM interface to niavely work with OpenAI.

Also added tracking tokens.

Author: wcjohns

InterSpec/LlmConversationHistory.h

@@ -28,6 +28,7 @@
 #include <vector>
 #include <string>
 #include <chrono>
+#include <optional>
 
 #include "external_libs/SpecUtils/3rdparty/nlohmann/json.hpp"
 
@@ -81,6 +82,13 @@ struct LlmConversationStart {
   std::chrono::system_clock::time_point timestamp;
   std::string conversationId;  // ID for the entire conversation thread
 
+  // Token usage tracking from LLM API responses
+  // Using optional<size_t> because some models/APIs don't provide token usage information,
+  // making it clear when this data is unavailable rather than defaulting to 0
+  std::optional<size_t> promptTokens;      // Tokens used in the input/prompt
+  std::optional<size_t> completionTokens;  // Tokens generated in the response
+  std::optional<size_t> totalTokens;       // Total tokens used (prompt + completion)
+  
   // Nested follow-up responses (assistant responses, tool calls, tool results)
   std::vector<LlmConversationResponse> responses;
 
@@ -131,6 +139,12 @@ class LlmConversationHistory {
   /** Add a follow-up response to a specific conversation by conversation ID */
   void addFollowUpResponse(const std::string& conversationId, const LlmConversationResponse& response);
 
+  /** Add token usage to a specific conversation by conversation ID (accumulates across API calls) */
+  void addTokenUsage(const std::string& conversationId, 
+                     std::optional<int> promptTokens,
+                     std::optional<int> completionTokens,
+                     std::optional<int> totalTokens);
+  
   /** Find a conversation start by conversation ID */
   LlmConversationStart* findConversationByConversationId(const std::string& conversationId);
 
@@ -185,4 +199,4 @@ class LlmConversationHistory {
   static LlmConversationResponse::Type stringToResponseType(const std::string& str);
 };
 
-#endif // LLM_CONVERSATION_HISTORY_H
+#endif // LLM_CONVERSATION_HISTORY_H

src/AnalystChecks.cpp

@@ -234,7 +234,7 @@ namespace AnalystChecks
 
     // get fit peak
 
-    if( options.source.has_value() )
+    if( options.source.has_value() && !options.source.value().empty() )
     {
       const string source = options.source.value();
 

src/LlmConversationHistory.cpp

@@ -147,6 +147,39 @@ void LlmConversationHistory::addFollowUpResponse(const std::string& conversation
   }
 }
 
+void LlmConversationHistory::addTokenUsage(const std::string& conversationId, 
+                                           std::optional<int> promptTokens,
+                                           std::optional<int> completionTokens,
+                                           std::optional<int> totalTokens) {
+  LlmConversationStart* conversation = findConversationByConversationId(conversationId);
+  if (conversation) {
+    // Accumulate token usage across API calls within this conversation
+    if (promptTokens.has_value() && (promptTokens.value() > 0) ) {
+      if (conversation->promptTokens.has_value()) {
+        conversation->promptTokens = conversation->promptTokens.value() + promptTokens.value();
+      } else {
+        conversation->promptTokens = static_cast<size_t>( promptTokens.value() );
+      }
+    }
+    
+    if (completionTokens.has_value() && (completionTokens.value() > 0)) {
+      if (conversation->completionTokens.has_value()) {
+        conversation->completionTokens = conversation->completionTokens.value() + completionTokens.value();
+      } else {
+        conversation->completionTokens = static_cast<size_t>( completionTokens.value() );
+      }
+    }
+    
+    if (totalTokens.has_value() && (totalTokens.value() > 0)) {
+      if (conversation->totalTokens.has_value()) {
+        conversation->totalTokens = conversation->totalTokens.value() + totalTokens.value();
+      } else {
+        conversation->totalTokens = static_cast<size_t>( totalTokens.value() );
+      }
+    }
+  }
+}
+
 LlmConversationStart* LlmConversationHistory::findConversationByConversationId(const std::string& conversationId) {
   for (auto& conv : *m_conversations) {
     if (conv.conversationId == conversationId) {
@@ -216,7 +249,8 @@ nlohmann::json LlmConversationHistory::toApiFormat() const {
             responseMsg["role"] = "assistant";
             responseMsg["tool_calls"] = json::array();
             json toolCall;
-            toolCall["id"] = conv.conversationId + ":" + response.invocationId;
+            // Use just the invocationId to keep within OpenAI's 40-character limit
+            toolCall["id"] = response.invocationId;
             toolCall["type"] = "function";
             toolCall["function"]["name"] = response.toolName;
             toolCall["function"]["arguments"] = response.toolParameters.dump();
@@ -226,7 +260,7 @@ nlohmann::json LlmConversationHistory::toApiFormat() const {
 
         case LlmConversationResponse::Type::ToolResult:
           responseMsg["role"] = "tool";
-          responseMsg["tool_call_id"] = conv.conversationId + ":" + response.invocationId;
+          responseMsg["tool_call_id"] = response.invocationId;
           responseMsg["content"] = response.content;
           break;
 
@@ -464,4 +498,4 @@ LlmConversationResponse::Type LlmConversationHistory::stringToResponseType(const
   return LlmConversationResponse::Type::Assistant; // Default fallback
 }
 
-#endif // USE_LLM_INTERFACE
+#endif // USE_LLM_INTERFACE

src/LlmInterface.cpp

@@ -387,6 +387,30 @@ void LlmInterface::handleApiResponse(const std::string& response) {
   try {
     json responseJson = json::parse(response);
 
+    // Parse and accumulate token usage information if available
+    if (responseJson.contains("usage") && m_history && !m_currentConversationId.empty()) {
+      const auto& usage = responseJson["usage"];
+      
+      std::optional<int> promptTokens, completionTokens, totalTokens;
+      if (usage.contains("prompt_tokens") && usage["prompt_tokens"].is_number())
+        promptTokens = usage["prompt_tokens"].get<int>();
+      if (usage.contains("completion_tokens") && usage["completion_tokens"].is_number())
+        completionTokens = usage["completion_tokens"].get<int>();
+      if (usage.contains("total_tokens") && usage["total_tokens"].is_number())
+        totalTokens = usage["total_tokens"].get<int>();
+      
+      // Accumulate token usage for this conversation
+      m_history->addTokenUsage(m_currentConversationId, promptTokens, completionTokens, totalTokens);
+      
+      if (completionTokens.has_value()) {
+        cout << "=== Token Usage This Call ===" << endl;
+        cout << "Prompt tokens: " << (promptTokens.has_value() ? std::to_string(promptTokens.value()) : "N/A") << endl;
+        cout << "Completion tokens: " << completionTokens.value() << endl;
+        cout << "Total tokens: " << (totalTokens.has_value() ? std::to_string(totalTokens.value()) : "N/A") << endl;
+        cout << "=============================" << endl;
+      }
+    }
+    
     if (responseJson.contains("choices") && !responseJson["choices"].empty()) {
       json choice = responseJson["choices"][0];
       if (choice.contains("message")) {
@@ -396,23 +420,15 @@ void LlmInterface::handleApiResponse(const std::string& response) {
         if( message.contains("content") && message["content"].is_string() )
           content = message["content"];
 
-
         if (role == "assistant") {
           // Extract thinking content and clean content
           auto [cleanContent, thinkingContent] = extractThinkingAndContent(content);
-                  
-                  cout
-                  << "=== Start Cleaned Response Content ===" << endl
-                  << cleanContent
-                  << "\n=== End Cleaned Response Content   ===" << endl
-                  << endl;
-
+          
           // Add assistant message to history with thinking content
           m_history->addAssistantMessageWithThinking(cleanContent, thinkingContent, m_currentConversationId);
 
           // Handle structured tool calls first (OpenAI format)
           if (message.contains("tool_calls")) {
-            cout << "Found structured tool_calls" << endl;
             executeToolCalls(message["tool_calls"]);
           } else {
             // Parse content for text-based tool requests (use cleaned content)
@@ -668,7 +684,15 @@ nlohmann::json LlmInterface::buildMessagesArray(const std::string& userMessage,
 
   json request;
   request["model"] = m_config->llmApi.model;
-  request["max_tokens"] = m_config->llmApi.maxTokens;
+  
+  // Use max_completion_tokens for newer OpenAI models, max_tokens for others
+  string modelName = m_config->llmApi.model;
+  if (modelName.find("gpt-4") != string::npos || modelName.find("gpt-3.5") != string::npos || 
+      modelName.find("o1") != string::npos || modelName.find("gpt-5") != string::npos) {
+    request["max_completion_tokens"] = m_config->llmApi.maxTokens;
+  } else {
+    request["max_tokens"] = m_config->llmApi.maxTokens;
+  }
 
   json messages = json::array();
 
@@ -684,8 +708,10 @@ nlohmann::json LlmInterface::buildMessagesArray(const std::string& userMessage,
         if (!m_history->isEmpty()) {
           json historyMessages = m_history->toApiFormat();
           cout << "=== Including " << historyMessages.size() << " history messages in request ===" << endl;
+          
           for (size_t i = 0; i < historyMessages.size(); ++i) {
             const auto& msg = historyMessages[i];
+            
             cout << "  " << i << ". " << msg["role"].get<string>() << ": " 
                  << (msg.contains("content") ? msg["content"].get<string>().substr(0, 50) + "..." : "tool_call") << endl;
             messages.push_back(msg);
@@ -754,7 +780,7 @@ void LlmInterface::setupJavaScriptBridge() {
   // Set up the JavaScript function to handle HTTP requests
   string jsCode = R"(
     window.llmHttpRequest = function(endpoint, requestJsonString, bearerToken, requestId) {
-      console.log('LLM HTTP Request to:', endpoint, 'For requestID', requestId);
+      //console.log('LLM HTTP Request to:', endpoint, 'For requestID', requestId);
       //console.log('Request data:', requestJsonString);
       
       var headers = {
@@ -779,12 +805,12 @@ void LlmInterface::setupJavaScriptBridge() {
         signal: controller.signal
       })
       .then(function(response) {
-        console.log('LLM Response status:', response.status);
+        //console.log('LLM Response status:', response.status);
         return response.text();
       })
       .then(function(responseText) {
         //console.log('LLM Response:', responseText);
-        console.log( 'Got LLM Response text' ); 
+        //console.log( 'Got LLM Response text', responseText ); 
         
         // Clear the timeout since we got a response
         clearTimeout(timeoutId);
@@ -820,7 +846,7 @@ void LlmInterface::setupJavaScriptBridge() {
   // Set up the response callback using JSignal to emit signal to C++
   string callbackJs = 
     "window.llmResponseCallback = function(response, requestId) { "
-    "console.log('Emitting signal to C++ with response length:', response.length, 'requestId:', requestId); "
+    //"console.log('Emitting signal to C++ with response length:', response.length, 'requestId:', requestId); "
     "" + m_responseSignal->createCall("response", "requestId") + ";"
     "};";
 
@@ -830,6 +856,8 @@ void LlmInterface::setupJavaScriptBridge() {
 }
 
 void LlmInterface::handleJavaScriptResponse(std::string response, int requestId) {
+  
+  cout << ":handleJavaScriptResponse: " << response << endl << endl << endl;
   try {
     // Find and remove the pending request
     PendingRequest pendingRequest;
@@ -842,7 +870,7 @@ void LlmInterface::handleJavaScriptResponse(std::string response, int requestId)
 
     // Check for errors first
     json responseJson = json::parse(response);
-    if (responseJson.contains("error")) {
+    if (responseJson.contains("error") && !responseJson["error"].is_null()) {
       string errorMsg = "LLM API Error: " + responseJson["error"].dump(2);
       cout << errorMsg << endl;