Skip to content

Commit 2f50901

Browse files
ericdalloericdallo
committed
Fix usage miscalculation.
1 parent b1ce583 commit 2f50901

File tree

5 files changed

+77
-71
lines changed

5 files changed

+77
-71
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## Unreleased
44

5+
- Fix usage miscalculation.
6+
57
## 0.14.3
68

79
- Fix reason-id on openai models afecting chat thoughts messages.

‎src/eca/features/chat.clj‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@
8383
[{:keys [input-tokens output-tokens
8484
input-cache-creation-tokens input-cache-read-tokens]}
8585
model
86-
{:keys [chat-id db*] :as chat-ctx}]
86+
{:keys [chat-id db*]}]
8787
(when (and output-tokens input-tokens)
8888
(swap! db* update-in [:chats chat-id :total-input-tokens] (fnil + 0) input-tokens)
8989
(swap! db* update-in [:chats chat-id :total-output-tokens] (fnil + 0) output-tokens)
@@ -98,13 +98,11 @@
9898
total-input-cache-read-tokens (get-in db [:chats chat-id :total-input-cache-read-tokens] nil)
9999
total-input-cache-tokens (or total-input-cache-creation-tokens 0)
100100
total-output-tokens (get-in db [:chats chat-id :total-output-tokens] 0)]
101-
(send-content! chat-ctx :system
102-
(assoc-some {:type :usage
103-
:message-output-tokens output-tokens
104-
:message-input-tokens (+ input-tokens message-input-cache-tokens)
105-
:session-tokens (+ total-input-tokens total-input-cache-tokens total-output-tokens)}
106-
:message-cost (tokens->cost input-tokens input-cache-creation-tokens input-cache-read-tokens output-tokens model db)
107-
:session-cost (tokens->cost total-input-tokens total-input-cache-creation-tokens total-input-cache-read-tokens total-output-tokens model db))))))
101+
(assoc-some {:message-output-tokens output-tokens
102+
:message-input-tokens (+ input-tokens message-input-cache-tokens)
103+
:session-tokens (+ total-input-tokens total-input-cache-tokens total-output-tokens)}
104+
:message-cost (tokens->cost input-tokens input-cache-creation-tokens input-cache-read-tokens output-tokens model db)
105+
:session-cost (tokens->cost total-input-tokens total-input-cache-creation-tokens total-input-cache-read-tokens total-output-tokens model db)))))
108106

109107
(defn ^:private message->decision [message]
110108
(let [slash? (string/starts-with? message "/")
@@ -170,6 +168,10 @@
170168
(send-content! chat-ctx :system {:type :progress
171169
:state :running
172170
:text "Generating"}))
171+
:on-usage-updated (fn [usage]
172+
(send-content! chat-ctx :system
173+
(merge {:type :usage}
174+
(usage-msg->usage usage model chat-ctx))))
173175
:on-message-received (fn [{:keys [type] :as msg}]
174176
(assert-chat-not-stopped! chat-ctx)
175177
(case type
@@ -188,10 +190,6 @@
188190
(finish-chat-prompt! :idle chat-ctx))
189191
:finish (do
190192
(add-to-history! {:role "assistant" :content @received-msgs*})
191-
(when-let [usage (usage-msg->usage (:usage msg) model chat-ctx)]
192-
(send-content! chat-ctx :system
193-
(merge usage
194-
{:type :usage})))
195193
(finish-chat-prompt! :idle chat-ctx))))
196194
:on-prepare-tool-call (fn [{:keys [id name arguments-text]}]
197195
(assert-chat-not-stopped! chat-ctx)

‎src/eca/llm_api.clj‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@
8181

8282
(defn complete!
8383
[{:keys [model model-config instructions reason? user-messages config on-first-response-received
84-
on-message-received on-error on-prepare-tool-call on-tool-called on-reason
84+
on-message-received on-error on-prepare-tool-call on-tool-called on-reason on-usage-updated
8585
past-messages tools]}]
8686
(let [first-response-received* (atom false)
8787
emit-first-message-fn (fn [& args]
@@ -114,7 +114,8 @@
114114
:on-error on-error-wrapper
115115
:on-prepare-tool-call on-prepare-tool-call-wrapper
116116
:on-tool-called on-tool-called
117-
:on-reason on-reason-wrapper}]
117+
:on-reason on-reason-wrapper
118+
:on-usage-updated on-usage-updated}]
118119
(cond
119120
(contains? #{"o4-mini"
120121
"o3"

‎src/eca/llm_providers/anthropic.clj‎

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@
9898
[{:keys [model user-messages temperature instructions max-output-tokens
9999
api-url api-key reason? reason-tokens past-messages tools web-search]
100100
:or {temperature 1.0}}
101-
{:keys [on-message-received on-error on-reason on-prepare-tool-call on-tool-called]}]
101+
{:keys [on-message-received on-error on-reason on-prepare-tool-call on-tool-called on-usage-updated]}]
102102
(let [messages (concat (normalize-messages past-messages)
103103
(normalize-messages user-messages))
104104
body (assoc-some
@@ -150,35 +150,38 @@
150150
:external-id (-> data :delta :signature)
151151
:id reason-id})
152152
nil)
153-
"message_delta" (case (-> data :delta :stop_reason)
154-
"tool_use" (doseq [content-block (vals @content-block*)]
155-
(when (= "tool_use" (:type content-block))
156-
(let [function-name (:name content-block)
157-
function-args (:input-json content-block)
158-
{:keys [new-messages]} (on-tool-called {:id (:id content-block)
159-
:name function-name
160-
:arguments (json/parse-string function-args)})
161-
messages (-> (normalize-messages new-messages)
162-
add-cache-to-last-message)]
163-
(base-request!
164-
{:rid (llm-util/gen-rid)
165-
:body (assoc body :messages messages)
166-
:api-url api-url
167-
:api-key api-key
168-
:content-block* (atom nil)
169-
:on-error on-error
170-
:on-response handle-response}))))
171-
"end_turn" (do
172-
(reset! content-block* {})
173-
(on-message-received {:type :finish
174-
:usage {:input-tokens (-> data :usage :input_tokens)
175-
:input-cache-creation-tokens (-> data :usage :cache_creation_input_tokens)
176-
:input-cache-read-tokens (-> data :usage :cache_read_input_tokens)
177-
:output-tokens (-> data :usage :output_tokens)}
178-
:finish-reason (-> data :delta :stop_reason)}))
179-
"max_tokens" (on-message-received {:type :limit-reached
180-
:tokens (:usage data)})
181-
nil)
153+
"message_delta" (do
154+
(when-let [usage (and (-> data :delta :stop_reason)
155+
(:usage data))]
156+
(on-usage-updated {:input-tokens (:input_tokens usage)
157+
:input-cache-creation-tokens (:cache_creation_input_tokens usage)
158+
:input-cache-read-tokens (:cache_read_input_tokens usage)
159+
:output-tokens (:output_tokens usage)}))
160+
(case (-> data :delta :stop_reason)
161+
"tool_use" (doseq [content-block (vals @content-block*)]
162+
(when (= "tool_use" (:type content-block))
163+
(let [function-name (:name content-block)
164+
function-args (:input-json content-block)
165+
{:keys [new-messages]} (on-tool-called {:id (:id content-block)
166+
:name function-name
167+
:arguments (json/parse-string function-args)})
168+
messages (-> (normalize-messages new-messages)
169+
add-cache-to-last-message)]
170+
(base-request!
171+
{:rid (llm-util/gen-rid)
172+
:body (assoc body :messages messages)
173+
:api-url api-url
174+
:api-key api-key
175+
:content-block* (atom nil)
176+
:on-error on-error
177+
:on-response handle-response}))))
178+
"end_turn" (do
179+
(reset! content-block* {})
180+
(on-message-received {:type :finish
181+
:finish-reason (-> data :delta :stop_reason)}))
182+
"max_tokens" (on-message-received {:type :limit-reached
183+
:tokens (:usage data)})
184+
nil))
182185
nil))]
183186
(base-request!
184187
{:rid (llm-util/gen-rid)

‎src/eca/llm_providers/openai.clj‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -41,31 +41,31 @@
4141
(on-error {:exception e})))))
4242

4343
(defn ^:private normalize-messages [past-messages]
44-
(keep-indexed (fn [i {:keys [role content] :as msg}]
45-
(case role
46-
"tool_call" {:type "function_call"
47-
:name (:name content)
48-
:call_id (:id content)
49-
:arguments (json/generate-string (:arguments content))}
50-
"tool_call_output"
51-
{:type "function_call_output"
52-
:call_id (:id content)
53-
:output (llm-util/stringfy-tool-result content)}
54-
;; TODO include reason blocks
55-
"reason" nil
56-
(update msg :content (fn [c]
57-
(if (string? c)
58-
c
59-
(mapv #(if (= "text" (name (:type %)))
60-
(assoc % :type (if (= "user" role)
61-
"input_text"
62-
"output_text"))
63-
%) c))))))
64-
past-messages))
44+
(keep (fn [{:keys [role content] :as msg}]
45+
(case role
46+
"tool_call" {:type "function_call"
47+
:name (:name content)
48+
:call_id (:id content)
49+
:arguments (json/generate-string (:arguments content))}
50+
"tool_call_output"
51+
{:type "function_call_output"
52+
:call_id (:id content)
53+
:output (llm-util/stringfy-tool-result content)}
54+
;; TODO include reason blocks
55+
"reason" nil
56+
(update msg :content (fn [c]
57+
(if (string? c)
58+
c
59+
(mapv #(if (= "text" (name (:type %)))
60+
(assoc % :type (if (= "user" role)
61+
"input_text"
62+
"output_text"))
63+
%) c))))))
64+
past-messages))
6565

6666
(defn completion! [{:keys [model user-messages instructions reason? temperature api-key api-url
6767
max-output-tokens past-messages tools web-search]}
68-
{:keys [on-message-received on-error on-prepare-tool-call on-tool-called on-reason]}]
68+
{:keys [on-message-received on-error on-prepare-tool-call on-tool-called on-reason on-usage-updated]}]
6969
(let [input (concat (normalize-messages past-messages)
7070
(normalize-messages user-messages))
7171
tools (cond-> tools
@@ -149,11 +149,13 @@
149149

150150
;; done
151151
"response.completed"
152-
(when-not (= "function_call" (-> data :response :output last :type))
153-
(on-message-received {:type :finish
154-
:usage {:input-tokens (-> data :response :usage :input_tokens)
155-
:output-tokens (-> data :response :usage :output_tokens)}
156-
:finish-reason (-> data :response :status)}))
152+
(do
153+
(on-usage-updated {:input-tokens (-> data :response :usage :input_tokens)
154+
:output-tokens (-> data :response :usage :output_tokens)})
155+
(when-not (= "function_call" (-> data :response :output last :type))
156+
(on-message-received {:type :finish
157+
158+
:finish-reason (-> data :response :status)})))
157159
nil))]
158160
(base-completion-request!
159161
{:rid (llm-util/gen-rid)

0 commit comments

Comments
 (0)