Get from Object Stores are extremely slow #703
Description
Observed behavior
Calling:
await obj.get(key)is extremely slow w.r.t. the same call in Go, or even with respect to a put in Python.
Moreover, during the get call, the CPU consumption of the Python process goes to 100%.
Expected behavior
No performance problems should be present.
Server and client version
NATS server: 2.11.4
NATS Python client version: 2.10.0
Host environment
I tried with multiple environments: a MacOS machine with both the server and the client, a NATS server on K8s with Python code deployed in other pods, and another Ubuntu VM on a proprietary infrastructure with both the server and the client.
Steps to reproduce
You need a NATS Server with JetStream enabled.
Run the following Python code:
import asyncio
import os
import time
import hashlib
import nats
def generate_blob(size_mb=200):
return os.urandom(size_mb * 1024 * 1024)
async def main():
key = "test-200mb"
nats_url = os.getenv("NATS_HOST", "nats://localhost:4222")
# Connect to NATS
nc = await nats.connect(servers=[nats_url])
js = nc.jetstream()
try:
obj = await js.object_store("benchmark1-files")
except nats.js.errors.BucketNotFoundError:
obj = await js.create_object_store("benchmark1-files")
# Generate data
data = generate_blob()
md5sum = hashlib.md5(data).hexdigest()
# Upload
start_put = time.time()
await obj.put(key, data)
elapsed_put = time.time() - start_put
size_mb = len(data) / (1024 * 1024)
speed_put = size_mb / elapsed_put
print(f"⬆️ Uploaded key: {key}")
print(f"Size: {size_mb:.2f} MB")
print(f"Duration: {elapsed_put:.2f} s")
print(f"Speed: {speed_put:.2f} MB/s")
print(f"MD5: {md5sum}")
# Sleep before download
await asyncio.sleep(3)
# Download
start_get = time.time()
result = await obj.get(key)
received = result.data
elapsed_get = time.time() - start_get
speed_get = len(received) / (1024 * 1024) / elapsed_get
# Validate
md5_check = hashlib.md5(received).hexdigest()
ok = md5_check == md5sum
print(f"\n⬇️ Downloaded key: {key}")
print(f"Duration: {elapsed_get:.2f} s")
print(f"Speed: {speed_get:.2f} MB/s")
print(f"MD5 OK: {ok}")
if __name__ == "__main__":
asyncio.run(main())You should observe that the download speed is extremely slow, I guess depending on the client CPU capabilities. Note that, during the processing, the CPU process of the Python process goes to 100%.
In contrast, run the following code with Go:
package main
import (
"crypto/md5"
"encoding/hex"
"fmt"
"math/rand"
"os"
"time"
"github.com/nats-io/nats.go"
)
func generateBlob(sizeMB int) []byte {
size := sizeMB * 1024 * 1024
buf := make([]byte, size)
_, _ = rand.Read(buf)
return buf
}
func main() {
key := "test-200mb"
natsURL := os.Getenv("NATS_HOST")
if natsURL == "" {
natsURL = "nats://localhost:4222"
}
// Connect
nc, err := nats.Connect(natsURL)
if err != nil {
panic(err)
}
js, err := nc.JetStream()
if err != nil {
panic(err)
}
// Access object store
obj, err := js.ObjectStore("benchmark1-files")
if err != nil {
panic(err)
}
// Generate data
data := generateBlob(200)
md5sum := md5.Sum(data)
md5hex := hex.EncodeToString(md5sum[:])
// Upload
startPut := time.Now()
_, err = obj.PutBytes(key, data)
if err != nil {
panic(err)
}
elapsedPut := time.Since(startPut).Seconds()
speedPut := float64(len(data)) / (1024 * 1024) / elapsedPut
fmt.Printf("⬆️ Uploaded key: %s\n", key)
fmt.Printf("Size: %.2f MB\n", float64(len(data))/(1024*1024))
fmt.Printf("Duration: %.2f s\n", elapsedPut)
fmt.Printf("Speed: %.2f MB/s\n", speedPut)
fmt.Printf("MD5: %s\n", md5hex)
// Sleep before download
time.Sleep(3 * time.Second)
// Download
startGet := time.Now()
received, err := obj.GetBytes(key)
if err != nil {
panic(err)
}
elapsedGet := time.Since(startGet).Seconds()
speedGet := float64(len(received)) / (1024 * 1024) / elapsedGet
// Validate
md5Check := md5.Sum(received)
ok := hex.EncodeToString(md5Check[:]) == md5hex
fmt.Printf("\n⬇️ Downloaded key: %s\n", key)
fmt.Printf("Duration: %.2f s\n", elapsedGet)
fmt.Printf("Speed: %.2f MB/s\n", speedGet)
fmt.Printf("MD5 OK: %v\n", ok)
}The speeds are much higher, with a particularly high difference in the download one.