implement iteragor for resources that support pagination

Author: vlboiko

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to this project will be documented in this file.
 
+## 0.104.0
+
+ENHANCEMENTS:
+
+* **Iterator Support**: Added comprehensive iterator functionality for handling both paginated and non-paginated API responses
+  * New `Iterator` interface with idiomatic Go methods: `Next() (RecordSet, error)`, `Previous() (RecordSet, error)`, `HasNext()`, `HasPrevious()`, `Reset() (RecordSet, error)`, `All() (RecordSet, error)`, `Count()`, `PageSize()`
+  * Added `GetIterator(params, pageSize)` and `GetIteratorWithContext(ctx, params, pageSize)` methods to all resources
+  * Automatic detection of paginated vs non-paginated responses
+  * Support for bidirectional navigation (forward and backward)
+  * Lazy loading of pages for memory efficiency
+  * Comprehensive documentation and examples in `docs/iterators.md`
+  * Pagination envelope unwrapping removed from interceptors - Iterator handles it directly
+  * `List()` now uses `GetIterator().All()` internally - Iterator is the base abstraction
+  * Global `PageSize` configuration in `VMSConfig` (default: 0 means no page_size param, let server decide)
+  * Can override page size per iterator, or use 0 to omit page_size parameter
+
 ## 0.103.0
 
 ENHANCEMENTS:

client.go

@@ -28,6 +28,12 @@ type (
 	// Renderable is the interface for Record, RecordSet, and EmptyRecord.
 	Renderable = core.Renderable
 
+	// DisplayableRecord is an interface for records that can be displayed.
+	DisplayableRecord = core.DisplayableRecord
+
+	// ApiError represents an error from the VAST API.
+	ApiError = core.ApiError
+
 	// TypedVMSRest is the strongly-typed client with compile-time type safety.
 	TypedVMSRest = rest.TypedVMSRest
 
@@ -44,6 +50,24 @@ type (
 	InterceptableVastResourceAPI = core.InterceptableVastResourceAPI
 )
 
+// Error handling functions
+var (
+	// IsNotFoundErr checks if the error is a 404 Not Found error.
+	IsNotFoundErr = core.IsNotFoundErr
+
+	// IgnoreNotFound returns nil if the error is a 404, otherwise returns the original error.
+	IgnoreNotFound = core.IgnoreNotFound
+
+	// IgnoreStatusCodes returns nil if the error is an API error with one of the specified status codes.
+	IgnoreStatusCodes = core.IgnoreStatusCodes
+
+	// ExpectStatusCodes checks if the error is an API error with one of the specified status codes.
+	ExpectStatusCodes = core.ExpectStatusCodes
+
+	// IsApiError checks if the error is an ApiError.
+	IsApiError = core.IsApiError
+)
+
 // NewTypedVMSRest creates a strongly-typed client with compile-time type safety.
 // Use when you need strict API contracts and IDE auto-completion.
 func NewTypedVMSRest(config *VMSConfig) (*TypedVMSRest, error) {

core/config.go

@@ -24,6 +24,7 @@ type VMSConfig struct {
 	MaxConnections int            // Maximum number of concurrent HTTP connections.
 	UserAgent      string         // Optional custom User-Agent header to use in HTTP requests. If empty, a default may be applied.
 	ApiVersion     string         // Optional API version
+	PageSize       int            // Default page size for iterators
 	// Context is an optional external context for controlling HTTP request lifecycle.
 	// When provided, it will be used as the parent context for all HTTP requests made by the client.
 	Context context.Context

core/interceptors.go

@@ -75,7 +75,7 @@ func (e *VastResource) doAfterRequest(ctx context.Context, response Renderable)
 	if !isDummyResource {
 		// Pre-normalization: attach @resourceType so resource hooks and user AfterRequestFn
 		// can rely on it for formatting/logging/branching even if later mutations change shape.
-		if err = SetResourceKey(response, resourceType); err != nil {
+		if err = setResourceKey(response, resourceType); err != nil {
 			return nil, err
 		}
 	}
@@ -103,7 +103,7 @@ func (e *VastResource) doAfterRequest(ctx context.Context, response Renderable)
 	// Post-normalization: re-attach @resourceType, because mutations can produce new
 	// Record/RecordSet instances which won't carry the earlier key.
 	if !isDummyResource {
-		if err = SetResourceKey(mutated, resourceType); err != nil {
+		if err = setResourceKey(mutated, resourceType); err != nil {
 			return nil, err
 		}
 	}
@@ -126,42 +126,8 @@ func defaultResponseMutations(response Renderable) (Renderable, error) {
 			}
 			return nil, fmt.Errorf("expected map[string]any under 'async_task', got %T", raw)
 		}
-		// Normalize pagination envelope when response is a single Record
-		if rs, matched, err := unwrapPaginationEnvelopeFromRecord(typed); matched {
-			if err != nil {
-				return nil, err
-			}
-			if rs != nil {
-				return rs, nil
-			}
-		}
 		return response, nil
 	case RecordSet:
-		// Normalize list responses that wrap data under a pagination envelope
-		// Expected keys: "results", "count", "next", "previous"
-		// Example payload returned as a single Record inside RecordSet: {"results": [...], "count": N, "next": ..., "previous": ...}
-		if len(typed) == 1 {
-			// Accept both Record and raw map[string]any as the envelope
-			if rec, ok := any(typed[0]).(Record); ok {
-				if rs, matched, err := unwrapPaginationEnvelopeFromRecord(rec); matched {
-					if err != nil {
-						return nil, err
-					}
-					if rs != nil {
-						return rs, nil
-					}
-				}
-			} else if raw, ok := any(typed[0]).(map[string]any); ok {
-				if rs, matched, err := unwrapPaginationEnvelopeFromRecord(Record(raw)); matched {
-					if err != nil {
-						return nil, err
-					}
-					if rs != nil {
-						return rs, nil
-					}
-				}
-			}
-		}
 		return typed, nil
 	case EmptyRecord:
 		// No op.
@@ -170,49 +136,6 @@ func defaultResponseMutations(response Renderable) (Renderable, error) {
 	return nil, fmt.Errorf("unsupported type %T for result", response)
 }
 
-// unwrapPaginationEnvelopeFromRecord attempts to detect and unwrap a standard pagination envelope
-// of the form {"results": [...], "count": N, "next": ..., "previous": ...} into a RecordSet.
-//
-// Returns:
-//   - (RecordSet, true, nil) when envelope matched and conversion succeeded
-//   - (nil, true, nil) when envelope matched but results are of unsupported type
-//   - (nil, false, nil) when envelope did not match
-//   - (nil, true, err) when envelope matched but conversion failed
-func unwrapPaginationEnvelopeFromRecord(rec Record) (RecordSet, bool, error) {
-	_, hasResults := rec["results"]
-	_, hasCount := rec["count"]
-	_, hasNext := rec["next"]
-	_, hasPrev := rec["previous"]
-	if !(hasResults && hasCount && hasNext && hasPrev) {
-		return nil, false, nil
-	}
-	inner := rec["results"]
-	// Prefer []map[string]any, but also handle []any of maps
-	if list, ok := inner.([]map[string]any); ok {
-		recordSet, err := ToRecordSet(list)
-		if err != nil {
-			return nil, true, err
-		}
-		return recordSet, true, nil
-	}
-	if anyList, ok := inner.([]any); ok {
-		converted := make([]map[string]any, 0, len(anyList))
-		for _, it := range anyList {
-			if m, ok := it.(map[string]any); ok {
-				converted = append(converted, m)
-			} else {
-				return nil, true, nil
-			}
-		}
-		recordSet, err := ToRecordSet(converted)
-		if err != nil {
-			return nil, true, err
-		}
-		return recordSet, true, nil
-	}
-	return nil, true, nil
-}
-
 // ######################################################
 //
 //	REQUEST/RESPONSE LOGGING

core/interceptors_test.go

@@ -0,0 +1,99 @@
+package core
+
+import (
+	"testing"
+)
+
+// Test that defaultResponseMutations no longer unwraps pagination envelopes
+func TestDefaultResponseMutations_PaginationNotUnwrapped(t *testing.T) {
+	// Create a pagination envelope
+	paginationEnvelope := Record{
+		"results": []any{
+			map[string]any{"id": float64(1), "name": "item1"},
+			map[string]any{"id": float64(2), "name": "item2"},
+		},
+		"count":    float64(2),
+		"next":     "https://example.com/api/v1/resources/?page=2",
+		"previous": nil,
+	}
+	
+	// Apply defaultResponseMutations
+	result, err := defaultResponseMutations(paginationEnvelope)
+	if err != nil {
+		t.Fatalf("Expected no error, got: %v", err)
+	}
+	
+	// Verify that the pagination envelope is NOT unwrapped
+	resultRecord, ok := result.(Record)
+	if !ok {
+		t.Fatalf("Expected Record, got %T", result)
+	}
+	
+	// Check that the envelope structure is preserved
+	if _, hasResults := resultRecord["results"]; !hasResults {
+		t.Error("Expected results field to be preserved")
+	}
+	
+	if _, hasCount := resultRecord["count"]; !hasCount {
+		t.Error("Expected count field to be preserved")
+	}
+	
+	if _, hasNext := resultRecord["next"]; !hasNext {
+		t.Error("Expected next field to be preserved")
+	}
+	
+	if _, hasPrevious := resultRecord["previous"]; !hasPrevious {
+		t.Error("Expected previous field to be preserved")
+	}
+}
+
+// Test that RecordSet is not altered by defaultResponseMutations
+func TestDefaultResponseMutations_RecordSetUnchanged(t *testing.T) {
+	recordSet := RecordSet{
+		{"id": float64(1), "name": "item1"},
+		{"id": float64(2), "name": "item2"},
+	}
+	
+	// Apply defaultResponseMutations
+	result, err := defaultResponseMutations(recordSet)
+	if err != nil {
+		t.Fatalf("Expected no error, got: %v", err)
+	}
+	
+	// Verify that RecordSet is unchanged
+	resultRecordSet, ok := result.(RecordSet)
+	if !ok {
+		t.Fatalf("Expected RecordSet, got %T", result)
+	}
+	
+	if len(resultRecordSet) != 2 {
+		t.Errorf("Expected 2 records, got %d", len(resultRecordSet))
+	}
+}
+
+// Test that async_task response mutation still works
+func TestDefaultResponseMutations_AsyncTask(t *testing.T) {
+	asyncTaskResponse := Record{
+		"async_task": map[string]any{
+			"id":    float64(12345),
+			"state": "RUNNING",
+		},
+	}
+	
+	// Apply defaultResponseMutations
+	result, err := defaultResponseMutations(asyncTaskResponse)
+	if err != nil {
+		t.Fatalf("Expected no error, got: %v", err)
+	}
+	
+	// Verify that async_task is normalized
+	resultRecord, ok := result.(Record)
+	if !ok {
+		t.Fatalf("Expected Record, got %T", result)
+	}
+	
+	// Check that ResourceTypeKey was added
+	if resourceType, ok := resultRecord[ResourceTypeKey]; !ok || resourceType != "VTask" {
+		t.Errorf("Expected ResourceTypeKey to be 'VTask', got: %v", resourceType)
+	}
+}

core/interfaces.go

@@ -23,6 +23,7 @@ type VastResourceAPI interface {
 	GetById(any) (Record, error)
 	Exists(Params) (bool, error)
 	MustExists(Params) bool
+	GetIterator(Params, int) Iterator
 	// Resource-level mutex lock for concurrent access control
 	Lock(...any) func()
 	// Internal methods
@@ -40,6 +41,7 @@ type VastResourceAPIWithContext interface {
 	GetByIdWithContext(context.Context, any) (Record, error)
 	ExistsWithContext(context.Context, Params) (bool, error)
 	MustExistsWithContext(context.Context, Params) bool
+	GetIteratorWithContext(context.Context, Params, int) Iterator
 }
 
 // InterceptableVastResourceAPI combines request interception with vast resource behavior.