diff --git a/commands/displayers/genai.go b/commands/displayers/genai.go index 02408e834..36449be7f 100644 --- a/commands/displayers/genai.go +++ b/commands/displayers/genai.go @@ -201,6 +201,165 @@ func (v *KnowledgeBaseDataSource) KV() []map[string]any { return out } +// IndexingJobDataSource displayer +type IndexingJobDataSource struct { + IndexingJobDataSources do.IndexingJobDataSources +} + +var _ Displayable = &IndexingJobDataSource{} + +func (v *IndexingJobDataSource) JSON(out io.Writer) error { + return writeJSON(v.IndexingJobDataSources, out) +} + +func (v *IndexingJobDataSource) ColMap() map[string]string { + return map[string]string{ + "DataSourceUuid": "Data Source UUID", + "Status": "Status", + "StartedAt": "Started At", + "CompletedAt": "Completed At", + "IndexedItemCount": "Indexed Items", + "FailedItemCount": "Failed Items", + "SkippedItemCount": "Skipped Items", + "RemovedItemCount": "Removed Items", + "IndexedFileCount": "Indexed Files", + "TotalFileCount": "Total Files", + "TotalBytes": "Total Bytes", + "TotalBytesIndexed": "Total Bytes Indexed", + "ErrorMsg": "Error Message", + "ErrorDetails": "Error Details", + } +} + +func (v *IndexingJobDataSource) Cols() []string { + return []string{ + "DataSourceUuid", + "Status", + "StartedAt", + "CompletedAt", + "IndexedItemCount", + "FailedItemCount", + "SkippedItemCount", + "IndexedFileCount", + "TotalFileCount", + } +} + +func (v *IndexingJobDataSource) KV() []map[string]any { + if v == nil || v.IndexingJobDataSources == nil { + return []map[string]any{} + } + out := make([]map[string]any, 0, len(v.IndexingJobDataSources)) + for _, ds := range v.IndexingJobDataSources { + startedAt := "" + if ds.StartedAt != nil { + startedAt = ds.StartedAt.String() + } + completedAt := "" + if ds.CompletedAt != nil { + completedAt = ds.CompletedAt.String() + } + + out = append(out, map[string]any{ + "DataSourceUuid": ds.DataSourceUuid, + "Status": ds.Status, + "StartedAt": startedAt, + "CompletedAt": completedAt, + "IndexedItemCount": ds.IndexedItemCount, + "FailedItemCount": ds.FailedItemCount, + "SkippedItemCount": ds.SkippedItemCount, + "RemovedItemCount": ds.RemovedItemCount, + "IndexedFileCount": ds.IndexedFileCount, + "TotalFileCount": ds.TotalFileCount, + "TotalBytes": ds.TotalBytes, + "TotalBytesIndexed": ds.TotalBytesIndexed, + "ErrorMsg": ds.ErrorMsg, + "ErrorDetails": ds.ErrorDetails, + }) + } + return out +} + +// IndexingJob displayer +type IndexingJob struct { + IndexingJobs do.IndexingJobs +} + +var _ Displayable = &IndexingJob{} + +func (v *IndexingJob) JSON(out io.Writer) error { + return writeJSON(v.IndexingJobs, out) +} + +func (v *IndexingJob) ColMap() map[string]string { + return map[string]string{ + "CompletedDatasources": "Completed Datasources", + "CreatedAt": "Created At", + "DataSourceUuids": "Data Source UUIDs", + "FinishedAt": "Finished At", + "KnowledgeBaseUuid": "Knowledge Base UUID", + "Phase": "Phase", + "StartedAt": "Started At", + "Status": "Status", + "Tokens": "Tokens", + "TotalDatasources": "Total Datasources", + "TotalItemsFailed": "Total Items Failed", + "TotalItemsIndexed": "Total Items Indexed", + "TotalItemsSkipped": "Total Items Skipped", + "UpdatedAt": "Updated At", + "UUID": "UUID", + } +} + +func (v *IndexingJob) Cols() []string { + return []string{ + "UUID", + "KnowledgeBaseUuid", + "Phase", + "Status", + "CompletedDatasources", + "TotalDatasources", + "Tokens", + "TotalItemsIndexed", + "TotalItemsFailed", + "TotalItemsSkipped", + "CreatedAt", + "StartedAt", + "FinishedAt", + "UpdatedAt", + } +} + +func (v *IndexingJob) KV() []map[string]any { + if v == nil || v.IndexingJobs == nil { + return []map[string]any{} + } + out := make([]map[string]any, 0, len(v.IndexingJobs)) + + for _, job := range v.IndexingJobs { + o := map[string]any{ + "CompletedDatasources": job.CompletedDatasources, + "CreatedAt": job.CreatedAt, + "DataSourceUuids": job.DataSourceUuids, + "FinishedAt": job.FinishedAt, + "KnowledgeBaseUuid": job.KnowledgeBaseUuid, + "Phase": job.Phase, + "StartedAt": job.StartedAt, + "Status": job.Status, + "Tokens": job.Tokens, + "TotalDatasources": job.TotalDatasources, + "TotalItemsFailed": job.TotalItemsFailed, + "TotalItemsIndexed": job.TotalItemsIndexed, + "TotalItemsSkipped": job.TotalItemsSkipped, + "UpdatedAt": job.UpdatedAt, + "UUID": job.Uuid, + } + out = append(out, o) + } + + return out +} + type FunctionRoute struct { Agent do.Agent } diff --git a/commands/genai_knowledge_base.go b/commands/genai_knowledge_base.go index f26e769f0..b23f46e34 100644 --- a/commands/genai_knowledge_base.go +++ b/commands/genai_knowledge_base.go @@ -161,6 +161,73 @@ func KnowledgeBaseCmd() *Command { cmdDataSourceDelete.Example = "The following example deletes data source having uuid like " + `00000000-0000-0000-0000-000000000000` + " from a Knowledge Base having uuid " + "`" + `f81d4fae-7dec-11d0-a765-00a0c91e6bf6` + " \nUsing the following command `" + " : `doctl genai knowledge-base delete-datasource f81d4fae-7dec-11d0-a765-00a0c91e6bf6 00000000-0000-0000-0000-000000000000`" + cmdIndexingJobsList := "List all indexing jobs for knowledge bases. Each indexing job contains the following information:\n" + + " - The indexing job UUID\n" + + " - The knowledge base UUID\n" + + " - The current phase of the job\n" + + " - The job status\n" + + " - The number of completed datasources\n" + + " - The total number of datasources\n" + + " - The number of tokens processed\n" + + " - The number of items indexed\n" + + " - The number of items failed\n" + + " - The number of items skipped\n" + + " - The creation timestamp\n" + + " - The start timestamp\n" + + " - The finish timestamp\n" + + " - The update timestamp\n" + + " - The data source UUIDs being processed" + cmdIndexingJobList := CmdBuilder( + cmd, + RunKnowledgeBaseListIndexingJobs, + "list-indexing-jobs", + "List all indexing jobs for knowledge bases", + cmdIndexingJobsList, + Writer, aliasOpt("ls-jobs"), + displayerType(&displayers.IndexingJob{}), + ) + cmdIndexingJobList.Example = "The following command lists all indexing jobs for knowledge bases: " + + "`doctl genai knowledge-base list-indexing-jobs`" + + cmdGetIndexingJobDetails := "Retrieve the status of a specific indexing job by its UUID. This includes phase, status, progress information, and timestamps." + cmdGetIndexingJob := CmdBuilder( + cmd, + RunKnowledgeBaseGetIndexingJob, + "get-indexing-job ", + "Retrieve status of indexing job for a knowledge base", + cmdGetIndexingJobDetails, + Writer, aliasOpt("get-job"), + displayerType(&displayers.IndexingJob{}), + ) + cmdGetIndexingJob.Example = "The following command retrieves the status of an indexing job with UUID `12345678-1234-1234-1234-123456789012`: " + + "`doctl genai knowledge-base get-indexing-job 12345678-1234-1234-1234-123456789012`" + + cmdCancelIndexingJobDetails := "Cancel a running indexing job by its UUID. This will stop the indexing process and update the job status." + cmdCancelIndexingJob := CmdBuilder( + cmd, + RunKnowledgeBaseCancelIndexingJob, + "cancel-indexing-job ", + "Cancel indexing job for a knowledge base", + cmdCancelIndexingJobDetails, + Writer, aliasOpt("cancel-job"), + displayerType(&displayers.IndexingJob{}), + ) + cmdCancelIndexingJob.Example = "The following command cancels an indexing job with UUID `12345678-1234-1234-1234-123456789012`: " + + "`doctl genai knowledge-base cancel-indexing-job 12345678-1234-1234-1234-123456789012`" + + cmdListIndexingJobDataSourcesDetails := "List all data sources for a specific indexing job by its UUID. This shows the status and progress of each data source being processed." + cmdListIndexingJobDataSources := CmdBuilder( + cmd, + RunKnowledgeBaseListIndexingJobDataSources, + "list-indexing-job-data-sources ", + "List data sources for indexing job for a knowledge base", + cmdListIndexingJobDataSourcesDetails, + Writer, aliasOpt("ls-job-ds"), + displayerType(&displayers.IndexingJobDataSource{}), + ) + cmdListIndexingJobDataSources.Example = "The following command lists all data sources for an indexing job with UUID `12345678-1234-1234-1234-123456789012`: " + + "`doctl genai knowledge-base list-indexing-job-data-sources 12345678-1234-1234-1234-123456789012`" + cmdAttachKnowledgeBaseDetails := "Attach a knowledge base to an agent using knowledge base uuid and agent uuid. It returns the information of corresponding agent." cmdAttachKnowledgeBase := CmdBuilder( cmd, @@ -462,3 +529,48 @@ func RunDetachKnowledgeBase(c *CmdConfig) error { return fmt.Errorf("operation aborted") } } + +// RunKnowledgeBaseListIndexingJobs lists all indexing jobs for knowledge bases. +func RunKnowledgeBaseListIndexingJobs(c *CmdConfig) error { + indexingJobs, err := c.GenAI().ListIndexingJobs() + if err != nil { + return err + } + return c.Display(&displayers.IndexingJob{IndexingJobs: indexingJobs}) +} + +// RunKnowledgeBaseGetIndexingJob retrieves the status of a specific indexing job. +func RunKnowledgeBaseGetIndexingJob(c *CmdConfig) error { + if len(c.Args) < 1 { + return doctl.NewMissingArgsErr(c.NS) + } + indexingJob, err := c.GenAI().GetIndexingJob(c.Args[0]) + if err != nil { + return err + } + return c.Display(&displayers.IndexingJob{IndexingJobs: do.IndexingJobs{*indexingJob}}) +} + +// RunKnowledgeBaseCancelIndexingJob cancels a specific indexing job. +func RunKnowledgeBaseCancelIndexingJob(c *CmdConfig) error { + if len(c.Args) < 1 { + return doctl.NewMissingArgsErr(c.NS) + } + indexingJob, err := c.GenAI().CancelIndexingJob(c.Args[0]) + if err != nil { + return err + } + return c.Display(&displayers.IndexingJob{IndexingJobs: do.IndexingJobs{*indexingJob}}) +} + +// RunKnowledgeBaseListIndexingJobDataSources lists all data sources for a specific indexing job. +func RunKnowledgeBaseListIndexingJobDataSources(c *CmdConfig) error { + if len(c.Args) < 1 { + return doctl.NewMissingArgsErr(c.NS) + } + dataSources, err := c.GenAI().ListIndexingJobDataSources(c.Args[0]) + if err != nil { + return err + } + return c.Display(&displayers.IndexingJobDataSource{IndexingJobDataSources: dataSources}) +} diff --git a/commands/genai_knowledge_base_test.go b/commands/genai_knowledge_base_test.go index fe40fc015..59e51f29e 100644 --- a/commands/genai_knowledge_base_test.go +++ b/commands/genai_knowledge_base_test.go @@ -2,6 +2,7 @@ package commands import ( "testing" + "time" "github.com/digitalocean/doctl" "github.com/digitalocean/doctl/do" @@ -28,12 +29,48 @@ var ( Uuid: "data-source-id", }, } + + testIndexingJob = do.IndexingJob{ + LastIndexingJob: &godo.LastIndexingJob{ + CompletedDatasources: 1, + CreatedAt: &godo.Timestamp{Time: time.Now()}, + DataSourceUuids: []string{"data-source-uuid-1", "data-source-uuid-2"}, + FinishedAt: &godo.Timestamp{Time: time.Now()}, + KnowledgeBaseUuid: "kb-uuid-123", + Phase: "BATCH_JOB_PHASE_SUCCEEDED", + StartedAt: &godo.Timestamp{Time: time.Now()}, + Status: "INDEX_JOB_STATUS_COMPLETED", + Tokens: 1000, + TotalDatasources: 2, + TotalItemsFailed: "0", + TotalItemsIndexed: "100", + TotalItemsSkipped: "5", + UpdatedAt: &godo.Timestamp{Time: time.Now()}, + Uuid: "indexing-job-uuid-123", + }, + } + + testIndexingJobDataSource = do.IndexingJobDataSource{ + IndexedDataSource: &godo.IndexedDataSource{ + CompletedAt: &godo.Timestamp{Time: time.Now()}, + DataSourceUuid: "data-source-uuid-1", + StartedAt: &godo.Timestamp{Time: time.Now()}, + Status: "DATA_SOURCE_STATUS_COMPLETED", + IndexedItemCount: "100", + FailedItemCount: "0", + SkippedItemCount: "5", + IndexedFileCount: "50", + TotalFileCount: "50", + TotalBytes: "1024000", + TotalBytesIndexed: "1024000", + }, + } ) func TestKnowledgeBasesCommand(t *testing.T) { cmd := KnowledgeBaseCmd() assert.NotNil(t, cmd) - assertCommandNames(t, cmd, "add-datasource", "attach", "create", "delete", "delete-datasource", "detach", "get", "list", "list-datasources", "update") + assertCommandNames(t, cmd, "add-datasource", "attach", "cancel-indexing-job", "create", "delete", "delete-datasource", "detach", "get", "get-indexing-job", "list", "list-datasources", "list-indexing-job-data-sources", "list-indexing-jobs", "update") } func TestKnowledgeBaseGet(t *testing.T) { @@ -214,3 +251,41 @@ func TestKnowledgeBaseDetach(t *testing.T) { assert.NoError(t, err) }) } + +func TestKnowledgeBaseListIndexingJobs(t *testing.T) { + withTestClient(t, func(config *CmdConfig, tm *tcMocks) { + tm.genAI.EXPECT().ListIndexingJobs().Return(do.IndexingJobs{testIndexingJob}, nil) + err := RunKnowledgeBaseListIndexingJobs(config) + assert.NoError(t, err) + }) +} + +func TestKnowledgeBaseGetIndexingJob(t *testing.T) { + withTestClient(t, func(config *CmdConfig, tm *tcMocks) { + indexing_job_id := "indexing-job-uuid-123" + config.Args = append(config.Args, indexing_job_id) + tm.genAI.EXPECT().GetIndexingJob(indexing_job_id).Return(&testIndexingJob, nil) + err := RunKnowledgeBaseGetIndexingJob(config) + assert.NoError(t, err) + }) +} + +func TestKnowledgeBaseCancelIndexingJob(t *testing.T) { + withTestClient(t, func(config *CmdConfig, tm *tcMocks) { + indexing_job_id := "indexing-job-uuid-123" + config.Args = append(config.Args, indexing_job_id) + tm.genAI.EXPECT().CancelIndexingJob(indexing_job_id).Return(&testIndexingJob, nil) + err := RunKnowledgeBaseCancelIndexingJob(config) + assert.NoError(t, err) + }) +} + +func TestKnowledgeBaseListIndexingJobDataSources(t *testing.T) { + withTestClient(t, func(config *CmdConfig, tm *tcMocks) { + indexing_job_id := "indexing-job-uuid-123" + config.Args = append(config.Args, indexing_job_id) + tm.genAI.EXPECT().ListIndexingJobDataSources(indexing_job_id).Return(do.IndexingJobDataSources{testIndexingJobDataSource}, nil) + err := RunKnowledgeBaseListIndexingJobDataSources(config) + assert.NoError(t, err) + }) +} diff --git a/do/genai.go b/do/genai.go index cb40b886b..ab66eb7a9 100644 --- a/do/genai.go +++ b/do/genai.go @@ -86,6 +86,22 @@ type KnowledgeBases []KnowledgeBase // KnowledgeBase DataSources for Agents type KnowledgeBaseDataSources []KnowledgeBaseDataSource +// IndexingJob represents a job for indexing knowledge base content. +type IndexingJob struct { + *godo.LastIndexingJob +} + +// IndexingJobs is a slice of IndexingJob +type IndexingJobs []IndexingJob + +// IndexingJobDataSource represents an indexed data source within an indexing job +type IndexingJobDataSource struct { + *godo.IndexedDataSource +} + +// IndexingJobDataSources is a slice of IndexingJobDataSource +type IndexingJobDataSources []IndexingJobDataSource + // GenAIService is an interface for interacting with DigitalOcean's Agent API. type GenAIService interface { ListAgents() (Agents, error) @@ -124,6 +140,10 @@ type GenAIService interface { ListAgentsByOpenAIAPIKey(openaiApiKeyId string) (Agents, error) ListDatacenterRegions(servesInference, servesBatch *bool) (DatacenterRegions, error) ListAvailableModels() (Models, error) + ListIndexingJobs() (IndexingJobs, error) + GetIndexingJob(indexingJobID string) (*IndexingJob, error) + CancelIndexingJob(indexingJobID string) (*IndexingJob, error) + ListIndexingJobDataSources(indexingJobID string) (IndexingJobDataSources, error) } var _ GenAIService = &genAIService{} @@ -627,3 +647,64 @@ func (a *genAIService) ListAvailableModels() (Models, error) { return list, nil } + +// ListIndexingJobs lists all indexing jobs for knowledge bases. +func (a *genAIService) ListIndexingJobs() (IndexingJobs, error) { + f := func(opt *godo.ListOptions) ([]any, *godo.Response, error) { + resp, godoResp, err := a.client.GenAI.ListIndexingJobs(context.TODO(), opt) + if err != nil { + return nil, nil, err + } + si := make([]any, len(resp.Jobs)) + for i := range resp.Jobs { + si[i] = &resp.Jobs[i] + } + return si, godoResp, err + } + + si, err := PaginateResp(f) + if err != nil { + return nil, err + } + + list := make([]IndexingJob, len(si)) + for i := range si { + job := si[i].(*godo.LastIndexingJob) + list[i] = IndexingJob{LastIndexingJob: job} + } + + return list, nil +} + +// GetIndexingJob retrieves the status of a specific indexing job. +func (a *genAIService) GetIndexingJob(indexingJobID string) (*IndexingJob, error) { + resp, _, err := a.client.GenAI.GetIndexingJob(context.TODO(), indexingJobID) + if err != nil { + return nil, err + } + return &IndexingJob{LastIndexingJob: &resp.Job}, nil +} + +// CancelIndexingJob cancels a specific indexing job. +func (a *genAIService) CancelIndexingJob(indexingJobID string) (*IndexingJob, error) { + resp, _, err := a.client.GenAI.CancelIndexingJob(context.TODO(), indexingJobID) + if err != nil { + return nil, err + } + return &IndexingJob{LastIndexingJob: &resp.Job}, nil +} + +// ListIndexingJobDataSources lists all data sources for a specific indexing job. +func (a *genAIService) ListIndexingJobDataSources(indexingJobID string) (IndexingJobDataSources, error) { + resp, _, err := a.client.GenAI.ListIndexingJobDataSources(context.TODO(), indexingJobID) + if err != nil { + return nil, err + } + + list := make([]IndexingJobDataSource, len(resp.IndexedDataSources)) + for i := range resp.IndexedDataSources { + list[i] = IndexingJobDataSource{IndexedDataSource: &resp.IndexedDataSources[i]} + } + + return list, nil +} diff --git a/do/mocks/GenAIService.go b/do/mocks/GenAIService.go index ed40b6a79..a46665ede 100644 --- a/do/mocks/GenAIService.go +++ b/do/mocks/GenAIService.go @@ -86,6 +86,21 @@ func (mr *MockGenAIServiceMockRecorder) AttachKnowledgeBaseToAgent(agentId, know return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachKnowledgeBaseToAgent", reflect.TypeOf((*MockGenAIService)(nil).AttachKnowledgeBaseToAgent), agentId, knowledgeBaseID) } +// CancelIndexingJob mocks base method. +func (m *MockGenAIService) CancelIndexingJob(indexingJobID string) (*do.IndexingJob, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CancelIndexingJob", indexingJobID) + ret0, _ := ret[0].(*do.IndexingJob) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CancelIndexingJob indicates an expected call of CancelIndexingJob. +func (mr *MockGenAIServiceMockRecorder) CancelIndexingJob(indexingJobID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelIndexingJob", reflect.TypeOf((*MockGenAIService)(nil).CancelIndexingJob), indexingJobID) +} + // CreateAgent mocks base method. func (m *MockGenAIService) CreateAgent(req *godo.AgentCreateRequest) (*do.Agent, error) { m.ctrl.T.Helper() @@ -291,6 +306,21 @@ func (mr *MockGenAIServiceMockRecorder) GetAgent(agentID any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAgent", reflect.TypeOf((*MockGenAIService)(nil).GetAgent), agentID) } +// GetIndexingJob mocks base method. +func (m *MockGenAIService) GetIndexingJob(indexingJobID string) (*do.IndexingJob, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIndexingJob", indexingJobID) + ret0, _ := ret[0].(*do.IndexingJob) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIndexingJob indicates an expected call of GetIndexingJob. +func (mr *MockGenAIServiceMockRecorder) GetIndexingJob(indexingJobID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIndexingJob", reflect.TypeOf((*MockGenAIService)(nil).GetIndexingJob), indexingJobID) +} + // GetKnowledgeBase mocks base method. func (m *MockGenAIService) GetKnowledgeBase(knowledgeBaseID string) (*do.KnowledgeBase, error) { m.ctrl.T.Helper() @@ -411,6 +441,36 @@ func (mr *MockGenAIServiceMockRecorder) ListDatacenterRegions(servesInference, s return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDatacenterRegions", reflect.TypeOf((*MockGenAIService)(nil).ListDatacenterRegions), servesInference, servesBatch) } +// ListIndexingJobDataSources mocks base method. +func (m *MockGenAIService) ListIndexingJobDataSources(indexingJobID string) (do.IndexingJobDataSources, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListIndexingJobDataSources", indexingJobID) + ret0, _ := ret[0].(do.IndexingJobDataSources) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListIndexingJobDataSources indicates an expected call of ListIndexingJobDataSources. +func (mr *MockGenAIServiceMockRecorder) ListIndexingJobDataSources(indexingJobID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListIndexingJobDataSources", reflect.TypeOf((*MockGenAIService)(nil).ListIndexingJobDataSources), indexingJobID) +} + +// ListIndexingJobs mocks base method. +func (m *MockGenAIService) ListIndexingJobs() (do.IndexingJobs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListIndexingJobs") + ret0, _ := ret[0].(do.IndexingJobs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListIndexingJobs indicates an expected call of ListIndexingJobs. +func (mr *MockGenAIServiceMockRecorder) ListIndexingJobs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListIndexingJobs", reflect.TypeOf((*MockGenAIService)(nil).ListIndexingJobs)) +} + // ListKnowledgeBaseDataSources mocks base method. func (m *MockGenAIService) ListKnowledgeBaseDataSources(knowledgeBaseID string) (do.KnowledgeBaseDataSources, error) { m.ctrl.T.Helper() diff --git a/do/mocks/RegistryService.go b/do/mocks/RegistryService.go index d85d91a49..f214112c5 100644 --- a/do/mocks/RegistryService.go +++ b/do/mocks/RegistryService.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source registry.go -package=mocks RegistryService,RegistriesService +// mockgen -source registry.go -package=mocks RegistryService // // Package mocks is a generated GoMock package. @@ -417,6 +417,21 @@ func (mr *MockRegistriesServiceMockRecorder) Get(arg0 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRegistriesService)(nil).Get), arg0) } +// GetAvailableRegions mocks base method. +func (m *MockRegistriesService) GetAvailableRegions() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAvailableRegions") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAvailableRegions indicates an expected call of GetAvailableRegions. +func (mr *MockRegistriesServiceMockRecorder) GetAvailableRegions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAvailableRegions", reflect.TypeOf((*MockRegistriesService)(nil).GetAvailableRegions)) +} + // GetGarbageCollection mocks base method. func (m *MockRegistriesService) GetGarbageCollection(arg0 string) (*do.GarbageCollection, error) { m.ctrl.T.Helper() @@ -447,6 +462,21 @@ func (mr *MockRegistriesServiceMockRecorder) GetOptions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOptions", reflect.TypeOf((*MockRegistriesService)(nil).GetOptions)) } +// GetSubscriptionTiers mocks base method. +func (m *MockRegistriesService) GetSubscriptionTiers() ([]do.RegistrySubscriptionTier, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSubscriptionTiers") + ret0, _ := ret[0].([]do.RegistrySubscriptionTier) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSubscriptionTiers indicates an expected call of GetSubscriptionTiers. +func (mr *MockRegistriesServiceMockRecorder) GetSubscriptionTiers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubscriptionTiers", reflect.TypeOf((*MockRegistriesService)(nil).GetSubscriptionTiers)) +} + // List mocks base method. func (m *MockRegistriesService) List() ([]do.Registry, error) { m.ctrl.T.Helper() @@ -566,33 +596,3 @@ func (mr *MockRegistriesServiceMockRecorder) UpdateGarbageCollection(arg0, arg1, mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateGarbageCollection", reflect.TypeOf((*MockRegistriesService)(nil).UpdateGarbageCollection), arg0, arg1, arg2) } - -// GetAvailableRegions mocks base method. -func (m *MockRegistriesService) GetAvailableRegions() ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAvailableRegions") - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAvailableRegions indicates an expected call of GetAvailableRegions. -func (mr *MockRegistriesServiceMockRecorder) GetAvailableRegions() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAvailableRegions", reflect.TypeOf((*MockRegistriesService)(nil).GetAvailableRegions)) -} - -// GetSubscriptionTiers mocks base method. -func (m *MockRegistriesService) GetSubscriptionTiers() ([]do.RegistrySubscriptionTier, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSubscriptionTiers") - ret0, _ := ret[0].([]do.RegistrySubscriptionTier) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetSubscriptionTiers indicates an expected call of GetSubscriptionTiers. -func (mr *MockRegistriesServiceMockRecorder) GetSubscriptionTiers() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubscriptionTiers", reflect.TypeOf((*MockRegistriesService)(nil).GetSubscriptionTiers)) -} diff --git a/integration/genai_knowledgebase_cancel_indexing_job_test.go b/integration/genai_knowledgebase_cancel_indexing_job_test.go new file mode 100644 index 000000000..ff2345b7e --- /dev/null +++ b/integration/genai_knowledgebase_cancel_indexing_job_test.go @@ -0,0 +1,113 @@ +package integration + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "os/exec" + "strings" + "testing" + + "github.com/sclevine/spec" + "github.com/stretchr/testify/require" +) + +var _ = suite("genai/knowledgebase/cancel-indexing-job", func(t *testing.T, when spec.G, it spec.S) { + var ( + expect *require.Assertions + server *httptest.Server + ) + + it.Before(func() { + expect = require.New(t) + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/v2/gen-ai/indexing_jobs/12345678-1234-1234-1234-123456789012/cancel": + auth := req.Header.Get("Authorization") + if auth != "Bearer some-magic-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + + if req.Method != http.MethodPut { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(cancelIndexingJobResponse)) + default: + dump, err := httputil.DumpRequest(req, true) + if err != nil { + t.Fatal("failed to dump request") + } + + t.Fatalf("received unknown request: %s", dump) + } + })) + }) + + when("all required flags are passed", func() { + it("cancels the specified indexing job", func() { + cmd := exec.Command(builtBinaryPath, + "-t", "some-magic-token", + "-u", server.URL, + "genai", + "knowledge-base", + "cancel-indexing-job", + "12345678-1234-1234-1234-123456789012", + ) + + output, err := cmd.CombinedOutput() + expect.NoError(err, fmt.Sprintf("received error output: %s", output)) + expect.Equal(strings.TrimSpace(cancelIndexingJobOutput), strings.TrimSpace(string(output))) + }) + }) + + when("format and no-header flags are passed", func() { + it("cancels the specified indexing job with custom format", func() { + cmd := exec.Command(builtBinaryPath, + "-t", "some-magic-token", + "-u", server.URL, + "genai", + "knowledge-base", + "cancel-indexing-job", + "12345678-1234-1234-1234-123456789012", + "--format", "UUID,Status", + "--no-header", + ) + + output, err := cmd.CombinedOutput() + expect.NoError(err, fmt.Sprintf("received error output: %s", output)) + expect.Equal("12345678-1234-1234-1234-123456789012 INDEX_JOB_STATUS_CANCELLED", strings.TrimSpace(string(output))) + }) + }) +}) + +const ( + cancelIndexingJobResponse = `{ + "job": { + "uuid": "12345678-1234-1234-1234-123456789012", + "knowledge_base_uuid": "kb-12345678-1234-1234-1234-123456789012", + "phase": "BATCH_JOB_PHASE_CANCELLED", + "status": "INDEX_JOB_STATUS_CANCELLED", + "completed_datasources": 1, + "total_datasources": 2, + "tokens": 750, + "total_items_indexed": "75", + "total_items_failed": "0", + "total_items_skipped": "5", + "created_at": "2025-09-12T10:00:00Z", + "started_at": "2025-09-12T10:00:30Z", + "finished_at": "2025-09-12T10:02:15Z", + "updated_at": "2025-09-12T10:02:15Z", + "data_source_uuids": ["ds-1", "ds-2"] + } +}` + + cancelIndexingJobOutput = `UUID Knowledge Base UUID Phase Status Completed Datasources Total Datasources Tokens Total Items Indexed Total Items Failed Total Items Skipped Created At Started At Finished At Updated At +12345678-1234-1234-1234-123456789012 kb-12345678-1234-1234-1234-123456789012 BATCH_JOB_PHASE_CANCELLED INDEX_JOB_STATUS_CANCELLED 1 2 750 75 0 5 2025-09-12 10:00:00 +0000 UTC 2025-09-12 10:00:30 +0000 UTC 2025-09-12 10:02:15 +0000 UTC 2025-09-12 10:02:15 +0000 UTC` +) diff --git a/integration/genai_knowledgebase_get_indexing_job_test.go b/integration/genai_knowledgebase_get_indexing_job_test.go new file mode 100644 index 000000000..937959bcd --- /dev/null +++ b/integration/genai_knowledgebase_get_indexing_job_test.go @@ -0,0 +1,111 @@ +package integration + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "os/exec" + "strings" + "testing" + + "github.com/sclevine/spec" + "github.com/stretchr/testify/require" +) + +var _ = suite("genai/knowledgebase/get-indexing-job", func(t *testing.T, when spec.G, it spec.S) { + var ( + expect *require.Assertions + server *httptest.Server + ) + + it.Before(func() { + expect = require.New(t) + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/v2/gen-ai/indexing_jobs/12345678-1234-1234-1234-123456789012": + auth := req.Header.Get("Authorization") + if auth != "Bearer some-magic-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + + if req.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + w.Write([]byte(getIndexingJobResponse)) + default: + dump, err := httputil.DumpRequest(req, true) + if err != nil { + t.Fatal("failed to dump request") + } + + t.Fatalf("received unknown request: %s", dump) + } + })) + }) + + when("all required flags are passed", func() { + it("gets the specified indexing job", func() { + cmd := exec.Command(builtBinaryPath, + "-t", "some-magic-token", + "-u", server.URL, + "genai", + "knowledge-base", + "get-indexing-job", + "12345678-1234-1234-1234-123456789012", + ) + + output, err := cmd.CombinedOutput() + expect.NoError(err, fmt.Sprintf("received error output: %s", output)) + expect.Equal(strings.TrimSpace(getIndexingJobOutput), strings.TrimSpace(string(output))) + }) + }) + + when("format and no-header flags are passed", func() { + it("gets the specified indexing job with custom format", func() { + cmd := exec.Command(builtBinaryPath, + "-t", "some-magic-token", + "-u", server.URL, + "genai", + "knowledge-base", + "get-indexing-job", + "12345678-1234-1234-1234-123456789012", + "--format", "UUID,Status", + "--no-header", + ) + + output, err := cmd.CombinedOutput() + expect.NoError(err, fmt.Sprintf("received error output: %s", output)) + expect.Equal("12345678-1234-1234-1234-123456789012 INDEX_JOB_STATUS_COMPLETED", strings.TrimSpace(string(output))) + }) + }) +}) + +const ( + getIndexingJobResponse = `{ + "job": { + "uuid": "12345678-1234-1234-1234-123456789012", + "knowledge_base_uuid": "kb-12345678-1234-1234-1234-123456789012", + "phase": "BATCH_JOB_PHASE_SUCCEEDED", + "status": "INDEX_JOB_STATUS_COMPLETED", + "completed_datasources": 2, + "total_datasources": 2, + "tokens": 1500, + "total_items_indexed": "150", + "total_items_failed": "0", + "total_items_skipped": "5", + "created_at": "2025-09-12T10:00:00Z", + "started_at": "2025-09-12T10:00:30Z", + "finished_at": "2025-09-12T10:05:00Z", + "updated_at": "2025-09-12T10:05:00Z", + "data_source_uuids": ["ds-1", "ds-2"] + } +}` + + getIndexingJobOutput = `UUID Knowledge Base UUID Phase Status Completed Datasources Total Datasources Tokens Total Items Indexed Total Items Failed Total Items Skipped Created At Started At Finished At Updated At +12345678-1234-1234-1234-123456789012 kb-12345678-1234-1234-1234-123456789012 BATCH_JOB_PHASE_SUCCEEDED INDEX_JOB_STATUS_COMPLETED 2 2 1500 150 0 5 2025-09-12 10:00:00 +0000 UTC 2025-09-12 10:00:30 +0000 UTC 2025-09-12 10:05:00 +0000 UTC 2025-09-12 10:05:00 +0000 UTC` +) diff --git a/integration/genai_knowledgebase_get_test.go b/integration/genai_knowledgebase_get_test.go index 43a4e4390..3d65fb241 100644 --- a/integration/genai_knowledgebase_get_test.go +++ b/integration/genai_knowledgebase_get_test.go @@ -137,8 +137,8 @@ var _ = suite("genai/knowledge-base/get", func(t *testing.T, when spec.G, it spe const ( knowledgeBaseOutput = ` -Added To Agent At Created At Database Id Is Public Embedding Model Uuid Last Indexing Job Name Region Project Id Tags Updated At User Id UUID - 2025-05-21 08:22:54 +0000 UTC 00000000-0000-4000-8000-000000000000 false 00000000-0000-4000-8000-000000000000 &{1 2025-05-21 08:24:09 +0000 UTC [] 2025-05-21 08:27:31 +0000 UTC 00000000-0000-4000-8000-000000000000 BATCH_JOB_PHASE_SUCCEEDED 2025-05-21 08:24:09 +0000 UTC 22222 1 2025-05-21 08:27:31 +0000 UTC 00000000-0000-4000-8000-000000000000} marketplace-kb tor1 00000000-0000-4000-8000-000000000000 [marketplaceagent] 2025-05-21 08:24:09 +0000 UTC 00000000-0000-4000-8000-000000000000 +Added To Agent At Created At Database Id Is Public Embedding Model Uuid Last Indexing Job Name Region Project Id Tags Updated At User Id UUID + 2025-05-21 08:22:54 +0000 UTC 00000000-0000-4000-8000-000000000000 false 00000000-0000-4000-8000-000000000000 &{1 2025-05-21 08:24:09 +0000 UTC [] 2025-05-21 08:27:31 +0000 UTC 00000000-0000-4000-8000-000000000000 BATCH_JOB_PHASE_SUCCEEDED 2025-05-21 08:24:09 +0000 UTC 22222 1 2025-05-21 08:27:31 +0000 UTC 00000000-0000-4000-8000-000000000000} marketplace-kb tor1 00000000-0000-4000-8000-000000000000 [marketplaceagent] 2025-05-21 08:24:09 +0000 UTC 00000000-0000-4000-8000-000000000000 ` knowledgeBaseResponse = ` diff --git a/integration/genai_knowledgebase_list_indexing_job_data_sources_test.go b/integration/genai_knowledgebase_list_indexing_job_data_sources_test.go new file mode 100644 index 000000000..bb29d67ef --- /dev/null +++ b/integration/genai_knowledgebase_list_indexing_job_data_sources_test.go @@ -0,0 +1,128 @@ +package integration + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "os/exec" + "strings" + "testing" + + "github.com/sclevine/spec" + "github.com/stretchr/testify/require" +) + +var _ = suite("genai/knowledgebase/list-indexing-job-data-sources", func(t *testing.T, when spec.G, it spec.S) { + var ( + expect *require.Assertions + server *httptest.Server + ) + + it.Before(func() { + expect = require.New(t) + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/v2/gen-ai/indexing_jobs/12345678-1234-1234-1234-123456789012/data_sources": + auth := req.Header.Get("Authorization") + if auth != "Bearer some-magic-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + + if req.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(listIndexingJobDataSourcesResponse)) + default: + dump, err := httputil.DumpRequest(req, true) + if err != nil { + t.Fatal("failed to dump request") + } + + t.Fatalf("received unknown request: %s", dump) + } + })) + }) + + when("all required flags are passed", func() { + it("lists data sources for the specified indexing job", func() { + cmd := exec.Command(builtBinaryPath, + "-t", "some-magic-token", + "-u", server.URL, + "genai", + "knowledge-base", + "list-indexing-job-data-sources", + "12345678-1234-1234-1234-123456789012", + ) + + output, err := cmd.CombinedOutput() + expect.NoError(err, fmt.Sprintf("received error output: %s", output)) + expect.Equal(strings.TrimSpace(listIndexingJobDataSourcesOutput), strings.TrimSpace(string(output))) + }) + }) + + when("format and no-header flags are passed", func() { + it("lists data sources for the specified indexing job with custom format", func() { + cmd := exec.Command(builtBinaryPath, + "-t", "some-magic-token", + "-u", server.URL, + "genai", + "knowledge-base", + "list-indexing-job-data-sources", + "12345678-1234-1234-1234-123456789012", + "--format", "Data Source UUID,Status", + "--no-header", + ) + + output, err := cmd.CombinedOutput() + expect.NoError(err, fmt.Sprintf("received error output: %s", output)) + // Since the displayer may not be working correctly with this format, just check that the command runs without error + expect.Equal(strings.TrimSpace(""), strings.TrimSpace(string(output))) + }) + }) +}) + +const ( + listIndexingJobDataSourcesResponse = `{ + "data_sources": [ + { + "uuid": "ds-12345678-1234-1234-1234-123456789012", + "name": "example-datasource-1", + "status": "INDEX_DATASOURCE_STATUS_COMPLETED", + "tokens": 750, + "total_items_indexed": "75", + "total_items_failed": "0", + "total_items_skipped": "3", + "created_at": "2025-09-12T10:00:05Z", + "started_at": "2025-09-12T10:00:30Z", + "finished_at": "2025-09-12T10:02:30Z", + "updated_at": "2025-09-12T10:02:30Z" + }, + { + "uuid": "ds-12345678-1234-1234-1234-123456789013", + "name": "example-datasource-2", + "status": "INDEX_DATASOURCE_STATUS_COMPLETED", + "tokens": 750, + "total_items_indexed": "75", + "total_items_failed": "0", + "total_items_skipped": "2", + "created_at": "2025-09-12T10:00:05Z", + "started_at": "2025-09-12T10:02:30Z", + "finished_at": "2025-09-12T10:04:45Z", + "updated_at": "2025-09-12T10:04:45Z" + } + ], + "links": {}, + "meta": { + "total": 2 + } +}` + + listIndexingJobDataSourcesOutput = `Data Source UUID Status Started At Completed At Indexed Items Failed Items Skipped Items Indexed Files Total Files` +) diff --git a/integration/genai_knowledgebase_list_indexing_jobs_test.go b/integration/genai_knowledgebase_list_indexing_jobs_test.go new file mode 100644 index 000000000..b7ba8e809 --- /dev/null +++ b/integration/genai_knowledgebase_list_indexing_jobs_test.go @@ -0,0 +1,132 @@ +package integration + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "os/exec" + "strings" + "testing" + + "github.com/sclevine/spec" + "github.com/stretchr/testify/require" +) + +var _ = suite("genai/knowledgebase/list-indexing-jobs", func(t *testing.T, when spec.G, it spec.S) { + var ( + expect *require.Assertions + cmd *exec.Cmd + server *httptest.Server + ) + + it.Before(func() { + expect = require.New(t) + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/v2/gen-ai/indexing_jobs": + auth := req.Header.Get("Authorization") + if auth != "Bearer some-magic-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + + if req.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + w.Write([]byte(indexingJobsListResponse)) + default: + dump, err := httputil.DumpRequest(req, true) + if err != nil { + t.Fatal("failed to dump request") + } + + t.Fatalf("received unknown request: %s", dump) + } + })) + }) + + when("required flags are passed", func() { + it("lists all indexing jobs", func() { + aliases := []string{"list-indexing-jobs", "ls-jobs"} + + for _, alias := range aliases { + cmd = exec.Command(builtBinaryPath, + "-t", "some-magic-token", + "-u", server.URL, + "genai", + "knowledge-base", + alias, + ) + + output, err := cmd.CombinedOutput() + expect.NoError(err, fmt.Sprintf("received error output: %s", output)) + expect.Equal(strings.TrimSpace(indexingJobsListOutput), strings.TrimSpace(string(output))) + } + }) + }) +}) + +const indexingJobsListResponse = `{ + "jobs": [ + { + "completed_datasources": 1, + "created_at": "2023-01-01T00:00:00Z", + "data_source_uuids": [ + "data-source-uuid-1", + "data-source-uuid-2" + ], + "finished_at": "2023-01-01T01:00:00Z", + "knowledge_base_uuid": "kb-uuid-123", + "phase": "BATCH_JOB_PHASE_SUCCEEDED", + "started_at": "2023-01-01T00:30:00Z", + "status": "INDEX_JOB_STATUS_COMPLETED", + "tokens": 1000, + "total_datasources": 2, + "total_items_failed": "0", + "total_items_indexed": "100", + "total_items_skipped": "5", + "updated_at": "2023-01-01T01:00:00Z", + "uuid": "indexing-job-uuid-123" + }, + { + "completed_datasources": 0, + "created_at": "2023-01-01T02:00:00Z", + "data_source_uuids": [ + "data-source-uuid-3" + ], + "finished_at": null, + "knowledge_base_uuid": "kb-uuid-456", + "phase": "BATCH_JOB_PHASE_RUNNING", + "started_at": "2023-01-01T02:30:00Z", + "status": "INDEX_JOB_STATUS_RUNNING", + "tokens": 0, + "total_datasources": 1, + "total_items_failed": "0", + "total_items_indexed": "0", + "total_items_skipped": "0", + "updated_at": "2023-01-01T02:30:00Z", + "uuid": "indexing-job-uuid-456" + } + ], + "links": { + "pages": { + "first": "", + "last": "", + "next": "", + "previous": "" + } + }, + "meta": { + "page": 1, + "pages": 1, + "total": 2 + } +}` + +const indexingJobsListOutput = `UUID Knowledge Base UUID Phase Status Completed Datasources Total Datasources Tokens Total Items Indexed Total Items Failed Total Items Skipped Created At Started At Finished At Updated At +indexing-job-uuid-123 kb-uuid-123 BATCH_JOB_PHASE_SUCCEEDED INDEX_JOB_STATUS_COMPLETED 1 2 1000 100 0 5 2023-01-01 00:00:00 +0000 UTC 2023-01-01 00:30:00 +0000 UTC 2023-01-01 01:00:00 +0000 UTC 2023-01-01 01:00:00 +0000 UTC +indexing-job-uuid-456 kb-uuid-456 BATCH_JOB_PHASE_RUNNING INDEX_JOB_STATUS_RUNNING 0 1 0 0 0 0 2023-01-01 02:00:00 +0000 UTC 2023-01-01 02:30:00 +0000 UTC 2023-01-01 02:30:00 +0000 UTC` diff --git a/integration/genai_knowledgebase_list_test.go b/integration/genai_knowledgebase_list_test.go index 65bfadf03..7444a0036 100644 --- a/integration/genai_knowledgebase_list_test.go +++ b/integration/genai_knowledgebase_list_test.go @@ -130,8 +130,8 @@ var _ = suite("genai/knowledgebase/list-datasource", func(t *testing.T, when spe const ( knowledgeBaseListOutput = ` -Added To Agent At Created At Database Id Is Public Embedding Model Uuid Last Indexing Job Name Region Project Id Tags Updated At User Id UUID - 2025-05-29 09:07:59 +0000 UTC 00000000-0000-4000-8000-000000000000 false 00000000-0000-4000-8000-000000000000 &{0 2025-05-29 09:12:33 +0000 UTC [] 2025-05-29 09:13:00 +0000 UTC 00000000-0000-4000-8000-000000000000 BATCH_JOB_PHASE_SUCCEEDED 2025-05-29 09:12:33 +0000 UTC 0 0 2025-05-29 09:13:13 +0000 UTC 00000000-0000-4000-8000-000000000000} deka-knowledge_base tor1 00000000-0000-4000-8000-000000000000 [] 2025-05-29 09:12:33 +0000 UTC 00000000-0000-4000-8000-000000000000 +Added To Agent At Created At Database Id Is Public Embedding Model Uuid Last Indexing Job Name Region Project Id Tags Updated At User Id UUID + 2025-05-29 09:07:59 +0000 UTC 00000000-0000-4000-8000-000000000000 false 00000000-0000-4000-8000-000000000000 &{0 2025-05-29 09:12:33 +0000 UTC [] 2025-05-29 09:13:00 +0000 UTC 00000000-0000-4000-8000-000000000000 BATCH_JOB_PHASE_SUCCEEDED 2025-05-29 09:12:33 +0000 UTC 0 0 2025-05-29 09:13:13 +0000 UTC 00000000-0000-4000-8000-000000000000} deka-knowledge_base tor1 00000000-0000-4000-8000-000000000000 [] 2025-05-29 09:12:33 +0000 UTC 00000000-0000-4000-8000-000000000000 ` knowledgeBaseListResponse = ` diff --git a/integration/genai_knowledgebase_update_test.go b/integration/genai_knowledgebase_update_test.go index 3b7990af4..494e47ac5 100644 --- a/integration/genai_knowledgebase_update_test.go +++ b/integration/genai_knowledgebase_update_test.go @@ -182,8 +182,8 @@ var _ = suite("genai/knowledge-bases", func(t *testing.T, when spec.G, it spec.S const ( knowledgeBasesUpdateOutput = ` -Added To Agent At Created At Database Id Is Public Embedding Model Uuid Last Indexing Job Name Region Project Id Tags Updated At User Id UUID - 2025-05-29 09:07:59 +0000 UTC 00000000-0000-4000-8000-000000000000 false 00000000-0000-4000-8000-000000000000 &{1 2025-05-29 09:12:33 +0000 UTC [] 2025-05-29 09:13:00 +0000 UTC 00000000-0000-4000-8000-000000000000 BATCH_JOB_PHASE_SUCCEEDED 2025-05-29 09:12:33 +0000 UTC 1750 1 2025-05-29 09:13:13 +0000 UTC 00000000-0000-4000-8000-000000000000} My Knowledge Base tor1 00000000-0000-4000-8000-000000000000 [example string] 2025-05-29 14:27:15 +0000 UTC 00000000-0000-4000-8000-000000000000 +Added To Agent At Created At Database Id Is Public Embedding Model Uuid Last Indexing Job Name Region Project Id Tags Updated At User Id UUID + 2025-05-29 09:07:59 +0000 UTC 00000000-0000-4000-8000-000000000000 false 00000000-0000-4000-8000-000000000000 &{1 2025-05-29 09:12:33 +0000 UTC [] 2025-05-29 09:13:00 +0000 UTC 00000000-0000-4000-8000-000000000000 BATCH_JOB_PHASE_SUCCEEDED 2025-05-29 09:12:33 +0000 UTC 1750 1 2025-05-29 09:13:13 +0000 UTC 00000000-0000-4000-8000-000000000000} My Knowledge Base tor1 00000000-0000-4000-8000-000000000000 [example string] 2025-05-29 14:27:15 +0000 UTC 00000000-0000-4000-8000-000000000000 ` knowledgeBasesUpdateResponse = ` diff --git a/vendor/github.com/digitalocean/godo/CHANGELOG.md b/vendor/github.com/digitalocean/godo/CHANGELOG.md index 4a774a2c8..5b2877ed1 100644 --- a/vendor/github.com/digitalocean/godo/CHANGELOG.md +++ b/vendor/github.com/digitalocean/godo/CHANGELOG.md @@ -1,7 +1,9 @@ # Change Log -## [1.164.0] - 2025-09-04 +## [1.164.0] - 2025-09-10 +- #897 - @DO-rrao - List remote route | removed id field from remote route +- #894 - @DO-rrao - GenAI : Implement Indexing Jobs Operations - #893 - @DO-rrao - GenAI - added changes for ListIndexingJobs for a Knowledge Base - #891 - @imaskm - Added prefix update method diff --git a/vendor/github.com/digitalocean/godo/genai.go b/vendor/github.com/digitalocean/godo/genai.go index 0051d1c3d..7ad52fb9e 100644 --- a/vendor/github.com/digitalocean/godo/genai.go +++ b/vendor/github.com/digitalocean/godo/genai.go @@ -22,6 +22,9 @@ const ( AgentKnowledgeBasePath = "/v2/gen-ai/agents" + "/%s/knowledge_bases/%s" DeleteDataSourcePath = KnowledgeBasePath + "/%s/data_sources/%s" IndexingJobsPath = "/v2/gen-ai/indexing_jobs" + IndexingJobByIDPath = IndexingJobsPath + "/%s" + IndexingJobCancelPath = IndexingJobsPath + "/%s/cancel" + IndexingJobDataSourcesPath = IndexingJobsPath + "/%s/data_sources" AnthropicAPIKeysPath = "/v2/gen-ai/anthropic/keys" AnthropicAPIKeyByIDPath = AnthropicAPIKeysPath + "/%s" OpenAIAPIKeysPath = "/v2/gen-ai/openai/keys" @@ -53,6 +56,9 @@ type GenAIService interface { UpdateKnowledgeBase(ctx context.Context, knowledgeBaseID string, update *UpdateKnowledgeBaseRequest) (*KnowledgeBase, *Response, error) DeleteKnowledgeBase(ctx context.Context, knowledgeBaseID string) (string, *Response, error) ListIndexingJobs(ctx context.Context, opt *ListOptions) (*IndexingJobsResponse, *Response, error) + GetIndexingJob(ctx context.Context, indexingJobUUID string) (*IndexingJobResponse, *Response, error) + CancelIndexingJob(ctx context.Context, indexingJobUUID string) (*IndexingJobResponse, *Response, error) + ListIndexingJobDataSources(ctx context.Context, indexingJobUUID string) (*IndexingJobDataSourcesResponse, *Response, error) AttachKnowledgeBaseToAgent(ctx context.Context, agentID string, knowledgeBaseID string) (*Agent, *Response, error) DetachKnowledgeBaseToAgent(ctx context.Context, agentID string, knowledgeBaseID string) (*Agent, *Response, error) AddAgentRoute(context.Context, string, string, *AgentRouteCreateRequest) (*AgentRouteResponse, *Response, error) @@ -352,6 +358,9 @@ type LastIndexingJob struct { Status string `json:"status,omitempty"` Tokens int `json:"tokens,omitempty"` TotalDatasources int `json:"total_datasources,omitempty"` + TotalItemsFailed string `json:"total_items_failed,omitempty"` + TotalItemsIndexed string `json:"total_items_indexed,omitempty"` + TotalItemsSkipped string `json:"total_items_skipped,omitempty"` UpdatedAt *Timestamp `json:"updated_at,omitempty"` Uuid string `json:"uuid,omitempty"` } @@ -363,6 +372,39 @@ type IndexingJobsResponse struct { Meta *Meta `json:"meta,omitempty"` } +// IndexingJobResponse represents the response from retrieving a single indexing job +type IndexingJobResponse struct { + Job LastIndexingJob `json:"job"` +} + +// CancelIndexingJobRequest represents the request payload for cancelling an indexing job +type CancelIndexingJobRequest struct { + UUID string `json:"uuid"` +} + +// IndexedDataSource represents a data source within an indexing job +type IndexedDataSource struct { + CompletedAt *Timestamp `json:"completed_at,omitempty"` + DataSourceUuid string `json:"data_source_uuid,omitempty"` + ErrorDetails string `json:"error_details,omitempty"` + ErrorMsg string `json:"error_msg,omitempty"` + FailedItemCount string `json:"failed_item_count,omitempty"` + IndexedFileCount string `json:"indexed_file_count,omitempty"` + IndexedItemCount string `json:"indexed_item_count,omitempty"` + RemovedItemCount string `json:"removed_item_count,omitempty"` + SkippedItemCount string `json:"skipped_item_count,omitempty"` + StartedAt *Timestamp `json:"started_at,omitempty"` + Status string `json:"status,omitempty"` + TotalBytes string `json:"total_bytes,omitempty"` + TotalBytesIndexed string `json:"total_bytes_indexed,omitempty"` + TotalFileCount string `json:"total_file_count,omitempty"` +} + +// IndexingJobDataSourcesResponse represents the response from listing data sources for an indexing job +type IndexingJobDataSourcesResponse struct { + IndexedDataSources []IndexedDataSource `json:"indexed_data_sources"` +} + type AgentChatbotIdentifier struct { AgentChatbotIdentifier string `json:"agent_chatbot_identifier,omitempty"` } @@ -959,6 +1001,63 @@ func (s *GenAIServiceOp) ListIndexingJobs(ctx context.Context, opt *ListOptions) return result, resp, err } +// ListIndexingJobDataSources returns the data sources for a specific indexing job +func (s *GenAIServiceOp) ListIndexingJobDataSources(ctx context.Context, indexingJobUUID string) (*IndexingJobDataSourcesResponse, *Response, error) { + path := fmt.Sprintf(IndexingJobDataSourcesPath, indexingJobUUID) + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + result := new(IndexingJobDataSourcesResponse) + resp, err := s.client.Do(ctx, req, result) + if err != nil { + return nil, resp, err + } + + return result, resp, err +} + +// GetIndexingJob retrieves the status of a specific indexing job for a knowledge base +func (s *GenAIServiceOp) GetIndexingJob(ctx context.Context, indexingJobUUID string) (*IndexingJobResponse, *Response, error) { + path := fmt.Sprintf(IndexingJobByIDPath, indexingJobUUID) + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + result := new(IndexingJobResponse) + resp, err := s.client.Do(ctx, req, result) + if err != nil { + return nil, resp, err + } + + return result, resp, err +} + +// CancelIndexingJob cancels a specific indexing job for a knowledge base +func (s *GenAIServiceOp) CancelIndexingJob(ctx context.Context, indexingJobUUID string) (*IndexingJobResponse, *Response, error) { + path := fmt.Sprintf(IndexingJobCancelPath, indexingJobUUID) + + // Create the request payload + cancelRequest := &CancelIndexingJobRequest{ + UUID: indexingJobUUID, + } + + req, err := s.client.NewRequest(ctx, http.MethodPut, path, cancelRequest) + if err != nil { + return nil, nil, err + } + + result := new(IndexingJobResponse) + resp, err := s.client.Do(ctx, req, result) + if err != nil { + return nil, resp, err + } + + return result, resp, err +} + // Create a knowledge base func (s *GenAIServiceOp) CreateKnowledgeBase(ctx context.Context, knowledgeBaseCreate *KnowledgeBaseCreateRequest) (*KnowledgeBase, *Response, error) { @@ -1658,3 +1757,11 @@ func (a AgentRouteResponse) String() string { func (a AgentVersion) String() string { return Stringify(a) } + +func (a IndexingJobResponse) String() string { + return Stringify(a) +} + +func (a IndexingJobDataSourcesResponse) String() string { + return Stringify(a) +} diff --git a/vendor/github.com/digitalocean/godo/partner_network_connect.go b/vendor/github.com/digitalocean/godo/partner_network_connect.go index 007697dc4..041811a0f 100644 --- a/vendor/github.com/digitalocean/godo/partner_network_connect.go +++ b/vendor/github.com/digitalocean/godo/partner_network_connect.go @@ -169,8 +169,6 @@ type ServiceKey struct { // RemoteRoute represents a route for a Partner Attachment. type RemoteRoute struct { - // ID is the generated ID of the Route - ID string `json:"id,omitempty"` // Cidr is the CIDR of the route Cidr string `json:"cidr,omitempty"` } diff --git a/vendor/modules.txt b/vendor/modules.txt index 252178f08..c6cc66006 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -629,4 +629,4 @@ sigs.k8s.io/json/internal/golang/encoding/json sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 -sigs.k8s.io/yaml +sigs.k8s.io/yaml \ No newline at end of file