fix(github): fix zerotime issues for GraphQL

Author: petkostas

backend/core/utils/time.go

@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import "time"
+
+// NilIfZeroTime returns nil if t is nil or represents the zero time (0001-01-01...).
+// Otherwise, it returns t unchanged.
+func NilIfZeroTime(t *time.Time) *time.Time {
+	if t == nil {
+		return nil
+	}
+	if t.IsZero() {
+		return nil
+	}
+	return t
+}

backend/core/utils/time_test.go

@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package utils
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNilIfZeroTime(t *testing.T) {
+	type args struct {
+		t *time.Time
+	}
+	tests := []struct {
+		name string
+		args args
+		want *time.Time
+	}{
+		{
+			name: "Empty date should be nil",
+			args: args{nil},
+			want: nil,
+		},
+		{
+			name: "Zero date should be nil",
+			args: args{&time.Time{}},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, NilIfZeroTime(tt.args.t), "NilIfZeroTime(%v)", tt.args.t)
+		})
+	}
+}

backend/plugins/github/models/release.go

@@ -25,25 +25,25 @@ import (
 
 type GithubRelease struct {
 	common.NoPKModel `json:"-" mapstructure:"-"`
-	ConnectionId     uint64    `json:"connection_id" gorm:"primaryKey"`
-	GithubId         int       `json:"github_id"`
-	Id               string    `json:"id" gorm:"type:varchar(255);primaryKey"`
-	AuthorName       string    `json:"authorName"`
-	AuthorID         string    `json:"authorId"`
-	CreatedAt        time.Time `json:"createdAt"`
-	DatabaseID       int       `json:"databaseId"`
-	Description      string    `json:"description"`
-	DescriptionHTML  string    `json:"descriptionHTML"`
-	IsDraft          bool      `json:"isDraft"`
-	IsLatest         bool      `json:"isLatest"`
-	IsPrerelease     bool      `json:"isPrerelease"`
-	Name             string    `json:"name"`
-	PublishedAt      time.Time `json:"publishedAt"`
-	ResourcePath     string    `json:"resourcePath"`
-	TagName          string    `json:"tagName"`
-	UpdatedAt        time.Time `json:"updatedAt"`
-	CommitSha        string    `json:"commit_sha"`
-	URL              string    `json:"url"`
+	ConnectionId     uint64     `json:"connection_id" gorm:"primaryKey"`
+	GithubId         int        `json:"github_id"`
+	Id               string     `json:"id" gorm:"type:varchar(255);primaryKey"`
+	AuthorName       string     `json:"authorName"`
+	AuthorID         string     `json:"authorId"`
+	CreatedAt        time.Time  `json:"createdAt"`
+	DatabaseID       int        `json:"databaseId"`
+	Description      string     `json:"description"`
+	DescriptionHTML  string     `json:"descriptionHTML"`
+	IsDraft          bool       `json:"isDraft"`
+	IsLatest         bool       `json:"isLatest"`
+	IsPrerelease     bool       `json:"isPrerelease"`
+	Name             string     `json:"name"`
+	PublishedAt      *time.Time `json:"publishedAt"`
+	ResourcePath     string     `json:"resourcePath"`
+	TagName          string     `json:"tagName"`
+	UpdatedAt        time.Time  `json:"updatedAt"`
+	CommitSha        string     `json:"commit_sha"`
+	URL              string     `json:"url"`
 }
 
 func (GithubRelease) TableName() string {

backend/plugins/github/tasks/release_convertor.go

@@ -73,7 +73,7 @@ func ConvertRelease(taskCtx plugin.SubTaskContext) errors.Error {
 				DomainEntity: domainlayer.DomainEntity{
 					Id: releaseIdGen.Generate(githubRelease.ConnectionId, githubRelease.Id),
 				},
-				PublishedAt:  githubRelease.PublishedAt,
+				PublishedAt:  *githubRelease.PublishedAt,
 				CicdScopeId:  releaseScopeIdGen.Generate(githubRelease.ConnectionId, githubRelease.GithubId),
 				Name:         githubRelease.Name,
 				DisplayTitle: githubRelease.Name,

backend/plugins/github_graphql/tasks/account_collector.go

@@ -40,14 +40,13 @@ type GraphqlQueryAccountWrapper struct {
 }
 
 type GraphqlQueryAccount struct {
-	Login     string
-	Id        int `graphql:"databaseId"`
-	Name      string
-	Company   string
-	Email     string
-	AvatarUrl string
-	HtmlUrl   string `graphql:"url"`
-	//Type      string
+	Login         string
+	Id            int `graphql:"databaseId"`
+	Name          string
+	Company       string
+	Email         string
+	AvatarUrl     string
+	HtmlUrl       string `graphql:"url"`
 	Organizations struct {
 		Nodes []struct {
 			Email      string

backend/plugins/github_graphql/tasks/account_graphql_pre_extractor.go

@@ -29,7 +29,6 @@ type GithubAccountEdge struct {
 	Email     string
 	AvatarUrl string
 	HtmlUrl   string `graphql:"url"`
-	//Type      string
 }
 type GraphqlInlineAccountQuery struct {
 	GithubAccountEdge `graphql:"... on User"`

backend/plugins/github_graphql/tasks/issue_collector.go

@@ -26,6 +26,7 @@ import (
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/github/models"
 	githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -134,6 +135,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
 			query := queryWrapper.(*GraphqlQueryIssueWrapper)
 			issues := query.Repository.IssueList.Issues
 			for _, rawL := range issues {
+				rawL.ClosedAt = utils.NilIfZeroTime(rawL.ClosedAt)
 				if since != nil && since.After(rawL.UpdatedAt) {
 					return messages, api.ErrFinishCollect
 				}

backend/plugins/github_graphql/tasks/issue_extractor.go

@@ -25,6 +25,7 @@ import (
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
 	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/github/models"
 	githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -68,6 +69,8 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
 			if err != nil {
 				return nil, err
 			}
+			// Normalize zero-date to nil for closedAt
+			issue.ClosedAt = utils.NilIfZeroTime(issue.ClosedAt)
 			results := make([]interface{}, 0, 1)
 			githubIssue, err := convertGithubIssue(milestoneMap, issue, data.Options.ConnectionId, data.Options.GithubId)
 			if err != nil {

backend/plugins/github_graphql/tasks/job_collector.go

@@ -19,13 +19,13 @@ package tasks
 
 import (
 	"encoding/json"
-	"fmt"
 	"reflect"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/github/models"
 	githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -202,14 +202,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 						RunId:                runId,
 						GraphqlQueryCheckRun: &checkRun,
 					}
-					// A checkRun without a startedAt time is a run that was never started (skipped)
-					// akwardly, GitHub assigns a completedAt time to such runs, which is the time when the run was skipped
-					// TODO: Decide if we want to skip those runs or should we assign the startedAt time to the completedAt time
-					if dbCheckRun.StartedAt == nil || dbCheckRun.StartedAt.IsZero() {
-						debug := fmt.Sprintf("collector: checkRun.StartedAt is nil or zero: %s", dbCheckRun.Id)
-						taskCtx.GetLogger().Debug(debug, "Collector: CheckRun started at is nil or zero")
-						continue
-					}
+					// A checkRun without a startedAt time is a run that was never started (skipped), GitHub returns
+					// a ZeroTime (Due to the GO implementation) for startedAt, so we need to check for that here.
+					dbCheckRun.StartedAt = utils.NilIfZeroTime(dbCheckRun.StartedAt)
+					dbCheckRun.CompletedAt = utils.NilIfZeroTime(dbCheckRun.CompletedAt)
 					updatedAt := dbCheckRun.StartedAt
 					if dbCheckRun.CompletedAt != nil {
 						updatedAt = dbCheckRun.CompletedAt

backend/plugins/github_graphql/tasks/job_extractor.go

@@ -24,6 +24,7 @@ import (
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
 	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/github/models"
 	githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
@@ -57,7 +58,9 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error {
 				return nil, err
 			}
 			results := make([]interface{}, 0, 1)
-
+			// Normalize zero-date times to nil
+			checkRun.StartedAt = utils.NilIfZeroTime(checkRun.StartedAt)
+			checkRun.CompletedAt = utils.NilIfZeroTime(checkRun.CompletedAt)
 			paramsBytes, marshalError := json.Marshal(checkRun.Steps.Nodes)
 			err = errors.Convert(marshalError)
 			if err != nil {