feat(gitlab): add toggle for collecting all users (#8528)

- Add GITLAB_SERVER_COLLECT_ALL_USERS configuration option
- Implement logic to collect all users when enabled
- Update task data and configuration reader to support new option
- Modify account collector to use new option

#8523

Author: narrowizard

backend/plugins/gitlab/impl/impl.go

@@ -207,7 +207,11 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
 	if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
 		return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
 	}
-
+	cfg := taskCtx.GetConfigReader()
+	op.CollectAllUsers = true
+	if cfg.IsSet("GITLAB_SERVER_COLLECT_ALL_USERS") {
+		op.CollectAllUsers = cfg.GetBool("GITLAB_SERVER_COLLECT_ALL_USERS")
+	}
 	taskData := tasks.GitlabTaskData{
 		Options:       op,
 		ApiClient:     apiClient,

backend/plugins/gitlab/tasks/account_collector.go

@@ -50,15 +50,15 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
 	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_TABLE)
 	logger := taskCtx.GetLogger()
 	logger.Info("collect gitlab users")
-
-	// it means we can not use /members/all to get the data
+	options := taskCtx.GetData().(*GitlabTaskData).Options
 	urlTemplate := "/projects/{{ .Params.ProjectId }}/members/all"
 	if semver.Compare(data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string), "v13.11") < 0 {
+		// it means we can not use /members/all to get the data
 		urlTemplate = "/projects/{{ .Params.ProjectId }}/members/"
 	}
 
-	// Collect all users if endpoint is private gitlab instance
-	if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") && !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://jihulab.com") {
+	// Collect all users if endpoint is private gitlab instance and GITLAB_SERVER_COLLECT_ALL_USERS
+	if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") && !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://jihulab.com") && options.CollectAllUsers {
 		urlTemplate = "/users"
 	}
 

backend/plugins/gitlab/tasks/task_data.go

@@ -24,11 +24,12 @@ import (
 )
 
 type GitlabOptions struct {
-	ConnectionId  uint64                    `mapstructure:"connectionId" json:"connectionId"`
-	ProjectId     int                       `mapstructure:"projectId" json:"projectId"`
-	FullName      string                    `mapstructure:"fullName" json:"fullName"`
-	ScopeConfigId uint64                    `mapstructure:"scopeConfigId" json:"scopeConfigId"`
-	ScopeConfig   *models.GitlabScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	ConnectionId    uint64                    `mapstructure:"connectionId" json:"connectionId"`
+	ProjectId       int                       `mapstructure:"projectId" json:"projectId"`
+	FullName        string                    `mapstructure:"fullName" json:"fullName"`
+	ScopeConfigId   uint64                    `mapstructure:"scopeConfigId" json:"scopeConfigId"`
+	ScopeConfig     *models.GitlabScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	CollectAllUsers bool
 }
 
 type GitlabTaskData struct {

env.example

@@ -69,6 +69,11 @@ ENDPOINT_CIDR_BLACKLIST=
 # Do not follow redirection when requesting data source APIs
 FORBID_REDIRECTION=false
 
+##########################
+# Plugin settings
+##########################
+GITLAB_SERVER_COLLECT_ALL_USERS=true
+
 ##########################
 # In plugin gitextractor, use go-git to collector repo's data
 ##########################