Merge branch 'dev'

Author: pylapp

.gitignore

@@ -1,2 +1,3 @@
 toolbox/github/data
 toolbox/diver/data
+toolbox/gitlab/data

CHANGELOG.md

@@ -1,10 +1,21 @@
 # CHANGELOG
 
+## Version 2.5.0
+
+### Features
+
+- [#32](https://github.com/Orange-OpenSource/floss-toolbox/issues/32) GitLab Auto Backup
+- [#49](https://github.com/Orange-OpenSource/floss-toolbox/issues/49) Look for leaks (GitLab)
+
+### Bugs
+
+- [#52](https://github.com/Orange-OpenSource/floss-toolbox/issues/52) Failure of git log if no commits
+
 ## Version 2.4.0
 
 ### Features
 
-- [#44](https://github.com/Orange-OpenSource/floss-toolbox/issues/44) Look for leaks
+- [#44](https://github.com/Orange-OpenSource/floss-toolbox/issues/44) Look for leaks (GitHub)
 - [#29](https://github.com/Orange-OpenSource/floss-toolbox/issues/29) Dry run
 
 ### Refactoring

README.md

@@ -1,4 +1,4 @@
-# floss-toolbox (version 2.4.0)
+# floss-toolbox (version 2.5.0)
 
 Toolbox to help developers and open source referents to have cleaner projects in GitHub organizations.
 
@@ -427,4 +427,67 @@ brew install gitleaks
 
 You need to define in the _configuration.rb_ files the Github organisation at **GITHUB_ORGANIZATION_NAME** and also your GitHub personal token at ** GITHUB_PERSONAL_ACCESS_TOKEN**.
 
-**You should also have your _git_ environment ready i.e. add your SSH private key if you clone by SSH for example. _gh_ must be installed, and _python3_ be ready. Obvisously _gitleaks_ must be installed**
+**You should also have your _git_ environment ready i.e. add your SSH private key if you clone by SSH for example. _gh_ must be installed, and _python3_ be ready. Obviously _gitleaks_ must be installed**
+
+# Play with GitLab web API
+
+## Prerequisites
+
+- Ruby Gem: `git 1.8.1`
+- Python3
+
+- Create a [GitLab personal token](https://gitlab.com/-/profile/personal_access_tokens) and define it in the _configuration.rb_ file for the `GILAB_PERSONAL_ACCESS_TOKEN` variable.
+- Define the GitLab organization id in the _configuration.rb_ file for the `GITLAB_ORGANIZATION_ID` variable. It will allow to send requests to query and modify your organization.
+
+## Prepare project
+
+```ruby
+gem install git
+```
+
+## Features
+
+### Make a backup of organization repositories
+
+_Keywords: #organisation #GitLab #repositories #clones #dump_
+
+This feature allows to clone all repositories of the defined GitLab organization (groups and subgroups incldued) and save them in a specific folder.
+
+Run the following command:
+```shell
+bash GitLabWizard.sh backup-all-repositories-from-org
+```
+
+This script will get configuation details picked from the Ruby configuration file; and triggers another Shell script to make a CURL request to the GitLab endpoint. A Python code will be called so as to extract repositories URLbefoire the cloning operation.
+
+You need to define in the _configuration.rb_ files the GitLab organisation ID at **GITLAB_ORGANIZATION_ID**.
+You have to also define the location to store clones at **REPOSITORIES_CLONE_LOCATION_PATH** and the access token at **GILAB_PERSONAL_ACCESS_TOKEN**.
+
+**You should also have your _git_ environment ready, i.e. add your SSH private key if you clone by SSH for example.**
+
+### Check if there are leaks in organisation repositories (using gitleaks)
+
+_Keywords: #organisation #GitLab #repositories #leaks #gitleaks_
+
+**Warning: This operation can take long time because of both Git histories and file trees parsing**
+
+This feature allows to check in all repositories of the GitHub organisation if there are leaks using the _gitleaks_ tool.
+
+Run the following command:
+```shell
+bash GitLabWizard.sh look-for-leaks
+```
+
+This script needs a GitLab personal access otken to make requests to GitLab API and also the GitLab group ID to use to get projects under it.
+The wizard Shell script will pick configuration details from the Ruby configuration file ; and triggers another Shell script for the data process. A Python code will be called too to process JSON sent by GItLab API..
+
+The [gitleaks](https://github.com/zricethezav/gitleaks) tool will be used to look inside the repository. To install it:
+
+```shell
+brew install gitleaks
+```
+
+You need to define in the _configuration.rb_ files the GitLab organisation ID at **GITLAB_ORGANIZATION_ID**.
+You have to also define the location to store clones at **REPOSITORIES_CLONE_LOCATION_PATH** and the access token at **GILAB_PERSONAL_ACCESS_TOKEN**.
+
+**You should also have your _git_ environment ready i.e. add your SSH private key if you clone by SSH for example. _gh_ must be installed, and _python3_ be ready. Obviously _gitleaks_ must be installed**

toolbox/diver/extract-emails-from-history.sh

@@ -7,7 +7,7 @@
 #
 # Author: Pierre-Yves LAPERSONNE <pierreyves(dot)lapersonne(at)orange(dot)com> et al.
 
-# Version.............: 1.0.1
+# Version.............: 1.0.2
 # Since...............: 06/10/2021
 # Description.........: Using the Git history, provide a list of contributors
 #
@@ -22,7 +22,7 @@
 
 set -euo pipefail
 
-VERSION="1.0.1"
+VERSION="1.0.2"
 SCRIPT_NAME="extract-emails-from-history"
 
 # -------------
@@ -216,6 +216,12 @@ fi
 
 touch "$git_log_file"
 
+if [ "$( git log --oneline -5 2>/dev/null | wc -l )" -eq 0 ]; then
+    echo "Warning: Project '$git_based_project' is a git repository without any commit, that's weird"
+    CleanFiles
+    NormalExit
+fi
+
 git log --since=$git_log_limit > $git_log_file
 
 if [ ! -s "$git_log_file" ]; then

toolbox/diver/find-contributors-in-git-logs.sh

@@ -7,7 +7,7 @@
 #
 # Author: Pierre-Yves LAPERSONNE <pierreyves(dot)lapersonne(at)orange(dot)com> et al.
 
-# Version.............: 1.2.0
+# Version.............: 1.2.1
 # Since...............: 11/05/2020
 # Description.........: Looks for words (defined in dedicated file) in git logs
 #
@@ -21,7 +21,7 @@
 #
 
 
-VERSION="1.2.0"
+VERSION="1.2.1"
 SCRIPT_NAME="find-contributors-in-git-logs"
 
 # -------------
@@ -234,6 +234,11 @@ if [ -f $git_log_file ]; then
     rm $git_log_file
 fi
 
+if [ "$( git log --oneline -5 2>/dev/null | wc -l )" -eq 0 ]; then
+    echo "Warning: Project '$git_based_project' is a git repository without any commit, that's weird"
+    NormalExit
+fi
+
 git log --since=$git_log_limit > $git_log_file
 
 if [ ! -s "$git_log_file" ]; then

toolbox/diver/find-missing-developers-in-git-commits.sh

@@ -7,7 +7,7 @@
 #
 # Author: Pierre-Yves LAPERSONNE <pierreyves(dot)lapersonne(at)orange(dot)com> et al.
 
-# Version.............: 1.1.0
+# Version.............: 1.1.1
 # Since...............: 12/05/2020
 # Description.........: Looks in git commits in the DCO has been used, i.e. if commits have been signed off.
 # Checks also if commits authors are defined.
@@ -21,7 +21,7 @@
 #       3 - problem with a command
 #
 
-VERSION="1.1.0"
+VERSION="1.1.1"
 SCRIPT_NAME="find-missing-developers-in-git-commits"
 
 # -------------
@@ -199,6 +199,11 @@ if [ -f $git_log_file ]; then
     rm $git_log_file
 fi
 
+if [ "$( git log --oneline -5 2>/dev/null | wc -l )" -eq 0 ]; then
+    echo "Warning: Project '$git_based_project' is a git repository without any commit, that's weird"
+    NormalExit
+fi
+
 git log --since=$git_log_limit > $git_log_file
 
 if [ ! -s "$git_log_file" ]; then

toolbox/diver/list-contributors-in-history.sh

@@ -7,7 +7,7 @@
 #
 # Author: Pierre-Yves LAPERSONNE <pierreyves(dot)lapersonne(at)orange(dot)com> et al.
 
-# Version.............: 1.0.0
+# Version.............: 1.0.2
 # Since...............: 24/02/2022
 # Description.........: Using the Git history, provide a list of contributors' email addresses
 #
@@ -22,7 +22,7 @@
 
 set -euo pipefail
 
-VERSION="1.0.1"
+VERSION="1.0.2"
 SCRIPT_NAME="list-contributors-in-history"
 
 # -------------
@@ -197,6 +197,11 @@ fi
 
 touch "$git_log_file"
 
+if [ "$( git log --oneline -5 2>/dev/null | wc -l )" -eq 0 ]; then
+    echo "Warning: Project '$git_based_project' is a git repository without any commit, that's weird"
+    NormalExit
+fi
+
 git log --since=$git_log_limit > $git_log_file
 
 if [ ! -s "$git_log_file" ]; then

toolbox/dry-run.sh

@@ -9,6 +9,7 @@
 
 # Since...............: 08/03/2021
 # Description.........:  Make a dry-run of the project to check if everything is ready to use
+# Version.............: 1.1.0
 
 # Couts
 # -----
@@ -128,6 +129,24 @@ CheckIfFileExists "github/utils/GitHubWrapper.rb"
 CheckIfFileExists "github/utils/GitWrapper.rb"
 CheckIfFileExists "github/utils/IO.rb"
 
+# GitLab features
+# ---------------
+
+echo -e "\n----------------------------------"
+echo "Assertions for the GITLAB features"
+echo "----------------------------------"
+
+echo -e "\nCheck if main folder exists..."
+CheckIfDirectoryExists "gitlab"
+CheckIfDirectoryExists "gitlab/data"
+
+echo -e "\nCheck files..."
+CheckIfFileExists "gitlab/configuration.rb"
+CheckIfFileExists "gitlab/GitLabWizard.sh"
+CheckIfFileExists "gitlab/utils/dump-git-repositories-from-gitlab.sh"
+CheckIfFileExists "github/utils/extract-repos-field-from-json.py" # Stored in github folder but used by dump-git-repositories-from-gitlab.sh
+CheckIfFileExists "github/utils/count-leaks-nodes.py" # Stored in github folder but used by check-leaks-from-gitlab.sh
+
 # Runtimes and tools
 # ------------------
 
@@ -164,6 +183,7 @@ echo "Assertions for configuration file"
 echo "---------------------------------"
 
 echo -e "\nCheck for entries in configuration file..."
+
 CheckIfConfigurationKeyDefined "github/configuration.rb" "GITHUB_PERSONAL_ACCESS_TOKEN"
 CheckIfConfigurationKeyDefined "github/configuration.rb" "GITHUB_ORGANIZATION_NAME"
 CheckIfConfigurationKeyDefined "github/configuration.rb" "GITHUB_ORGANIZATION_ADMINS"
@@ -183,6 +203,13 @@ CheckIfConfigurationKeyDefined "github/configuration.rb" "FILENAME_PROJECTS_WITH
 CheckIfConfigurationKeyDefined "github/configuration.rb" "FILENAME_EMPTY_PROJECTS"
 CheckIfConfigurationKeyDefined "github/configuration.rb" "REPOSITORIES_CLONE_LOCATION_PATH"
 CheckIfConfigurationKeyDefined "github/configuration.rb" "REPOSITORIES_CLONE_URL_JSON_KEY"
+
+CheckIfConfigurationKeyDefined "gitlab/configuration.rb" "GILAB_PERSONAL_ACCESS_TOKEN"
+CheckIfConfigurationKeyDefined "gitlab/configuration.rb" "GITLAB_ORGANIZATION_ID"
+CheckIfConfigurationKeyDefined "gitlab/configuration.rb" "RESULTS_PER_PAGE"
+CheckIfConfigurationKeyDefined "gitlab/configuration.rb" "REPOSITORIES_CLONE_LOCATION_PATH"
+CheckIfConfigurationKeyDefined "gitlab/configuration.rb" "REPOSITORIES_CLONE_URL_JSON_KEY"
+
 echo -e "🔎  I hope configuration entries are - well - defined, be sure of that"
 
 # Conclusion

toolbox/github/configuration.rb

@@ -9,8 +9,8 @@
 # Allow debug message or not
 $LOG_DEBUG = false
 
-# API, tokens, GitHub organization
-# --------------------------------
+# GitHub organization
+# -------------------
 
 # You have to create a personal access token on: https://github.com/settings/tokens
 $GITHUB_PERSONAL_ACCESS_TOKEN = ""

toolbox/github/utils/check-leaks-from-github.sh

@@ -75,14 +75,14 @@ if [ -z "$organisation_name" -o "$organisation_name" == "" ]; then
 fi
 
 cloning_url_key=$2
-if [ -z "$cloning_url_key" -o "$organisation_name" == "" ]; then
+if [ -z "$cloning_url_key" -o "$cloning_url_key" == "" ]; then
 	echo "ERROR: No JSON key for URL. Exits now."
     UsageAndExit
 	exit $EXIT_BAD_ARGUMENTS
 fi
 
 dump_folder_name=$3
-if [ -z "$dump_folder_name" -o "$organisation_name" == "" ]; then
+if [ -z "$dump_folder_name" -o "$dump_folder_name" == "" ]; then
 	echo "ERROR: No dump folder name defined. Exits now."
     UsageAndExit
 	exit $EXIT_BAD_ARGUMENTS
@@ -195,7 +195,7 @@ while read url_line; do
 
 done < "$dir_before_dump/$url_for_cloning"
 
-echo "Looking done!"
+echo "Scanning done!"
 
 # Step 6 - Clean up