Skip to content

Commit 31927be

Browse files
data-douserdata-douser
committed
WIP make the CDS extractor platform independent
Adds .cmd script equivalents for autobuild.sh, index-files.sh, and pre-finalize.sh scripts -- with the (unverified) intention being that the CDS extractor should be able to run on Linux, Mac, or Windows. Migrates most of the `index-files` script logic from shell script to javascript. Adds a `package.json` file to assist with managing dependencies for the new `extractors/cds/tools/index-files.js` script.
1 parent fe6937a commit 31927be

File tree

8 files changed

+197
-98
lines changed

8 files changed

+197
-98
lines changed

‎extractors/cds/tools/autobuild.cmd‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
@echo off
2+
3+
type NUL && "%CODEQL_DIST%\codeql" database index-files ^
4+
--include-extension=.cds ^
5+
--language cds ^
6+
--prune **\node_modules\**\* ^
7+
--prune **\.eslint\**\* ^
8+
--total-size-limit=10m ^
9+
-- ^
10+
"%CODEQL_EXTRACTOR_CDS_WIP_DATABASE%"
11+
12+
exit /b %ERRORLEVEL%

‎extractors/cds/tools/autobuild.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/sh
1+
#!/usr/bin/env bash
22

33
set -eu
44

@@ -9,9 +9,10 @@ set -eu
99
# Any changes should be synchronized between these three places.
1010

1111
exec "${CODEQL_DIST}/codeql" database index-files \
12-
--language cds \
13-
--total-size-limit 10m \
1412
--include-extension=.cds \
13+
--language cds \
1514
--prune **/node_modules/**/* \
1615
--prune **/.eslint/**/* \
16+
--total-size-limit=10m \
17+
-- \
1718
"$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"

‎extractors/cds/tools/index-files.cmd‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
@echo off
2+
3+
if "%~1"=="" (
4+
echo Usage: %0 ^<response_file_path^>
5+
exit /b 1
6+
)
7+
8+
where node >nul 2>nul
9+
if %ERRORLEVEL% neq 0 (
10+
echo node executable is required (in PATH) to run the 'index-files.js' script. Please install Node.js and try again.
11+
exit /b 2
12+
)
13+
14+
where npm >nul 2>nul
15+
if %ERRORLEVEL% neq 0 (
16+
echo npm executable is required (in PATH) to install the dependencies for the 'index-files.js' script.
17+
exit /b 3
18+
)
19+
20+
set "_response_file_path=%~1"
21+
22+
echo Checking response file for CDS files to index
23+
24+
if not exist "%_response_file_path%" (
25+
echo 'codeql database index-files --language cds' command terminated early as response file '%_response_file_path%' does not exist or is empty. This is because no CDS files were selected or found.
26+
exit /b 0
27+
)
28+
29+
echo Installing node package dependencies and running the 'index-files.js' script
30+
npm install --quiet --no-audit --no-fund --no-package-json && ^
31+
node "%~dp0index-files.js" "%_response_file_path%"
32+
33+
exit /b %ERRORLEVEL%

‎extractors/cds/tools/index-files.js‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
const { existsSync, readFileSync, statSync } = require('fs');
2+
const { execSync, spawnSync } = require('child_process');
3+
const { dirname, join, resolve } = require('path');
4+
5+
console.log('Indexing CDS files');
6+
7+
const responseFile = process.argv[2];
8+
9+
const npmInstallCmdWithArgs = 'npm install --quiet --no-audit --no-fund --no-package-lock';
10+
11+
if (!existsSync(responseFile)) {
12+
console.log(`'codeql database index-files --language cds' terminated early as response file '${responseFile}' does not exist. This is because no CDS files were selected or found.`);
13+
process.exit(0);
14+
}
15+
16+
if (statSync(responseFile).size === 0) {
17+
console.log(`'codeql database index-files --language cds' terminated early as response file '${responseFile}' is empty. This is because no CDS files were selected or found.`);
18+
process.exit(0);
19+
}
20+
21+
let cdsCommand = 'cds';
22+
try {
23+
execSync('cds --version', { stdio: 'ignore' });
24+
} catch {
25+
console.log('Pre-installing cds compiler');
26+
const responseFiles = readFileSync(responseFile, 'utf-8').split('\n').filter(Boolean);
27+
const packageJsonDirs = new Set();
28+
29+
responseFiles.forEach(file => {
30+
let dir = dirname(file);
31+
while (dir !== resolve(dir, '..')) {
32+
if (existsSync(join(dir, 'package.json')) && readFileSync(join(dir, 'package.json'), 'utf-8').includes('@sap/cds')) {
33+
packageJsonDirs.add(dir);
34+
break;
35+
}
36+
dir = resolve(dir, '..');
37+
}
38+
});
39+
40+
packageJsonDirs.forEach(dir => {
41+
console.log(`Installing @sap/cds-dk into ${dir} to enable CDS compilation.`);
42+
execSync(`${npmInstallCmdWithArgs} @sap/cds-dk`, { cwd: dir });
43+
execSync(npmInstallCmdWithArgs, { cwd: dir });
44+
});
45+
46+
cdsCommand = 'npx -y --package @sap/cds-dk cds';
47+
}
48+
49+
console.log('Processing CDS files to JSON');
50+
51+
const responseFiles = readFileSync(responseFile, 'utf-8').split('\n').filter(Boolean);
52+
responseFiles.forEach(cdsFile => {
53+
const cdsJsonFile = `${cdsFile}.json`;
54+
console.log(`Processing CDS file ${cdsFile} to: ${cdsJsonFile}`);
55+
const result = spawnSync(cdsCommand, ['compile', cdsFile, '-2', 'json', '-o', cdsJsonFile, '--locations'], { shell: true });
56+
if (result.error || result.status !== 0) {
57+
const stderrTruncated = result.stderr.toString().split('\n').filter(line => line.startsWith('[ERROR]')).slice(-4).join('\n');
58+
const errorMessage = `Could not compile the file ${cdsFile}.\nReported error(s):\n\`\`\`\n${stderrTruncated}\n\`\`\``;
59+
console.log(errorMessage);
60+
execSync(`${process.env.CODEQL_DIST}/codeql database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to compile one or more SAP CAP CDS files" --severity error --markdown-message "${errorMessage}" --file-path "${cdsFile}" "${process.env.CODEQL_EXTRACTOR_CDS_WIP_DATABASE}"`);
61+
}
62+
});
63+
64+
if (!process.env.CODEQL_EXTRACTOR_JAVASCRIPT_ROOT) {
65+
process.env.CODEQL_EXTRACTOR_JAVASCRIPT_ROOT = execSync(`${process.env.CODEQL_DIST}/codeql resolve extractor --language=javascript`).toString().trim();
66+
process.env.CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE = process.env.CODEQL_EXTRACTOR_CDS_WIP_DATABASE;
67+
process.env.CODEQL_EXTRACTOR_JAVASCRIPT_DIAGNOSTIC_DIR = process.env.CODEQL_EXTRACTOR_CDS_DIAGNOSTIC_DIR;
68+
process.env.CODEQL_EXTRACTOR_JAVASCRIPT_LOG_DIR = process.env.CODEQL_EXTRACTOR_CDS_LOG_DIR;
69+
process.env.CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR = process.env.CODEQL_EXTRACTOR_CDS_SCRATCH_DIR;
70+
process.env.CODEQL_EXTRACTOR_JAVASCRIPT_TRAP_DIR = process.env.CODEQL_EXTRACTOR_CDS_TRAP_DIR;
71+
process.env.CODEQL_EXTRACTOR_JAVASCRIPT_SOURCE_ARCHIVE_DIR = process.env.CODEQL_EXTRACTOR_CDS_SOURCE_ARCHIVE_DIR;
72+
}
73+
74+
let excludeFilters = '';
75+
if (process.env.LGTM_INDEX_FILTERS) {
76+
console.log(`Found $LGTM_INDEX_FILTERS already set to:\n${process.env.LGTM_INDEX_FILTERS}`);
77+
excludeFilters = '\n' + process.env.LGTM_INDEX_FILTERS.split('\n').filter(line => line.startsWith('exclude') && !['exclude:**/*', 'exclude:**/*.*'].includes(line)).join('\n');
78+
}
79+
80+
process.env.LGTM_INDEX_FILTERS = `exclude:**/*.*\ninclude:**/*.cds.json\ninclude:**/*.cds\nexclude:**/node_modules/**/*.*${excludeFilters}`;
81+
console.log(`Setting $LGTM_INDEX_FILTERS to:\n${process.env.LGTM_INDEX_FILTERS}`);
82+
process.env.LGTM_INDEX_TYPESCRIPT = 'NONE';
83+
process.env.LGTM_INDEX_FILETYPES = '.cds:JSON';
84+
delete process.env.LGTM_INDEX_INCLUDE;
85+
86+
console.log('Extracting the cds.json files');
87+
88+
execSync(`${process.env.CODEQL_EXTRACTOR_JAVASCRIPT_ROOT}/tools/autobuild.sh`, { stdio: 'inherit' });

‎extractors/cds/tools/index-files.sh‎

Lines changed: 26 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -1,104 +1,40 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
22

33
set -eu
44

5-
echo "Indexing CDS files"
6-
7-
# Check if the list of files is empty
8-
response_file="$1"
9-
10-
# If the response_file doesn't exist, terminate:
11-
if [ ! -f "$response_file" ]; then
12-
echo "codeql database index-files --language cds terminated early as response file '$response_file' does not exist. This is because no CDS files were selected or found."
13-
exit 0
5+
if [ $# -ne 1 ]
6+
then
7+
echo "Usage: $0 <response_file_path>"
8+
exit 1
149
fi
1510

16-
# If the response_file is empty, terminate
17-
if [ ! -s "$response_file" ]; then
18-
echo "codeql database index-files --language cds terminated early as response file '$response_file' is empty. This is because no CDS files were selected or found."
19-
exit 0
11+
if ! command -v node > /dev/null
12+
then
13+
echo "node executable is required (in PATH) to run the 'index-files.js' script. Please install Node.js and try again."
14+
exit 2
2015
fi
2116

22-
# Determine if we have the cds command available, and if not, install the cds development kit
23-
# in the appropriate directories
24-
if ! command -v cds &> /dev/null
17+
if ! command -v npm > /dev/null
2518
then
26-
echo "Pre-installing cds compiler"
27-
28-
# Find all the directories containing a package.json with a dependency on @sap/cds, where
29-
# the directory contains at least one of the files listed in the response file (e.g. the
30-
# cds files we want to extract).
31-
#
32-
# We then install the cds development kit (@sap/cds-dk) in each directory, which makes the
33-
# `cds` command usable from the npx command within that directory.
34-
#
35-
# Nested package.json files simply cause the package to be installed in the parent node_modules
36-
# directory.
37-
#
38-
# We also ensure we skip node_modules, as we can end up in a recursive loop
39-
find . -type d -name node_modules -prune -false -o -type f \( -iname 'package.json' \) -exec grep -ql '@sap/cds' {} \; -execdir bash -c "grep -q \"^\$(pwd)\(/\|$\)\" \"$response_file\"" \; -execdir bash -c "echo \"Installing @sap/cds-dk into \$(pwd) to enable CDS compilation.\"" \; -execdir npm install --silent @sap/cds-dk \; -execdir npm install --silent \;
40-
41-
# Use the npx command to dynamically install the cds development kit (@sap/cds-dk) package if necessary,
42-
# which then provides the cds command line tool in directories which are not covered by the package.json
43-
# install command approach above
44-
cds_command="npx -y --package @sap/cds-dk cds"
45-
else
46-
cds_command="cds"
19+
echo "npm executable is required (in PATH) to install the dependencies for the 'index-files.js' script."
20+
exit 3
4721
fi
4822

49-
echo "Processing CDS files to JSON"
23+
_response_file_path="$1"
5024

51-
# Run the cds compile command on each file in the response file, outputting the compiled JSON to a file with
52-
# the same name
53-
while IFS= read -r cds_file; do
54-
echo "Processing CDS file $cds_file to:"
55-
if ! $cds_command compile "$cds_file" -2 json -o "$cds_file.json" --locations 2> "$cds_file.err"; then
56-
stderr_truncated=`grep "^\[ERROR\]" "$cds_file.err" | tail -n 4`
57-
error_message=$'Could not compile the file '"$cds_file"$'.\nReported error(s):\n```\n'"$stderr_truncated"$'\n```'
58-
echo "$error_message"
59-
# Log an error diagnostic which appears on the status page
60-
"$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to compile one or more SAP CAP CDS files" --severity error --markdown-message "$error_message" --file-path "$cds_file" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
61-
fi
62-
done < "$response_file"
25+
echo "Checking response file for CDS files to index"
6326

64-
# Check if the JS extractor variables are set, and set them if not
65-
if [ -z "${CODEQL_EXTRACTOR_JAVASCRIPT_ROOT:-}" ]; then
66-
# Find the JavaScript extractor location
67-
export CODEQL_EXTRACTOR_JAVASCRIPT_ROOT="$("$CODEQL_DIST/codeql" resolve extractor --language=javascript)"
68-
69-
# Set the JAVASCRIPT extractor environment variables to the same as the CDS extractor environment variables
70-
# so that the JS extractor will write to the CDS database
71-
export CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE="$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
72-
export CODEQL_EXTRACTOR_JAVASCRIPT_DIAGNOSTIC_DIR="$CODEQL_EXTRACTOR_CDS_DIAGNOSTIC_DIR"
73-
export CODEQL_EXTRACTOR_JAVASCRIPT_LOG_DIR="$CODEQL_EXTRACTOR_CDS_LOG_DIR"
74-
export CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR="$CODEQL_EXTRACTOR_CDS_SCRATCH_DIR"
75-
export CODEQL_EXTRACTOR_JAVASCRIPT_TRAP_DIR="$CODEQL_EXTRACTOR_CDS_TRAP_DIR"
76-
export CODEQL_EXTRACTOR_JAVASCRIPT_SOURCE_ARCHIVE_DIR="$CODEQL_EXTRACTOR_CDS_SOURCE_ARCHIVE_DIR"
77-
fi
78-
79-
# Check if LGTM_INDEX_FILTERS is already set
80-
# This typically happens if "paths" or "paths-ignore" are set in the LGTM.yml file
81-
if [ -z "${LGTM_INDEX_FILTERS:-}" ]; then
82-
exclude_filters=""
83-
else
84-
echo $'Found \$LGTM_INDEX_FILTERS already set to:\n'"$LGTM_INDEX_FILTERS"
85-
# If it is set, we will try to honour the paths-ignore filter
86-
# Split by \n and find all the entries that start with exclude, excluding "exclude:**/*" and "exclude:**/*.*"
87-
# and then join them back together with \n
88-
exclude_filters=$'\n'"$(echo "$LGTM_INDEX_FILTERS" | grep '^exclude' | grep -v 'exclude:\*\*/\*\|exclude:\*\*/\*\.\*')"
27+
# Terminate early if the _response_file_path doesn't exist or is empty,
28+
# which indicates that no CDS files were selected or found.
29+
if [ ! -f "$_response_file_path" ] || [ ! -s "$_response_file_path" ]
30+
then
31+
echo "'codeql database index-files --language cds' command terminated early as response file '$_response_file_path' does not exist or is empty. This is because no CDS files were selected or found."
32+
# Exit without error to avoid failing any calling (javascript)
33+
# extractor, and llow the tool the report the lack of coverage
34+
# for CDS files.
35+
exit 0
8936
fi
9037

91-
# Enable extraction of the cds.json files only
92-
export LGTM_INDEX_FILTERS=$'exclude:**/*.*\ninclude:**/*.cds.json\ninclude:**/*.cds\nexclude:**/node_modules/**/*.*'"$exclude_filters"
93-
echo "Setting \$LGTM_INDEX_FILTERS to:\n$LGTM_INDEX_FILTERS"
94-
export LGTM_INDEX_TYPESCRIPT="NONE"
95-
# Configure to copy over the CDS files as well, by pretending they are JSON
96-
export LGTM_INDEX_FILETYPES=".cds:JSON"
97-
# Ignore the LGTM_INDEX_INCLUDE variable for this purpose, as it may
98-
# refer explicitly to .ts or .js files
99-
unset LGTM_INDEX_INCLUDE
100-
101-
echo "Extracting the cds.json files"
102-
103-
# Invoke the JavaScript autobuilder to index the .cds.json files only
104-
"$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT"/tools/autobuild.sh
38+
echo "Installing node package dependencies and running the 'index-files.js' script"
39+
npm install --quiet --no-audit --no-fund --no-package-json && \
40+
node "$(dirname "$0")/index-files.js" "$_response_file_path"

‎extractors/cds/tools/package.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "@advanced-security/codeql-sap-js_index-cds-files",
3+
"version": "1.0.0",
4+
"description": "CodeQL extractor for DB indexing of .cds.json files produced by the 'cds' compiler.",
5+
"main": "index-files.js",
6+
"dependencies": {
7+
"@sap/cds-dk": "^4.0.0",
8+
"child_process": "^1.0.2",
9+
"fs": "^0.0.1-security",
10+
"path": "^0.12.7"
11+
}
12+
}

‎extractors/javascript/tools/pre-finalize.cmd‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
@echo off
2+
3+
if not defined CODEQL_EXTRACTOR_CDS_SKIP_EXTRACTION (
4+
type NUL && "%CODEQL_DIST%\codeql" database index-files ^
5+
--include-extension=.cds ^
6+
--language cds ^
7+
--prune **\node_modules\**\* ^
8+
--prune **\.eslint\**\* ^
9+
--total-size-limit=10m ^
10+
-- ^
11+
"%CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE%"
12+
)
13+
14+
exit /b %ERRORLEVEL%

‎extractors/javascript/tools/pre-finalize.sh‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
22

33
set -eu
44

@@ -8,14 +8,17 @@ set -eu
88
# - extractors/javascript/tools/pre-finalize.sh (here)
99
# Any changes should be synchronized between these three places.
1010

11-
# Do not extract CDS files if the CODEQL_EXTRACTOR_CDS_SKIP_EXTRACTION environment variable is set
11+
# Do not extract CDS files if the
12+
# CODEQL_EXTRACTOR_CDS_SKIP_EXTRACTION
13+
# environment variable is set.
1214
if [ -z "${CODEQL_EXTRACTOR_CDS_SKIP_EXTRACTION:-}" ]; then
1315
# Call the index-files command with the CDS extractor
14-
"$CODEQL_DIST/codeql" database index-files \
15-
--language cds \
16-
--total-size-limit 10m \
16+
"${CODEQL_DIST}/codeql" database index-files \
1717
--include-extension=.cds \
18+
--language cds \
1819
--prune **/node_modules/**/* \
1920
--prune **/.eslint/**/* \
21+
--total-size-limit=10m \
22+
-- \
2023
"$CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE"
2124
fi

0 commit comments

Comments
 (0)