Refactor and improve test infrastructure, starting with automated version validation tests (#4126)

* renamed to end-to-end-tests and moved backend unit tests to unit-tests folder

* create folder for kits code, refactor imports to use @cmt and @test

* move variants into the kits folder

* refactor into better folder structure, still more to go

* Initial refactor making presets validation unit testing possible.

This seems to work well so far, I have v1-v3 presets tested, with some
more refactors regarding error reporting to come. However, the
design/infrastructure refactor seems to be allowing us to unit test
validation.

* implemented Presets version validation automated testing

* Added version testing for presets v1-9.

This adds testing for our version validation for all Presets versions
currently supported.
It creates presets on the fly and ensures that we correctly get errors,
or don't, depending on the setup and what fields are present with what
versions.

* fix workflows

* Ensure that we correctly initialize the presetsParser class

* refactor imports

* refactor imports

* more import fixes

* imports in projectStatus

* missed one

* workflow imports

* added comment about definite assignment

* add comments

* fix launch file to make sure breakpoints happen

* fix syntax error

Author: gcampbell-msft

.eslintignore

@@ -1,3 +1,3 @@
 *.js
-test/extension-tests/successful-build/project-folder/build
+test/end-to-end-tests/successful-build/project-folder/build
 tools/prepublish.js

.github/workflows/ci-main-linux.yml

@@ -59,17 +59,17 @@ jobs:
       - name: Run successful-build test
         uses: GabrielBB/[email protected]
         with:
-          run: yarn extensionTestsSuccessfulBuild
+          run: yarn endToEndTestsSuccessfulBuild
 
       - name: Run single root test
         uses: GabrielBB/[email protected]
         with:
-          run: yarn extensionTestsSingleRoot
+          run: yarn endToEndTestsSingleRoot
 
       - name: Run multi root test
         uses: GabrielBB/[email protected]
         with:
-          run: yarn extensionTestsMultiRoot
+          run: yarn endToEndTestsMultiRoot
 
       - name: Upload test logs if tests failed
         if: failure()

.github/workflows/ci-main-mac.yml

@@ -46,7 +46,7 @@ jobs:
 
 
       - name: Run successful-build test
-        run: yarn extensionTestsSuccessfulBuild
+        run: yarn endToEndTestsSuccessfulBuild
 
       - name: Run backend tests
         run: yarn backendTests
@@ -58,10 +58,10 @@ jobs:
         run: yarn unitTests
 
       - name: Run single root test
-        run: yarn extensionTestsSingleRoot
+        run: yarn endToEndTestsSingleRoot
 
       - name: Run multi root test
-        run: yarn extensionTestsMultiRoot
+        run: yarn endToEndTestsMultiRoot
 
       - name: Upload test logs if tests failed
         if: failure()

.github/workflows/ci-main.win.yml

@@ -55,13 +55,13 @@ jobs:
         run: yarn unitTests
 
       - name: Run successful-build test
-        run: yarn extensionTestsSuccessfulBuild
+        run: yarn endToEndTestsSuccessfulBuild
 
       - name: Run single root test
-        run: yarn extensionTestsSingleRoot
+        run: yarn endToEndTestsSingleRoot
 
       - name: Run multi root test
-        run: yarn extensionTestsMultiRoot
+        run: yarn endToEndTestsMultiRoot
 
       - name: Upload test logs if tests failed
         if: failure()

.vscode/launch.json

@@ -33,9 +33,9 @@
             "sourceMaps": true,
             "outFiles": [
                 "${workspaceFolder}/out/*",
-                "${workspaceFolder}/out/src/*",
+                "${workspaceFolder}/out/src/**",
                 "${workspaceFolder}/out/test/*",
-                "${workspaceFolder}/out/test/unit-tests/*"
+                "${workspaceFolder}/out/test/unit-tests/**",
             ],
             "env": {
                 "CMT_TESTING": "1",
@@ -50,17 +50,17 @@
             "request": "launch",
             "runtimeExecutable": "${execPath}",
             "args": [
-                "${workspaceFolder}/test/extension-tests/successful-build/project-folder",
+                "${workspaceFolder}/test/end-to-end-tests/successful-build/project-folder",
                 "--extensionDevelopmentPath=${workspaceFolder}",
-                "--extensionTestsPath=${workspaceFolder}/out/test/extension-tests/successful-build/index"
+                "--extensionTestsPath=${workspaceFolder}/out/test/end-to-end-tests/successful-build/index"
             ],
             "sourceMaps": true,
             "outFiles": [
                 "${workspaceFolder}/out/*",
-                "${workspaceFolder}/out/src/*",
+                "${workspaceFolder}/out/src/**",
                 "${workspaceFolder}/out/test/*",
-                "${workspaceFolder}/out/test/extension-tests/successful-build/*",
-                "${workspaceFolder}/out/test/extension-tests/successful-build/test/*"
+                "${workspaceFolder}/out/test/end-to-end-tests/successful-build/*",
+                "${workspaceFolder}/out/test/end-to-end-tests/successful-build/test/*"
             ],
             "preLaunchTask": "Pretest",
             "env": {
@@ -75,17 +75,17 @@
             "request": "launch",
             "runtimeExecutable": "${execPath}",
             "args": [
-                "${workspaceFolder}/test/extension-tests/single-root-UI/project-folder",
+                "${workspaceFolder}/test/end-to-end-tests/single-root-UI/project-folder",
                 "--extensionDevelopmentPath=${workspaceFolder}",
-                "--extensionTestsPath=${workspaceFolder}/out/test/extension-tests/single-root-UI/index"
+                "--extensionTestsPath=${workspaceFolder}/out/test/end-to-end-tests/single-root-UI/index"
             ],
             "sourceMaps": true,
             "outFiles": [
                 "${workspaceFolder}/out/*",
-                "${workspaceFolder}/out/src/*",
+                "${workspaceFolder}/out/src/**",
                 "${workspaceFolder}/out/test/*",
-                "${workspaceFolder}/out/test/extension-tests/single-root-UI/*",
-                "${workspaceFolder}/out/test/extension-tests/single-root-UI/test/*"
+                "${workspaceFolder}/out/test/end-to-end-tests/single-root-UI/*",
+                "${workspaceFolder}/out/test/end-to-end-tests/single-root-UI/test/*"
             ],
             "preLaunchTask": "Pretest",
             "env": {
@@ -100,17 +100,17 @@
             "request": "launch",
             "runtimeExecutable": "${execPath}",
             "args": [
-                "${workspaceFolder}/test/extension-tests/multi-root-UI/project-workspace.code-workspace",
+                "${workspaceFolder}/test/end-to-end-tests/multi-root-UI/project-workspace.code-workspace",
                 "--extensionDevelopmentPath=${workspaceFolder}",
-                "--extensionTestsPath=${workspaceFolder}/out/test/extension-tests/multi-root-UI/index"
+                "--extensionTestsPath=${workspaceFolder}/out/test/end-to-end-tests/multi-root-UI/index"
             ],
             "sourceMaps": true,
             "outFiles": [
                 "${workspaceFolder}/out/*",
-                "${workspaceFolder}/out/src/*",
+                "${workspaceFolder}/out/src/**",
                 "${workspaceFolder}/out/test/*",
-                "${workspaceFolder}/out/test/extension-tests/multi-root-UI/*",
-                "${workspaceFolder}/out/test/extension-tests/multi-root-UI/test/*"
+                "${workspaceFolder}/out/test/end-to-end-tests/multi-root-UI/*",
+                "${workspaceFolder}/out/test/end-to-end-tests/multi-root-UI/test/*"
             ],
             "preLaunchTask": "Pretest",
             "env": {
@@ -122,7 +122,7 @@
         {
             "type": "node",
             "request": "launch",
-            "name": "Run Backend tests",
+            "name": "Run Backend Unit tests",
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",
             "args": [
                 "-u",
@@ -134,7 +134,7 @@
                 "ts-node/register",
                 "-r",
                 "tsconfig-paths/register",
-                "${workspaceFolder}/test/backend-unit-tests/**/*.test.ts"
+                "${workspaceFolder}/test/unit-tests/backend/**/*.test.ts"
             ],
             "internalConsoleOptions": "openOnSessionStart",
             "env": {

package.json

@@ -3701,10 +3701,10 @@
     "lint": "gulp lint",
     "smokeTests": "yarn run pretest && node ./out/test/smoke/badProject/runTest.js && node ./out/test/smoke/goodProject/runTest.js && node ./out/test/smoke/noCtest/runTest.js",
     "unitTests": "yarn run pretest && node ./out/test/unit-tests/runTest.js",
-    "extensionTestsSuccessfulBuild": "yarn run pretest && node ./out/test/extension-tests/successful-build/runTest.js",
-    "extensionTestsSingleRoot": "yarn run pretest && node ./out/test/extension-tests/single-root-UI/runTest.js",
-    "extensionTestsMultiRoot": "yarn run pretest && node ./out/test/extension-tests/multi-root-UI/runTest.js",
-    "backendTests": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 999999 --colors -r ts-node/register -r tsconfig-paths/register ./test/backend-unit-tests/**/*.test.ts",
+    "endToEndTestsSuccessfulBuild": "yarn run pretest && node ./out/test/end-to-end-tests/successful-build/runTest.js",
+    "endToEndTestsSingleRoot": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-UI/runTest.js",
+    "endToEndTestsMultiRoot": "yarn run pretest && node ./out/test/end-to-end-tests/multi-root-UI/runTest.js",
+    "backendTests": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 999999 --colors -r ts-node/register -r tsconfig-paths/register ./test/unit-tests/backend/**/*.test.ts",
     "build-product-icon-font": "yarn --cwd ./tools/product-icon-font-generator/ install && yarn --cwd ./tools/product-icon-font-generator/ build && node ./tools/product-icon-font-generator/dist/index.js --source-directory ./res/product-icons/ --output-directory ./res/ --woff2"
   },
   "devDependencies": {

src/cache.ts

@@ -2,10 +2,10 @@
  * Module for reading from the CMake cache
  */ /** */
 
-import * as logging from './logging';
-import { fs } from './pr';
-import rollbar from './rollbar';
-import * as util from './util';
+import * as logging from '@cmt/logging';
+import { fs } from '@cmt/pr';
+import rollbar from '@cmt/rollbar';
+import * as util from '@cmt/util';
 import * as nls from 'vscode-nls';
 
 nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })();

src/cmake/cmakeExecutable.ts renamed to src/cmakeExecutable.ts

@@ -1,6 +1,6 @@
-import * as proc from '../proc';
-import * as util from '../util';
-import {setContextAndStore} from '../extension';
+import * as proc from '@cmt/proc';
+import * as util from '@cmt/util';
+import {setContextAndStore} from '@cmt/extension';
 import * as logging from '@cmt/logging';
 import * as nls from 'vscode-nls';
 

src/cmakeProject.ts

@@ -1,7 +1,7 @@
 import { CMakeCache } from '@cmt/cache';
-import { CMakeExecutable, getCMakeExecutableInformation } from '@cmt/cmake/cmakeExecutable';
+import { CMakeExecutable, getCMakeExecutableInformation } from '@cmt/cmakeExecutable';
 import { CompilationDatabase } from '@cmt/compilationDatabase';
-import * as debuggerModule from '@cmt/debugger';
+import * as debuggerModule from '@cmt/debug/debugger';
 import collections from '@cmt/diagnostics/collections';
 import * as shlex from '@cmt/shlex';
 import { Strand } from '@cmt/strand';
@@ -23,34 +23,34 @@ import {
     ExecutableTarget,
     NoGeneratorError
 } from '@cmt/drivers/drivers';
-import { CTestDriver } from './ctest';
-import { CPackDriver } from './cpack';
-import { WorkflowDriver } from './workflow';
-import { CMakeBuildConsumer } from './diagnostics/build';
-import { CMakeOutputConsumer } from './diagnostics/cmake';
-import { FileDiagnostic, populateCollection } from './diagnostics/util';
-import { expandStrings, expandString, ExpansionOptions } from './expand';
-import { CMakeGenerator, Kit, SpecialKits } from './kit';
-import * as logging from './logging';
-import { fs } from './pr';
+import { CTestDriver } from '@cmt/ctest';
+import { CPackDriver } from '@cmt/cpack';
+import { WorkflowDriver } from '@cmt/workflow';
+import { CMakeBuildConsumer } from '@cmt/diagnostics/build';
+import { CMakeOutputConsumer } from '@cmt/diagnostics/cmake';
+import { FileDiagnostic, populateCollection } from '@cmt/diagnostics/util';
+import { expandStrings, expandString, ExpansionOptions } from '@cmt/expand';
+import { CMakeGenerator, Kit, SpecialKits } from '@cmt/kits/kit';
+import * as logging from '@cmt/logging';
+import { fs } from '@cmt/pr';
 import { buildCmdStr, DebuggerEnvironmentVariable, ExecutionResult, ExecutionOptions } from './proc';
-import { FireLate, Property } from './prop';
-import rollbar from './rollbar';
-import * as telemetry from './telemetry';
-import { VariantManager } from './variant';
+import { FireLate, Property } from '@cmt/prop';
+import rollbar from '@cmt/rollbar';
+import * as telemetry from '@cmt/telemetry';
+import { VariantManager } from '@cmt/kits/variant';
 import * as nls from 'vscode-nls';
-import { ConfigurationWebview } from './cacheView';
-import { enableFullFeatureSet, extensionManager, updateFullFeatureSet, setContextAndStore } from './extension';
-import { CMakeCommunicationMode, ConfigurationReader, OptionConfig, UseCMakePresets, checkConfigureOverridesPresent } from './config';
-import * as preset from '@cmt/preset';
+import { ConfigurationWebview } from '@cmt/ui/cacheView';
+import { enableFullFeatureSet, extensionManager, updateFullFeatureSet, setContextAndStore } from '@cmt/extension';
+import { CMakeCommunicationMode, ConfigurationReader, OptionConfig, UseCMakePresets, checkConfigureOverridesPresent } from '@cmt/config';
+import * as preset from '@cmt/presets/preset';
 import * as util from '@cmt/util';
-import { Environment, EnvironmentUtils } from './environmentVariables';
-import { KitsController } from './kitsController';
-import { PresetsController } from './presetsController';
-import paths from './paths';
-import { ProjectController } from './projectController';
+import { Environment, EnvironmentUtils } from '@cmt/environmentVariables';
+import { KitsController } from '@cmt/kits/kitsController';
+import { PresetsController } from '@cmt/presets/presetsController';
+import paths from '@cmt/paths';
+import { ProjectController } from '@cmt/projectController';
 import { MessageItem } from 'vscode';
-import { DebugTrackerFactory, DebuggerInformation, getDebuggerPipeName } from './debug/debuggerConfigureDriver';
+import { DebugTrackerFactory, DebuggerInformation, getDebuggerPipeName } from '@cmt/debug/cmakeDebugger/debuggerConfigureDriver';
 import { ConfigurationType } from 'vscode-cmake-tools';
 
 nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })();

src/cmakeTaskProvider.ts

@@ -10,7 +10,7 @@ import { Environment, EnvironmentUtils } from './environmentVariables';
 import * as logging from './logging';
 import { extensionManager, getActiveProject } from './extension';
 import { CMakeProject, ConfigureTrigger } from './cmakeProject';
-import * as preset from '@cmt/preset';
+import * as preset from '@cmt/presets/preset';
 import { UseCMakePresets } from './config';
 import * as telemetry from '@cmt/telemetry';
 import * as util from '@cmt/util';