diff --git a/tested/configs.py b/tested/configs.py index ea7515396..3d287e4b7 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -130,7 +130,7 @@ class Bundle: language: "Language" global_config: GlobalConfig out: IO - preprocessor_messages: list[ExtendedMessage] = [] + messages: set[ExtendedMessage] = set() @property def config(self) -> DodonaConfig: @@ -209,7 +209,7 @@ def create_bundle( output: IO, suite: Suite, language: str | None = None, - preprocessor_messages: list[ExtendedMessage] | None = None, + messages: set[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -219,8 +219,7 @@ def create_bundle( :param suite: The test suite. :param language: Optional programming language. If None, the one from the Dodona configuration will be used. - :param preprocessor_messages: Indicator that the natural language translator - for the DSL key that was not defined in any translations map. + :param messages: Messages generated out of the preprocessor and the translate parser. :return: The configuration bundle. """ @@ -237,12 +236,13 @@ def create_bundle( suite=suite, ) lang_config = langs.get_language(global_config, language) - if preprocessor_messages is None: - preprocessor_messages = [] + + if messages is None: + messages = set() return Bundle( language=lang_config, global_config=global_config, out=output, - preprocessor_messages=preprocessor_messages, + messages=messages, ) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 5690f12c4..aeb89bc17 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -83,11 +83,11 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: rendered_dsl = self.render_children(element) # Parse the DSL - parsed_dsl = parse_dsl(rendered_dsl) + parsed_dsl_with_messages = parse_dsl(rendered_dsl) # Get all actual tests tests = [] - for tab in parsed_dsl.tabs: + for tab in parsed_dsl_with_messages.data.tabs: for context in tab.contexts: for testcase in context.testcases: tests.append(testcase) diff --git a/tested/dodona.py b/tested/dodona.py index 90e5f7683..83a4fcd90 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -27,7 +27,7 @@ class Permission(StrEnum): ZEUS = auto() -@define +@define(frozen=True) class ExtendedMessage: description: str format: str = "text" diff --git a/tested/dsl/dsl_errors.py b/tested/dsl/dsl_errors.py index f5949729f..84cd1a796 100644 --- a/tested/dsl/dsl_errors.py +++ b/tested/dsl/dsl_errors.py @@ -102,3 +102,14 @@ def build_preprocessor_messages( ) for key in translations_missing_key ] + + +def build_deprecated_language_message() -> ExtendedMessage: + """ + Builds a message for not using the '!programming_language' tag in the DSL. + :return: The deprecation message. + """ + return ExtendedMessage( + f"WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!", + permission=Permission.STAFF, + ) diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index 167e0e147..51f0647c2 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -1266,6 +1266,32 @@ "type" : "string", "description" : "A language-specific literal, which will be used verbatim." } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!programming_language" + }, + "value":{ + "type": "object", + "description" : "Programming-language-specific statement or expression.", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } } ] }, @@ -1313,6 +1339,57 @@ ] } }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!programming_language" + }, + "value":{ + "type": "object", + "description" : "Programming-language-specific statement or expression.", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "oneOf" : [ + { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + } + } + } + }, { "type" : "object", "required": [ diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index d9e3f4074..0471d175d 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -464,6 +464,14 @@ }, { "description" : "Programming-language-specific statement or expression.", + "anyOf": [ + { + "type": "object" + }, + { + "type": "programming_language" + } + ], "type" : "object", "minProperties" : 1, "propertyNames" : { @@ -873,7 +881,8 @@ "not" : { "type" : [ "oracle", - "expression" + "expression", + "programming_language" ] } }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 1e26a991e..35d746279 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -30,7 +30,11 @@ ) from tested.dodona import ExtendedMessage from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string -from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error +from tested.dsl.dsl_errors import ( + build_deprecated_language_message, + handle_dsl_validation_errors, + raise_yaml_error, +) from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( BooleanType, @@ -67,7 +71,7 @@ TextOutputChannel, ValueOutputChannel, ) -from tested.utils import get_args, recursive_dict_merge +from tested.utils import DataWithMessage, get_args, recursive_dict_merge YamlDict = dict[str, "YamlObject"] @@ -86,9 +90,22 @@ class ReturnOracle(dict): pass +class ProgrammingLanguageMap(dict): + pass + + OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle + YamlDict + | list + | bool + | float + | int + | str + | None + | ExpressionString + | ReturnOracle + | ProgrammingLanguageMap ) @@ -136,6 +153,16 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: return ReturnOracle(result) +def _return_programming_language_map( + loader: yaml.Loader, node: yaml.Node +) -> ProgrammingLanguageMap: + result = _parse_yaml_value(loader, node) + assert isinstance( + result, dict + ), f"A programming language map must be an object, got {result} which is a {type(result)}." + return ProgrammingLanguageMap(result) + + def _parse_yaml(yaml_stream: str) -> YamlObject: """ Parse a string or stream to YAML. @@ -146,6 +173,9 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) + yaml.add_constructor( + "!programming_language", _return_programming_language_map, loader + ) try: return yaml.load(yaml_stream, loader) @@ -161,6 +191,10 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) +def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, ProgrammingLanguageMap) + + def load_schema_validator( dsl_object: YamlObject = None, file: str = "schema-strict.json" ) -> Validator: @@ -189,9 +223,11 @@ def validate_tested_dsl_expression(value: object) -> bool: schema_object = json.load(schema_file) original_validator: Type[Validator] = validator_for(schema_object) - type_checker = original_validator.TYPE_CHECKER.redefine( - "oracle", is_oracle - ).redefine("expression", is_expression) + type_checker = ( + original_validator.TYPE_CHECKER.redefine("oracle", is_oracle) + .redefine("expression", is_expression) + .redefine("programming_language", is_programming_language_map) + ) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)( validate_tested_dsl_expression @@ -495,7 +531,10 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A statement cannot have an expected return value.") -def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: +def _convert_testcase( + testcase: YamlDict, context: DslContext +) -> DataWithMessage[Testcase]: + messages = set() context = context.deepen_context(testcase) # This is backwards compatability to some extend. @@ -511,6 +550,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: the_dict = {context.language: expr_stmt} else: assert isinstance(expr_stmt, dict) + if not isinstance(expr_stmt, ProgrammingLanguageMap): + messages.add(build_deprecated_language_message()) the_dict = expr_stmt the_dict = {SupportedLanguage(l): cast(str, v) for l, v in the_dict.items()} if "statement" in testcase: @@ -587,24 +628,32 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: else: the_description = None - return Testcase( - description=the_description, - input=the_input, - output=output, - link_files=context.files, - line_comment=line_comment, + return DataWithMessage( + data=Testcase( + description=the_description, + input=the_input, + output=output, + link_files=context.files, + line_comment=line_comment, + ), + messages=messages, ) -def _convert_context(context: YamlDict, dsl_context: DslContext) -> Context: +def _convert_context( + context: YamlDict, dsl_context: DslContext +) -> DataWithMessage[Context]: dsl_context = dsl_context.deepen_context(context) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) testcases = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) - return Context(testcases=testcases) + return DataWithMessage( + data=Context(testcases=testcases.data), + messages=testcases.messages, + ) -def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: +def _convert_tab(tab: YamlDict, context: DslContext) -> DataWithMessage[Tab]: """ Translate a DSL tab to a full test suite tab. @@ -619,44 +668,65 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts = _convert_dsl_list(tab["contexts"], context, _convert_context) + contexts_with_messages = _convert_dsl_list( + tab["contexts"], context, _convert_context + ) + contexts = contexts_with_messages.data + messages = contexts_with_messages.messages elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts = _convert_dsl_list(tab["cases"], context, _convert_context) + contexts_with_messages = _convert_dsl_list( + tab["cases"], context, _convert_context + ) + contexts = contexts_with_messages.data + messages = contexts_with_messages.messages elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) testcases = _convert_dsl_list(tab["testcases"], context, _convert_testcase) - contexts = [Context(testcases=[t]) for t in testcases] + messages = testcases.messages + contexts = [Context(testcases=[t]) for t in testcases.data] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) testcases = _convert_dsl_list(tab["scripts"], context, _convert_testcase) - contexts = [Context(testcases=[t]) for t in testcases] + messages = testcases.messages + contexts = [Context(testcases=[t]) for t in testcases.data] - return Tab(name=name, contexts=contexts) + return DataWithMessage( + data=Tab(name=name, contexts=contexts), + messages=messages, + ) T = TypeVar("T") def _convert_dsl_list( - dsl_list: list, context: DslContext, converter: Callable[[YamlDict, DslContext], T] -) -> list[T]: + dsl_list: list, + context: DslContext, + converter: Callable[[YamlDict, DslContext], DataWithMessage[T]], +) -> DataWithMessage[list[T]]: """ Convert a list of YAML objects into a test suite object. """ objects = [] + messages = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - objects.append(converter(dsl_object, context)) - return objects + obj = converter(dsl_object, context) + messages.update(obj.messages) + objects.append(obj.data) + return DataWithMessage( + data=objects, + messages=messages, + ) -def _convert_dsl(dsl_object: YamlObject) -> Suite: +def _convert_dsl(dsl_object: YamlObject) -> DataWithMessage[Suite]: """ Translate a DSL test suite into a full test suite. @@ -683,12 +753,15 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, namespace=namespace) + return DataWithMessage( + data=Suite(tabs=tabs.data, namespace=namespace), + messages=tabs.messages, + ) else: - return Suite(tabs=tabs) + return DataWithMessage(data=Suite(tabs=tabs.data), messages=tabs.messages) -def parse_dsl(dsl_string: str) -> Suite: +def parse_dsl(dsl_string: str) -> DataWithMessage[Suite]: """ Parse a string containing a DSL test suite into our representation, a test suite. @@ -708,5 +781,5 @@ def translate_to_test_suite(dsl_string: str) -> str: :param dsl_string: The DSL. :return: The test suite. """ - suite = parse_dsl(dsl_string) - return suite_to_json(suite) + suite_with_message = parse_dsl(dsl_string) + return suite_to_json(suite_with_message.data) diff --git a/tested/judge/core.py b/tested/judge/core.py index 811b21d79..4147aff59 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -116,8 +116,9 @@ def judge(bundle: Bundle): # Do the set-up for the judgement. collector = OutputManager(bundle.out) collector.add(StartJudgement()) - if bundle.preprocessor_messages: - collector.add_messages(bundle.preprocessor_messages) + if bundle.messages: + collector.add_messages(bundle.messages) + max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() diff --git a/tested/main.py b/tested/main.py index e72df435d..0246d5c43 100644 --- a/tested/main.py +++ b/tested/main.py @@ -19,7 +19,7 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): :param judge_output: Where the judge output will be written to. :param language: The language to use to translate the test-suite. """ - messages = [] + messages = set() try: with open(f"{config.resources}/{config.test_suite}", "r") as t: textual_suite = t.read() @@ -34,12 +34,20 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): is_yaml = ext.lower() in (".yaml", ".yml") if is_yaml: if language: - textual_suite, messages = apply_translations(textual_suite, language) - suite = parse_dsl(textual_suite) + textual_suite, new_messages = apply_translations(textual_suite, language) + messages.update(new_messages) + suite_with_messages = parse_dsl(textual_suite) + messages.update(suite_with_messages.messages) + suite = suite_with_messages.data else: suite = parse_test_suite(textual_suite) - pack = create_bundle(config, judge_output, suite, preprocessor_messages=messages) + pack = create_bundle( + config, + judge_output, + suite, + messages=messages, + ) from .judge import judge judge(pack) diff --git a/tested/utils.py b/tested/utils.py index e913c8b6f..6b5c838e1 100644 --- a/tested/utils.py +++ b/tested/utils.py @@ -7,9 +7,13 @@ from collections.abc import Callable, Iterable from itertools import zip_longest from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, TypeGuard, TypeVar +from typing import IO, TYPE_CHECKING, Any, Generic, TypeGuard, TypeVar from typing import get_args as typing_get_args +from attr import define + +from tested.dodona import ExtendedMessage + if TYPE_CHECKING: from tested.serialisation import Assignment @@ -56,6 +60,12 @@ def get_identifier() -> str: T = TypeVar("T") +@define +class DataWithMessage(Generic[T]): + data: T + messages: set[ExtendedMessage] + + def get_args(type_: Any) -> tuple[Any, ...]: """ Get the args of a type or the type itself. diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index aecf78846..d0bcac1a6 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -19,6 +19,7 @@ SequenceTypes, StringTypes, ) +from tested.dodona import Permission from tested.dsl import parse_dsl, translate_to_test_suite from tested.dsl.translate_parser import load_schema_validator from tested.serialisation import ( @@ -1327,6 +1328,101 @@ def test_programming_language_can_be_globally_configured(): assert testcase.input.literals.keys() == {"java"} +def test_deprecated_programming_language_map_gives_warning(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + data_w_messages = parse_dsl(yaml_str) + messages = data_w_messages.messages + assert messages + message = list(messages)[0] + assert ( + message.description + == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + ) + assert message.permission == Permission.STAFF + assert message.format == "text" + + +def test_programming_language_with_and_without_generate_same(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + print(suite) + + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: !programming_language + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + json_str = translate_to_test_suite(yaml_str) + suite2 = parse_test_suite(json_str) + print(suite2) + assert suite == suite2 + + +def test_programming_language_tag_gives_no_warning(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: !programming_language + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + data_w_messages = parse_dsl(yaml_str) + messages = data_w_messages.messages + assert not messages + + +def test_deprecated_programming_language_map_not_duplicate(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + - unit: "cube list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 3, 6, 9, 12] + """ + data_w_messages = parse_dsl(yaml_str) + messages = data_w_messages.messages + assert len(messages) == 1 + message = list(messages)[0] + assert ( + message.description + == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + ) + assert message.permission == Permission.STAFF + assert message.format == "text" + + def test_strict_json_schema_is_valid(): path_to_schema = Path(__file__).parent / "tested-draft7.json" with open(path_to_schema, "r") as schema_file: diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 34bb52270..6a2ed7225 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -151,7 +151,7 @@ def test_nat_lang_and_prog_lang_combination(): en: "tests(11)" nl: "testen(11)" return: 11 - - expression: + - expression: !programming_language javascript: "{{animal}}_javascript(1 + 1)" typescript: "{{animal}}_typescript(1 + 1)" java: "Submission.{{animal}}_java(1 + 1)" @@ -166,7 +166,7 @@ def test_nat_lang_and_prog_lang_combination(): testcases: - expression: tests(11) return: 11 - - expression: + - expression: !programming_language javascript: animals_javascript(1 + 1) typescript: animals_typescript(1 + 1) java: Submission.animals_java(1 + 1) diff --git a/tests/tested-draft7.json b/tests/tested-draft7.json index 1e49ec747..e995bcbf4 100644 --- a/tests/tested-draft7.json +++ b/tests/tested-draft7.json @@ -28,7 +28,8 @@ "object", "string", "oracle", - "expression" + "expression", + "programming_language" ] }, "stringArray": {