From f9e118f7336f4e4cb7c2e3a04192b67801449ed3 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 31 Mar 2025 15:59:29 +0200 Subject: [PATCH 01/19] Added the possibility for using the programming_language key --- tested/dsl/schema-strict-nat-translation.json | 24 ++++++++++++++++++ tested/dsl/schema-strict.json | 11 +++++++- tested/dsl/translate_parser.py | 25 +++++++++++++------ tested/nat_translation.py | 2 +- tests/test_preprocess_dsl.py | 5 ++-- 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index 4f23a30f0..e598cc2be 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -1252,6 +1252,18 @@ "type" : "string", "description" : "A statement of expression in Python-like syntax as YAML string." }, + { + "type": "object", + "description" : "Programming-language-specific statement or expression.", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + }, { "type" : "object", "required": [ @@ -1286,6 +1298,18 @@ "type" : "string", "description" : "A statement of expression in Python-like syntax as YAML string." }, + { + "type": "object", + "description" : "Programming-language-specific statement or expression.", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + }, { "type" : "object", "required": [ diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 8ec453be2..0729d67d6 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -464,6 +464,14 @@ }, { "description" : "Programming-language-specific statement or expression.", + "oneOf": [ + { + "type": "object" + }, + { + "type": "programming_language" + } + ], "type" : "object", "minProperties" : 1, "propertyNames" : { @@ -872,7 +880,8 @@ "not" : { "type" : [ "oracle", - "expression" + "expression", + "programming_language" ] } }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index a7b05504e..f070838c5 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -1,6 +1,4 @@ import json -import sys -import textwrap from collections.abc import Callable from decimal import Decimal from pathlib import Path @@ -9,7 +7,6 @@ import yaml from attrs import define, evolve, field from jsonschema import TypeChecker -from jsonschema.exceptions import ValidationError from jsonschema.protocols import Validator from jsonschema.validators import extend as extend_validator from jsonschema.validators import validator_for @@ -89,9 +86,13 @@ class ReturnOracle(dict): pass +class ProgrammingLanguageMap(dict): + pass + + OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle + YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | ProgrammingLanguageMap ) @@ -138,6 +139,13 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: ), f"A custom oracle must be an object, got {result} which is a {type(result)}." return ReturnOracle(result) +def _return_programming_language_map(loader: yaml.Loader, node: yaml.Node) -> ProgrammingLanguageMap: + result = _parse_yaml_value(loader, node) + assert isinstance( + result, dict + ), f"A programming language map must be an object, got {result} which is a {type(result)}." + return ProgrammingLanguageMap(result) + def _parse_yaml(yaml_stream: str) -> YamlObject: """ @@ -149,6 +157,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) + yaml.add_constructor("!programming_language", _return_programming_language_map, loader) try: return yaml.load(yaml_stream, loader) @@ -163,6 +172,8 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool: def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) +def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, ProgrammingLanguageMap) def test(value: object) -> bool: if not isinstance(value, str): @@ -184,7 +195,7 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator: original_validator: Type[Validator] = validator_for(schema_object) type_checker = original_validator.TYPE_CHECKER.redefine( "oracle", is_oracle - ).redefine("expression", is_expression) + ).redefine("expression", is_expression).redefine("programming_language", is_programming_language_map) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)(test) tested_validator = extend_validator(original_validator, type_checker=type_checker) @@ -500,11 +511,11 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: line_comment = "" _validate_testcase_combinations(testcase) if (expr_stmt := testcase.get("statement", testcase.get("expression"))) is not None: - if isinstance(expr_stmt, dict) or context.language != "tested": + if isinstance(expr_stmt, dict | ProgrammingLanguageMap) or context.language != "tested": if isinstance(expr_stmt, str): the_dict = {context.language: expr_stmt} else: - assert isinstance(expr_stmt, dict) + assert isinstance(expr_stmt, dict | ProgrammingLanguageMap) the_dict = expr_stmt the_dict = {SupportedLanguage(l): cast(str, v) for l, v in the_dict.items()} if "statement" in testcase: diff --git a/tested/nat_translation.py b/tested/nat_translation.py index d409fd4eb..a49047f7b 100644 --- a/tested/nat_translation.py +++ b/tested/nat_translation.py @@ -41,7 +41,7 @@ def represent_with_tag(self, tag, value): @staticmethod def custom_representer(dumper, data): if "__tag__" in data: - if data["__tag__"] != "!programming_language": + if data["__tag__"]: return dumper.represent_with_tag(data["__tag__"], data["value"]) else: data = data["value"] diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index ba5f93bb5..0ee661dbb 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -25,7 +25,6 @@ def validate_natural_translate(yaml_str: str, translated_yaml_str: str): yaml_object = parse_yaml(yaml_str) translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment) translated_yaml = convert_to_yaml(translated_dsl) - print(translated_yaml) assert translated_yaml.strip() == translated_yaml_str @@ -138,7 +137,7 @@ def test_return(): validate_natural_translate(yaml_str, translated_yaml_str) -def test_nat_lang_and_prog_lang_combo(): +def test_nat_lang_and_prog_lang_combination(): yaml_str = """ translations: animal: @@ -166,7 +165,7 @@ def test_nat_lang_and_prog_lang_combo(): testcases: - expression: tests(11) return: 11 - - expression: + - expression: !programming_language javascript: animals_javascript(1 + 1) typescript: animals_typescript(1 + 1) java: Submission.animals_java(1 + 1) From 3836d62739122005a367d1a9ce246fafdbca97a3 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 31 Mar 2025 16:02:00 +0200 Subject: [PATCH 02/19] Fixed some linting issues --- tested/dsl/translate_parser.py | 35 +++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index f070838c5..52dee9c85 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -92,7 +92,16 @@ class ProgrammingLanguageMap(dict): OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | ProgrammingLanguageMap + YamlDict + | list + | bool + | float + | int + | str + | None + | ExpressionString + | ReturnOracle + | ProgrammingLanguageMap ) @@ -139,7 +148,10 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: ), f"A custom oracle must be an object, got {result} which is a {type(result)}." return ReturnOracle(result) -def _return_programming_language_map(loader: yaml.Loader, node: yaml.Node) -> ProgrammingLanguageMap: + +def _return_programming_language_map( + loader: yaml.Loader, node: yaml.Node +) -> ProgrammingLanguageMap: result = _parse_yaml_value(loader, node) assert isinstance( result, dict @@ -157,7 +169,9 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) - yaml.add_constructor("!programming_language", _return_programming_language_map, loader) + yaml.add_constructor( + "!programming_language", _return_programming_language_map, loader + ) try: return yaml.load(yaml_stream, loader) @@ -172,9 +186,11 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool: def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) + def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ProgrammingLanguageMap) + def test(value: object) -> bool: if not isinstance(value, str): return False @@ -193,9 +209,11 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator: schema_object = json.load(schema_file) original_validator: Type[Validator] = validator_for(schema_object) - type_checker = original_validator.TYPE_CHECKER.redefine( - "oracle", is_oracle - ).redefine("expression", is_expression).redefine("programming_language", is_programming_language_map) + type_checker = ( + original_validator.TYPE_CHECKER.redefine("oracle", is_oracle) + .redefine("expression", is_expression) + .redefine("programming_language", is_programming_language_map) + ) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)(test) tested_validator = extend_validator(original_validator, type_checker=type_checker) @@ -511,7 +529,10 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: line_comment = "" _validate_testcase_combinations(testcase) if (expr_stmt := testcase.get("statement", testcase.get("expression"))) is not None: - if isinstance(expr_stmt, dict | ProgrammingLanguageMap) or context.language != "tested": + if ( + isinstance(expr_stmt, dict | ProgrammingLanguageMap) + or context.language != "tested" + ): if isinstance(expr_stmt, str): the_dict = {context.language: expr_stmt} else: From 537f0e84fe38e13670e2d7da586ee1db918782c7 Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 3 Apr 2025 09:26:27 +0200 Subject: [PATCH 03/19] Made some changes --- tested/nat_translation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tested/nat_translation.py b/tested/nat_translation.py index a49047f7b..4cb2c03c9 100644 --- a/tested/nat_translation.py +++ b/tested/nat_translation.py @@ -90,7 +90,7 @@ def translate_yaml( } elif isinstance(data, list): return [translate_yaml(item, translations, language, env) for item in data] - elif isinstance(data, str): + elif isinstance(data, str) and translations: try: result = env.from_string(data).render(translations) return result @@ -164,6 +164,7 @@ def run_translation( _, ext = os.path.splitext(path) assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}." parsed_yaml = parse_yaml(yaml_stream) + print(parsed_yaml) validate_pre_dsl(parsed_yaml) enviroment = create_enviroment() From 6675fa8dbdc3d13f25e6a4c67fbcbbf0f4c6490b Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 7 Apr 2025 12:24:56 +0200 Subject: [PATCH 04/19] Fixed tests --- tested/nat_translation.py | 6 ++---- tests/test_preprocess_dsl.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tested/nat_translation.py b/tested/nat_translation.py index 34176620e..43ce4c60c 100644 --- a/tested/nat_translation.py +++ b/tested/nat_translation.py @@ -90,10 +90,9 @@ def translate_yaml( } elif isinstance(data, list): return [translate_yaml(item, translations, language, env) for item in data] - elif isinstance(data, str) and translations: + elif isinstance(data, str): try: - result = env.from_string(data).render(translations) - return result + return env.from_string(data).render(translations) except TemplateSyntaxError: return data return data @@ -164,7 +163,6 @@ def run_translation( _, ext = os.path.splitext(path) assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}." parsed_yaml = parse_yaml(yaml_stream) - print(parsed_yaml) validate_pre_dsl(parsed_yaml) enviroment = create_enviroment() diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index c879c375f..c00d5fa64 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -150,7 +150,7 @@ def test_nat_lang_and_prog_lang_combination(): en: "tests(11)" nl: "testen(11)" return: 11 - - expression: + - expression: !programming_language javascript: "{{animal}}_javascript(1 + 1)" typescript: "{{animal}}_typescript(1 + 1)" java: "Submission.{{animal}}_java(1 + 1)" From 23dafd36564e80e081a26368fdfc536cac440389 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 7 Apr 2025 14:17:19 +0200 Subject: [PATCH 05/19] Fixed dsl tests and added warning --- tested/dsl/schema-strict.json | 2 +- tested/dsl/translate_parser.py | 49 +++++++++++++++++++++------------- tested/judge/core.py | 13 ++++++++- tested/testsuite.py | 1 + tests/tested-draft7.json | 3 ++- 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 1ff4bbf71..0471d175d 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -464,7 +464,7 @@ }, { "description" : "Programming-language-specific statement or expression.", - "oneOf": [ + "anyOf": [ { "type": "object" }, diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 52dee9c85..2709dcd23 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -10,6 +10,7 @@ from jsonschema.protocols import Validator from jsonschema.validators import extend as extend_validator from jsonschema.validators import validator_for +from typing_extensions import deprecated from tested.datatypes import ( AdvancedNumericTypes, @@ -518,7 +519,8 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A statement cannot have an expected return value.") -def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: +def _convert_testcase(testcase: YamlDict, context: DslContext) -> tuple[Testcase, bool]: + using_deprecated_prog_lang = False context = context.deepen_context(testcase) # This is backwards compatability to some extend. @@ -536,7 +538,9 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if isinstance(expr_stmt, str): the_dict = {context.language: expr_stmt} else: - assert isinstance(expr_stmt, dict | ProgrammingLanguageMap) + assert isinstance(expr_stmt, dict) + if not isinstance(expr_stmt, ProgrammingLanguageMap): + using_deprecated_prog_lang = True the_dict = expr_stmt the_dict = {SupportedLanguage(l): cast(str, v) for l, v in the_dict.items()} if "statement" in testcase: @@ -619,18 +623,18 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: output=output, link_files=context.files, line_comment=line_comment, - ) + ), using_deprecated_prog_lang -def _convert_context(context: YamlDict, dsl_context: DslContext) -> Context: +def _convert_context(context: YamlDict, dsl_context: DslContext) -> tuple[Context, bool]: dsl_context = dsl_context.deepen_context(context) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) - return Context(testcases=testcases) + testcases, deprecated_prog = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) + return Context(testcases=testcases), deprecated_prog -def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: +def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, bool]: """ Translate a DSL tab to a full test suite tab. @@ -641,45 +645,52 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: context = context.deepen_context(tab) name = tab.get("unit", tab.get("tab")) assert isinstance(name, str) + deprecated_prog = False # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts = _convert_dsl_list(tab["contexts"], context, _convert_context) + contexts, val = _convert_dsl_list(tab["contexts"], context, _convert_context) + elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts = _convert_dsl_list(tab["cases"], context, _convert_context) + contexts, val = _convert_dsl_list(tab["cases"], context, _convert_context) + deprecated_prog = deprecated_prog or val elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases = _convert_dsl_list(tab["testcases"], context, _convert_testcase) + testcases, val = _convert_dsl_list(tab["testcases"], context, _convert_testcase) contexts = [Context(testcases=[t]) for t in testcases] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases = _convert_dsl_list(tab["scripts"], context, _convert_testcase) + testcases, val = _convert_dsl_list(tab["scripts"], context, _convert_testcase) contexts = [Context(testcases=[t]) for t in testcases] - return Tab(name=name, contexts=contexts) + deprecated_prog = deprecated_prog or val + return Tab(name=name, contexts=contexts), deprecated_prog T = TypeVar("T") def _convert_dsl_list( - dsl_list: list, context: DslContext, converter: Callable[[YamlDict, DslContext], T] -) -> list[T]: + dsl_list: list, context: DslContext, converter: Callable[[YamlDict, DslContext], tuple[T, bool]] +) -> tuple[list[T], bool]: """ Convert a list of YAML objects into a test suite object. """ + deprecated_prog = False objects = [] for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - objects.append(converter(dsl_object, context)) - return objects + obj, val = converter(dsl_object, context) + deprecated_prog = deprecated_prog or val + objects.append(obj) + return objects, deprecated_prog def _convert_dsl(dsl_object: YamlObject) -> Suite: @@ -705,13 +716,13 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs = _convert_dsl_list(tab_list, context, _convert_tab) + tabs, deprecated_prog = _convert_dsl_list(tab_list, context, _convert_tab) if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, namespace=namespace) + return Suite(tabs=tabs, namespace=namespace, using_deprecated_prog_languages=deprecated_prog) else: - return Suite(tabs=tabs) + return Suite(tabs=tabs, using_deprecated_prog_languages=deprecated_prog) def parse_dsl(dsl_string: str) -> Suite: diff --git a/tested/judge/core.py b/tested/judge/core.py index a2353eb5c..047924799 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -123,11 +123,22 @@ def judge(bundle: Bundle): collector.add_messages( [ ExtendedMessage( - f"The natural translator found the key {key}, that was not defined in the corresponding translations maps!", + f"The natural translator found the key '{key}', that was not defined in the corresponding translations maps!", permission=Permission.STAFF, ) ] ) + + if bundle.suite.using_deprecated_prog_languages: + collector.add_messages( + [ + ExtendedMessage( + f"WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!", + permission=Permission.STAFF, + ) + ] + ) + max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() diff --git a/tested/testsuite.py b/tested/testsuite.py index 4ef5f0aac..b3b563f58 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -763,6 +763,7 @@ class Suite(WithFeatures, WithFunctions): tabs: list[Tab] = field(factory=list) namespace: str = "submission" + using_deprecated_prog_languages: bool = False def get_used_features(self) -> FeatureSet: """ diff --git a/tests/tested-draft7.json b/tests/tested-draft7.json index 1e49ec747..e995bcbf4 100644 --- a/tests/tested-draft7.json +++ b/tests/tested-draft7.json @@ -28,7 +28,8 @@ "object", "string", "oracle", - "expression" + "expression", + "programming_language" ] }, "stringArray": { From 214c5ee851633bc44bb5ea337b753c52a05c0e06 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 7 Apr 2025 15:52:34 +0200 Subject: [PATCH 06/19] fix linting --- tested/dsl/translate_parser.py | 35 +++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 2709dcd23..9f6821de8 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -617,20 +617,27 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> tuple[Testcase else: the_description = None - return Testcase( - description=the_description, - input=the_input, - output=output, - link_files=context.files, - line_comment=line_comment, - ), using_deprecated_prog_lang + return ( + Testcase( + description=the_description, + input=the_input, + output=output, + link_files=context.files, + line_comment=line_comment, + ), + using_deprecated_prog_lang, + ) -def _convert_context(context: YamlDict, dsl_context: DslContext) -> tuple[Context, bool]: +def _convert_context( + context: YamlDict, dsl_context: DslContext +) -> tuple[Context, bool]: dsl_context = dsl_context.deepen_context(context) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases, deprecated_prog = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) + testcases, deprecated_prog = _convert_dsl_list( + raw_testcases, dsl_context, _convert_testcase + ) return Context(testcases=testcases), deprecated_prog @@ -678,7 +685,9 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, bool]: def _convert_dsl_list( - dsl_list: list, context: DslContext, converter: Callable[[YamlDict, DslContext], tuple[T, bool]] + dsl_list: list, + context: DslContext, + converter: Callable[[YamlDict, DslContext], tuple[T, bool]], ) -> tuple[list[T], bool]: """ Convert a list of YAML objects into a test suite object. @@ -720,7 +729,11 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, namespace=namespace, using_deprecated_prog_languages=deprecated_prog) + return Suite( + tabs=tabs, + namespace=namespace, + using_deprecated_prog_languages=deprecated_prog, + ) else: return Suite(tabs=tabs, using_deprecated_prog_languages=deprecated_prog) From d98aa8216586a885276b9040d5041a4d915b7a06 Mon Sep 17 00:00:00 2001 From: breblanc Date: Thu, 10 Apr 2025 10:09:32 +0200 Subject: [PATCH 07/19] removed default --- tested/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tested/__main__.py b/tested/__main__.py index 72f9f0850..ba8779bb1 100644 --- a/tested/__main__.py +++ b/tested/__main__.py @@ -36,7 +36,6 @@ "--translate", type=str, help="Specifies the language to translate translate the dsl to.", - default="-", ) parser = parser.parse_args() From 202168bceadf2e5840b702fa675308ba1ebb1bed Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 14 Apr 2025 16:14:54 +0200 Subject: [PATCH 08/19] added extra bit to the schema --- tested/dsl/schema-strict-nat-translation.json | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index fe5de34be..51f0647c2 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -1359,8 +1359,33 @@ "$ref" : "#/definitions/programmingLanguage" }, "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "oneOf" : [ + { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] } } } From c4c2fe445ed3a8dae6641c273975eefa156878f0 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 28 Apr 2025 20:25:24 +0200 Subject: [PATCH 09/19] using messages instead of boolean in parser --- tested/configs.py | 10 ++++- tested/descriptions/renderer.py | 2 +- tested/dodona.py | 2 +- tested/dsl/__main__.py | 2 +- tested/dsl/dsl_errors.py | 24 +++------- tested/dsl/translate_parser.py | 65 ++++++++++++--------------- tested/judge/core.py | 8 +--- tested/main.py | 6 +-- tested/testsuite.py | 1 - tests/test_dsl_yaml.py | 78 ++++++++++++++++----------------- 10 files changed, 91 insertions(+), 107 deletions(-) diff --git a/tested/configs.py b/tested/configs.py index ea7515396..86a2169e9 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -131,6 +131,7 @@ class Bundle: global_config: GlobalConfig out: IO preprocessor_messages: list[ExtendedMessage] = [] + parser_messages: list[ExtendedMessage] = [] @property def config(self) -> DodonaConfig: @@ -210,6 +211,7 @@ def create_bundle( suite: Suite, language: str | None = None, preprocessor_messages: list[ExtendedMessage] | None = None, + parser_messages: list[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -219,8 +221,8 @@ def create_bundle( :param suite: The test suite. :param language: Optional programming language. If None, the one from the Dodona configuration will be used. - :param preprocessor_messages: Indicator that the natural language translator - for the DSL key that was not defined in any translations map. + :param preprocessor_messages: Messages generated out of the preprocessor. + :param parser_messages: Messages generated out of the DSL-parser. :return: The configuration bundle. """ @@ -240,9 +242,13 @@ def create_bundle( if preprocessor_messages is None: preprocessor_messages = [] + if parser_messages is None: + parser_messages = [] + return Bundle( language=lang_config, global_config=global_config, out=output, preprocessor_messages=preprocessor_messages, + parser_messages=parser_messages ) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 5690f12c4..974e95023 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -83,7 +83,7 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: rendered_dsl = self.render_children(element) # Parse the DSL - parsed_dsl = parse_dsl(rendered_dsl) + parsed_dsl, _ = parse_dsl(rendered_dsl) # Get all actual tests tests = [] diff --git a/tested/dodona.py b/tested/dodona.py index 90e5f7683..83a4fcd90 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -27,7 +27,7 @@ class Permission(StrEnum): ZEUS = auto() -@define +@define(frozen=True) class ExtendedMessage: description: str format: str = "text" diff --git a/tested/dsl/__main__.py b/tested/dsl/__main__.py index 6066d9742..864950b5c 100644 --- a/tested/dsl/__main__.py +++ b/tested/dsl/__main__.py @@ -24,7 +24,7 @@ with smart_close(parser.input) as input_file: dsl = input_file.read() -suite = translate_to_test_suite(dsl) +suite, _ = translate_to_test_suite(dsl) with smart_close(parser.output) as output_file: output_file.write(suite) diff --git a/tested/dsl/dsl_errors.py b/tested/dsl/dsl_errors.py index a506c90aa..84cd1a796 100644 --- a/tested/dsl/dsl_errors.py +++ b/tested/dsl/dsl_errors.py @@ -104,22 +104,12 @@ def build_preprocessor_messages( ] -def build_translate_parser_messages( - using_deprecated_prog_languages: bool, -) -> list[ExtendedMessage]: +def build_deprecated_language_message() -> ExtendedMessage: """ - Build the translate parser messages from the missing keys. - - :param translations_missing_key: The missing keys. - :return: The translate parser messages. + Builds a message for not using the '!programming_language' tag in the DSL. + :return: The deprecation message. """ - messages = [] - - if using_deprecated_prog_languages: - messages.append( - ExtendedMessage( - f"WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!", - permission=Permission.STAFF, - ) - ) - return messages + return ExtendedMessage( + f"WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!", + permission=Permission.STAFF, + ) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index dfd9d5c28..b8150684e 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -30,7 +30,8 @@ ) from tested.dodona import ExtendedMessage from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string -from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error +from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error, \ + build_deprecated_language_message from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( BooleanType, @@ -527,8 +528,8 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A statement cannot have an expected return value.") -def _convert_testcase(testcase: YamlDict, context: DslContext) -> tuple[Testcase, bool]: - using_deprecated_prog_lang = False +def _convert_testcase(testcase: YamlDict, context: DslContext) -> tuple[Testcase, set[ExtendedMessage]]: + messages = set() context = context.deepen_context(testcase) # This is backwards compatability to some extend. @@ -548,7 +549,7 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> tuple[Testcase else: assert isinstance(expr_stmt, dict) if not isinstance(expr_stmt, ProgrammingLanguageMap): - using_deprecated_prog_lang = True + messages.add(build_deprecated_language_message()) the_dict = expr_stmt the_dict = {SupportedLanguage(l): cast(str, v) for l, v in the_dict.items()} if "statement" in testcase: @@ -633,23 +634,23 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> tuple[Testcase link_files=context.files, line_comment=line_comment, ), - using_deprecated_prog_lang, + messages, ) def _convert_context( context: YamlDict, dsl_context: DslContext -) -> tuple[Context, bool]: +) -> tuple[Context, set[ExtendedMessage]]: dsl_context = dsl_context.deepen_context(context) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases, deprecated_prog = _convert_dsl_list( + testcases, messages = _convert_dsl_list( raw_testcases, dsl_context, _convert_testcase ) - return Context(testcases=testcases), deprecated_prog + return Context(testcases=testcases), messages -def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, bool]: +def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, set[ExtendedMessage]]: """ Translate a DSL tab to a full test suite tab. @@ -660,33 +661,29 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, bool]: context = context.deepen_context(tab) name = tab.get("unit", tab.get("tab")) assert isinstance(name, str) - deprecated_prog = False # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts, val = _convert_dsl_list(tab["contexts"], context, _convert_context) - + contexts, messages = _convert_dsl_list(tab["contexts"], context, _convert_context) elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts, val = _convert_dsl_list(tab["cases"], context, _convert_context) - deprecated_prog = deprecated_prog or val + contexts, messages = _convert_dsl_list(tab["cases"], context, _convert_context) elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases, val = _convert_dsl_list(tab["testcases"], context, _convert_testcase) + testcases, messages = _convert_dsl_list(tab["testcases"], context, _convert_testcase) contexts = [Context(testcases=[t]) for t in testcases] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases, val = _convert_dsl_list(tab["scripts"], context, _convert_testcase) + testcases, messages = _convert_dsl_list(tab["scripts"], context, _convert_testcase) contexts = [Context(testcases=[t]) for t in testcases] - deprecated_prog = deprecated_prog or val - return Tab(name=name, contexts=contexts), deprecated_prog + return Tab(name=name, contexts=contexts), messages T = TypeVar("T") @@ -695,22 +692,22 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, bool]: def _convert_dsl_list( dsl_list: list, context: DslContext, - converter: Callable[[YamlDict, DslContext], tuple[T, bool]], -) -> tuple[list[T], bool]: + converter: Callable[[YamlDict, DslContext], tuple[T, set[ExtendedMessage]]], +) -> tuple[list[T], set[ExtendedMessage]]: """ Convert a list of YAML objects into a test suite object. """ - deprecated_prog = False objects = [] + messages = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - obj, val = converter(dsl_object, context) - deprecated_prog = deprecated_prog or val + obj, new_messages = converter(dsl_object, context) + messages.update(new_messages) objects.append(obj) - return objects, deprecated_prog + return objects, messages -def _convert_dsl(dsl_object: YamlObject) -> Suite: +def _convert_dsl(dsl_object: YamlObject) -> tuple[Suite, set[ExtendedMessage]]: """ Translate a DSL test suite into a full test suite. @@ -733,20 +730,16 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs, deprecated_prog = _convert_dsl_list(tab_list, context, _convert_tab) + tabs, messages = _convert_dsl_list(tab_list, context, _convert_tab) if namespace: assert isinstance(namespace, str) - return Suite( - tabs=tabs, - namespace=namespace, - using_deprecated_prog_languages=deprecated_prog, - ) + return Suite(tabs=tabs, namespace=namespace), messages else: - return Suite(tabs=tabs, using_deprecated_prog_languages=deprecated_prog) + return Suite(tabs=tabs), messages -def parse_dsl(dsl_string: str) -> Suite: +def parse_dsl(dsl_string: str) -> tuple[Suite, set[ExtendedMessage]]: """ Parse a string containing a DSL test suite into our representation, a test suite. @@ -759,12 +752,12 @@ def parse_dsl(dsl_string: str) -> Suite: return _convert_dsl(dsl_object) -def translate_to_test_suite(dsl_string: str) -> str: +def translate_to_test_suite(dsl_string: str) -> tuple[str, set[ExtendedMessage]]: """ Convert a DSL to a test suite. :param dsl_string: The DSL. :return: The test suite. """ - suite = parse_dsl(dsl_string) - return suite_to_json(suite) + suite, messages = parse_dsl(dsl_string) + return suite_to_json(suite), messages diff --git a/tested/judge/core.py b/tested/judge/core.py index 7c682c508..c1c648873 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -18,7 +18,6 @@ StatusMessage, report_update, ) -from tested.dsl.dsl_errors import build_translate_parser_messages from tested.features import is_supported from tested.internationalization import get_i18n_string, set_locale from tested.judge.collector import OutputManager @@ -120,11 +119,8 @@ def judge(bundle: Bundle): if bundle.preprocessor_messages: collector.add_messages(bundle.preprocessor_messages) - # TODO: In the PR of adding file usage to the DSL, other deprecation warnings were added during parsing. - # So I think it's a nice split to have messages from parser and from the preprocessor. - collector.add_messages( - build_translate_parser_messages(bundle.suite.using_deprecated_prog_languages) - ) + if bundle.parser_messages: + collector.add_messages(bundle.parser_messages) max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() diff --git a/tested/main.py b/tested/main.py index e72df435d..b9574e758 100644 --- a/tested/main.py +++ b/tested/main.py @@ -35,11 +35,11 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): if is_yaml: if language: textual_suite, messages = apply_translations(textual_suite, language) - suite = parse_dsl(textual_suite) + suite, parse_messages = parse_dsl(textual_suite) else: - suite = parse_test_suite(textual_suite) + suite, parse_messages = parse_test_suite(textual_suite) - pack = create_bundle(config, judge_output, suite, preprocessor_messages=messages) + pack = create_bundle(config, judge_output, suite, preprocessor_messages=messages, parser_messages=parse_messages) from .judge import judge judge(pack) diff --git a/tested/testsuite.py b/tested/testsuite.py index b3b563f58..4ef5f0aac 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -763,7 +763,6 @@ class Suite(WithFeatures, WithFunctions): tabs: list[Tab] = field(factory=list) namespace: str = "submission" - using_deprecated_prog_languages: bool = False def get_used_features(self) -> FeatureSet: """ diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index aecf78846..726150f48 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -57,7 +57,7 @@ def test_parse_one_tab_ctx(): stderr: "Error string" exit_code: 1 """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert suite.namespace == "solution" assert len(suite.tabs) == 1 @@ -89,7 +89,7 @@ def test_parse_ctx_exception(): - arguments: [ "--arg", "error" ] exception: "Error" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 2 tab = suite.tabs[0] @@ -138,7 +138,7 @@ def test_parse_ctx_exception_types(): csharp: "System.DivideByZeroException" python: "ZeroDivisionError" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 2 tab = suite.tabs[0] @@ -205,7 +205,7 @@ def test_parse_ctx_with_config(): data: " Fail " """ args = ["-a", "2.125", "1.212"] - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -278,7 +278,7 @@ def test_statements(): - expression: 'safe.content()' return: !expression 'uint8(5)' """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -315,7 +315,7 @@ def test_expression_and_main(): - expression: 'add(5, 7)' return: 12 """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -341,7 +341,7 @@ def test_expression(): - expression: "heir(8, 3)" return: [ 3, 6, 9, 12, 15, 2, 7, 1, 13, 8, 16, 10, 14, 4, 11, 5 ] """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -366,7 +366,7 @@ def test_expression_with_explicit_language(): - expression: "heir(8, 3)" return: [ 3, 6, 9, 12, 15, 2, 7, 1, 13, 8, 16, 10, 14, 4, 11, 5 ] """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -418,7 +418,7 @@ def test_statement_with_yaml_dict(): alpha: 5 beta: 6 """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -451,7 +451,7 @@ def test_global_definition_config_trickles_down(): stderr: "Error string" exit_code: 1 """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout assert isinstance(stdout.oracle, GenericTextOracle) @@ -481,7 +481,7 @@ def test_global_config_trickles_down(): stderr: "Error string" exit_code: 1 """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout assert isinstance(stdout.oracle, GenericTextOracle) @@ -509,7 +509,7 @@ def test_tab_config_trickles_down_stdout(): stderr: "Error string" exit_code: 1 """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout assert isinstance(stdout.oracle, GenericTextOracle) @@ -537,7 +537,7 @@ def test_tab_config_trickles_down_stderr(): stderr: "Error string" exit_code: 1 """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr assert isinstance(stderr.oracle, GenericTextOracle) @@ -556,7 +556,7 @@ def test_expression_raw_return(): - expression: 'test()' return: !expression '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]' """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -588,7 +588,7 @@ def test_empty_constructor(function_name, result): - expression: 'test()' return: !expression '{function_name}()' """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -618,7 +618,7 @@ def test_empty_constructor_with_param(function_name, result): - expression: 'test()' return: !expression '{function_name}([])' """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -640,7 +640,7 @@ def test_text_built_in_checks_implied(): stdout: data: "hallo" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -664,7 +664,7 @@ def test_text_built_in_checks_explicit(): data: "hallo" oracle: "builtin" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -691,7 +691,7 @@ def test_text_custom_checks_correct(): name: "evaluate_test" arguments: [!expression "'yes'", 5, !expression "set([5, 5])"] """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -728,7 +728,7 @@ def test_value_built_in_checks_implied(): return: !oracle value: !expression "'hallo'" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -757,7 +757,7 @@ def test_file_custom_check_correct(): name: "evaluate_test" file: "test.py" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -785,7 +785,7 @@ def test_value_built_in_checks_explicit(): value: "hallo" oracle: "builtin" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -814,7 +814,7 @@ def test_value_custom_checks_correct(): name: "evaluate_test" arguments: ['yes', 5, !expression "set([5, 5])"] """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -863,7 +863,7 @@ def test_value_specific_checks_correct(): javascript: - "js" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -931,7 +931,7 @@ def test_yaml_set_tag_is_supported(): - expression: 'test()' return: !!set {5, 6} """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -974,7 +974,7 @@ def test_yaml_custom_tags_are_supported(all_types, value): - expression: 'test()' return: !{the_type} {json_type} """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -1000,7 +1000,7 @@ def test_global_language_literals(): javascript: "hello()" python: "hello_2()" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -1030,7 +1030,7 @@ def test_one_language_literal(): javascript: "hello()" python: "hello_2()" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -1147,9 +1147,9 @@ def test_one_language_literal(): ], ) def test_old_and_new_names_work(old, new): - old_json = translate_to_test_suite(old) + old_json, _ = translate_to_test_suite(old) old_suite = parse_test_suite(old_json) - new_json = translate_to_test_suite(new) + new_json, _ = translate_to_test_suite(new) new_suite = parse_test_suite(new_json) assert old_suite == new_suite @@ -1186,7 +1186,7 @@ def test_files_are_propagated(): - name: "test" url: "twooo.md" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) tab = suite.tabs[0] ctx0, ctx1 = tab.contexts @@ -1209,7 +1209,7 @@ def test_newlines_are_added_to_stdout(): config: tryFloatingPoint: true """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.data assert actual_stdout == "12\n" @@ -1221,7 +1221,7 @@ def test_newlines_are_added_to_stdout(): - arguments: [ '-a', '5', '7' ] stdout: "hello" """ - json_str = translate_to_test_suite(yaml_str2) + json_str, _ = translate_to_test_suite(yaml_str2) suite = parse_test_suite(json_str) actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.data assert actual_stdout == "hello\n" @@ -1238,7 +1238,7 @@ def test_newlines_are_added_to_stderr(): config: tryFloatingPoint: true """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "12\n" @@ -1250,7 +1250,7 @@ def test_newlines_are_added_to_stderr(): - arguments: [ '-a', '5', '7' ] stderr: "hello" """ - json_str = translate_to_test_suite(yaml_str2) + json_str, _ = translate_to_test_suite(yaml_str2) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "hello\n" @@ -1266,7 +1266,7 @@ def test_no_duplicate_newlines_are_added(): hello world """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual = suite.tabs[0].contexts[0].testcases[0].output.stdout.data assert actual == "hello\nworld\n" @@ -1284,7 +1284,7 @@ def test_can_disable_normalizing_newlines(): tryFloatingPoint: true normalizeTrailingNewlines: false """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "12" @@ -1298,7 +1298,7 @@ def test_empty_text_data_newlines(): - arguments: [ '-a', '5', '7' ] stderr: "" """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "" @@ -1314,7 +1314,7 @@ def test_programming_language_can_be_globally_configured(): - expression: "Numbers.oddValues(new int[]{1, 2, 3, 4, 5, 6, 7, 8})" return: [1, 3, 5, 7] """ - json_str = translate_to_test_suite(yaml_str) + json_str, _ = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] From 8d5f70021df090a0b9406858c2aec167b29bd934 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 28 Apr 2025 20:52:02 +0200 Subject: [PATCH 10/19] hotfix --- tested/configs.py | 8 ++++---- tested/dsl/__main__.py | 2 +- tested/dsl/translate_parser.py | 33 +++++++++++++++++++++++---------- tested/main.py | 19 ++++++++++++++----- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/tested/configs.py b/tested/configs.py index 86a2169e9..c7c5919ac 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -131,7 +131,7 @@ class Bundle: global_config: GlobalConfig out: IO preprocessor_messages: list[ExtendedMessage] = [] - parser_messages: list[ExtendedMessage] = [] + parser_messages: set[ExtendedMessage] = set() @property def config(self) -> DodonaConfig: @@ -211,7 +211,7 @@ def create_bundle( suite: Suite, language: str | None = None, preprocessor_messages: list[ExtendedMessage] | None = None, - parser_messages: list[ExtendedMessage] | None = None, + parser_messages: set[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -243,12 +243,12 @@ def create_bundle( preprocessor_messages = [] if parser_messages is None: - parser_messages = [] + parser_messages = set() return Bundle( language=lang_config, global_config=global_config, out=output, preprocessor_messages=preprocessor_messages, - parser_messages=parser_messages + parser_messages=parser_messages, ) diff --git a/tested/dsl/__main__.py b/tested/dsl/__main__.py index 864950b5c..6066d9742 100644 --- a/tested/dsl/__main__.py +++ b/tested/dsl/__main__.py @@ -24,7 +24,7 @@ with smart_close(parser.input) as input_file: dsl = input_file.read() -suite, _ = translate_to_test_suite(dsl) +suite = translate_to_test_suite(dsl) with smart_close(parser.output) as output_file: output_file.write(suite) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index b8150684e..235bc1777 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -30,8 +30,11 @@ ) from tested.dodona import ExtendedMessage from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string -from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error, \ - build_deprecated_language_message +from tested.dsl.dsl_errors import ( + build_deprecated_language_message, + handle_dsl_validation_errors, + raise_yaml_error, +) from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( BooleanType, @@ -528,7 +531,9 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A statement cannot have an expected return value.") -def _convert_testcase(testcase: YamlDict, context: DslContext) -> tuple[Testcase, set[ExtendedMessage]]: +def _convert_testcase( + testcase: YamlDict, context: DslContext +) -> tuple[Testcase, set[ExtendedMessage]]: messages = set() context = context.deepen_context(testcase) @@ -650,7 +655,9 @@ def _convert_context( return Context(testcases=testcases), messages -def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, set[ExtendedMessage]]: +def _convert_tab( + tab: YamlDict, context: DslContext +) -> tuple[Tab, set[ExtendedMessage]]: """ Translate a DSL tab to a full test suite tab. @@ -665,7 +672,9 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, set[ExtendedM # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts, messages = _convert_dsl_list(tab["contexts"], context, _convert_context) + contexts, messages = _convert_dsl_list( + tab["contexts"], context, _convert_context + ) elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. @@ -675,12 +684,16 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> tuple[Tab, set[ExtendedM # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases, messages = _convert_dsl_list(tab["testcases"], context, _convert_testcase) + testcases, messages = _convert_dsl_list( + tab["testcases"], context, _convert_testcase + ) contexts = [Context(testcases=[t]) for t in testcases] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases, messages = _convert_dsl_list(tab["scripts"], context, _convert_testcase) + testcases, messages = _convert_dsl_list( + tab["scripts"], context, _convert_testcase + ) contexts = [Context(testcases=[t]) for t in testcases] return Tab(name=name, contexts=contexts), messages @@ -752,12 +765,12 @@ def parse_dsl(dsl_string: str) -> tuple[Suite, set[ExtendedMessage]]: return _convert_dsl(dsl_object) -def translate_to_test_suite(dsl_string: str) -> tuple[str, set[ExtendedMessage]]: +def translate_to_test_suite(dsl_string: str) -> str: """ Convert a DSL to a test suite. :param dsl_string: The DSL. :return: The test suite. """ - suite, messages = parse_dsl(dsl_string) - return suite_to_json(suite), messages + suite, _ = parse_dsl(dsl_string) + return suite_to_json(suite) diff --git a/tested/main.py b/tested/main.py index b9574e758..308f52941 100644 --- a/tested/main.py +++ b/tested/main.py @@ -19,7 +19,8 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): :param judge_output: Where the judge output will be written to. :param language: The language to use to translate the test-suite. """ - messages = [] + preprocessor_messages = [] + parse_messages = set() try: with open(f"{config.resources}/{config.test_suite}", "r") as t: textual_suite = t.read() @@ -34,12 +35,20 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): is_yaml = ext.lower() in (".yaml", ".yml") if is_yaml: if language: - textual_suite, messages = apply_translations(textual_suite, language) + textual_suite, preprocessor_messages = apply_translations( + textual_suite, language + ) suite, parse_messages = parse_dsl(textual_suite) else: - suite, parse_messages = parse_test_suite(textual_suite) - - pack = create_bundle(config, judge_output, suite, preprocessor_messages=messages, parser_messages=parse_messages) + suite = parse_test_suite(textual_suite) + + pack = create_bundle( + config, + judge_output, + suite, + preprocessor_messages=preprocessor_messages, + parser_messages=parse_messages, + ) from .judge import judge judge(pack) From ffc1d3a943a83db7e5c46fe1d9df4b3577e97633 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 28 Apr 2025 20:58:59 +0200 Subject: [PATCH 11/19] revert changes in tests --- tests/test_dsl_yaml.py | 78 +++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 726150f48..aecf78846 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -57,7 +57,7 @@ def test_parse_one_tab_ctx(): stderr: "Error string" exit_code: 1 """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert suite.namespace == "solution" assert len(suite.tabs) == 1 @@ -89,7 +89,7 @@ def test_parse_ctx_exception(): - arguments: [ "--arg", "error" ] exception: "Error" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 2 tab = suite.tabs[0] @@ -138,7 +138,7 @@ def test_parse_ctx_exception_types(): csharp: "System.DivideByZeroException" python: "ZeroDivisionError" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 2 tab = suite.tabs[0] @@ -205,7 +205,7 @@ def test_parse_ctx_with_config(): data: " Fail " """ args = ["-a", "2.125", "1.212"] - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -278,7 +278,7 @@ def test_statements(): - expression: 'safe.content()' return: !expression 'uint8(5)' """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -315,7 +315,7 @@ def test_expression_and_main(): - expression: 'add(5, 7)' return: 12 """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -341,7 +341,7 @@ def test_expression(): - expression: "heir(8, 3)" return: [ 3, 6, 9, 12, 15, 2, 7, 1, 13, 8, 16, 10, 14, 4, 11, 5 ] """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -366,7 +366,7 @@ def test_expression_with_explicit_language(): - expression: "heir(8, 3)" return: [ 3, 6, 9, 12, 15, 2, 7, 1, 13, 8, 16, 10, 14, 4, 11, 5 ] """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -418,7 +418,7 @@ def test_statement_with_yaml_dict(): alpha: 5 beta: 6 """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -451,7 +451,7 @@ def test_global_definition_config_trickles_down(): stderr: "Error string" exit_code: 1 """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout assert isinstance(stdout.oracle, GenericTextOracle) @@ -481,7 +481,7 @@ def test_global_config_trickles_down(): stderr: "Error string" exit_code: 1 """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout assert isinstance(stdout.oracle, GenericTextOracle) @@ -509,7 +509,7 @@ def test_tab_config_trickles_down_stdout(): stderr: "Error string" exit_code: 1 """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout assert isinstance(stdout.oracle, GenericTextOracle) @@ -537,7 +537,7 @@ def test_tab_config_trickles_down_stderr(): stderr: "Error string" exit_code: 1 """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr assert isinstance(stderr.oracle, GenericTextOracle) @@ -556,7 +556,7 @@ def test_expression_raw_return(): - expression: 'test()' return: !expression '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]' """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -588,7 +588,7 @@ def test_empty_constructor(function_name, result): - expression: 'test()' return: !expression '{function_name}()' """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -618,7 +618,7 @@ def test_empty_constructor_with_param(function_name, result): - expression: 'test()' return: !expression '{function_name}([])' """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -640,7 +640,7 @@ def test_text_built_in_checks_implied(): stdout: data: "hallo" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -664,7 +664,7 @@ def test_text_built_in_checks_explicit(): data: "hallo" oracle: "builtin" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -691,7 +691,7 @@ def test_text_custom_checks_correct(): name: "evaluate_test" arguments: [!expression "'yes'", 5, !expression "set([5, 5])"] """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -728,7 +728,7 @@ def test_value_built_in_checks_implied(): return: !oracle value: !expression "'hallo'" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -757,7 +757,7 @@ def test_file_custom_check_correct(): name: "evaluate_test" file: "test.py" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -785,7 +785,7 @@ def test_value_built_in_checks_explicit(): value: "hallo" oracle: "builtin" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -814,7 +814,7 @@ def test_value_custom_checks_correct(): name: "evaluate_test" arguments: ['yes', 5, !expression "set([5, 5])"] """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -863,7 +863,7 @@ def test_value_specific_checks_correct(): javascript: - "js" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -931,7 +931,7 @@ def test_yaml_set_tag_is_supported(): - expression: 'test()' return: !!set {5, 6} """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -974,7 +974,7 @@ def test_yaml_custom_tags_are_supported(all_types, value): - expression: 'test()' return: !{the_type} {json_type} """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -1000,7 +1000,7 @@ def test_global_language_literals(): javascript: "hello()" python: "hello_2()" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -1030,7 +1030,7 @@ def test_one_language_literal(): javascript: "hello()" python: "hello_2()" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] @@ -1147,9 +1147,9 @@ def test_one_language_literal(): ], ) def test_old_and_new_names_work(old, new): - old_json, _ = translate_to_test_suite(old) + old_json = translate_to_test_suite(old) old_suite = parse_test_suite(old_json) - new_json, _ = translate_to_test_suite(new) + new_json = translate_to_test_suite(new) new_suite = parse_test_suite(new_json) assert old_suite == new_suite @@ -1186,7 +1186,7 @@ def test_files_are_propagated(): - name: "test" url: "twooo.md" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) tab = suite.tabs[0] ctx0, ctx1 = tab.contexts @@ -1209,7 +1209,7 @@ def test_newlines_are_added_to_stdout(): config: tryFloatingPoint: true """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.data assert actual_stdout == "12\n" @@ -1221,7 +1221,7 @@ def test_newlines_are_added_to_stdout(): - arguments: [ '-a', '5', '7' ] stdout: "hello" """ - json_str, _ = translate_to_test_suite(yaml_str2) + json_str = translate_to_test_suite(yaml_str2) suite = parse_test_suite(json_str) actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.data assert actual_stdout == "hello\n" @@ -1238,7 +1238,7 @@ def test_newlines_are_added_to_stderr(): config: tryFloatingPoint: true """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "12\n" @@ -1250,7 +1250,7 @@ def test_newlines_are_added_to_stderr(): - arguments: [ '-a', '5', '7' ] stderr: "hello" """ - json_str, _ = translate_to_test_suite(yaml_str2) + json_str = translate_to_test_suite(yaml_str2) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "hello\n" @@ -1266,7 +1266,7 @@ def test_no_duplicate_newlines_are_added(): hello world """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual = suite.tabs[0].contexts[0].testcases[0].output.stdout.data assert actual == "hello\nworld\n" @@ -1284,7 +1284,7 @@ def test_can_disable_normalizing_newlines(): tryFloatingPoint: true normalizeTrailingNewlines: false """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "12" @@ -1298,7 +1298,7 @@ def test_empty_text_data_newlines(): - arguments: [ '-a', '5', '7' ] stderr: "" """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data assert actual_stderr == "" @@ -1314,7 +1314,7 @@ def test_programming_language_can_be_globally_configured(): - expression: "Numbers.oddValues(new int[]{1, 2, 3, 4, 5, 6, 7, 8})" return: [1, 3, 5, 7] """ - json_str, _ = translate_to_test_suite(yaml_str) + json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) assert len(suite.tabs) == 1 tab = suite.tabs[0] From c562b910937c31f85688a0c46b643f8678bf1535 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 28 Apr 2025 21:54:00 +0200 Subject: [PATCH 12/19] added the tests for programming_language tag in the parser --- tests/test_dsl_yaml.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index aecf78846..d14e66640 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -19,6 +19,7 @@ SequenceTypes, StringTypes, ) +from tested.dodona import Permission from tested.dsl import parse_dsl, translate_to_test_suite from tested.dsl.translate_parser import load_schema_validator from tested.serialisation import ( @@ -1326,6 +1327,59 @@ def test_programming_language_can_be_globally_configured(): assert testcase.input.type == "expression" assert testcase.input.literals.keys() == {"java"} +def test_deprecated_programming_language_map_gives_warning(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + _, messages = parse_dsl(yaml_str) + assert messages + message = list(messages)[0] + assert message.description == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + assert message.permission == Permission.STAFF + assert message.format == "text" + +def test_programming_language_tag_gives_no_warning(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: !programming_language + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + _, messages = parse_dsl(yaml_str) + assert not messages + +def test_deprecated_programming_language_map_not_duplicate(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + - unit: "cube list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 3, 6, 9, 12] + """ + _, messages = parse_dsl(yaml_str) + assert len(messages) == 1 + message = list(messages)[0] + assert message.description == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + assert message.permission == Permission.STAFF + assert message.format == "text" def test_strict_json_schema_is_valid(): path_to_schema = Path(__file__).parent / "tested-draft7.json" From 71dfcc40d4985d883788f0b49e4d0217c1b2b144 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 15:02:15 +0200 Subject: [PATCH 13/19] fix linting --- tests/test_dsl_yaml.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index d14e66640..ecf767a3a 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1327,6 +1327,7 @@ def test_programming_language_can_be_globally_configured(): assert testcase.input.type == "expression" assert testcase.input.literals.keys() == {"java"} + def test_deprecated_programming_language_map_gives_warning(): yaml_str = """ - unit: "square list" @@ -1340,9 +1341,13 @@ def test_deprecated_programming_language_map_gives_warning(): _, messages = parse_dsl(yaml_str) assert messages message = list(messages)[0] - assert message.description == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + assert ( + message.description + == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + ) assert message.permission == Permission.STAFF - assert message.format == "text" + assert message.format == "text" + def test_programming_language_tag_gives_no_warning(): yaml_str = """ @@ -1357,6 +1362,7 @@ def test_programming_language_tag_gives_no_warning(): _, messages = parse_dsl(yaml_str) assert not messages + def test_deprecated_programming_language_map_not_duplicate(): yaml_str = """ - unit: "square list" @@ -1377,10 +1383,14 @@ def test_deprecated_programming_language_map_not_duplicate(): _, messages = parse_dsl(yaml_str) assert len(messages) == 1 message = list(messages)[0] - assert message.description == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + assert ( + message.description + == "WARNING: You are using YAML syntax to specify statements or expressions in multiple programming languages without the `!programming_language` tag. This usage is deprecated!" + ) assert message.permission == Permission.STAFF assert message.format == "text" + def test_strict_json_schema_is_valid(): path_to_schema = Path(__file__).parent / "tested-draft7.json" with open(path_to_schema, "r") as schema_file: From e9ba1ef8119a1d6866fccc09163c222baf342b34 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 2 May 2025 12:00:45 +0200 Subject: [PATCH 14/19] changed some of the names --- tested/nat_translation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tested/nat_translation.py b/tested/nat_translation.py index 560e9f50b..ecff42c6b 100644 --- a/tested/nat_translation.py +++ b/tested/nat_translation.py @@ -33,7 +33,7 @@ def validate_pre_dsl(yaml_object: Any): handle_dsl_validation_errors(errors) -class CustomDumper(yaml.SafeDumper): +class CustomTagFormatDumper(yaml.SafeDumper): def represent_with_tag(self, tag, value): if isinstance(value, dict): @@ -44,7 +44,7 @@ def represent_with_tag(self, tag, value): return self.represent_scalar(tag, value) @staticmethod - def custom_representer(dumper, data): + def custom_tag_format_representer(dumper, data): """ Will turn the given object back into YAML. @@ -59,7 +59,7 @@ def custom_representer(dumper, data): ) -def construct_custom(loader, tag_suffix, node): +def construct_custom_tag_format(loader, tag_suffix, node): """ This constructor will turn the given YAML into an object that can be used for translation. @@ -159,9 +159,9 @@ def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str): def convert_to_yaml(translated_data: Any) -> str: - CustomDumper.add_representer(dict, CustomDumper.custom_representer) + CustomTagFormatDumper.add_representer(dict, CustomTagFormatDumper.custom_tag_format_representer) return yaml.dump( - translated_data, Dumper=CustomDumper, allow_unicode=True, sort_keys=False + translated_data, Dumper=CustomTagFormatDumper, allow_unicode=True, sort_keys=False ) @@ -170,7 +170,7 @@ def parse_yaml(yaml_stream: str) -> Any: Parse a string or stream to YAML. """ loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.SafeLoader) - yaml.add_multi_constructor("", construct_custom, loader) + yaml.add_multi_constructor("", construct_custom_tag_format, loader) try: return yaml.load(yaml_stream, loader) From 8be8e8311b4e2b16ef913fa3e953d54d41c53767 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 2 May 2025 13:29:23 +0200 Subject: [PATCH 15/19] merged messages and made extra test --- tested/configs.py | 18 ++++++------------ tested/main.py | 11 ++++------- tests/test_dsl_yaml.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/tested/configs.py b/tested/configs.py index c7c5919ac..3d287e4b7 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -130,8 +130,7 @@ class Bundle: language: "Language" global_config: GlobalConfig out: IO - preprocessor_messages: list[ExtendedMessage] = [] - parser_messages: set[ExtendedMessage] = set() + messages: set[ExtendedMessage] = set() @property def config(self) -> DodonaConfig: @@ -210,8 +209,7 @@ def create_bundle( output: IO, suite: Suite, language: str | None = None, - preprocessor_messages: list[ExtendedMessage] | None = None, - parser_messages: set[ExtendedMessage] | None = None, + messages: set[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -221,8 +219,7 @@ def create_bundle( :param suite: The test suite. :param language: Optional programming language. If None, the one from the Dodona configuration will be used. - :param preprocessor_messages: Messages generated out of the preprocessor. - :param parser_messages: Messages generated out of the DSL-parser. + :param messages: Messages generated out of the preprocessor and the translate parser. :return: The configuration bundle. """ @@ -239,16 +236,13 @@ def create_bundle( suite=suite, ) lang_config = langs.get_language(global_config, language) - if preprocessor_messages is None: - preprocessor_messages = [] - if parser_messages is None: - parser_messages = set() + if messages is None: + messages = set() return Bundle( language=lang_config, global_config=global_config, out=output, - preprocessor_messages=preprocessor_messages, - parser_messages=parser_messages, + messages=messages, ) diff --git a/tested/main.py b/tested/main.py index 308f52941..ca383bdd0 100644 --- a/tested/main.py +++ b/tested/main.py @@ -19,8 +19,7 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): :param judge_output: Where the judge output will be written to. :param language: The language to use to translate the test-suite. """ - preprocessor_messages = [] - parse_messages = set() + messages = set() try: with open(f"{config.resources}/{config.test_suite}", "r") as t: textual_suite = t.read() @@ -35,10 +34,9 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): is_yaml = ext.lower() in (".yaml", ".yml") if is_yaml: if language: - textual_suite, preprocessor_messages = apply_translations( - textual_suite, language - ) + textual_suite, messages = apply_translations(textual_suite, language) suite, parse_messages = parse_dsl(textual_suite) + messages.update(parse_messages) else: suite = parse_test_suite(textual_suite) @@ -46,8 +44,7 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): config, judge_output, suite, - preprocessor_messages=preprocessor_messages, - parser_messages=parse_messages, + messages=messages, ) from .judge import judge diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index ecf767a3a..842052e0e 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1349,6 +1349,35 @@ def test_deprecated_programming_language_map_gives_warning(): assert message.format == "text" +def test_programming_language_with_and_without_generate_same(): + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + print(suite) + + yaml_str = """ + - unit: "square list" + cases: + - script: + - expression: !programming_language + python: "[x * 2 for x in range(5)]" + javascript: "[...Array(5).keys()].map(x => x * 2)" + return: [0, 2, 4, 6, 8] + """ + json_str = translate_to_test_suite(yaml_str) + suite2 = parse_test_suite(json_str) + print(suite2) + assert suite == suite2 + + def test_programming_language_tag_gives_no_warning(): yaml_str = """ - unit: "square list" From 0f2e397a39458f103e4b29dface6c335bcfe6480 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 2 May 2025 13:43:33 +0200 Subject: [PATCH 16/19] Fixed crash --- tested/judge/core.py | 7 ++----- tested/main.py | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tested/judge/core.py b/tested/judge/core.py index c1c648873..4147aff59 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -116,11 +116,8 @@ def judge(bundle: Bundle): # Do the set-up for the judgement. collector = OutputManager(bundle.out) collector.add(StartJudgement()) - if bundle.preprocessor_messages: - collector.add_messages(bundle.preprocessor_messages) - - if bundle.parser_messages: - collector.add_messages(bundle.parser_messages) + if bundle.messages: + collector.add_messages(bundle.messages) max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() diff --git a/tested/main.py b/tested/main.py index ca383bdd0..f94343364 100644 --- a/tested/main.py +++ b/tested/main.py @@ -34,7 +34,8 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): is_yaml = ext.lower() in (".yaml", ".yml") if is_yaml: if language: - textual_suite, messages = apply_translations(textual_suite, language) + textual_suite, new_messages = apply_translations(textual_suite, language) + messages.update(new_messages) suite, parse_messages = parse_dsl(textual_suite) messages.update(parse_messages) else: From d8b39eb4df6d45a4c750dbef553b68436ff26380 Mon Sep 17 00:00:00 2001 From: breblanc Date: Fri, 9 May 2025 16:51:47 +0200 Subject: [PATCH 17/19] Use Data class instead of tuple --- tested/descriptions/renderer.py | 4 +- tested/dsl/translate_parser.py | 78 +++++++++++++++++++-------------- tested/main.py | 5 ++- tested/utils.py | 10 ++++- tests/test_dsl_yaml.py | 9 ++-- 5 files changed, 64 insertions(+), 42 deletions(-) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 974e95023..2f5cc010f 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -83,11 +83,11 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: rendered_dsl = self.render_children(element) # Parse the DSL - parsed_dsl, _ = parse_dsl(rendered_dsl) + parsed_dsl_w_messages = parse_dsl(rendered_dsl) # Get all actual tests tests = [] - for tab in parsed_dsl.tabs: + for tab in parsed_dsl_w_messages.data.tabs: for context in tab.contexts: for testcase in context.testcases: tests.append(testcase) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 235bc1777..c750148be 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -71,7 +71,7 @@ TextOutputChannel, ValueOutputChannel, ) -from tested.utils import get_args, recursive_dict_merge +from tested.utils import DataWithMessage, get_args, recursive_dict_merge YamlDict = dict[str, "YamlObject"] @@ -533,7 +533,7 @@ def _validate_testcase_combinations(testcase: YamlDict): def _convert_testcase( testcase: YamlDict, context: DslContext -) -> tuple[Testcase, set[ExtendedMessage]]: +) -> DataWithMessage[Testcase, ExtendedMessage]: messages = set() context = context.deepen_context(testcase) @@ -631,33 +631,34 @@ def _convert_testcase( else: the_description = None - return ( - Testcase( + return DataWithMessage( + data=Testcase( description=the_description, input=the_input, output=output, link_files=context.files, line_comment=line_comment, ), - messages, + messages=messages, ) def _convert_context( context: YamlDict, dsl_context: DslContext -) -> tuple[Context, set[ExtendedMessage]]: +) -> DataWithMessage[Context, ExtendedMessage]: dsl_context = dsl_context.deepen_context(context) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases, messages = _convert_dsl_list( - raw_testcases, dsl_context, _convert_testcase + data_w_messages = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) + return DataWithMessage( + data=Context(testcases=data_w_messages.data), + messages=data_w_messages.messages, ) - return Context(testcases=testcases), messages def _convert_tab( tab: YamlDict, context: DslContext -) -> tuple[Tab, set[ExtendedMessage]]: +) -> DataWithMessage[Tab, ExtendedMessage]: """ Translate a DSL tab to a full test suite tab. @@ -672,31 +673,32 @@ def _convert_tab( # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts, messages = _convert_dsl_list( - tab["contexts"], context, _convert_context - ) + data_w_messages = _convert_dsl_list(tab["contexts"], context, _convert_context) + contexts = data_w_messages.data elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts, messages = _convert_dsl_list(tab["cases"], context, _convert_context) + data_w_messages = _convert_dsl_list(tab["cases"], context, _convert_context) + contexts = data_w_messages.data elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases, messages = _convert_dsl_list( + data_w_messages = _convert_dsl_list( tab["testcases"], context, _convert_testcase ) - contexts = [Context(testcases=[t]) for t in testcases] + contexts = [Context(testcases=[t]) for t in data_w_messages.data] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases, messages = _convert_dsl_list( - tab["scripts"], context, _convert_testcase - ) - contexts = [Context(testcases=[t]) for t in testcases] + data_w_messages = _convert_dsl_list(tab["scripts"], context, _convert_testcase) + contexts = [Context(testcases=[t]) for t in data_w_messages.data] - return Tab(name=name, contexts=contexts), messages + return DataWithMessage( + data=Tab(name=name, contexts=contexts), + messages=data_w_messages.messages, + ) T = TypeVar("T") @@ -705,8 +707,8 @@ def _convert_tab( def _convert_dsl_list( dsl_list: list, context: DslContext, - converter: Callable[[YamlDict, DslContext], tuple[T, set[ExtendedMessage]]], -) -> tuple[list[T], set[ExtendedMessage]]: + converter: Callable[[YamlDict, DslContext], DataWithMessage[T, ExtendedMessage]], +) -> DataWithMessage[list[T], ExtendedMessage]: """ Convert a list of YAML objects into a test suite object. """ @@ -714,13 +716,16 @@ def _convert_dsl_list( messages = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - obj, new_messages = converter(dsl_object, context) - messages.update(new_messages) - objects.append(obj) - return objects, messages + data_w_messages = converter(dsl_object, context) + messages.update(data_w_messages.messages) + objects.append(data_w_messages.data) + return DataWithMessage( + data=objects, + messages=messages, + ) -def _convert_dsl(dsl_object: YamlObject) -> tuple[Suite, set[ExtendedMessage]]: +def _convert_dsl(dsl_object: YamlObject) -> DataWithMessage[Suite, ExtendedMessage]: """ Translate a DSL test suite into a full test suite. @@ -743,16 +748,21 @@ def _convert_dsl(dsl_object: YamlObject) -> tuple[Suite, set[ExtendedMessage]]: if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs, messages = _convert_dsl_list(tab_list, context, _convert_tab) + data_w_messages = _convert_dsl_list(tab_list, context, _convert_tab) if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, namespace=namespace), messages + return DataWithMessage( + data=Suite(tabs=data_w_messages.data, namespace=namespace), + messages=data_w_messages.messages, + ) else: - return Suite(tabs=tabs), messages + return DataWithMessage( + data=Suite(tabs=data_w_messages.data), messages=data_w_messages.messages + ) -def parse_dsl(dsl_string: str) -> tuple[Suite, set[ExtendedMessage]]: +def parse_dsl(dsl_string: str) -> DataWithMessage[Suite, ExtendedMessage]: """ Parse a string containing a DSL test suite into our representation, a test suite. @@ -772,5 +782,5 @@ def translate_to_test_suite(dsl_string: str) -> str: :param dsl_string: The DSL. :return: The test suite. """ - suite, _ = parse_dsl(dsl_string) - return suite_to_json(suite) + suite_w_message = parse_dsl(dsl_string) + return suite_to_json(suite_w_message.data) diff --git a/tested/main.py b/tested/main.py index f94343364..5c0f137c3 100644 --- a/tested/main.py +++ b/tested/main.py @@ -36,8 +36,9 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): if language: textual_suite, new_messages = apply_translations(textual_suite, language) messages.update(new_messages) - suite, parse_messages = parse_dsl(textual_suite) - messages.update(parse_messages) + suite_w_messages = parse_dsl(textual_suite) + messages.update(suite_w_messages.messages) + suite = suite_w_messages.data else: suite = parse_test_suite(textual_suite) diff --git a/tested/utils.py b/tested/utils.py index e913c8b6f..47ca8db53 100644 --- a/tested/utils.py +++ b/tested/utils.py @@ -7,9 +7,11 @@ from collections.abc import Callable, Iterable from itertools import zip_longest from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, TypeGuard, TypeVar +from typing import IO, TYPE_CHECKING, Any, Generic, TypeGuard, TypeVar from typing import get_args as typing_get_args +from attr import define + if TYPE_CHECKING: from tested.serialisation import Assignment @@ -56,6 +58,12 @@ def get_identifier() -> str: T = TypeVar("T") +@define +class DataWithMessage(Generic[T, K]): + data: T + messages: set[K] + + def get_args(type_: Any) -> tuple[Any, ...]: """ Get the args of a type or the type itself. diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 842052e0e..d0bcac1a6 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1338,7 +1338,8 @@ def test_deprecated_programming_language_map_gives_warning(): javascript: "[...Array(5).keys()].map(x => x * 2)" return: [0, 2, 4, 6, 8] """ - _, messages = parse_dsl(yaml_str) + data_w_messages = parse_dsl(yaml_str) + messages = data_w_messages.messages assert messages message = list(messages)[0] assert ( @@ -1388,7 +1389,8 @@ def test_programming_language_tag_gives_no_warning(): javascript: "[...Array(5).keys()].map(x => x * 2)" return: [0, 2, 4, 6, 8] """ - _, messages = parse_dsl(yaml_str) + data_w_messages = parse_dsl(yaml_str) + messages = data_w_messages.messages assert not messages @@ -1409,7 +1411,8 @@ def test_deprecated_programming_language_map_not_duplicate(): javascript: "[...Array(5).keys()].map(x => x * 2)" return: [0, 3, 6, 9, 12] """ - _, messages = parse_dsl(yaml_str) + data_w_messages = parse_dsl(yaml_str) + messages = data_w_messages.messages assert len(messages) == 1 message = list(messages)[0] assert ( From 544d4df202df382e4de26b168af5f5de5ec02abe Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 12 May 2025 13:40:54 +0200 Subject: [PATCH 18/19] changed a few variable names + few small changes --- tested/descriptions/renderer.py | 4 +- tested/dsl/translate_parser.py | 71 ++++++++++++++++----------------- tested/utils.py | 6 ++- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 2f5cc010f..aeb89bc17 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -83,11 +83,11 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: rendered_dsl = self.render_children(element) # Parse the DSL - parsed_dsl_w_messages = parse_dsl(rendered_dsl) + parsed_dsl_with_messages = parse_dsl(rendered_dsl) # Get all actual tests tests = [] - for tab in parsed_dsl_w_messages.data.tabs: + for tab in parsed_dsl_with_messages.data.tabs: for context in tab.contexts: for testcase in context.testcases: tests.append(testcase) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index c750148be..6397e5669 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -533,7 +533,7 @@ def _validate_testcase_combinations(testcase: YamlDict): def _convert_testcase( testcase: YamlDict, context: DslContext -) -> DataWithMessage[Testcase, ExtendedMessage]: +) -> DataWithMessage[Testcase]: messages = set() context = context.deepen_context(testcase) @@ -545,10 +545,7 @@ def _convert_testcase( line_comment = "" _validate_testcase_combinations(testcase) if (expr_stmt := testcase.get("statement", testcase.get("expression"))) is not None: - if ( - isinstance(expr_stmt, dict | ProgrammingLanguageMap) - or context.language != "tested" - ): + if isinstance(expr_stmt, dict) or context.language != "tested": if isinstance(expr_stmt, str): the_dict = {context.language: expr_stmt} else: @@ -645,20 +642,18 @@ def _convert_testcase( def _convert_context( context: YamlDict, dsl_context: DslContext -) -> DataWithMessage[Context, ExtendedMessage]: +) -> DataWithMessage[Context]: dsl_context = dsl_context.deepen_context(context) raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - data_w_messages = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) + testcases = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) return DataWithMessage( - data=Context(testcases=data_w_messages.data), - messages=data_w_messages.messages, + data=Context(testcases=testcases.data), + messages=testcases.messages, ) -def _convert_tab( - tab: YamlDict, context: DslContext -) -> DataWithMessage[Tab, ExtendedMessage]: +def _convert_tab(tab: YamlDict, context: DslContext) -> DataWithMessage[Tab]: """ Translate a DSL tab to a full test suite tab. @@ -673,31 +668,37 @@ def _convert_tab( # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - data_w_messages = _convert_dsl_list(tab["contexts"], context, _convert_context) - contexts = data_w_messages.data + contexts_with_messages = _convert_dsl_list( + tab["contexts"], context, _convert_context + ) + contexts = contexts_with_messages.data + messages = contexts_with_messages.messages elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - data_w_messages = _convert_dsl_list(tab["cases"], context, _convert_context) - contexts = data_w_messages.data + contexts_with_messages = _convert_dsl_list( + tab["cases"], context, _convert_context + ) + contexts = contexts_with_messages.data + messages = contexts_with_messages.messages elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - data_w_messages = _convert_dsl_list( - tab["testcases"], context, _convert_testcase - ) - contexts = [Context(testcases=[t]) for t in data_w_messages.data] + testcases = _convert_dsl_list(tab["testcases"], context, _convert_testcase) + messages = testcases.messages + contexts = [Context(testcases=[t]) for t in testcases.data] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - data_w_messages = _convert_dsl_list(tab["scripts"], context, _convert_testcase) - contexts = [Context(testcases=[t]) for t in data_w_messages.data] + testcases = _convert_dsl_list(tab["scripts"], context, _convert_testcase) + messages = testcases.messages + contexts = [Context(testcases=[t]) for t in testcases.data] return DataWithMessage( data=Tab(name=name, contexts=contexts), - messages=data_w_messages.messages, + messages=messages, ) @@ -707,8 +708,8 @@ def _convert_tab( def _convert_dsl_list( dsl_list: list, context: DslContext, - converter: Callable[[YamlDict, DslContext], DataWithMessage[T, ExtendedMessage]], -) -> DataWithMessage[list[T], ExtendedMessage]: + converter: Callable[[YamlDict, DslContext], DataWithMessage[T]], +) -> DataWithMessage[list[T]]: """ Convert a list of YAML objects into a test suite object. """ @@ -716,16 +717,16 @@ def _convert_dsl_list( messages = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - data_w_messages = converter(dsl_object, context) - messages.update(data_w_messages.messages) - objects.append(data_w_messages.data) + obj = converter(dsl_object, context) + messages.update(obj.messages) + objects.append(obj.data) return DataWithMessage( data=objects, messages=messages, ) -def _convert_dsl(dsl_object: YamlObject) -> DataWithMessage[Suite, ExtendedMessage]: +def _convert_dsl(dsl_object: YamlObject) -> DataWithMessage[Suite]: """ Translate a DSL test suite into a full test suite. @@ -748,21 +749,19 @@ def _convert_dsl(dsl_object: YamlObject) -> DataWithMessage[Suite, ExtendedMessa if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - data_w_messages = _convert_dsl_list(tab_list, context, _convert_tab) + tabs = _convert_dsl_list(tab_list, context, _convert_tab) if namespace: assert isinstance(namespace, str) return DataWithMessage( - data=Suite(tabs=data_w_messages.data, namespace=namespace), - messages=data_w_messages.messages, + data=Suite(tabs=tabs.data, namespace=namespace), + messages=tabs.messages, ) else: - return DataWithMessage( - data=Suite(tabs=data_w_messages.data), messages=data_w_messages.messages - ) + return DataWithMessage(data=Suite(tabs=tabs.data), messages=tabs.messages) -def parse_dsl(dsl_string: str) -> DataWithMessage[Suite, ExtendedMessage]: +def parse_dsl(dsl_string: str) -> DataWithMessage[Suite]: """ Parse a string containing a DSL test suite into our representation, a test suite. diff --git a/tested/utils.py b/tested/utils.py index 47ca8db53..6b5c838e1 100644 --- a/tested/utils.py +++ b/tested/utils.py @@ -12,6 +12,8 @@ from attr import define +from tested.dodona import ExtendedMessage + if TYPE_CHECKING: from tested.serialisation import Assignment @@ -59,9 +61,9 @@ def get_identifier() -> str: @define -class DataWithMessage(Generic[T, K]): +class DataWithMessage(Generic[T]): data: T - messages: set[K] + messages: set[ExtendedMessage] def get_args(type_: Any) -> tuple[Any, ...]: From 4e9b9bdada541328f6c11fbf461cf43d6e212a2a Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 12 May 2025 16:49:34 +0200 Subject: [PATCH 19/19] changed variable names --- tested/dsl/translate_parser.py | 4 ++-- tested/main.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 6397e5669..35d746279 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -781,5 +781,5 @@ def translate_to_test_suite(dsl_string: str) -> str: :param dsl_string: The DSL. :return: The test suite. """ - suite_w_message = parse_dsl(dsl_string) - return suite_to_json(suite_w_message.data) + suite_with_message = parse_dsl(dsl_string) + return suite_to_json(suite_with_message.data) diff --git a/tested/main.py b/tested/main.py index 5c0f137c3..0246d5c43 100644 --- a/tested/main.py +++ b/tested/main.py @@ -36,9 +36,9 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None): if language: textual_suite, new_messages = apply_translations(textual_suite, language) messages.update(new_messages) - suite_w_messages = parse_dsl(textual_suite) - messages.update(suite_w_messages.messages) - suite = suite_w_messages.data + suite_with_messages = parse_dsl(textual_suite) + messages.update(suite_with_messages.messages) + suite = suite_with_messages.data else: suite = parse_test_suite(textual_suite)