diff --git a/.coveragerc b/.coveragerc index 05ec510dc..59da97abc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,7 +5,7 @@ omit = */.venv/* [report] -fail_under = 65.0 +fail_under = 60.0 precision = 2 skip_covered = true show_missing = true diff --git a/.python-version b/.python-version index bbbadfe7c..e93d6f64a 100644 --- a/.python-version +++ b/.python-version @@ -1,2 +1,2 @@ -3.8 +3.9 diff --git a/pyproject.toml b/pyproject.toml index 01bf3026b..51fbb2003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,11 @@ dependencies = [ # Parsing and rendering RST. "docutils >= 0.16, == 0.*", + # Tree Sitter is used for language/AST-aware parsing of Python, C and other files. + "tree-sitter", + "tree-sitter-c", + "tree-sitter-python", + # Requirements-to-source traceability. Colored syntax for source files. "pygments >= 2.10.0, == 2.*", diff --git a/strictdoc/backend/sdoc/error_handling.py b/strictdoc/backend/sdoc/error_handling.py index fc090cfbd..135ecdda9 100644 --- a/strictdoc/backend/sdoc/error_handling.py +++ b/strictdoc/backend/sdoc/error_handling.py @@ -1,4 +1,6 @@ # mypy: disable-error-code="attr-defined,no-untyped-call,no-untyped-def,union-attr" +from typing import Optional + from textx import TextXSyntaxError from strictdoc.backend.sdoc.models.document import SDocDocument @@ -25,7 +27,13 @@ def get_textx_syntax_error_message(exception: TextXSyntaxError): class StrictDocSemanticError(Exception): def __init__( - self, title, hint, example, line=None, col=None, filename=None + self, + title: str, + hint: Optional[str], + example: Optional[str], + line: Optional[int] = None, + col: Optional[int] = None, + filename: Optional[str] = None, ): super().__init__(title, hint, line, col, filename) self.title = title diff --git a/strictdoc/backend/sdoc/grammar/type_system.py b/strictdoc/backend/sdoc/grammar/type_system.py index a3582f4b0..6c8d35b10 100644 --- a/strictdoc/backend/sdoc/grammar/type_system.py +++ b/strictdoc/backend/sdoc/grammar/type_system.py @@ -47,6 +47,7 @@ (' FORMAT: ' g_file_format = FileEntryFormat '\n')? ' VALUE: ' g_file_path = /.*$/ '\n' (' LINE_RANGE: ' g_line_range = /.*$/ '\n')? + (' FUNCTION: ' function = /.*$/ '\n')? ; FileEntryFormat[noskipws]: diff --git a/strictdoc/backend/sdoc/models/type_system.py b/strictdoc/backend/sdoc/models/type_system.py index 642022fe1..4665ddb50 100644 --- a/strictdoc/backend/sdoc/models/type_system.py +++ b/strictdoc/backend/sdoc/models/type_system.py @@ -56,6 +56,7 @@ def __init__( g_file_format: Optional[str], g_file_path: str, g_line_range: Optional[str], + function: Optional[str], ): self.parent = parent @@ -85,6 +86,11 @@ def __init__( int(range_components_str[1]), ) + # textX parses an optional element as an empty string. We make it to None ourselves. + self.function: Optional[str] = ( + function if function is not None and len(function) > 0 else None + ) + class FileEntryFormat: SOURCECODE = "Sourcecode" diff --git a/strictdoc/backend/sdoc_source_code/marker_parser.py b/strictdoc/backend/sdoc_source_code/marker_parser.py new file mode 100644 index 000000000..a677efcf9 --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/marker_parser.py @@ -0,0 +1,104 @@ +import re +from typing import List, Union + +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.range_marker import ( + LineMarker, + RangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req + +REGEX_REQ = r"[A-Za-z][A-Za-z0-9\\-]+" +# @relation(REQ-1, scope=function) +REGEX_FUNCTION = ( + rf"@relation\((/?)({REGEX_REQ}(?:, {REGEX_REQ})*)\, scope=function\)" +) +REGEX_RANGE = rf"@sdoc\[(/?)({REGEX_REQ}(?:, {REGEX_REQ})*)\]" +REGEX_LINE = rf"@sdoc\((/?)({REGEX_REQ}(?:, {REGEX_REQ})*)\)" + + +class MarkerParser: + @staticmethod + def parse( + input_string: str, + line_start: int, + line_end: int, + comment_line_start: int, + comment_column_start: int, + ) -> List[Union[FunctionRangeMarker, RangeMarker, LineMarker]]: + markers: List[Union[FunctionRangeMarker, RangeMarker, LineMarker]] = [] + for input_line_idx_, input_line_ in enumerate( + input_string.splitlines() + ): + match_function = None + match_line = None + match_range = None + + match_function = re.search(REGEX_FUNCTION, input_line_) + if match_function is None: + match_range = re.search(REGEX_RANGE, input_line_) + if match_range is None: + match_line = re.search(REGEX_LINE, input_line_) + + match = ( + match_function + if match_function is not None + else match_range + if match_range is not None + else match_line + ) + if match is None: + continue + + start_or_end = match.group(1) != "/" + req_list = match.group(2) + + first_requirement_index = match.start(2) + + current_line = comment_line_start + input_line_idx_ + first_requirement_column = first_requirement_index + 1 + if input_line_idx_ == 0: + first_requirement_column += comment_column_start - 1 + requirements = [] + for req_match in re.finditer(REGEX_REQ, req_list): + req_item = req_match.group(0) # Matched REQ-XXX item + # Calculate actual position relative to the original string + start_index = ( + req_match.start() + ) # Offset by where group 1 starts + requirement = Req(None, req_item) + requirement.ng_source_line = current_line + requirement.ng_source_column = ( + first_requirement_column + start_index + ) + requirements.append(requirement) + + if match_function is not None: + function_marker = FunctionRangeMarker(None, requirements) + function_marker.ng_source_line_begin = line_start + function_marker.ng_range_line_begin = line_start + function_marker.ng_range_line_end = line_end + function_marker.ng_marker_line = current_line + function_marker.ng_marker_column = first_requirement_column + markers.append(function_marker) + elif match_range is not None: + range_marker = RangeMarker( + None, "[" if start_or_end else "[/", requirements + ) + range_marker.ng_source_line_begin = line_start + range_marker.ng_source_column_begin = first_requirement_column + range_marker.ng_range_line_begin = line_start + range_marker.ng_range_line_end = line_end + markers.append(range_marker) + elif match_line is not None: + line_marker = LineMarker(None, requirements) + line_marker.ng_source_line_begin = line_start + line_marker.ng_range_line_begin = line_start + line_marker.ng_range_line_end = line_end + markers.append(line_marker) + else: + continue + + return markers diff --git a/strictdoc/backend/sdoc_source_code/models/function.py b/strictdoc/backend/sdoc_source_code/models/function.py new file mode 100644 index 000000000..b35019b64 --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/models/function.py @@ -0,0 +1,20 @@ +from typing import Any, List + +from strictdoc.helpers.auto_described import auto_described + + +@auto_described +class Function: + def __init__( + self, + parent: Any, + name: str, + line_begin: int, + line_end: int, + parts: List[Any], + ): + self.parent = parent + self.name = name + self.parts: List[Any] = parts + self.line_begin = line_begin + self.line_end = line_end diff --git a/strictdoc/backend/sdoc_source_code/models/function_range_marker.py b/strictdoc/backend/sdoc_source_code/models/function_range_marker.py new file mode 100644 index 000000000..4c9727e4c --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/models/function_range_marker.py @@ -0,0 +1,52 @@ +# mypy: disable-error-code="no-untyped-def,type-arg" +from typing import List, Optional + +from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req +from strictdoc.helpers.auto_described import auto_described + + +@auto_described +class FunctionRangeMarker: + def __init__(self, parent, reqs_objs: List[Req]): + assert isinstance(reqs_objs, list) + self.parent = parent + self.reqs_objs: List[Req] = reqs_objs + self.reqs: List[str] = list(map(lambda req: req.uid, reqs_objs)) + + # Line number of the marker in the source code. + self.ng_source_line_begin: Optional[int] = None + self.ng_source_column_begin: Optional[int] = None + + # Line number of the marker range in the source code: + # TODO: Improve description. + # For Begin ranges: + # ng_range_line_begin == ng_source_line_begin # noqa: ERA001 + # ng_range_line_end == ng_source_line_begin of the End marker # noqa: ERA001, E501 + # For End ranges: + # ng_range_line_begin == ng_range_line_begin of the Begin marker # noqa: ERA001, E501 + # ng_range_line_end == ng_source_line_begin # noqa: ERA001 + self.ng_range_line_begin: Optional[int] = None + self.ng_range_line_end: Optional[int] = None + + self.ng_marker_line: Optional[int] = None + self.ng_marker_column: Optional[int] = None + + self.ng_is_nodoc = "nosdoc" in self.reqs + + def is_range_marker(self) -> bool: + return True + + def is_line_marker(self) -> bool: + return False + + def is_begin(self) -> bool: + return True + + def is_end(self) -> bool: + return False + + +@auto_described +class ForwardFunctionRangeMarker(FunctionRangeMarker): + def __init__(self, parent, reqs_objs: List[Req]): + super().__init__(parent, reqs_objs) diff --git a/strictdoc/backend/sdoc_source_code/models/range_marker.py b/strictdoc/backend/sdoc_source_code/models/range_marker.py index c74fca42e..289aedda3 100644 --- a/strictdoc/backend/sdoc_source_code/models/range_marker.py +++ b/strictdoc/backend/sdoc_source_code/models/range_marker.py @@ -1,5 +1,5 @@ # mypy: disable-error-code="no-untyped-def,type-arg" -from typing import List +from typing import Any, List, Optional from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req from strictdoc.helpers.auto_described import auto_described @@ -7,15 +7,16 @@ @auto_described class RangeMarker: - def __init__(self, parent, begin_or_end, reqs_objs: List[Req]): + def __init__(self, parent: Any, begin_or_end: str, reqs_objs: List[Req]): assert isinstance(reqs_objs, list) - self.parent = parent - self.begin_or_end = begin_or_end + self.parent: Any = parent + self.begin_or_end: str = begin_or_end self.reqs_objs: List[Req] = reqs_objs self.reqs: List[str] = list(map(lambda req: req.uid, reqs_objs)) # Line number of the marker in the source code. - self.ng_source_line_begin = None + self.ng_source_line_begin: Optional[int] = None + self.ng_source_column_begin: Optional[int] = None # Line number of the marker range in the source code: # TODO: Improve description. @@ -25,50 +26,51 @@ def __init__(self, parent, begin_or_end, reqs_objs: List[Req]): # For End ranges: # ng_range_line_begin == ng_range_line_begin of the Begin marker # noqa: ERA001, E501 # ng_range_line_end == ng_source_line_begin # noqa: ERA001 - self.ng_range_line_begin = None - self.ng_range_line_end = None + self.ng_range_line_begin: Optional[int] = None + self.ng_range_line_end: Optional[int] = None self.ng_is_nodoc = "nosdoc" in self.reqs - def is_begin(self): + def is_begin(self) -> bool: return self.begin_or_end == "[" - def is_end(self): + def is_end(self) -> bool: return self.begin_or_end == "[/" - def is_range_marker(self): + def is_range_marker(self) -> bool: return True - def is_line_marker(self): + def is_line_marker(self) -> bool: return False @auto_described class LineMarker: - def __init__(self, parent, reqs_objs): + def __init__(self, parent: Any, reqs_objs: List[Req]) -> None: assert isinstance(reqs_objs, list) self.parent = parent self.reqs_objs = reqs_objs self.reqs = list(map(lambda req: req.uid, reqs_objs)) # Line number of the marker in the source code. - self.ng_source_line_begin = None + self.ng_source_line_begin: Optional[int] = None + self.ng_source_column_begin: Optional[int] = None - self.ng_range_line_begin = None - self.ng_range_line_end = None + self.ng_range_line_begin: Optional[int] = None + self.ng_range_line_end: Optional[int] = None self.ng_is_nodoc = "nosdoc" in self.reqs - def is_begin(self): + def is_begin(self) -> bool: return True - def is_end(self): + def is_end(self) -> bool: return False - def is_range_marker(self): + def is_range_marker(self) -> bool: return False - def is_line_marker(self): + def is_line_marker(self) -> bool: return True @@ -81,19 +83,19 @@ def __init__(self, start_or_end: bool, reqs_objs: List): self.reqs_objs = reqs_objs # Line number of the marker in the source code. - self.ng_source_line_begin = None + self.ng_source_line_begin: Optional[int] = None - self.ng_range_line_begin = None - self.ng_range_line_end = None + self.ng_range_line_begin: Optional[int] = None + self.ng_range_line_end: Optional[int] = None - def is_begin(self): + def is_begin(self) -> bool: return self.start_or_end - def is_end(self): + def is_end(self) -> bool: return not self.start_or_end - def is_range_marker(self): + def is_range_marker(self) -> bool: return True - def is_line_marker(self): + def is_line_marker(self) -> bool: return False diff --git a/strictdoc/backend/sdoc_source_code/models/requirement_marker.py b/strictdoc/backend/sdoc_source_code/models/requirement_marker.py index d562c0b76..c7a9e2371 100644 --- a/strictdoc/backend/sdoc_source_code/models/requirement_marker.py +++ b/strictdoc/backend/sdoc_source_code/models/requirement_marker.py @@ -1,15 +1,16 @@ -# mypy: disable-error-code="no-untyped-def" +from typing import Any, Optional + from strictdoc.helpers.auto_described import auto_described @auto_described class Req: - def __init__(self, parent, uid: str): + def __init__(self, parent: Any, uid: str): assert isinstance(uid, str) assert len(uid) > 0 self.parent = parent self.uid: str = uid - self.ng_source_line = None - self.ng_source_column = None + self.ng_source_line: Optional[int] = None + self.ng_source_column: Optional[int] = None diff --git a/strictdoc/backend/sdoc_source_code/models/source_file_info.py b/strictdoc/backend/sdoc_source_code/models/source_file_info.py index 7998385d4..8c50a9ee7 100644 --- a/strictdoc/backend/sdoc_source_code/models/source_file_info.py +++ b/strictdoc/backend/sdoc_source_code/models/source_file_info.py @@ -1,6 +1,10 @@ # mypy: disable-error-code="no-untyped-def,type-arg,var-annotated" from typing import List, Union +from strictdoc.backend.sdoc_source_code.models.function import Function +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) from strictdoc.backend.sdoc_source_code.models.range_marker import ( ForwardRangeMarker, LineMarker, @@ -20,14 +24,7 @@ def __init__(self, parts: List): """ self.parts: List = parts - - """ - { - 2: RangeMarker(...), - 4: RangeMarker(...), - } - """ - self.ng_map_lines_to_markers = {} + self.functions: List[Function] = [] """ { @@ -39,15 +36,17 @@ def __init__(self, parts: List): self.ng_lines_total = 0 self.ng_lines_covered = 0 - self._coverage = 0 + self._coverage: float = 0 self.markers: List[ - Union[LineMarker, RangeMarker, ForwardRangeMarker] + Union[ + FunctionRangeMarker, LineMarker, RangeMarker, ForwardRangeMarker + ] ] = [] def get_coverage(self): return self._coverage - def set_coverage_stats(self, lines_total, lines_covered): + def set_coverage_stats(self, lines_total: int, lines_covered: int) -> None: self.ng_lines_total = lines_total self.ng_lines_covered = lines_covered self._coverage = round(lines_covered / lines_total * 100, 1) diff --git a/strictdoc/backend/sdoc_source_code/parse_context.py b/strictdoc/backend/sdoc_source_code/parse_context.py new file mode 100644 index 000000000..a486a7a9c --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/parse_context.py @@ -0,0 +1,12 @@ +from typing import Any, Dict, List + +from strictdoc.backend.sdoc_source_code.models.range_marker import RangeMarker + + +class ParseContext: + def __init__(self, filename: str, lines_total: int) -> None: + self.filename: str = filename + self.lines_total: int = lines_total + self.markers: List[Any] = [] + self.marker_stack: List[RangeMarker] = [] + self.map_reqs_to_markers: Dict[str, Any] = {} diff --git a/strictdoc/backend/sdoc_source_code/processors/general_language_marker_processors.py b/strictdoc/backend/sdoc_source_code/processors/general_language_marker_processors.py new file mode 100644 index 000000000..a5496275d --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/processors/general_language_marker_processors.py @@ -0,0 +1,254 @@ +from typing import Any, List, Optional, Tuple, Union + +from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.range_marker import ( + ForwardRangeMarker, + LineMarker, + RangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.source_file_info import ( + SourceFileTraceabilityInfo, +) +from strictdoc.backend.sdoc_source_code.parse_context import ParseContext +from strictdoc.helpers.cast import assert_cast + + +def source_file_traceability_info_processor( + source_file_traceability_info: SourceFileTraceabilityInfo, + parse_context: ParseContext, +) -> None: + if len(parse_context.marker_stack) > 0: + raise create_unmatch_range_error( + parse_context.marker_stack, parse_context.filename + ) + source_file_traceability_info.markers = parse_context.markers + + # Finding how many lines are covered by the requirements in the file. + # Quick and dirty: https://stackoverflow.com/a/15273749/598057 + merged_ranges: List[List[Any]] = [] + marker: Union[ + FunctionRangeMarker, LineMarker, RangeMarker, ForwardRangeMarker + ] + for marker in source_file_traceability_info.markers: + # At this point, we don't have any ForwardRangeMarkers because they + # come from Requirements, not from source code. + assert isinstance( + marker, (FunctionRangeMarker, RangeMarker, LineMarker) + ), marker + if marker.ng_is_nodoc: + continue + if not marker.is_begin(): + continue + begin, end = ( + assert_cast(marker.ng_range_line_begin, int), + assert_cast(marker.ng_range_line_end, int), + ) + if merged_ranges and merged_ranges[-1][1] >= (begin - 1): + merged_ranges[-1][1] = max(merged_ranges[-1][1], end) + else: + merged_ranges.append([begin, end]) + coverage = 0 + for merged_range in merged_ranges: + coverage += merged_range[1] - merged_range[0] + 1 + source_file_traceability_info.set_coverage_stats( + parse_context.lines_total, coverage + ) + + +def create_begin_end_range_reqs_mismatch_error( + filename: str, + line: int, + col: int, + lhs_marker_reqs: List[str], + rhs_marker_reqs: List[str], +) -> StrictDocSemanticError: + lhs_marker_reqs_str = ", ".join(lhs_marker_reqs) + rhs_marker_reqs_str = ", ".join(rhs_marker_reqs) + + return StrictDocSemanticError( + "STRICTDOC RANGE: BEGIN and END requirements mismatch", + ( + "STRICT RANGE marker should START and END " + "with the same requirement(s): " + f"'{lhs_marker_reqs_str}' != '{rhs_marker_reqs_str}'." + ), + # @sdoc[nosdoc] # noqa: ERA001 + """ +# [REQ-001] +Content... +# [/REQ-001] + """.lstrip(), + # @sdoc[/nosdoc] # noqa: ERA001 + line=line, + col=col, + filename=filename, + ) + + +def create_end_without_begin_error( + filename: str, line: int, col: int +) -> StrictDocSemanticError: + return StrictDocSemanticError( + "STRICTDOC RANGE: END marker without preceding BEGIN marker", + ( + "STRICT RANGE shall be opened with " + "START marker and ended with END marker." + ), + # @sdoc[nosdoc] # noqa: ERA001 + """ +# [REQ-001] +Content... +# [/REQ-001] + """.lstrip(), + # @sdoc[/nosdoc] # noqa: ERA001 + line=line, + col=col, + filename=filename, + ) + + +def create_unmatch_range_error( + unmatched_ranges: List[RangeMarker], filename: str +) -> StrictDocSemanticError: + assert isinstance(unmatched_ranges, list) + assert len(unmatched_ranges) > 0 + range_locations: List[Tuple[int, int]] = [] + for unmatched_range_ in unmatched_ranges: + assert unmatched_range_.ng_source_line_begin is not None + assert unmatched_range_.ng_source_column_begin is not None + range_locations.append( + ( + unmatched_range_.ng_source_line_begin, + unmatched_range_.ng_source_column_begin, + ) + ) + first_location = range_locations[0] + hint: Optional[str] = None + if len(unmatched_ranges) > 1: + range_lines = range_locations[1:] + hint = f"The @sdoc keywords are also unmatched on lines: {range_lines}." + + return StrictDocSemanticError( + "Unmatched @sdoc keyword found in source file.", + hint=hint, + # @sdoc[nosdoc] + example=( + "Each @sdoc keyword must be matched with a closing keyword. " + "Example:\n" + "@sdoc[REQ-001]\n" + "...\n" + "@sdoc[/REQ-001]" + ), + # @sdoc[/nosdoc] + line=first_location[0], + col=first_location[1], + filename=filename, + ) + + +def function_range_marker_processor( + marker: FunctionRangeMarker, parse_context: ParseContext +) -> None: + if marker.ng_is_nodoc: + return + + if ( + len(parse_context.marker_stack) > 0 + and parse_context.marker_stack[-1].ng_is_nodoc + ): + # This marker is within a "nosdoc" block, so we ignore it. + return + + parse_context.markers.append(marker) + + assert marker.ng_source_line_begin is not None + for req in marker.reqs: + markers = parse_context.map_reqs_to_markers.setdefault(req, []) + markers.append(marker) + + +def range_marker_processor( + marker: RangeMarker, parse_context: ParseContext +) -> None: + current_top_marker: RangeMarker + + if marker.ng_is_nodoc: + if marker.is_begin(): + parse_context.marker_stack.append(marker) + elif marker.is_end(): + try: + current_top_marker = parse_context.marker_stack.pop() + if ( + not current_top_marker.ng_is_nodoc + or current_top_marker.is_end() + ): + raise create_begin_end_range_reqs_mismatch_error( + "FIXME", -1, -1, current_top_marker.reqs, marker.reqs + ) + except IndexError: + raise create_end_without_begin_error("FIXME", -1, -1) from None + return + + if ( + len(parse_context.marker_stack) > 0 + and parse_context.marker_stack[-1].ng_is_nodoc + ): + # This marker is within a "nosdoc" block, so we ignore it. + return + + parse_context.markers.append(marker) + + assert marker.ng_source_line_begin is not None + + if marker.is_begin(): + marker.ng_range_line_begin = marker.ng_source_line_begin + parse_context.marker_stack.append(marker) + assert marker.ng_source_line_begin is not None + for req in marker.reqs: + markers = parse_context.map_reqs_to_markers.setdefault(req, []) + markers.append(marker) + + elif marker.is_end(): + try: + current_top_marker = parse_context.marker_stack.pop() + if marker.reqs != current_top_marker.reqs: + assert marker.ng_source_line_begin is not None + raise create_begin_end_range_reqs_mismatch_error( + "FIXME", + marker.ng_source_line_begin, + -1, + current_top_marker.reqs, + marker.reqs, + ) + + current_top_marker.ng_range_line_end = marker.ng_source_line_begin + + marker.ng_range_line_end = marker.ng_range_line_begin + marker.ng_range_line_begin = current_top_marker.ng_range_line_begin + + except IndexError: + raise create_end_without_begin_error("FIXME", 1, 1) from None + else: + raise NotImplementedError + + +def line_marker_processor( + line_marker: LineMarker, parse_context: ParseContext +) -> None: + if ( + len(parse_context.marker_stack) > 0 + and parse_context.marker_stack[-1].ng_is_nodoc + ): + # This marker is within a "nosdoc" block, so we ignore it. + return + + parse_context.markers.append(line_marker) + + assert line_marker.ng_source_line_begin is not None + + for req in line_marker.reqs: + markers = parse_context.map_reqs_to_markers.setdefault(req, []) + markers.append(line_marker) diff --git a/strictdoc/backend/sdoc_source_code/reader.py b/strictdoc/backend/sdoc_source_code/reader.py index 3119eb199..961e3bdc3 100644 --- a/strictdoc/backend/sdoc_source_code/reader.py +++ b/strictdoc/backend/sdoc_source_code/reader.py @@ -8,6 +8,9 @@ from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError from strictdoc.backend.sdoc_source_code.grammar import SOURCE_FILE_GRAMMAR +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) from strictdoc.backend.sdoc_source_code.models.range_marker import ( ForwardRangeMarker, LineMarker, @@ -27,7 +30,6 @@ def __init__(self, lines_total): self.lines_total = lines_total self.markers = [] self.marker_stack: List[RangeMarker] = [] - self.map_lines_to_markers = {} self.map_reqs_to_markers = {} @@ -53,10 +55,14 @@ def source_file_traceability_info_processor( # Finding how many lines are covered by the requirements in the file. # Quick and dirty: https://stackoverflow.com/a/15273749/598057 merged_ranges = [] - marker: Union[LineMarker, RangeMarker, ForwardRangeMarker] + marker: Union[ + FunctionRangeMarker, LineMarker, RangeMarker, ForwardRangeMarker + ] for marker in source_file_traceability_info.markers: # At this point, we don't have any ForwardRangeMarkers because they - # come from Requirements, not from source code. + # come from Requirements, not from source code. We also don't have + # function range markers because this general reader does not support + # parsing them. assert isinstance(marker, (RangeMarker, LineMarker)), marker if marker.ng_is_nodoc: continue @@ -187,12 +193,10 @@ def range_marker_processor(marker: RangeMarker, parse_context: ParseContext): parse_context.markers.append(marker) marker.ng_source_line_begin = line - parse_context.map_lines_to_markers[line] = marker if marker.is_begin(): marker.ng_range_line_begin = line parse_context.marker_stack.append(marker) - parse_context.map_lines_to_markers[line] = marker for req in marker.reqs: markers = parse_context.map_reqs_to_markers.setdefault(req, []) markers.append(marker) @@ -231,8 +235,6 @@ def line_marker_processor(line_marker: LineMarker, parse_context: ParseContext): line_marker.ng_range_line_begin = line line_marker.ng_range_line_end = line - parse_context.map_lines_to_markers[line] = line_marker - for req in line_marker.reqs: markers = parse_context.map_reqs_to_markers.setdefault(req, []) markers.append(line_marker) @@ -289,13 +291,9 @@ def read(self, input_string, file_path=None): input_string, file_name=file_path ) ) - if source_file_traceability_info: - source_file_traceability_info.ng_map_lines_to_markers = ( - parse_context.map_lines_to_markers - ) - source_file_traceability_info.ng_map_reqs_to_markers = ( - parse_context.map_reqs_to_markers - ) + source_file_traceability_info.ng_map_reqs_to_markers = ( + parse_context.map_reqs_to_markers + ) except StrictDocSemanticError as exc: raise exc diff --git a/strictdoc/backend/sdoc_source_code/reader_c.py b/strictdoc/backend/sdoc_source_code/reader_c.py new file mode 100644 index 000000000..510226207 --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/reader_c.py @@ -0,0 +1,175 @@ +# mypy: disable-error-code="no-redef,no-untyped-call,no-untyped-def,type-arg,var-annotated" +import sys +import traceback +from typing import List, Union + +import tree_sitter_c +from tree_sitter import Language, Parser + +from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError +from strictdoc.backend.sdoc_source_code.marker_parser import MarkerParser +from strictdoc.backend.sdoc_source_code.models.function import Function +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.range_marker import ( + LineMarker, + RangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.source_file_info import ( + SourceFileTraceabilityInfo, +) +from strictdoc.backend.sdoc_source_code.parse_context import ParseContext +from strictdoc.backend.sdoc_source_code.processors.general_language_marker_processors import ( + function_range_marker_processor, + line_marker_processor, + range_marker_processor, + source_file_traceability_info_processor, +) +from strictdoc.backend.sdoc_source_code.tree_sitter_helpers import traverse_tree +from strictdoc.helpers.string import get_lines_count + + +class SourceFileTraceabilityReader_C: + def read(self, input_buffer: bytes, file_path=None): + assert isinstance(input_buffer, bytes) + + file_size = len(input_buffer) + + traceability_info = SourceFileTraceabilityInfo([]) + + if file_size == 0: + return traceability_info + + length = get_lines_count(input_buffer) + parse_context = ParseContext(file_path, length) + + # Works since Python 3.9 but we also lint this with mypy from Python 3.8. + language_arg = tree_sitter_c.language() + py_language = Language( # type: ignore[call-arg, unused-ignore] + language_arg + ) + parser = Parser(py_language) # type: ignore[call-arg, unused-ignore] + + tree = parser.parse(input_buffer) + + nodes = traverse_tree(tree) + for node_ in nodes: + if node_.type == "function_definition": + function_name: str = "" + + for child_ in node_.children: + if child_.type == "function_declarator": + assert child_.children[0].type == "identifier" + assert child_.children[0].text + + function_name = child_.children[0].text.decode("utf8") + assert function_name is not None, "Function name" + + function_comment_text = None + if ( + node_.prev_sibling is not None + and node_.prev_sibling.type == "comment" + ): + function_comment_node = node_.prev_sibling + assert function_comment_node.text is not None + function_comment_text = function_comment_node.text.decode( + "utf8" + ) + + function_last_line = node_.end_point[0] + 1 + + markers: List[ + Union[FunctionRangeMarker, RangeMarker, LineMarker] + ] = MarkerParser.parse( + function_comment_text, + function_comment_node.start_point[0] + 1, + function_last_line, + function_comment_node.start_point[0] + 1, + function_comment_node.start_point[1] + 1, + ) + for marker_ in markers: + if isinstance(marker_, FunctionRangeMarker) and ( + function_range_marker_ := marker_ + ): + function_range_marker_processor( + function_range_marker_, parse_context + ) + traceability_info.markers.append( + function_range_marker_ + ) + traceability_info.parts.append( + function_range_marker_ + ) + + new_function = Function( + parent=None, + name=function_name, + line_begin=node_.range.start_point[0] + 1, + line_end=node_.range.end_point[0] + 1, + parts=[], + ) + traceability_info.functions.append(new_function) + elif node_.type == "comment": + # A marker example: + # @sdoc[REQ-001] + if node_.text is None: + raise NotImplementedError("Comment without a text") + + node_text_string = node_.text.decode("utf8") + + markers: List[ + Union[FunctionRangeMarker, RangeMarker, LineMarker] + ] = MarkerParser.parse( + node_text_string, + node_.start_point[0] + 1, + node_.end_point[0] + 1, + node_.start_point[0] + 1, + node_.start_point[1] + 1, + ) + for marker_ in markers: + if isinstance(marker_, RangeMarker) and ( + range_marker_ := marker_ + ): + range_marker_processor(range_marker_, parse_context) + traceability_info.parts.append(range_marker_) + elif isinstance(marker_, LineMarker) and ( + line_marker_ := marker_ + ): + line_marker_processor(line_marker_, parse_context) + traceability_info.parts.append(line_marker_) + else: + continue + else: + pass + + source_file_traceability_info_processor( + traceability_info, parse_context + ) + + traceability_info.ng_map_reqs_to_markers = ( + parse_context.map_reqs_to_markers + ) + + return traceability_info + + def read_from_file(self, file_path): + try: + with open(file_path, "rb") as file: + sdoc_content = file.read() + sdoc = self.read(sdoc_content, file_path=file_path) + return sdoc + except NotImplementedError: + traceback.print_exc() + sys.exit(1) + except StrictDocSemanticError as exc: + print(exc.to_print_message()) # noqa: T201 + sys.exit(1) + except Exception as exc: # pylint: disable=broad-except + print( # noqa: T201 + f"error: SourceFileTraceabilityReader_Python: could not parse file: " + f"{file_path}.\n{exc.__class__.__name__}: {exc}" + ) + # TODO: when --debug is provided + # traceback.print_exc() # noqa: ERA001 + sys.exit(1) diff --git a/strictdoc/backend/sdoc_source_code/reader_python.py b/strictdoc/backend/sdoc_source_code/reader_python.py new file mode 100644 index 000000000..d0496d9d3 --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/reader_python.py @@ -0,0 +1,226 @@ +# mypy: disable-error-code="no-redef,no-untyped-call,no-untyped-def,type-arg,var-annotated" +import sys +import traceback +from typing import List, Optional, Union + +import tree_sitter_python +from tree_sitter import Language, Node, Parser + +from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError +from strictdoc.backend.sdoc_source_code.marker_parser import MarkerParser +from strictdoc.backend.sdoc_source_code.models.function import Function +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.range_marker import ( + LineMarker, + RangeMarker, +) +from strictdoc.backend.sdoc_source_code.models.source_file_info import ( + SourceFileTraceabilityInfo, +) +from strictdoc.backend.sdoc_source_code.parse_context import ParseContext +from strictdoc.backend.sdoc_source_code.processors.general_language_marker_processors import ( + function_range_marker_processor, + line_marker_processor, + range_marker_processor, + source_file_traceability_info_processor, +) +from strictdoc.backend.sdoc_source_code.tree_sitter_helpers import traverse_tree +from strictdoc.helpers.string import get_lines_count + + +class SourceFileTraceabilityReader_Python: + def read(self, input_buffer: bytes, file_path=None): + assert isinstance(input_buffer, bytes) + + file_size = len(input_buffer) + + traceability_info = SourceFileTraceabilityInfo([]) + + if file_size == 0: + return traceability_info + + length = get_lines_count(input_buffer) + parse_context = ParseContext(file_path, length) + + # Works since Python 3.9 but we also lint this with mypy from Python 3.8. + language_arg = tree_sitter_python.language() + py_language = Language( # type: ignore[call-arg, unused-ignore] + language_arg + ) + parser = Parser(py_language) # type: ignore[call-arg, unused-ignore] + + tree = parser.parse(input_buffer) + + functions_stack: List[Function] = [] + + nodes = traverse_tree(tree) + map_function_to_node = {} + for node_ in nodes: + if node_.type == "module": + function = Function( + parent=None, + name="module", + line_begin=node_.start_point[0] + 1, + line_end=node_.end_point[0] + 1, + parts=[], + ) + functions_stack.append(function) + map_function_to_node[function] = node_ + elif node_.type == "function_definition": + function_name: str = "" + function_block: Optional[Node] = None + + # assert 0, node_.children + for child_ in node_.children: + if child_.type == "identifier": + if child_.text is not None: + function_name = child_.text.decode("utf-8") + if child_.type == "function_declarator": + # FIXME + # assert 0, child_.children + function_name = "FOO" + if child_.type == "block": + function_block = child_ + + assert function_name is not None, "Function name" + + block_comment = None + if ( + function_block is not None + and len(function_block.children) > 0 + and function_block.children[0].type + == "expression_statement" + ): + if len(function_block.children[0].children) > 0: + if ( + function_block.children[0].children[0].type + == "string" + ): + block_comment = function_block.children[0].children[ + 0 + ] + # string contains of three parts: + # string_start string_content string_end + string_content = block_comment.children[1] + assert string_content.text is not None + + block_comment_text = string_content.text.decode( + "utf-8" + ) + markers = MarkerParser.parse( + block_comment_text, + node_.start_point[0] + 1, + node_.end_point[0] + 1, + string_content.start_point[0] + 1, + string_content.start_point[1] + 1, + ) + for marker_ in markers: + if isinstance( + marker_, FunctionRangeMarker + ) and (function_range_marker_ := marker_): + function_range_marker_processor( + function_range_marker_, parse_context + ) + traceability_info.markers.append( + function_range_marker_ + ) + + # FIXME: This look more complex than needed but can't make mypy happy. + cursor_: Optional[Node] = node_ + while cursor_ is not None and (cursor_ := cursor_.parent): + if cursor_ == map_function_to_node[functions_stack[-1]]: + break + if cursor_.type == "function_definition": + functions_stack.pop() + assert len(functions_stack) > 0 + else: + # This is counterintuitive: + # The top-level functions don't have the top-level module set + # as their parent, so in this branch, we simply clear the whole + # function stack, leaving the top module only. + functions_stack = functions_stack[:1] + + new_function = Function( + parent=None, + name=function_name, + line_begin=node_.range.start_point[0] + 1, + line_end=node_.range.end_point[0] + 1, + parts=[], + ) + map_function_to_node[new_function] = node_ + + parent_function = functions_stack[-1] + + parent_function.parts.append(new_function) + functions_stack.append(new_function) + traceability_info.functions.append(new_function) + elif node_.type == "comment": + # A marker example: + # @sdoc[REQ-001] + if node_.text is None: + raise NotImplementedError("Comment without a text") + + node_text_string = node_.text.decode("utf8") + + markers: List[ + Union[FunctionRangeMarker, RangeMarker, LineMarker] + ] = MarkerParser.parse( + node_text_string, + node_.start_point[0] + 1, + node_.end_point[0] + 1, + node_.start_point[0] + 1, + node_.start_point[1] + 1, + ) + for marker_ in markers: + if isinstance(marker_, RangeMarker) and ( + range_marker := marker_ + ): + range_marker_processor(range_marker, parse_context) + elif isinstance(marker_, LineMarker) and ( + line_marker := marker_ + ): + line_marker_processor(line_marker, parse_context) + else: + continue + else: + pass + + assert ( + functions_stack[0].name == "module" + or functions_stack[0].name == "translation_unit" + ) + + traceability_info.parts = functions_stack[0].parts + + source_file_traceability_info_processor( + traceability_info, parse_context + ) + + traceability_info.ng_map_reqs_to_markers = ( + parse_context.map_reqs_to_markers + ) + + return traceability_info + + def read_from_file(self, file_path): + try: + with open(file_path, "rb") as file: + sdoc_content = file.read() + sdoc = self.read(sdoc_content, file_path=file_path) + return sdoc + except NotImplementedError: + traceback.print_exc() + sys.exit(1) + except StrictDocSemanticError as exc: + print(exc.to_print_message()) # noqa: T201 + sys.exit(1) + except Exception as exc: # pylint: disable=broad-except + print( # noqa: T201 + f"error: SourceFileTraceabilityReader_Python: could not parse file: " + f"{file_path}.\n{exc.__class__.__name__}: {exc}" + ) + # TODO: when --debug is provided + # traceback.print_exc() # noqa: ERA001 + sys.exit(1) diff --git a/strictdoc/backend/sdoc_source_code/tree_sitter_helpers.py b/strictdoc/backend/sdoc_source_code/tree_sitter_helpers.py new file mode 100644 index 000000000..cadf4e8f6 --- /dev/null +++ b/strictdoc/backend/sdoc_source_code/tree_sitter_helpers.py @@ -0,0 +1,18 @@ +from typing import Generator + +from tree_sitter import Node, Tree + + +def traverse_tree(tree: Tree) -> Generator[Node, None, None]: + cursor = tree.walk() + + visited_children = False + while True: + if not visited_children: + yield cursor.node + if not cursor.goto_first_child(): + visited_children = True + elif cursor.goto_next_sibling(): + visited_children = False + elif not cursor.goto_parent(): + break diff --git a/strictdoc/core/file_traceability_index.py b/strictdoc/core/file_traceability_index.py index bb4174246..8dbb87e79 100644 --- a/strictdoc/core/file_traceability_index.py +++ b/strictdoc/core/file_traceability_index.py @@ -1,8 +1,11 @@ # mypy: disable-error-code="arg-type,attr-defined,no-any-return,no-untyped-def" -from typing import Any, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple from strictdoc.backend.sdoc.models.node import SDocNode from strictdoc.backend.sdoc.models.reference import FileReference, Reference +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + ForwardFunctionRangeMarker, +) from strictdoc.backend.sdoc_source_code.models.range_marker import ( ForwardRangeMarker, RangeMarker, @@ -12,12 +15,13 @@ SourceFileTraceabilityInfo, ) from strictdoc.helpers.exception import StrictDocException +from strictdoc.helpers.ordered_set import OrderedSet class FileTraceabilityIndex: def __init__(self): # "file.py" -> List[SDocNode] - self.map_paths_to_reqs: Dict[str, List[SDocNode]] = {} + self.map_paths_to_reqs: Dict[str, OrderedSet[SDocNode]] = {} # "REQ-001" -> List[FileReference] self.map_reqs_uids_to_paths: Dict[str, List[FileReference]] = {} @@ -27,7 +31,12 @@ def __init__(self): str, SourceFileTraceabilityInfo ] = {} - self.map_reqs_uids_to_line_range_file_refs: Dict[str, List[Any]] = {} + self.map_reqs_uids_to_line_range_file_refs: Dict[ + str, List[Tuple[str, Tuple[int, int]]] + ] = {} + self.map_file_function_names_to_reqs_uids: Dict[ + str, Dict[str, List[str]] + ] = {} # "file.py" -> ( # general_requirements: [SDocNode], # noqa: ERA001 @@ -183,9 +192,9 @@ def validate(self): source_file_info.markers.append(start_marker) source_file_info.markers.append(end_marker) - # assert 0, self.map_reqs_uids_to_line_range_file_refs - def create_requirement(self, requirement: SDocNode) -> None: + assert requirement.reserved_uid is not None + # A requirement can have multiple File references, and this function is # called for every File reference. if requirement.reserved_uid in self.map_reqs_uids_to_paths: @@ -196,22 +205,35 @@ def create_requirement(self, requirement: SDocNode) -> None: if isinstance(ref, FileReference): file_reference: FileReference = ref requirements = self.map_paths_to_reqs.setdefault( - file_reference.get_posix_path(), [] + file_reference.get_posix_path(), OrderedSet() ) - requirements.append(requirement) + requirements.add(requirement) paths = self.map_reqs_uids_to_paths.setdefault( requirement.reserved_uid, [] ) paths.append(ref) - if file_reference.g_file_entry.line_range is not None: - requirements = ( + if file_reference.g_file_entry.function is not None: + one_file_function_name_to_reqs_uids = ( + self.map_file_function_names_to_reqs_uids.setdefault( + file_reference.get_posix_path(), {} + ) + ) + function_name_to_reqs_uids = ( + one_file_function_name_to_reqs_uids.setdefault( + file_reference.g_file_entry.function, [] + ) + ) + function_name_to_reqs_uids.append(requirement.reserved_uid) + elif file_reference.g_file_entry.line_range is not None: + assert requirement.reserved_uid is not None + req_uid_to_line_range_file_refs = ( self.map_reqs_uids_to_line_range_file_refs.setdefault( requirement.reserved_uid, [] ) ) - requirements.append( + req_uid_to_line_range_file_refs.append( ( file_reference.get_posix_path(), file_reference.g_file_entry.line_range, @@ -227,3 +249,39 @@ def create_traceability_info( self.map_paths_to_source_file_traceability_info[ source_file_rel_path ] = traceability_info + + for function_ in traceability_info.functions: + if ( + source_file_rel_path + not in self.map_file_function_names_to_reqs_uids + ): + continue + + reqs_uids = self.map_file_function_names_to_reqs_uids[ + source_file_rel_path + ].get(function_.name, None) + if reqs_uids is None: + continue + + reqs = [] + for req_uid_ in reqs_uids: + req = Req(None, req_uid_) + reqs.append(req) + + function_marker = ForwardFunctionRangeMarker( + parent=None, reqs_objs=reqs + ) + function_marker.ng_source_line_begin = function_.line_begin + function_marker.ng_source_column_begin = 1 + function_marker.ng_range_line_begin = function_.line_begin + function_marker.ng_range_line_end = function_.line_end + function_marker.ng_marker_line = function_.line_begin + function_marker.ng_marker_column = 1 + + for req_uid_ in reqs_uids: + markers = traceability_info.ng_map_reqs_to_markers.setdefault( + req_uid_, [] + ) + markers.append(function_marker) + + traceability_info.markers.append(function_marker) diff --git a/strictdoc/core/project_config.py b/strictdoc/core/project_config.py index d98bedfc8..7e865c2b9 100644 --- a/strictdoc/core/project_config.py +++ b/strictdoc/core/project_config.py @@ -47,6 +47,7 @@ class ProjectFeature(str, Enum): STANDALONE_DOCUMENT_SCREEN = "STANDALONE_DOCUMENT_SCREEN" TRACEABILITY_MATRIX_SCREEN = "TRACEABILITY_MATRIX_SCREEN" REQUIREMENT_TO_SOURCE_TRACEABILITY = "REQUIREMENT_TO_SOURCE_TRACEABILITY" + SOURCE_FILE_LANGUAGE_PARSERS = "SOURCE_FILE_LANGUAGE_PARSERS" MERMAID = "MERMAID" RAPIDOC = "RAPIDOC" @@ -277,6 +278,11 @@ def is_activated_mermaid(self) -> bool: def is_activated_rapidoc(self) -> bool: return ProjectFeature.RAPIDOC in self.project_features + def is_activated_source_file_language_parsers(self) -> bool: + return ( + ProjectFeature.SOURCE_FILE_LANGUAGE_PARSERS in self.project_features + ) + def get_strictdoc_root_path(self) -> str: return self.environment.path_to_strictdoc diff --git a/strictdoc/core/traceability_index_builder.py b/strictdoc/core/traceability_index_builder.py index 55afad643..a38639f28 100644 --- a/strictdoc/core/traceability_index_builder.py +++ b/strictdoc/core/traceability_index_builder.py @@ -23,6 +23,12 @@ from strictdoc.backend.sdoc_source_code.reader import ( SourceFileTraceabilityReader, ) +from strictdoc.backend.sdoc_source_code.reader_c import ( + SourceFileTraceabilityReader_C, +) +from strictdoc.backend.sdoc_source_code.reader_python import ( + SourceFileTraceabilityReader_Python, +) from strictdoc.core.document_finder import DocumentFinder from strictdoc.core.document_iterator import DocumentCachingIterator from strictdoc.core.document_meta import DocumentMeta @@ -178,10 +184,36 @@ def create( if not is_source_file_referenced: continue source_file.is_referenced = True - traceability_reader = SourceFileTraceabilityReader() - traceability_info = traceability_reader.read_from_file( - source_file.full_path - ) + + # FIXME: It should be possible to simplify this branching. + if project_config.is_activated_source_file_language_parsers(): + if source_file.full_path.endswith(".py"): + traceability_reader_python = ( + SourceFileTraceabilityReader_Python() + ) + traceability_info = ( + traceability_reader_python.read_from_file( + source_file.full_path + ) + ) + elif source_file.full_path.endswith(".c"): + traceability_reader_c = SourceFileTraceabilityReader_C() + traceability_info = ( + traceability_reader_c.read_from_file( + source_file.full_path + ) + ) + else: + traceability_reader = SourceFileTraceabilityReader() + traceability_info = traceability_reader.read_from_file( + source_file.full_path + ) + else: + traceability_reader = SourceFileTraceabilityReader() + traceability_info = traceability_reader.read_from_file( + source_file.full_path + ) + if traceability_info: traceability_index.create_traceability_info( source_file.in_doctree_source_file_rel_path_posix, diff --git a/strictdoc/export/html/form_objects/requirement_form_object.py b/strictdoc/export/html/form_objects/requirement_form_object.py index a39b55f97..24acf0750 100644 --- a/strictdoc/export/html/form_objects/requirement_form_object.py +++ b/strictdoc/export/html/form_objects/requirement_form_object.py @@ -533,6 +533,7 @@ def get_requirement_relations( g_file_format=FileEntryFormat.SOURCECODE, g_file_path=reference_field.field_value, g_line_range="", + function=None, ) references.append( FileReference( diff --git a/strictdoc/export/html/generators/source_file_view_generator.py b/strictdoc/export/html/generators/source_file_view_generator.py index abee52949..1e8bbe0a8 100644 --- a/strictdoc/export/html/generators/source_file_view_generator.py +++ b/strictdoc/export/html/generators/source_file_view_generator.py @@ -15,6 +15,10 @@ from pygments.lexers.templates import HtmlDjangoLexer from pygments.util import ClassNotFound +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + ForwardFunctionRangeMarker, + FunctionRangeMarker, +) from strictdoc.backend.sdoc_source_code.models.range_marker import ( ForwardRangeMarker, LineMarker, @@ -196,8 +200,18 @@ def get_pygmented_source_lines( ) continue - source_line = source_file_lines[marker_line - 1] + if isinstance(marker, ForwardFunctionRangeMarker): + before_line = pygmented_source_file_line.rstrip("\n") + " " + pygmented_source_file_lines[marker_line - 1] = ( + SourceMarkerTuple(Markup(before_line), Markup("\n"), marker) + ) + continue + if isinstance(marker, FunctionRangeMarker): + # FIXME + marker_line = marker.ng_marker_line + + source_line = source_file_lines[marker_line - 1] assert len(marker.reqs_objs) > 0 before_line = source_line[ : marker.reqs_objs[0].ng_source_column - 1 @@ -205,6 +219,8 @@ def get_pygmented_source_lines( closing_bracket_index = ( source_line.index("]") if isinstance(marker, RangeMarker) + else source_line.index(", scope") + if isinstance(marker, FunctionRangeMarker) else source_line.index(")") if isinstance(marker, LineMarker) else None diff --git a/strictdoc/export/html/generators/view_objects/source_file_view_object.py b/strictdoc/export/html/generators/view_objects/source_file_view_object.py index bf466a380..ce9d41dfb 100644 --- a/strictdoc/export/html/generators/view_objects/source_file_view_object.py +++ b/strictdoc/export/html/generators/view_objects/source_file_view_object.py @@ -6,6 +6,9 @@ from markupsafe import Markup from strictdoc import __version__ +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) from strictdoc.backend.sdoc_source_code.models.range_marker import ( ForwardRangeMarker, LineMarker, @@ -23,7 +26,9 @@ class SourceMarkerTuple(NamedTuple): before_line: Markup after_line: Markup - marker: Union[ForwardRangeMarker, LineMarker, RangeMarker] + marker: Union[ + FunctionRangeMarker, ForwardRangeMarker, LineMarker, RangeMarker + ] SourceLineEntry = Union[ diff --git a/strictdoc/export/spdx/spdx_to_sdoc_converter.py b/strictdoc/export/spdx/spdx_to_sdoc_converter.py index feb90ed47..dd8e59f75 100644 --- a/strictdoc/export/spdx/spdx_to_sdoc_converter.py +++ b/strictdoc/export/spdx/spdx_to_sdoc_converter.py @@ -283,6 +283,7 @@ def _convert_file( g_file_format=None, g_file_path=file.name, g_line_range=None, + function=None, ), ) ] @@ -338,6 +339,7 @@ def _convert_snippet( g_file_format=None, g_file_path=spdx_file.name, g_line_range=f"{snippet.line_range.begin}, {snippet.line_range.end - 1}", + function=None, ), ) ] diff --git a/strictdoc/helpers/string.py b/strictdoc/helpers/string.py index 1bc2dca23..3bc5fcfa3 100644 --- a/strictdoc/helpers/string.py +++ b/strictdoc/helpers/string.py @@ -1,17 +1,16 @@ # mypy: disable-error-code="no-untyped-def" import re -from typing import Optional +from typing import Optional, Union REGEX_TRAILING_WHITESPACE_SINGLELINE = re.compile(r"\s{2,}") REGEX_TRAILING_WHITESPACE_MULTILINE = re.compile(r" +\n") -def get_lines_count(string): - # TODO: Windows strings - count = string.count("\n") - if string[-1] != "\n": - count += 1 - return count +def get_lines_count(string_or_bytes: Union[str, bytes]): + string = string_or_bytes + if isinstance(string_or_bytes, bytes): + string = string_or_bytes.decode("utf-8") + return len(string.splitlines()) # WIP: Check if this is used. diff --git a/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/file.py b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/file.py new file mode 100644 index 000000000..9f69869aa --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/file.py @@ -0,0 +1,4 @@ +# @sdoc[REQ-1] +def hello_world(): + print("hello world") # noqa: T201 +# @sdoc[/REQ-1] diff --git a/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/input.sdoc b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/input.sdoc new file mode 100644 index 000000000..a4dc62e0b --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/input.sdoc @@ -0,0 +1,10 @@ +[DOCUMENT] +TITLE: Hello world doc + +[REQUIREMENT] +UID: REQ-1 +TITLE: Requirement Title +STATEMENT: Requirement Statement +RELATIONS: +- TYPE: File + VALUE: file.py diff --git a/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/strictdoc.toml b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/strictdoc.toml new file mode 100644 index 000000000..f5502ad57 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/strictdoc.toml @@ -0,0 +1,7 @@ +[project] + +features = [ + "REQUIREMENT_TO_SOURCE_TRACEABILITY", + "SOURCE_FILE_LANGUAGE_PARSERS", + "PROJECT_STATISTICS_SCREEN" +] diff --git a/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/test.itest b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/test.itest new file mode 100644 index 000000000..3dd5e4487 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/01_c_range/test.itest @@ -0,0 +1,12 @@ +REQUIRES: PYTHON_39_OR_HIGHER + +RUN: %strictdoc export %S --output-dir Output | filecheck %s --dump-input=fail +CHECK: Published: Hello world doc + +RUN: %check_exists --file "%S/Output/html/_source_files/file.py.html" + +RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-HTML +CHECK-HTML: +RUN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SOURCE-FILE +CHECK-SOURCE-FILE: +CHECK-SOURCE-FILE: # noqa: T201 diff --git a/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/file.c b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/file.c new file mode 100644 index 000000000..b9e59d17d --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/file.c @@ -0,0 +1,38 @@ +#include + +/** + * Some text. + * + * @relation(REQ-1, scope=function) + */ +void hello_world(void) { + print("hello world\n"); +} + +/** + * Some text. + * + * @relation(REQ-2, scope=function) + */ +void hello_world_2(void) { + print("hello world\n"); +} + +/** + * Some text. + * + * @relation(REQ-1, REQ-2, scope=function) + */ +void hello_world_3(void) { + print("hello world\n"); +} + +/** + * Some text. + * + * @relation(REQ-1, scope=function) + * @relation(REQ-2, scope=function) + */ +void hello_world_4(void) { + print("hello world\n"); +} diff --git a/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/input.sdoc b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/input.sdoc new file mode 100644 index 000000000..773cd8938 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/input.sdoc @@ -0,0 +1,18 @@ +[DOCUMENT] +TITLE: Hello world doc + +[REQUIREMENT] +UID: REQ-1 +TITLE: Requirement Title +STATEMENT: Requirement Statement +RELATIONS: +- TYPE: File + VALUE: file.c + +[REQUIREMENT] +UID: REQ-2 +TITLE: Requirement Title #2 +STATEMENT: Requirement Statement #2 +RELATIONS: +- TYPE: File + VALUE: file.c diff --git a/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/strictdoc.toml b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/strictdoc.toml new file mode 100644 index 000000000..f5502ad57 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/strictdoc.toml @@ -0,0 +1,7 @@ +[project] + +features = [ + "REQUIREMENT_TO_SOURCE_TRACEABILITY", + "SOURCE_FILE_LANGUAGE_PARSERS", + "PROJECT_STATISTICS_SCREEN" +] diff --git a/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/test.itest b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/test.itest new file mode 100644 index 000000000..bc322b1a0 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/02_c_function/test.itest @@ -0,0 +1,21 @@ +REQUIRES: PYTHON_39_OR_HIGHER + +RUN: %strictdoc export %S --output-dir Output | filecheck %s --dump-input=fail +CHECK: Published: Hello world doc + +RU: %check_exists --file "%S/Output/html/_source_files/file.py.html" + +RN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-HTML +CHECK-HTML: +CHECK-HTML: +CHECK-HTML: + +UN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SOURCE-FILE +CHECK-SOURCE-FILE: 
    @relation(REQ-1, scope=function)
6
diff --git a/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/file.c b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/file.c new file mode 100644 index 000000000..b9e59d17d --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/file.c @@ -0,0 +1,38 @@ +#include + +/** + * Some text. + * + * @relation(REQ-1, scope=function) + */ +void hello_world(void) { + print("hello world\n"); +} + +/** + * Some text. + * + * @relation(REQ-2, scope=function) + */ +void hello_world_2(void) { + print("hello world\n"); +} + +/** + * Some text. + * + * @relation(REQ-1, REQ-2, scope=function) + */ +void hello_world_3(void) { + print("hello world\n"); +} + +/** + * Some text. + * + * @relation(REQ-1, scope=function) + * @relation(REQ-2, scope=function) + */ +void hello_world_4(void) { + print("hello world\n"); +} diff --git a/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/input.sdoc b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/input.sdoc new file mode 100644 index 000000000..d6d8b0491 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/input.sdoc @@ -0,0 +1,26 @@ +[DOCUMENT] +TITLE: Hello world doc + +[REQUIREMENT] +UID: REQ-1 +TITLE: Requirement Title +STATEMENT: Requirement Statement +RELATIONS: +- TYPE: File + VALUE: file.c + FUNCTION: hello_world +- TYPE: File + VALUE: file.c + FUNCTION: hello_world_2 + +[REQUIREMENT] +UID: REQ-2 +TITLE: Requirement Title #2 +STATEMENT: Requirement Statement #2 +RELATIONS: +- TYPE: File + VALUE: file.c + FUNCTION: hello_world +- TYPE: File + VALUE: file.c + FUNCTION: hello_world_2 diff --git a/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/strictdoc.toml b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/strictdoc.toml new file mode 100644 index 000000000..f5502ad57 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/strictdoc.toml @@ -0,0 +1,7 @@ +[project] + +features = [ + "REQUIREMENT_TO_SOURCE_TRACEABILITY", + "SOURCE_FILE_LANGUAGE_PARSERS", + "PROJECT_STATISTICS_SCREEN" +] diff --git a/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/test.itest b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/test.itest new file mode 100644 index 000000000..bc322b1a0 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/c/03_c_forward_function/test.itest @@ -0,0 +1,21 @@ +REQUIRES: PYTHON_39_OR_HIGHER + +RUN: %strictdoc export %S --output-dir Output | filecheck %s --dump-input=fail +CHECK: Published: Hello world doc + +RU: %check_exists --file "%S/Output/html/_source_files/file.py.html" + +RN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-HTML +CHECK-HTML: +CHECK-HTML: +CHECK-HTML: + +UN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SOURCE-FILE +CHECK-SOURCE-FILE: 
    @relation(REQ-1, scope=function)
6
diff --git a/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/file.py b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/file.py new file mode 100644 index 000000000..9f69869aa --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/file.py @@ -0,0 +1,4 @@ +# @sdoc[REQ-1] +def hello_world(): + print("hello world") # noqa: T201 +# @sdoc[/REQ-1] diff --git a/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/input.sdoc b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/input.sdoc new file mode 100644 index 000000000..a4dc62e0b --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/input.sdoc @@ -0,0 +1,10 @@ +[DOCUMENT] +TITLE: Hello world doc + +[REQUIREMENT] +UID: REQ-1 +TITLE: Requirement Title +STATEMENT: Requirement Statement +RELATIONS: +- TYPE: File + VALUE: file.py diff --git a/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/strictdoc.toml b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/strictdoc.toml new file mode 100644 index 000000000..f5502ad57 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/strictdoc.toml @@ -0,0 +1,7 @@ +[project] + +features = [ + "REQUIREMENT_TO_SOURCE_TRACEABILITY", + "SOURCE_FILE_LANGUAGE_PARSERS", + "PROJECT_STATISTICS_SCREEN" +] diff --git a/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/test.itest b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/test.itest new file mode 100644 index 000000000..3dd5e4487 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/01_python_range/test.itest @@ -0,0 +1,12 @@ +REQUIRES: PYTHON_39_OR_HIGHER + +RUN: %strictdoc export %S --output-dir Output | filecheck %s --dump-input=fail +CHECK: Published: Hello world doc + +RUN: %check_exists --file "%S/Output/html/_source_files/file.py.html" + +RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-HTML +CHECK-HTML: +RUN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SOURCE-FILE +CHECK-SOURCE-FILE: +CHECK-SOURCE-FILE: # noqa: T201 diff --git a/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/file.py b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/file.py new file mode 100644 index 000000000..8553be5b0 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/file.py @@ -0,0 +1,13 @@ +def hello_world(): + """ + Some text. + + @relation(REQ-1, scope=function) + """ + # @sdoc[REQ-1] + print("hello world") # noqa: T201 + # @sdoc[/REQ-1] + print("hello world") # noqa: T201 + # @sdoc[REQ-1] + print("hello world") # noqa: T201 + # @sdoc[/REQ-1] diff --git a/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/input.sdoc b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/input.sdoc new file mode 100644 index 000000000..a4dc62e0b --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/input.sdoc @@ -0,0 +1,10 @@ +[DOCUMENT] +TITLE: Hello world doc + +[REQUIREMENT] +UID: REQ-1 +TITLE: Requirement Title +STATEMENT: Requirement Statement +RELATIONS: +- TYPE: File + VALUE: file.py diff --git a/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/strictdoc.toml b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/strictdoc.toml new file mode 100644 index 000000000..f5502ad57 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/strictdoc.toml @@ -0,0 +1,7 @@ +[project] + +features = [ + "REQUIREMENT_TO_SOURCE_TRACEABILITY", + "SOURCE_FILE_LANGUAGE_PARSERS", + "PROJECT_STATISTICS_SCREEN" +] diff --git a/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/test.itest b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/test.itest new file mode 100644 index 000000000..8577a8e34 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/02_python_function/test.itest @@ -0,0 +1,21 @@ +REQUIRES: PYTHON_39_OR_HIGHER + +RUN: %strictdoc export %S --output-dir Output | filecheck %s --dump-input=fail +CHECK: Published: Hello world doc + +RUN: %check_exists --file "%S/Output/html/_source_files/file.py.html" + +RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-HTML +CHECK-HTML: +CHECK-HTML: +CHECK-HTML: + +RUN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SOURCE-FILE +CHECK-SOURCE-FILE: 
    @relation(REQ-1, scope=function)
6
diff --git a/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/file.py b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/file.py new file mode 100644 index 000000000..11d2d7ca4 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/file.py @@ -0,0 +1,10 @@ +def hello_world(): + """ + @relation(REQ-1, scope=function) + """ + # @sdoc[REQ-1] + print("hello world") # noqa: T201 + # @sdoc(REQ-1) + print("hello world") # noqa: T201 + print("hello world") # noqa: T201 + # @sdoc[/REQ-1] diff --git a/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/input.sdoc b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/input.sdoc new file mode 100644 index 000000000..c8d5f7480 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/input.sdoc @@ -0,0 +1,20 @@ +[DOCUMENT] +TITLE: Hello world doc + +[REQUIREMENT] +UID: REQ-1 +TITLE: Requirement Title +STATEMENT: Requirement Statement +RELATIONS: +- TYPE: File + VALUE: file.py + FUNCTION: hello_world + +[REQUIREMENT] +UID: REQ-2 +TITLE: Requirement Title +STATEMENT: Requirement Statement +RELATIONS: +- TYPE: File + VALUE: file.py + FUNCTION: hello_world diff --git a/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/strictdoc.toml b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/strictdoc.toml new file mode 100644 index 000000000..f5502ad57 --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/strictdoc.toml @@ -0,0 +1,7 @@ +[project] + +features = [ + "REQUIREMENT_TO_SOURCE_TRACEABILITY", + "SOURCE_FILE_LANGUAGE_PARSERS", + "PROJECT_STATISTICS_SCREEN" +] diff --git a/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/test.itest b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/test.itest new file mode 100644 index 000000000..85d137a1a --- /dev/null +++ b/tests/integration/features/file_traceability/_language_parsers/python/03_python_forward_function/test.itest @@ -0,0 +1,19 @@ +REQUIRES: PYTHON_39_OR_HIGHER + +RUN: %strictdoc export %S --output-dir Output | filecheck %s --dump-input=fail +CHECK: Published: Hello world doc + +RUN: %check_exists --file "%S/Output/html/_source_files/file.py.html" + +RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-HTML +CHECK-HTML: +CHECK-HTML: +CHECK-HTML: +CHECK-HTML: + +RUN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SOURCE-FILE +CHECK-SOURCE-FILE: def hello_world +CHECK-SOURCE-FILE: href="../_source_files/file.py.html#REQ-1#1#10" +CHECK-SOURCE-FILE: href="../_source_files/file.py.html#REQ-2#1#10" +CHECK-SOURCE-FILE:
+CHECK-SOURCE-FILE: href="../_source_files/file.py.html#REQ-1#1#10" diff --git a/tests/integration/lit.cfg b/tests/integration/lit.cfg index 508b5f3dc..641b68f23 100644 --- a/tests/integration/lit.cfg +++ b/tests/integration/lit.cfg @@ -1,5 +1,6 @@ import os import subprocess +import sys import lit.formats @@ -34,7 +35,10 @@ config.substitutions.append(('%mkdir', 'python \"{}/tests/integration/mkdir.py\" config.substitutions.append(('%rm', 'python \"{}/tests/integration/rm.py\"'.format(current_dir))) config.substitutions.append(('%touch', 'python \"{}/tests/integration/touch.py\"'.format(current_dir))) -config.suffixes = ['.itest', '.c'] +config.suffixes = ['.itest'] + +if sys.version_info.major == 3 and sys.version_info.minor >= 9: + config.available_features.add('PYTHON_39_OR_HIGHER') config.is_windows = lit_config.isWindows if not lit_config.isWindows: diff --git a/tests/unit/strictdoc/backend/sdoc_source_code/test_dsl_source_file_syntax_c.py b/tests/unit/strictdoc/backend/sdoc_source_code/test_dsl_source_file_syntax_c.py new file mode 100644 index 000000000..63120881d --- /dev/null +++ b/tests/unit/strictdoc/backend/sdoc_source_code/test_dsl_source_file_syntax_c.py @@ -0,0 +1,87 @@ +import sys + +import pytest + +from strictdoc.backend.sdoc_source_code.models.source_file_info import ( + SourceFileTraceabilityInfo, +) +from strictdoc.backend.sdoc_source_code.reader_c import ( + SourceFileTraceabilityReader_C, +) +from strictdoc.backend.sdoc_source_code.reader_python import ( + SourceFileTraceabilityReader_Python, +) + +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 9), reason="Requires Python 3.9 or higher" +) + + +def test_00_empty_file(): + input_string = b"""""" + + reader = SourceFileTraceabilityReader_Python() + + info = reader.read(input_string) + + assert isinstance(info, SourceFileTraceabilityInfo) + assert len(info.markers) == 0 + + +def test_01_single_string(): + input_string = b"""\ +// Unimportant comment. +""" + + reader = SourceFileTraceabilityReader_C() + + info = reader.read(input_string) + + assert isinstance(info, SourceFileTraceabilityInfo) + assert len(info.parts) == 0 + assert len(info.markers) == 0 + + +def test_02_functions(): + input_string = b"""\ +#include + +/** + * Some text. + * + * @relation(REQ-1, scope=function) + */ +void hello_world(void) { + print("hello world\\n"); +} + +/** + * Some text. + * + * @relation(REQ-2, scope=function) + */ +void hello_world_2(void) { + print("hello world\\n"); +} +""" + + reader = SourceFileTraceabilityReader_C() + + info: SourceFileTraceabilityInfo = reader.read( + input_string, file_path="foo.c" + ) + + assert isinstance(info, SourceFileTraceabilityInfo) + assert len(info.parts) == 2 + assert len(info.markers) == 2 + assert info.markers[0].ng_source_line_begin == 3 + assert info.markers[0].ng_range_line_begin == 3 + assert info.markers[0].ng_range_line_end == 10 + assert info.markers[0].reqs_objs[0].ng_source_line == 6 + assert info.markers[0].reqs_objs[0].ng_source_column == 14 + + assert info.markers[1].ng_source_line_begin == 12 + assert info.markers[1].ng_range_line_begin == 12 + assert info.markers[1].ng_range_line_end == 19 + assert info.markers[1].reqs_objs[0].ng_source_line == 15 + assert info.markers[1].reqs_objs[0].ng_source_column == 14 diff --git a/tests/unit/strictdoc/backend/sdoc_source_code/test_dsl_source_file_syntax_python.py b/tests/unit/strictdoc/backend/sdoc_source_code/test_dsl_source_file_syntax_python.py new file mode 100644 index 000000000..892d822ed --- /dev/null +++ b/tests/unit/strictdoc/backend/sdoc_source_code/test_dsl_source_file_syntax_python.py @@ -0,0 +1,453 @@ +import sys +from typing import List + +import pytest + +from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError +from strictdoc.backend.sdoc_source_code.models.function import Function +from strictdoc.backend.sdoc_source_code.models.range_marker import RangeMarker +from strictdoc.backend.sdoc_source_code.models.source_file_info import ( + SourceFileTraceabilityInfo, +) +from strictdoc.backend.sdoc_source_code.reader_python import ( + SourceFileTraceabilityReader_Python, +) + +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 9), reason="Requires Python 3.9 or higher" +) + + +def test_00_empty_file(): + input_string = b"""""" + + reader = SourceFileTraceabilityReader_Python() + + info = reader.read(input_string) + + assert isinstance(info, SourceFileTraceabilityInfo) + assert len(info.markers) == 0 + + +def test_01_single_string(): + input_string = b"""\ +# Hello +""" + + reader = SourceFileTraceabilityReader_Python() + + info = reader.read(input_string) + + assert isinstance(info, SourceFileTraceabilityInfo) + assert len(info.parts) == 0 + assert len(info.markers) == 0 + + +def test_02_functions(): + input_string = b"""\ +def hello_1(): + print("1") + def hello_1_1(): + print("1_1") + def hello_1_1_1(): + print("1_1_1") + print("1_1_1 E") + print("1_1 E") + print("1 E") +def hello_2(): + print("2") + def hello_2_1(): + print("2_1") + def hello_2_1_1(): + print("2_1_1") + print("2_1_1 E") + print("2_1 E") + print("2 E") +def hello_3(): + print("3") + def hello_3_1(): + print("3_1") + def hello_3_1_1(): + print("3_1_1") + print("3_1_1 E") + print("3_1 E") + print("3 E") +""" + + reader = SourceFileTraceabilityReader_Python() + + info: SourceFileTraceabilityInfo = reader.read(input_string) + + assert isinstance(info, SourceFileTraceabilityInfo) + assert len(info.parts) == 3 + + function_1 = info.parts[0] + assert isinstance(function_1, Function) + assert function_1.name == "hello_1" + assert len(function_1.parts) == 1 + + function_1_1 = function_1.parts[0] + assert isinstance(function_1_1, Function) + assert function_1_1.name == "hello_1_1" + assert len(function_1_1.parts) == 1 + + function_1_1_1 = function_1_1.parts[0] + assert isinstance(function_1_1_1, Function) + assert function_1_1_1.name == "hello_1_1_1" + assert len(function_1_1_1.parts) == 0 + + function_2 = info.parts[1] + assert isinstance(function_2, Function) + assert function_2.name == "hello_2" + assert len(function_2.parts) == 1 + + function_2_1 = function_2.parts[0] + assert isinstance(function_2_1, Function) + assert function_2_1.name == "hello_2_1" + assert len(function_2_1.parts) == 1 + + function_2_1_1 = function_2_1.parts[0] + assert isinstance(function_2_1_1, Function) + assert function_2_1_1.name == "hello_2_1_1" + assert len(function_2_1_1.parts) == 0 + + function_3 = info.parts[2] + assert isinstance(function_3, Function) + assert function_3.name == "hello_3" + assert len(function_3.parts) == 1 + + function_3_1 = function_3.parts[0] + assert isinstance(function_3_1, Function) + assert function_3_1.name == "hello_3_1" + assert len(function_3_1.parts) == 1 + + function_3_1_1 = function_3_1.parts[0] + assert isinstance(function_3_1_1, Function) + assert function_3_1_1.name == "hello_3_1_1" + assert len(function_3_1_1.parts) == 0 + + +def test_001_one_range_marker(): + source_input = b""" +# @sdoc[REQ-001, REQ-002, REQ-003] +print("Hello world") +# @sdoc[/REQ-001, REQ-002, REQ-003] +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + info: SourceFileTraceabilityInfo = reader.read(source_input) + markers = info.markers + assert markers[0].reqs == ["REQ-001", "REQ-002", "REQ-003"] + assert markers[0].begin_or_end == "[" + assert markers[0].ng_source_line_begin == 1 + assert markers[0].ng_range_line_begin == 1 + assert markers[0].ng_range_line_end == 3 + + assert info.ng_lines_total == 3 + assert info.ng_lines_covered == 3 + assert info.get_coverage() == 100 + + +def test_002_two_range_markers(): + source_input = b""" +# @sdoc[REQ-001] +CONTENT 1 +CONTENT 2 +CONTENT 3 +# @sdoc[/REQ-001] +# @sdoc[REQ-002] +CONTENT 4 +CONTENT 5 +CONTENT 6 +# @sdoc[/REQ-002] +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + document = reader.read(source_input) + markers = document.markers + assert len(markers) == 4 + marker_1 = markers[0] + marker_2 = markers[1] + marker_3 = markers[2] + marker_4 = markers[3] + assert marker_1.reqs == ["REQ-001"] + assert marker_2.reqs == ["REQ-001"] + assert marker_3.reqs == ["REQ-002"] + assert marker_4.reqs == ["REQ-002"] + + assert marker_1.ng_source_line_begin == 1 + assert marker_2.ng_source_line_begin == 5 + assert marker_3.ng_source_line_begin == 6 + assert marker_4.ng_source_line_begin == 10 + + assert marker_1.ng_range_line_begin == 1 + assert marker_2.ng_range_line_begin == 1 + assert marker_3.ng_range_line_begin == 6 + assert marker_4.ng_range_line_begin == 6 + + +def test_008_three_nested_range_markers(): + source_input = b""" +CONTENT 1 +# @sdoc[REQ-001] +CONTENT 2 +# @sdoc[REQ-002] +CONTENT 3 +# @sdoc[REQ-003] +CONTENT 4 +# @sdoc[/REQ-003] +CONTENT 5 +# @sdoc[/REQ-002] +CONTENT 6 +# @sdoc[/REQ-001] +CONTENT 7 +# @sdoc[REQ-001] +CONTENT 8 +# @sdoc[/REQ-001] +CONTENT 9 +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + document = reader.read(source_input) + markers = document.markers + assert len(markers) == 8 + marker_1 = markers[0] + marker_2 = markers[1] + marker_3 = markers[2] + marker_4 = markers[3] + marker_5 = markers[4] + marker_6 = markers[5] + marker_7 = markers[6] + marker_8 = markers[7] + assert marker_1.reqs == ["REQ-001"] + assert marker_2.reqs == ["REQ-002"] + assert marker_3.reqs == ["REQ-003"] + assert marker_4.reqs == ["REQ-003"] + assert marker_5.reqs == ["REQ-002"] + assert marker_6.reqs == ["REQ-001"] + assert marker_7.reqs == ["REQ-001"] + assert marker_8.reqs == ["REQ-001"] + + assert marker_1.ng_source_line_begin == 2 + assert marker_2.ng_source_line_begin == 4 + assert marker_3.ng_source_line_begin == 6 + assert marker_4.ng_source_line_begin == 8 + assert marker_5.ng_source_line_begin == 10 + assert marker_6.ng_source_line_begin == 12 + assert marker_7.ng_source_line_begin == 14 + assert marker_8.ng_source_line_begin == 16 + + assert marker_1.ng_range_line_begin == 2 + assert marker_2.ng_range_line_begin == 4 + assert marker_3.ng_range_line_begin == 6 + assert marker_4.ng_range_line_begin == 6 + assert marker_5.ng_range_line_begin == 4 + assert marker_6.ng_range_line_begin == 2 + assert marker_7.ng_range_line_begin == 14 + assert marker_8.ng_range_line_begin == 14 + + assert document.ng_lines_total == 17 + assert document.ng_lines_covered == 14 + assert document.get_coverage() == 82.4 + + +def test_010_nosdoc_keyword(): + source_input = b""" +# @sdoc[nosdoc] +# @sdoc[REQ-001] +CONTENT 1 +CONTENT 2 +CONTENT 3 +# @sdoc[/REQ-001] +# @sdoc[/nosdoc] +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + document = reader.read(source_input) + assert len(document.markers) == 0 + + +def test_011_nosdoc_keyword_then_normal_marker(): + source_input = b""" +# @sdoc[nosdoc] +# @sdoc[REQ-001] +CONTENT 1 +CONTENT 2 +CONTENT 3 +# @sdoc[/REQ-001] +# @sdoc[/nosdoc] +# @sdoc[REQ-001] +CONTENT 1 +CONTENT 2 +CONTENT 3 +# @sdoc[/REQ-001] +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + document = reader.read(source_input) + assert len(document.markers) == 2 + + +def test_011_nosdoc_keyword_then_normal_marker_4spaces_indent(): + source_input = b""" + # @sdoc[nosdoc] + # @sdoc[REQ-001] + CONTENT 1 + CONTENT 2 + CONTENT 3 + # @sdoc[/REQ-001] + # @sdoc[/nosdoc] + # @sdoc[REQ-001] + CONTENT 1 + CONTENT 2 + CONTENT 3 + # @sdoc[/REQ-001] +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + document = reader.read(source_input) + assert len(document.markers) == 2 + + +# Testing that correct line location is assigned when the marker is not on the +# first line. +def test_012_marker_not_first_line(): + source_input = b""" + + +# @sdoc[REQ-001] +# CONTENT 1 +# CONTENT 2 +# CONTENT 3 +# @sdoc[/REQ-001] + + + +""" + + reader = SourceFileTraceabilityReader_Python() + + document = reader.read(source_input) + + markers: List[RangeMarker] = document.markers + assert markers[0].reqs == ["REQ-001"] + assert markers[0].begin_or_end == "[" + assert markers[0].ng_source_line_begin == 4 + assert markers[0].ng_range_line_begin == 4 + assert markers[0].ng_range_line_end == 8 + + assert markers[1].reqs == ["REQ-001"] + assert markers[1].begin_or_end == "[/" + assert markers[1].ng_source_line_begin == 8 + assert markers[1].ng_range_line_begin == 4 + assert markers[1].ng_range_line_end == 8 + + assert document.ng_lines_total == 11 + assert document.ng_lines_covered == 5 + assert document.get_coverage() == 45.5 + + +""" +LINE markers. +""" + + +def test_050_line_marker(): + source_input = b""" +# @sdoc(REQ-001) +CONTENT 1 +# @sdoc(REQ-002) +CONTENT 2 +# @sdoc(REQ-003) +CONTENT 3 +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + document = reader.read(source_input) + markers = document.markers + assert markers[0].reqs == ["REQ-001"] + assert markers[0].ng_source_line_begin == 1 + assert markers[0].ng_range_line_begin == 1 + assert markers[0].ng_range_line_end == 1 + assert markers[1].reqs == ["REQ-002"] + assert markers[1].ng_source_line_begin == 3 + assert markers[1].ng_range_line_begin == 3 + assert markers[1].ng_range_line_end == 3 + assert markers[2].reqs == ["REQ-003"] + assert markers[2].ng_source_line_begin == 5 + assert markers[2].ng_range_line_begin == 5 + assert markers[2].ng_range_line_end == 5 + + assert document.ng_lines_total == 6 + assert document.ng_lines_covered == 3 + assert document.get_coverage() == 50.0 + + +def test_validation_01_one_range_marker_begin_req_not_equal_to_end_req(): + source_input = b""" +# @sdoc[REQ-001] +CONTENT 1 +CONTENT 2 +CONTENT 3 +# @sdoc[/REQ-002] +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + with pytest.raises(Exception) as exc_info: + _ = reader.read(source_input) + + assert exc_info.type is StrictDocSemanticError + assert ( + exc_info.value.args[0] + == "STRICTDOC RANGE: BEGIN and END requirements mismatch" + ) + + +def test_validation_02_one_range_marker_end_without_begin(): + source_input = b""" +# @sdoc[/REQ-002] +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + with pytest.raises(Exception) as exc_info: + _ = reader.read(source_input) + + assert exc_info.type is StrictDocSemanticError + assert ( + exc_info.value.args[0] + == "STRICTDOC RANGE: END marker without preceding BEGIN marker" + ) + + +def test_validation_03_range_start_without_range_end(): + source_input = b""" +# @sdoc[REQ-001] +# @sdoc[REQ-002] +CONTENT 1 +CONTENT 2 +CONTENT 3 +""".lstrip() + + reader = SourceFileTraceabilityReader_Python() + + with pytest.raises(Exception) as exc_info: + _ = reader.read(source_input) + + assert exc_info.type is StrictDocSemanticError + assert ( + exc_info.value.args[0] + == "Unmatched @sdoc keyword found in source file." + ) + assert ( + exc_info.value.args[1] + == "The @sdoc keywords are also unmatched on lines: [(2, 9)]." + ) diff --git a/tests/unit/strictdoc/backend/sdoc_source_code/test_marker_parser.py b/tests/unit/strictdoc/backend/sdoc_source_code/test_marker_parser.py new file mode 100644 index 000000000..ebd86794d --- /dev/null +++ b/tests/unit/strictdoc/backend/sdoc_source_code/test_marker_parser.py @@ -0,0 +1,104 @@ +import sys + +import pytest + +from strictdoc.backend.sdoc_source_code.marker_parser import MarkerParser +from strictdoc.backend.sdoc_source_code.models.function_range_marker import ( + FunctionRangeMarker, +) + +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 9), reason="Requires Python 3.9 or higher" +) + + +def test_01_(): + input_string = """\ +@relation(REQ-1, scope=function) +""" + + function_ranges = MarkerParser.parse(input_string, 1, 1, 1, 1) + function_range = function_ranges[0] + assert isinstance(function_range, FunctionRangeMarker) + assert function_range.ng_source_line_begin == 1 + assert function_range.ng_range_line_begin == 1 + assert function_range.ng_range_line_end == 1 + assert function_range.reqs_objs[0].ng_source_line == 1 + assert function_range.reqs_objs[0].ng_source_column == 11 + + +def test_02_(): + input_string = """\ + + +@relation(REQ-1, scope=function) +""" + + function_ranges = MarkerParser.parse(input_string, 1, 5, 1, 1) + function_range = function_ranges[0] + + assert isinstance(function_range, FunctionRangeMarker) + assert function_range.ng_source_line_begin == 1 + assert function_range.ng_range_line_begin == 1 + assert function_range.ng_range_line_end == 5 + assert function_range.reqs_objs[0].ng_source_line == 3 + assert function_range.reqs_objs[0].ng_source_column == 11 + + +def test_03_(): + input_string = """\ + + + @relation(REQ-1, scope=function) +""" + + function_ranges = MarkerParser.parse(input_string, 1, 5, 4, 4) + function_range = function_ranges[0] + + assert isinstance(function_range, FunctionRangeMarker) + assert function_range.ng_source_line_begin == 1 + assert function_range.ng_range_line_begin == 1 + assert function_range.ng_range_line_end == 5 + assert function_range.reqs_objs[0].ng_source_line == (4 - 1) + 3 + assert function_range.reqs_objs[0].ng_source_column == 15 + + +def test_04_(): + input_string = """\ +/** + * Some text. + * + * @relation(REQ-1, scope=function) + */ +""" + + function_ranges = MarkerParser.parse(input_string, 1, 5, 1, 1) + function_range = function_ranges[0] + + assert isinstance(function_range, FunctionRangeMarker) + assert function_range.ng_source_line_begin == 1 + assert function_range.ng_range_line_begin == 1 + assert function_range.ng_range_line_end == 5 + assert function_range.reqs_objs[0].ng_source_line == 4 + assert function_range.reqs_objs[0].ng_source_column == 14 + + +def test_05_(): + input_string = """\ +/** + * Some text. + * + * @relation(REQ-1, scope=function) + * @relation(REQ-2, scope=function) + */ +""" + + function_ranges = MarkerParser.parse(input_string, 1, 6, 1, 1) + function_range = function_ranges[0] + + assert isinstance(function_range, FunctionRangeMarker) + assert function_range.ng_source_line_begin == 1 + assert function_range.ng_range_line_begin == 1 + assert function_range.ng_range_line_end == 6 + assert function_range.reqs_objs[0].ng_source_line == 4 + assert function_range.reqs_objs[0].ng_source_column == 14