Merge pull request #2566 from strictdoc-project/stanislaw/node_field_origin

feat(commands/manage_autouid): rework the generation of MID/HASH

Author: stanislaw

strictdoc/backend/sdoc/models/document_config.py

@@ -102,10 +102,10 @@ def __init__(
             if enable_mid == "True"
             else (False if root == "False" else None)
         )
-        self.relation_field: str = (
+        self.relation_field: Optional[str] = (
             relation_field
             if relation_field is not None and len(relation_field) > 0
-            else "UID"
+            else None
         )
 
         self.markup: Optional[str] = markup
@@ -168,6 +168,9 @@ def get_prefix(self) -> str:
             return self.requirement_prefix
         return "REQ-"
 
+    def get_relation_field(self) -> str:
+        return self.relation_field or "UID"
+
     def has_meta(self) -> bool:
         # TODO: When OPTIONS are not provided to a document, the self.number and
         # self.version are both None. Otherwise, they become empty strings "".

strictdoc/backend/sdoc/models/node.py

@@ -3,6 +3,7 @@
 """
 
 from collections import OrderedDict
+from enum import Enum
 from typing import Any, Generator, List, Optional, Tuple, Union
 
 from strictdoc.backend.sdoc.document_reference import DocumentReference
@@ -41,6 +42,15 @@ def __init__(self) -> None:
         self.title_number_string: Optional[str] = None
 
 
+class SDocNodeFieldOrigin(str, Enum):
+    DOCUMENT = "DOCUMENT"
+    SOURCE = "SOURCE"
+
+    @staticmethod
+    def all() -> List[str]:  # noqa: A003
+        return list(map(lambda c: c.value, SDocNodeFieldOrigin))
+
+
 @auto_described
 class SDocNodeField:
     def __init__(
@@ -54,6 +64,7 @@ def __init__(
         self.field_name: str = field_name
         self.parts: List[Any] = parts
         self.multiline: bool = multiline__ is not None and len(multiline__) > 0
+        self.origin: SDocNodeFieldOrigin = SDocNodeFieldOrigin.DOCUMENT
 
     @staticmethod
     def create_from_string(
@@ -98,6 +109,12 @@ def get_text_value(self) -> str:
                 raise NotImplementedError(part)  # pragma: no cover
         return text
 
+    def is_document_origin(self) -> bool:
+        return self.origin == SDocNodeFieldOrigin.DOCUMENT
+
+    def mark_as_source_origin(self) -> None:
+        self.origin = SDocNodeFieldOrigin.SOURCE
+
 
 @auto_described
 class SDocNode(SDocNodeIF):
@@ -310,7 +327,7 @@ def reserved_uid(self) -> Optional[str]:
         config = assert_cast(document.config, DocumentConfig)
 
         return self._get_cached_field(
-            config.relation_field, singleline_only=True
+            config.get_relation_field(), singleline_only=True
         )
 
     @reserved_uid.setter
@@ -319,7 +336,7 @@ def reserved_uid(self, uid: Optional[str]) -> None:
         config = assert_cast(document.config, DocumentConfig)
 
         self.set_field_value(
-            field_name=config.relation_field,
+            field_name=config.get_relation_field(),
             form_field_index=0,
             value=uid,
         )

strictdoc/backend/sdoc/writer.py

@@ -123,6 +123,7 @@ def write_with_fragments(
                 output += "\n"
 
             enable_mid = document_config.enable_mid
+            relation_field = document_config.relation_field
             markup = document_config.markup
             auto_levels_specified = document_config.ng_auto_levels_specified
             layout = document_config.layout
@@ -132,6 +133,7 @@ def write_with_fragments(
 
             if (
                 enable_mid is not None
+                or relation_field is not None
                 or markup is not None
                 or auto_levels_specified
                 or layout is not None
@@ -147,6 +149,11 @@ def write_with_fragments(
                     output += "True" if enable_mid else "False"
                     output += "\n"
 
+                if relation_field is not None:
+                    output += "  RELATION_FIELD: "
+                    output += relation_field
+                    output += "\n"
+
                 if markup is not None:
                     output += "  MARKUP: "
                     output += markup
@@ -408,7 +415,11 @@ def _print_requirement_fields(
             ):
                 continue
             fields = section_content.ordered_fields_lookup[field_name]
+
             for field in fields:
+                if not field.is_document_origin():
+                    continue
+
                 field_value = field.get_text_value()
                 assert len(field_value) > 0
 

strictdoc/backend/sdoc_source_code/models/source_node.py

@@ -3,7 +3,7 @@
 """
 
 from dataclasses import dataclass, field
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
 from strictdoc.backend.sdoc_source_code.models.function import Function
 from strictdoc.backend.sdoc_source_code.models.function_range_marker import (
@@ -35,6 +35,8 @@ class SourceNode:
     fields: dict[str, str] = field(default_factory=dict)
     fields_locations: dict[str, tuple[int, int]] = field(default_factory=dict)
     function: Optional[Function] = None
+    # FIXME: Adding SDocNode here causes circular import problem.
+    sdoc_node: Optional[Any] = None
 
     def get_sdoc_field(
         self,

strictdoc/commands/manage_autouid_command.py

@@ -1,6 +1,8 @@
 import sys
+from typing import Dict, Optional
 
 from strictdoc.backend.sdoc.errors.document_tree_error import DocumentTreeError
+from strictdoc.backend.sdoc.models.node import SDocNode
 from strictdoc.backend.sdoc.writer import SDWriter
 from strictdoc.backend.sdoc_source_code.marker_writer import MarkerWriter
 from strictdoc.backend.sdoc_source_code.models.source_file_info import (
@@ -16,7 +18,7 @@
 from strictdoc.core.traceability_index import TraceabilityIndex
 from strictdoc.core.traceability_index_builder import TraceabilityIndexBuilder
 from strictdoc.helpers.parallelizer import Parallelizer
-from strictdoc.helpers.sha256 import get_random_sha256, get_sha256
+from strictdoc.helpers.sha256 import get_random_sha256, get_sha256, is_sha256
 from strictdoc.helpers.string import (
     create_safe_acronym,
 )
@@ -97,6 +99,13 @@ def execute(
                 )
                 next_number += 1
 
+        for (
+            trace_info_
+        ) in traceability_index.get_file_traceability_index().trace_infos:
+            ManageAutoUIDCommand._rewrite_source_file(
+                trace_info_, project_config
+            )
+
         for document in traceability_index.document_tree.document_list:
             assert document.meta is not None
 
@@ -116,21 +125,15 @@ def execute(
             ) as output_file:
                 output_file.write(document_content)
 
-        for (
-            trace_info_
-        ) in traceability_index.get_file_traceability_index().trace_infos:
-            ManageAutoUIDCommand._rewrite_source_file(
-                trace_info_, project_config
-            )
-
     @staticmethod
     def _rewrite_source_file(
-        trace_info: SourceFileTraceabilityInfo, project_config: ProjectConfig
+        trace_info: SourceFileTraceabilityInfo,
+        project_config: ProjectConfig,
     ) -> None:
         """
         NOTE: This only updates the source code with the new calculated value.
-              All links in the graph database and links in the search index are
-              not modified for now.
+              All links in the graph database and links in the search index
+              ARE NOT! modified for now.
         """
 
         assert trace_info.source_file is not None
@@ -146,7 +149,19 @@ def _rewrite_source_file(
         with open(trace_info.source_file.full_path, "rb") as source_file_:
             file_bytes = source_file_.read()
 
-        rewrites = {}
+        field_remapped_mid = "MID"
+
+        relevant_source_node_config = (
+            project_config.get_relevant_source_nodes_entry(
+                trace_info.source_file.in_doctree_source_file_rel_path_posix
+            )
+        )
+        if relevant_source_node_config is not None:
+            field_remapped_mid = (
+                relevant_source_node_config.sdoc_to_source_map.get("MID", "MID")
+            )
+
+        file_rewrites = {}
         for source_node_ in trace_info.source_nodes:
             function = source_node_.function
             if function is None or function.code_byte_range is None:
@@ -157,50 +172,101 @@ def _rewrite_source_file(
             if source_node_.comment_byte_range is None:
                 continue
 
-            # FILE_PATH: The file the code resides in, relative to the root of the project repository.
-            file_path = bytes(
-                trace_info.source_file.in_doctree_source_file_rel_path_posix,
-                encoding="utf8",
-            )
+            if field_remapped_mid not in source_node_.fields:
+                continue
 
-            # INSTANCE:	The requirement template instance, minus tags with hash strings.
-            instance_bytes = bytearray()
-            for field_name_, field_value_ in source_node_.fields.items():
-                if field_name_ in ("SPDX-Req-ID", "SPDX-Req-HKey"):
-                    continue
-                instance_bytes += bytes(field_value_, encoding="utf8")
-
-            # CODE: The code that the SPDX-Req applies to.
-            code = file_bytes[
-                function.code_byte_range.start : function.code_byte_range.end
-            ]
-
-            # This is important for Windows. Otherwise, the hash key will be calculated incorrectly.
-            instance_bytes = instance_bytes.replace(b"\r\n", b"\n")
-            code = code.replace(b"\r\n", b"\n")
-
-            hash_spdx_id = bytes(get_random_sha256(), encoding="utf8")
-            hash_spdx_hash = generate_code_hash(
-                project=bytes(project_config.project_title, encoding="utf8"),
-                file_path=file_path,
-                instance=bytes(instance_bytes),
-                code=code,
-            )
-            patched_node = MarkerWriter().write(
-                source_node_,
-                rewrites={
-                    "SPDX-Req-ID": hash_spdx_id,
-                    "SPDX-Req-HKey": hash_spdx_hash,
-                },
-                comment_file_bytes=file_bytes[
-                    source_node_.comment_byte_range.start : source_node_.comment_byte_range.end
-                ],
-            )
-            rewrites[source_node_] = patched_node
+            node_rewrites: Dict[str, bytes] = {}
+
+            # If the source node has the MID (SPDX-REQ-ID), but it is not yet a
+            # valid SHA256 identifier, create one and patch the node.
+            existing_req_id = source_node_.fields[field_remapped_mid]
+            if not is_sha256(existing_req_id):
+                hash_spdx_id_str = get_random_sha256()
+                hash_spdx_id = bytes(hash_spdx_id_str, encoding="utf8")
+
+                if (sdoc_node_ := source_node_.sdoc_node) is not None:
+                    sdoc_node_.set_field_value(
+                        field_name="MID",
+                        form_field_index=0,
+                        value=hash_spdx_id_str,
+                    )
+                node_rewrites[field_remapped_mid] = hash_spdx_id
+
+                patched_node = MarkerWriter().write(
+                    source_node_,
+                    rewrites=node_rewrites,
+                    comment_file_bytes=file_bytes[
+                        source_node_.comment_byte_range.start : source_node_.comment_byte_range.end
+                    ],
+                )
+                file_rewrites[source_node_] = patched_node
+
+            # If a source node has no sidecar SDoc node attached, there is
+            # nothing else to do.
+            if source_node_.sdoc_node is None:
+                continue
+
+            #
+            # The following is only applicable to the Linux Kernel Requirements
+            # Template proposal:
+            #
+            # Generate HASH field if it is not present. The HASH field is only
+            # generated for SDoc nodes, the source code nodes are not modified.
+            #
+            sdoc_node: SDocNode = source_node_.sdoc_node
+
+            existing_req_hash: Optional[str] = None
+            if "HASH" in sdoc_node.ordered_fields_lookup:
+                hash_field = sdoc_node.get_field_by_name("HASH")
+                existing_req_hash = hash_field.get_text_value()
+
+            if existing_req_hash is None or not is_sha256(existing_req_hash):
+                # FILE_PATH: The file the code resides in, relative to the root of the project repository.
+                file_path = bytes(
+                    trace_info.source_file.in_doctree_source_file_rel_path_posix,
+                    encoding="utf8",
+                )
+
+                # INSTANCE:	The requirement template instance, minus tags with hash strings.
+                instance_bytes = bytearray()
+                for (
+                    field_name_,
+                    field_values_,
+                ) in sdoc_node.ordered_fields_lookup.items():
+                    if field_name_ in ("MID", "HASH"):
+                        continue
+                    for field_value_ in field_values_:
+                        instance_bytes += bytes(
+                            field_value_.get_text_value(), encoding="utf8"
+                        )
+
+                # CODE: The code that the node hash applies to.
+                code = file_bytes[
+                    function.code_byte_range.start : function.code_byte_range.end
+                ]
+
+                # This is important for Windows. Otherwise, the hash key will be calculated incorrectly.
+                instance_bytes = instance_bytes.replace(b"\r\n", b"\n")
+                code = code.replace(b"\r\n", b"\n")
+
+                hash_spdx_hash = generate_code_hash(
+                    project=bytes(
+                        project_config.project_title, encoding="utf8"
+                    ),
+                    file_path=file_path,
+                    instance=bytes(instance_bytes),
+                    code=code,
+                )
+                hash_spdx_hash_str = hash_spdx_hash.decode("utf8")
+                sdoc_node.set_field_value(
+                    field_name="HASH",
+                    form_field_index=0,
+                    value=hash_spdx_hash_str,
+                )
 
         source_writer = SourceWriter()
         output_string = source_writer.write(
-            trace_info, rewrites=rewrites, file_bytes=file_bytes
+            trace_info, rewrites=file_rewrites, file_bytes=file_bytes
         )
 
         with open(trace_info.source_file.full_path, "wb") as source_file_: