Fix file watcher infinite loops and improve logging for local invoke

samcli/commands/local/cli_common/invoke_context.py

@@ -103,6 +103,7 @@ def __init__(
         invoke_images: Optional[str] = None,
         mount_symlinks: Optional[bool] = False,
         no_mem_limit: Optional[bool] = False,
+        no_watch: Optional[bool] = False,
     ) -> None:
         """
         Initialize the context
@@ -204,6 +205,7 @@ def __init__(
 
         self._mount_symlinks: Optional[bool] = mount_symlinks
         self._no_mem_limit = no_mem_limit
+        self._no_watch = no_watch
 
         # Note(xinhol): despite self._function_provider and self._stacks are initialized as None
         # they will be assigned with a non-None value in __enter__() and
@@ -234,16 +236,18 @@ def __enter__(self) -> "InvokeContext":
             ContainersMode.WARM: RefreshableSamFunctionProvider,
             ContainersMode.COLD: SamFunctionProvider,
         }
-
         _function_providers_args: Dict[ContainersMode, List[Any]] = {
             ContainersMode.WARM: [self._stacks, self._parameter_overrides, self._global_parameter_overrides],
             ContainersMode.COLD: [self._stacks],
         }
-
         # don't resolve the code URI immediately if we passed in docker vol by passing True for use_raw_codeuri
         # this way at the end the code URI will get resolved against the basedir option
         if self._docker_volume_basedir:
             _function_providers_args[self._containers_mode].append(True)
+            if self._no_watch:
+                _function_providers_args[self._containers_mode].extend([False, True])
+        elif self._no_watch:
+            _function_providers_args[self._containers_mode].extend([False, False, True])
 
         self._function_provider = _function_providers_class[self._containers_mode](
             *_function_providers_args[self._containers_mode]
@@ -415,6 +419,7 @@ def lambda_runtime(self) -> LambdaRuntime:
                     image_builder,
                     mount_symlinks=self._mount_symlinks,
                     no_mem_limit=self._no_mem_limit,
+                    no_watch=self._no_watch,
                 ),
                 ContainersMode.COLD: LambdaRuntime(
                     self._container_manager,

samcli/commands/local/start_api/cli.py

@@ -95,6 +95,12 @@
     required_param_lists=[["ssl_cert_file"]],
     help="Path to SSL key file (default: None)",
 )
+@click.option(
+    "--no-watch",
+    is_flag=True,
+    default=False,
+    help="Disable file watching. Local code changes will not reset running docker container.",
+)
 @invoke_common_options
 @warm_containers_common_options
 @local_common_options
@@ -141,6 +147,7 @@ def cli(
     terraform_plan_file,
     ssl_cert_file,
     ssl_key_file,
+    no_watch,
     no_memory_limit,
 ):
     """
@@ -178,6 +185,7 @@ def cli(
         ssl_cert_file,
         ssl_key_file,
         no_memory_limit,
+        no_watch,
     )  # pragma: no cover
 
 
@@ -211,6 +219,7 @@ def do_cli(  # pylint: disable=R0914
     ssl_cert_file,
     ssl_key_file,
     no_mem_limit,
+    no_watch,
 ):
     """
     Implementation of the ``cli`` method, just separated out for unit testing purposes
@@ -257,6 +266,7 @@ def do_cli(  # pylint: disable=R0914
             invoke_images=processed_invoke_images,
             add_host=add_host,
             no_mem_limit=no_mem_limit,
+            no_watch=no_watch,
         ) as invoke_context:
             ssl_context = (ssl_cert_file, ssl_key_file) if ssl_cert_file else None
             service = LocalApiService(

samcli/commands/local/start_api/core/options.py

@@ -43,6 +43,7 @@
     "add_host",
     "invoke_image",
     "disable_authorizer",
+    "no_watch",
 ]
 
 CONFIGURATION_OPTION_NAMES: List[str] = ["config_env", "config_file"] + SAVE_PARAMS_OPTIONS

samcli/commands/local/start_lambda/cli.py

@@ -59,6 +59,12 @@
 )
 @skip_prepare_infra_option
 @service_common_options(3001)
+@click.option(
+    "--no-watch",
+    is_flag=True,
+    default=False,
+    help="Disable file watching. Local code changes will not reset running docker container.",
+)
 @invoke_common_options
 @warm_containers_common_options
 @local_common_options
@@ -101,6 +107,7 @@ def cli(
     hook_name,
     skip_prepare_infra,
     terraform_plan_file,
+    no_watch,
     no_memory_limit,
 ):
     """
@@ -134,6 +141,7 @@ def cli(
         invoke_image,
         hook_name,
         no_memory_limit,
+        no_watch,
     )  # pragma: no cover
 
 
@@ -163,6 +171,7 @@ def do_cli(  # pylint: disable=R0914
     invoke_image,
     hook_name,
     no_mem_limit,
+    no_watch,
 ):
     """
     Implementation of the ``cli`` method, just separated out for unit testing purposes
@@ -209,6 +218,7 @@ def do_cli(  # pylint: disable=R0914
             add_host=add_host,
             invoke_images=processed_invoke_images,
             no_mem_limit=no_mem_limit,
+            no_watch=no_watch,
         ) as invoke_context:
             service = LocalLambdaService(lambda_invoke_context=invoke_context, port=port, host=host)
             service.start()

samcli/commands/local/start_lambda/core/options.py

@@ -38,6 +38,7 @@
     "add_host",
     "invoke_image",
     "no_memory_limit",
+    "no_watch",
 ]
 
 ARTIFACT_LOCATION_OPTIONS: List[str] = [

samcli/lib/providers/sam_function_provider.py

@@ -800,6 +800,7 @@ def __init__(
         global_parameter_overrides: Optional[Dict] = None,
         use_raw_codeuri: bool = False,
         ignore_code_extraction_warnings: bool = False,
+        no_watch: Optional[bool] = False,
     ) -> None:
         """
         Initialize the class with SAM template data. The SAM template passed to this provider is assumed
@@ -831,9 +832,14 @@ def __init__(
                 self.parent_templates_paths.append(stack.location)
 
         self.is_changed = False
-        self._observer = FileObserver(self._set_templates_changed)
-        self._observer.start()
-        self._watch_stack_templates(stacks)
+
+        # Only initialize file watcher if no_watch is False
+        if not no_watch:
+            self._observer: Optional[FileObserver] = FileObserver(self._set_templates_changed)
+            self._observer.start()
+            self._watch_stack_templates(stacks)
+        else:
+            self._observer = None
 
     @property
     def stacks(self) -> List[Stack]:
@@ -896,15 +902,17 @@ def _set_templates_changed(self, paths: List[str]) -> None:
             ", ".join(paths),
         )
         self.is_changed = True
-        for stack in self._stacks:
-            self._observer.unwatch(stack.location)
+        if self._observer:
+            for stack in self._stacks:
+                self._observer.unwatch(stack.location)
 
     def _watch_stack_templates(self, stacks: List[Stack]) -> None:
         """
         initialize the list of stack template watchers
         """
-        for stack in stacks:
-            self._observer.watch(stack.location)
+        if self._observer:
+            for stack in stacks:
+                self._observer.watch(stack.location)
 
     def _refresh_loaded_functions(self) -> None:
         """
@@ -934,4 +942,5 @@ def stop_observer(self) -> None:
         """
         Stop Observing.
         """
-        self._observer.stop()
+        if self._observer:
+            self._observer.stop()

samcli/lib/utils/file_observer.py

@@ -183,11 +183,22 @@ def _on_change(self, resources: List[str], package_type: str) -> None:
         package_type: str
             determine if the changed resource is a source code path or an image name
         """
+        LOG.debug(
+            "Acquiring lock for _on_change to process %s %s resource changes: %s",
+            len(resources),
+            package_type,
+            resources,
+        )
         with self._watch_lock:
             changed_functions: List[FunctionConfig] = []
             for resource in resources:
                 if self._observed_functions[package_type].get(resource, None):
                     changed_functions += self._observed_functions[package_type][resource]
+            LOG.debug(
+                "Acquired lock and processing %s changed functions from %s resources",
+                len(changed_functions),
+                len(resources),
+            )
             self._input_on_change(changed_functions)
 
     def watch(self, function_config: FunctionConfig) -> None:
@@ -204,9 +215,18 @@ def watch(self, function_config: FunctionConfig) -> None:
         ObserverException:
             if not able to observe the input function source path/image
         """
+        LOG.debug(
+            "Acquiring lock for watch to observe %s function: %s", function_config.packagetype, function_config.name
+        )
         with self._watch_lock:
             if self.get_resources.get(function_config.packagetype, None):
                 resources = self.get_resources[function_config.packagetype](function_config)
+                LOG.debug(
+                    "Acquired lock for watch, observing %s resources for function %s: %s",
+                    len(resources),
+                    function_config.name,
+                    resources,
+                )
                 for resource in resources:
                     functions = self._observed_functions[function_config.packagetype].get(resource, [])
                     functions += [function_config]
@@ -424,6 +444,7 @@ def __init__(self) -> None:
         self._observed_watches: Dict[str, ObservedWatch] = {}
         self._watch_dog_observed_paths: Dict[str, List[str]] = {}
         self._observer: BaseObserver = Observer()
+
         self._code_modification_handler: PatternMatchingEventHandler = PatternMatchingEventHandler(
             patterns=["*"], ignore_patterns=[], ignore_directories=False
         )

samcli/local/lambdafn/runtime.py

@@ -395,7 +395,9 @@ class WarmLambdaRuntime(LambdaRuntime):
     warm containers life cycle.
     """
 
-    def __init__(self, container_manager, image_builder, observer=None, mount_symlinks=False, no_mem_limit=False):
+    def __init__(
+        self, container_manager, image_builder, observer=None, mount_symlinks=False, no_mem_limit=False, no_watch=False
+    ):
         """
         Initialize the Local Lambda runtime
 
@@ -405,13 +407,23 @@ def __init__(self, container_manager, image_builder, observer=None, mount_symlin
             Instance of the ContainerManager class that can run a local Docker container
         image_builder samcli.local.docker.lambda_image.LambdaImage
             Instance of the LambdaImage class that can create am image
-        warm_containers bool
-            Determines if the warm containers is enabled or not.
+        observer
+            Optional observer for file watching
+        mount_symlinks bool
+            Optional. True if symlinks should be mounted in the container
+        no_mem_limit bool
+            Optional. True if memory limit should be disabled
+        no_watch bool
+            Optional. True if file watching should be disabled
         """
         self._function_configs = {}
         self._containers = {}
+        self._no_watch = no_watch
 
-        self._observer = observer if observer else LambdaFunctionObserver(self._on_code_change)
+        if no_watch:
+            self._observer = None
+        else:
+            self._observer = observer if observer else LambdaFunctionObserver(self._on_code_change)
 
         super().__init__(container_manager, image_builder, mount_symlinks=mount_symlinks, no_mem_limit=no_mem_limit)
 
@@ -453,7 +465,8 @@ def create(
             if container:
                 self._container_manager.stop(container)
                 self._containers.pop(exist_function_config.full_path, None)
-            self._observer.unwatch(exist_function_config)
+            if self._observer is not None:
+                self._observer.unwatch(exist_function_config)
         elif container and container.is_created():
             LOG.info("Reuse the created warm container for Lambda function '%s'", function_config.full_path)
             return container
@@ -468,8 +481,10 @@ def create(
             )
             debug_context = None
 
-        self._observer.watch(function_config)
-        self._observer.start()
+        # Only watch and start observer if file watching is enabled
+        if self._observer is not None:
+            self._observer.watch(function_config)
+            self._observer.start()
 
         container = super().create(
             function_config, debug_context, container_host, container_host_interface, extra_hosts
@@ -545,7 +560,8 @@ def clean_running_containers_and_related_resources(self):
             LOG.debug("Terminate running warm container for Lambda Function '%s'", function_name)
             self._container_manager.stop(container)
         self._clean_decompressed_paths()
-        self._observer.stop()
+        if self._observer is not None:
+            self._observer.stop()
 
     def _on_code_change(self, functions):
         """
@@ -566,7 +582,8 @@ def _on_code_change(self, functions):
                 function_full_path,
                 resource,
             )
-            self._observer.unwatch(function_config)
+            if self._observer is not None:
+                self._observer.unwatch(function_config)
             self._function_configs.pop(function_full_path, None)
             container = self._containers.get(function_full_path, None)
             if container: