Merge pull request #3364 from Azure/release-2.13.1.0

Release 2.13.1.1 (branch release-2.13.1.0) to master

Author: narrieta

.github/workflows/ci_pr.yml

@@ -20,7 +20,7 @@ jobs:
     name: "Python ${{ matrix.python-version }} Unit Tests"
     runs-on: ubuntu-20.04
     container:
-      image: ubuntu:16.04
+      image: ubuntu:24.04
       volumes:
         - /home/waagent:/home/waagent
     defaults:
@@ -29,25 +29,32 @@ jobs:
 
     env:
       NOSEOPTS: "--verbose"
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-    
+
     steps:
     - uses: actions/checkout@v3
 
-    - name: Install Python ${{ matrix.python-version }}
+    - name: Install Python ${{ matrix.python-version }} Virtual Environment
       run: |
         apt-get update
-        apt-get install -y curl bzip2 sudo python3
-        curl https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2 -o python-${{ matrix.python-version }}.tar.bz2
-        sudo tar xjvf python-${{ matrix.python-version }}.tar.bz2 --directory /
+        apt-get install -y curl bzip2 sudo
+        curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
+        sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
+        #
+        # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
+        #
+        if [[ "${{ matrix.python-version }}" == "2.6" ]]; then
+            apt-get -y install python3
+        fi
+        #
+        # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
+        #
+        if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then
+          ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
+        fi
 
-    - name: Test with nosetests
+    - name: Execute Tests
       run: |
-        if [[ ${{ matrix.python-version }} == "2.6" ]]; then
-          source /home/waagent/virtualenv/python2.6.9/bin/activate
-        else
-          source /home/waagent/virtualenv/python3.4.8/bin/activate
-        fi
+        source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
         ./ci/nosetests.sh
         exit $?
 

azurelinuxagent/agent.py

@@ -34,17 +34,18 @@
 from azurelinuxagent.ga import logcollector, cgroupconfigurator
 from azurelinuxagent.ga.cgroupcontroller import AGENT_LOG_COLLECTOR
 from azurelinuxagent.ga.cpucontroller import _CpuController
-from azurelinuxagent.ga.cgroupapi import get_cgroup_api, log_cgroup_warning, InvalidCgroupMountpointException
+from azurelinuxagent.ga.cgroupapi import create_cgroup_api, InvalidCgroupMountpointException
+from azurelinuxagent.ga.firewall_manager import FirewallManager
 
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.event as event
 import azurelinuxagent.common.logger as logger
+from azurelinuxagent.common.event import WALAEventOperation
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.ga.logcollector import LogCollector, OUTPUT_RESULTS_FILE_PATH
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.utils import fileutil, textutil
 from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
-from azurelinuxagent.common.utils.networkutil import AddFirewallRules
 from azurelinuxagent.common.version import AGENT_NAME, AGENT_LONG_VERSION, AGENT_VERSION, \
     DISTRO_NAME, DISTRO_VERSION, \
     PY_VERSION_MAJOR, PY_VERSION_MINOR, \
@@ -208,28 +209,34 @@ def collect_logs(self, is_full_mode):
         else:
             logger.info("Running log collector mode normal")
 
+        LogCollector.initialize_telemetry()
+
         # Check the cgroups unit
         log_collector_monitor = None
         tracked_controllers = []
         if CollectLogsHandler.is_enabled_monitor_cgroups_check():
             try:
-                cgroup_api = get_cgroup_api()
+                cgroup_api = create_cgroup_api()
+                logger.info("Using cgroup {0} for resource enforcement and monitoring".format(cgroup_api.get_cgroup_version()))
             except InvalidCgroupMountpointException as e:
-                log_cgroup_warning("The agent does not support cgroups if the default systemd mountpoint is not being used: {0}".format(ustr(e)), send_event=True)
+                event.warn(WALAEventOperation.LogCollection, "The agent does not support cgroups if the default systemd mountpoint is not being used: {0}", ustr(e))
                 sys.exit(logcollector.INVALID_CGROUPS_ERRCODE)
             except CGroupsException as e:
-                log_cgroup_warning("Unable to determine which cgroup version to use: {0}".format(ustr(e)), send_event=True)
+                event.warn(WALAEventOperation.LogCollection, "Unable to determine which cgroup version to use: {0}", ustr(e))
                 sys.exit(logcollector.INVALID_CGROUPS_ERRCODE)
 
             log_collector_cgroup = cgroup_api.get_process_cgroup(process_id="self", cgroup_name=AGENT_LOG_COLLECTOR)
             tracked_controllers = log_collector_cgroup.get_controllers()
+            for controller in tracked_controllers:
+                logger.info("{0} controller for cgroup: {1}".format(controller.get_controller_type(), controller))
 
             if len(tracked_controllers) != len(log_collector_cgroup.get_supported_controller_names()):
-                log_cgroup_warning("At least one required controller is missing. The following controllers are required for the log collector to run: {0}".format(log_collector_cgroup.get_supported_controller_names()))
+                event.warn(WALAEventOperation.LogCollection, "At least one required controller is missing. The following controllers are required for the log collector to run: {0}", log_collector_cgroup.get_supported_controller_names())
                 sys.exit(logcollector.INVALID_CGROUPS_ERRCODE)
 
-            if not log_collector_cgroup.check_in_expected_slice(cgroupconfigurator.LOGCOLLECTOR_SLICE):
-                log_cgroup_warning("The Log Collector process is not in the proper cgroups", send_event=False)
+            expected_slice = cgroupconfigurator.LOGCOLLECTOR_SLICE
+            if not log_collector_cgroup.check_in_expected_slice(expected_slice):
+                event.warn(WALAEventOperation.LogCollection, "The Log Collector process is not in the proper cgroups. Expected slice: {0}", expected_slice)
                 sys.exit(logcollector.INVALID_CGROUPS_ERRCODE)
 
         try:
@@ -270,15 +277,16 @@ def collect_logs(self, is_full_mode):
                 log_collector_monitor.stop()
 
     @staticmethod
-    def setup_firewall(firewall_metadata):
-
-        print("Setting up firewall for the WALinux Agent with args: {0}".format(firewall_metadata))
+    def setup_firewall(endpoint):
+        logger.set_prefix("Firewall")
+        threading.current_thread().name = "Firewall"
+        event.info(event.WALAEventOperation.Firewall, "Setting up firewall after boot. Endpoint: {0}", ustr(endpoint))
         try:
-            AddFirewallRules.add_iptables_rules(firewall_metadata['wait'], firewall_metadata['dst_ip'],
-                                                firewall_metadata['uid'])
-            print("Successfully set the firewall rules")
+            firewall_manager = FirewallManager.create(endpoint)
+            firewall_manager.setup()
+            event.info(event.WALAEventOperation.Firewall, "Successfully set the firewall rules")
         except Exception as error:
-            print("Unable to add firewall rules. Error: {0}".format(ustr(error)))
+            event.error(event.WALAEventOperation.Firewall, "Unable to add firewall rules. Error: {0}", ustr(error))
             sys.exit(1)
 
 
@@ -291,7 +299,7 @@ def main(args=None):
         args = []
     if len(args) <= 0:
         args = sys.argv[1:]
-    command, force, verbose, debug, conf_file_path, log_collector_full_mode, firewall_metadata = parse_args(args)
+    command, force, verbose, debug, conf_file_path, log_collector_full_mode, firewall_endpoint = parse_args(args)
     if command == AgentCommands.Version:
         version()
     elif command == AgentCommands.Help:
@@ -318,7 +326,7 @@ def main(args=None):
             elif command == AgentCommands.CollectLogs:
                 agent.collect_logs(log_collector_full_mode)
             elif command == AgentCommands.SetupFirewall:
-                agent.setup_firewall(firewall_metadata)
+                agent.setup_firewall(firewall_endpoint)
         except Exception as e:
             logger.error(u"Failed to run '{0}': {1}",
                          command,
@@ -335,11 +343,7 @@ def parse_args(sys_args):
     debug = False
     conf_file_path = None
     log_collector_full_mode = False
-    firewall_metadata = {
-        "dst_ip": None,
-        "uid": None,
-        "wait": ""
-    }
+    endpoint = None
 
     regex_cmd_format = "^([-/]*){0}"
 
@@ -383,20 +387,17 @@ def parse_args(sys_args):
             cmd = AgentCommands.CollectLogs
         elif re.match(regex_cmd_format.format("full"), arg):
             log_collector_full_mode = True
-        elif re.match(regex_cmd_format.format(AgentCommands.SetupFirewall), arg):
-            cmd = AgentCommands.SetupFirewall
-        elif re.match(regex_cmd_format.format("dst_ip=(?P<dst_ip>[\\d.]{7,})"), arg):
-            firewall_metadata['dst_ip'] = re.match(regex_cmd_format.format("dst_ip=(?P<dst_ip>[\\d.]{7,})"), arg).group(
-                'dst_ip')
-        elif re.match(regex_cmd_format.format("uid=(?P<uid>[\\d]+)"), arg):
-            firewall_metadata['uid'] = re.match(regex_cmd_format.format("uid=(?P<uid>[\\d]+)"), arg).group('uid')
-        elif re.match(regex_cmd_format.format("(w|wait)$"), arg):
-            firewall_metadata['wait'] = "-w"
         else:
-            cmd = AgentCommands.Help
-            break
+            regex_cmd = regex_cmd_format.format("{0}=(?P<endpoint>[\\d.]{{7,}})".format(AgentCommands.SetupFirewall))
+            match = re.match(regex_cmd, arg)
+            if match is not None:
+                cmd = AgentCommands.SetupFirewall
+                endpoint = match.group('endpoint')
+            else:
+                cmd = AgentCommands.Help
+                break
 
-    return cmd, force, verbose, debug, conf_file_path, log_collector_full_mode, firewall_metadata
+    return cmd, force, verbose, debug, conf_file_path, log_collector_full_mode, endpoint
 
 
 def version():
@@ -416,11 +417,11 @@ def usage():
     """
     Return agent usage message
     """
-    s  = "\n"
+    s = "\n"
     s += ("usage: {0} [-verbose] [-force] [-help] "
            "-configuration-path:<path to configuration file>" 
            "-deprovision[+user]|-register-service|-version|-daemon|-start|"
-           "-run-exthandlers|-show-configuration|-collect-logs [-full]|-setup-firewall [-dst_ip=<IP> -uid=<UID> [-w/--wait]]"
+           "-run-exthandlers|-show-configuration|-collect-logs [-full]|-setup-firewall=<IP>]"
            "").format(sys.argv[0])
     s += "\n"
     return s

azurelinuxagent/common/conf.py

@@ -147,7 +147,8 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
     "Debug.EnableAgentMemoryUsageCheck": False,
     "Debug.EnableFastTrack": True,
     "Debug.EnableGAVersioning": True,
-    "Debug.EnableCgroupV2ResourceLimiting": False
+    "Debug.EnableCgroupV2ResourceLimiting": False,
+    "Debug.EnableExtensionPolicy": False
 }
 
 
@@ -170,8 +171,7 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
     "ResourceDisk.MountOptions": None,
     "ResourceDisk.Filesystem": "ext3",
     "AutoUpdate.GAFamily": "Prod",
-    "Debug.CgroupMonitorExpiryTime": "2022-03-31",
-    "Debug.CgroupMonitorExtensionName": "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent",
+    "Policy.PolicyFilePath": "/etc/waagent_policy.json"
 }
 
 
@@ -316,6 +316,10 @@ def get_agent_log_file():
     return "/var/log/waagent.log"
 
 
+def get_policy_file_path(conf=__conf__):
+    return conf.get("Policy.PolicyFilePath", "/etc/waagent_policy.json")
+
+
 def get_fips_enabled(conf=__conf__):
     return conf.get_switch("OS.EnableFIPS", False)
 
@@ -615,25 +619,6 @@ def get_enable_agent_memory_usage_check(conf=__conf__):
     """
     return conf.get_switch("Debug.EnableAgentMemoryUsageCheck", False)
 
-
-def get_cgroup_monitor_expiry_time(conf=__conf__):
-    """
-    cgroups monitoring for pilot extensions disabled after expiry time
-
-    NOTE: This option is experimental and may be removed in later versions of the Agent.
-    """
-    return conf.get("Debug.CgroupMonitorExpiryTime", "2022-03-31")
-
-
-def get_cgroup_monitor_extension_name (conf=__conf__):
-    """
-    cgroups monitoring extension name
-
-    NOTE: This option is experimental and may be removed in later versions of the Agent.
-    """
-    return conf.get("Debug.CgroupMonitorExtensionName", "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent")
-
-
 def get_enable_fast_track(conf=__conf__):
     """
     If True, the agent use FastTrack when retrieving goal states
@@ -683,13 +668,21 @@ def get_firewall_rules_log_period(conf=__conf__):
     """
     return conf.get_int("Debug.FirewallRulesLogPeriod", 86400)
 
+  
+def get_extension_policy_enabled(conf=__conf__):
+    """
+    Determine whether extension policy is enabled. If true, policy will be enforced before installing any extensions.
+    NOTE: This option is experimental and may be removed in later versions of the Agent.
+    """
+    return conf.get_switch("Debug.EnableExtensionPolicy", False)
 
+  
 def get_enable_cgroup_v2_resource_limiting(conf=__conf__):
     """
     If True, the agent will enable resource monitoring and enforcement for the log collector on machines using cgroup v2.
     NOTE: This option is experimental and may be removed in later versions of the Agent.
     """
-    return conf.get_switch("Debug.EnableCgroupV2ResourceLimiting", False)
+    return conf.get_switch("Debug.EnableCgroupV2ResourceLimiting", True)
 
 
 def get_log_collector_initial_delay(conf=__conf__):