Merge pull request #3421 from Azure/release-2.14.0.0

Release 2.14.0.1 (branch release-2.14.0.0) to master

Author: maddieford

.github/workflows/ci_pr.yml

@@ -8,132 +8,118 @@ on:
   workflow_dispatch:
 
 jobs:
-  test-python-2_6-and-3_4-versions:
-
+  execute-tests:
+    name: "Python ${{ matrix.python-version }} Unit Tests"
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
         include:
+          #
+          # Some of the Python versions we test are not supported by the setup-python Github Action. For those versions, we use a
+          # pre-built virtual environment.
+          #
           - python-version: "2.6"
+            use_virtual_environment: true
+          - python-version: "2.7"
+            use_virtual_environment: true
           - python-version: "3.4"
-
-    name: "Python ${{ matrix.python-version }} Unit Tests"
-    runs-on: ubuntu-20.04
-    container:
-      image: ubuntu:24.04
-      volumes:
-        - /home/waagent:/home/waagent
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    env:
-      NOSEOPTS: "--verbose"
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install Python ${{ matrix.python-version }} Virtual Environment
-      run: |
-        apt-get update
-        apt-get install -y curl bzip2 sudo
-        curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
-        sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
-        #
-        # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
-        #
-        if [[ "${{ matrix.python-version }}" == "2.6" ]]; then
-            apt-get -y install python3
-        fi
-        #
-        # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
-        #
-        if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then
-          ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
-        fi
-
-    - name: Execute Tests
-      run: |
-        source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
-        ./ci/nosetests.sh
-        exit $?
-
-  test-python-2_7:
-
-    strategy:
-      fail-fast: false
-
-    name: "Python 2.7 Unit Tests"
-    runs-on: ubuntu-20.04
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    env:
-      NOSEOPTS: "--verbose"
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install Python 2.7
-      run: |
-        apt-get update
-        apt-get install -y curl bzip2 sudo
-        curl https://dcrdata.blob.core.windows.net/python/python-2.7.tar.bz2 -o python-2.7.tar.bz2
-        sudo tar xjvf python-2.7.tar.bz2 --directory /
-
-    - name: Test with nosetests
-      run: |
-        source /home/waagent/virtualenv/python2.7.16/bin/activate
-        ./ci/nosetests.sh
-        exit $?
-
-  test-current-python-versions:
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
+            use_virtual_environment: true
           - python-version: "3.5"
-            # workaround found in https://github.com/actions/setup-python/issues/866
-            # for issue "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:728)" on Python 3.5
-            pip_trusted_host: "pypi.python.org pypi.org files.pythonhosted.org"
+            use_virtual_environment: true
           - python-version: "3.6"
+            use_virtual_environment: true
           - python-version: "3.7"
+            use_virtual_environment: true
           - python-version: "3.8"
           - python-version: "3.9"
             additional-nose-opts: "--with-coverage --cover-erase --cover-inclusive --cover-branches --cover-package=azurelinuxagent"
           - python-version: "3.10"
           - python-version: "3.11"
-
-    name: "Python ${{ matrix.python-version }} Unit Tests"
-    runs-on: ubuntu-20.04
-
-    env:
-      NOSEOPTS: "--with-timer ${{ matrix.additional-nose-opts }}"
+          - python-version: "3.12"
 
     steps:
-
-    - name: Checkout WALinuxAgent repo
+    - name: Checkout WALinuxAgent
       uses: actions/checkout@v3
-
+    #
+    # We either install Python and the test dependencies, or download a pre-built virtual environment, depending on the
+    # use_virtual_environment flag.
+    #
     - name: Setup Python ${{ matrix.python-version }}
+      if: (!matrix.use_virtual_environment)
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-      env:
-        PIP_TRUSTED_HOST: ${{ matrix.pip_trusted_host }}
 
     - name: Install dependencies
+      if: (!matrix.use_virtual_environment)
       id: install-dependencies
       run: |
         sudo env "PATH=$PATH" python -m pip install --upgrade pip
         sudo env "PATH=$PATH" pip install -r requirements.txt
         sudo env "PATH=$PATH" pip install -r test-requirements.txt
         sudo env "PATH=$PATH" pip install --upgrade pylint
 
+    - name: Setup Python ${{ matrix.python-version }} Virtual Environment
+      if: matrix.use_virtual_environment
+      id: install-venv
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y curl bzip2 sudo
+        curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
+        sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
+        #
+        # The virtual environments have dependencies on old versions of OpenSSL (e.g 1.0/1.1) which are not available on Ubuntu 24. We use this script to patch the environments.
+        #
+        if [[ "${{ matrix.use_virtual_environment}}" == "true" ]]; then
+          sudo ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
+        fi
+
+    #
+    # Execute the tests
+    #
+    - name: Execute Unit Tests
+      run: |
+        if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then
+          #
+          # Use pytest
+          #
+          ./ci/pytest.sh
+        else
+          #
+          # Use nosetests
+          #
+          if [[ "${{ matrix.use_virtual_environment}}" == "true" ]]; then # the pytest version on the venvs does not support the --with-timer option
+            export NOSEOPTS="--verbose ${{ matrix.additional-nose-opts }}"
+          else
+            export NOSEOPTS="--verbose --with-timer ${{ matrix.additional-nose-opts }}"
+          fi
+
+          #
+          # If using a venv, activate it.
+          #
+          if [[ "${{ matrix.use_virtual_environment}}" == "true"  ]]; then
+            source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
+          fi
+        
+          ./ci/nosetests.sh
+        fi
+
+    #
+    # Execute pylint even when the tests fail (but only if the dependencies were installed successfully)
+    #
+    # The virtual environments for 2.6, 2.7, and 3.4 do not include pylint, so we skip those Python versions.
+    #
     - name: Run pylint
+      if: (!contains(fromJSON('["2.6", "2.7", "3.4"]'),  matrix.python-version) && (success() || (failure() && steps.install-dependencies.outcome == 'success')))
       run: |
+        #
+        # If using a venv, activate it.
+        #
+        if [[ "${{ matrix.use_virtual_environment}}" == "true"  ]]; then
+              source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
+        fi
+
         #
         # List of files/directories to be checked by pylint.
         # The end-to-end tests run only on Python 3.9 and we lint them only on that version.
@@ -153,15 +139,15 @@ jobs:
         # * 'no-self-use' ("R0201: Method could be a function") was moved to an optional extension on 3.8 and is no longer used by default. It needs
         #    to be suppressed for previous versions (3.0-3.7), though.
         # * 'contextmanager-generator-missing-cleanup' are false positives if yield is used inside an if-else block for contextmanager generator functions.
-        #   (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html). 
+        #   (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html).
         #   This is not implemented on versions (3.0-3.7) Bad option value 'contextmanager-generator-missing-cleanup' (bad-option-value)
-        # * 3.9-3.11 will produce "too-many-positional-arguments" for several methods that are having more than 5 args, so we suppress that warning.
+        # * >= 3.9 will produce "too-many-positional-arguments" for several methods that are having more than 5 args, so we suppress that warning.
         #  (R0917: Too many positional arguments (8/5) (too-many-positional-arguments))
         PYLINT_OPTIONS="--rcfile=ci/pylintrc --jobs=0"
         if [[ "${{ matrix.python-version }}" == "3.9" ]]; then
           PYLINT_OPTIONS="$PYLINT_OPTIONS --disable=no-member,too-many-positional-arguments --ignore=main.py"
         fi
-        if [[ "${{ matrix.python-version }}" =~ ^3\.(10|11)$ ]]; then
+        if [[ "${{ matrix.python-version }}" =~ ^3\.(10|11|12)$ ]]; then
           PYLINT_OPTIONS="$PYLINT_OPTIONS --disable=too-many-positional-arguments"
         fi
         if [[ "${{ matrix.python-version }}" =~ ^3\.[0-7]$ ]]; then
@@ -173,16 +159,10 @@ jobs:
 
         pylint $PYLINT_OPTIONS $PYLINT_FILES
 
-    - name: Execute Unit Tests
-      if: success() || (failure() && steps.install-dependencies.outcome == 'success')
-      run: |
-        if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then
-          ./ci/pytest.sh
-        else
-          ./ci/nosetests.sh
-        fi
-
-    - name: Compile Coverage
+    #
+    # Lastly, compile code coverage
+    #
+    - name: Compile Code Coverage
       if: matrix.python-version == '3.9'
       run: |
         echo looking for coverage files :
@@ -191,7 +171,7 @@ jobs:
         sudo env "PATH=$PATH" coverage xml
         sudo env "PATH=$PATH" coverage report
 
-    - name: Upload Coverage
+    - name: Upload Code Coverage
       if: matrix.python-version ==  '3.9'
       uses: codecov/codecov-action@v3
       with:

README.md

@@ -3,8 +3,8 @@
 
 ## Linux distributions support
 
-Our daily automation tests most of the [Linux distributions supported by Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/endorsed-distros); the Agent can be
-used on other distributions as well, but development, testing and support for those are done by the open source community.
+The list of distros we officially support is maintained at: [Linux distributions supported by Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/endorsed-distros). Our daily automation tests most of these distributions. The Agent can be
+used on other distributions as well, but development, testing and support for those are done by the open source community. This repo contains community-driven support for some distributions which are not officially supported by Azure.
 
 Testing is done using the develop branch, which can be unstable. For a stable build please use the master branch instead.
 
@@ -653,6 +653,15 @@ _Default: customscript,runcommand_
 
 The list of extensions which will be excluded from cgroups limits. This should be comma separated. 
 
+#### __Protocol.EndpointDiscovery__
+
+_Type: String_  
+_Default: dhcp_
+
+Determines how the agent will discover the [WireServer endpoint](https://learn.microsoft.com/en-us/azure/virtual-network/what-is-ip-address-168-63-129-16). 
+Agent will use DHCP by default to discover the WireServer endpoint, but if this setting is 'static' the agent will use the known WireServer address (168.63.129.16).
+Possible options are "dhcp" (default) or "static".
+
 ### Telemetry
 
 WALinuxAgent collects usage data and sends it to Microsoft to help improve our products and services. The data collected is used to track service health and

azurelinuxagent/agent.py

@@ -247,6 +247,7 @@ def collect_logs(self, is_full_mode):
                 for controller in tracked_controllers:
                     if isinstance(controller, _CpuController):
                         controller.initialize_cpu_usage()
+                        controller.track_throttle_time(True)
                         break
                 log_collector_monitor = get_log_collector_monitor_handler(tracked_controllers)
                 log_collector_monitor.run()

azurelinuxagent/common/conf.py

@@ -171,7 +171,8 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
     "ResourceDisk.MountOptions": None,
     "ResourceDisk.Filesystem": "ext3",
     "AutoUpdate.GAFamily": "Prod",
-    "Policy.PolicyFilePath": "/etc/waagent_policy.json"
+    "Policy.PolicyFilePath": "/etc/waagent_policy.json",
+    "Protocol.EndpointDiscovery": "dhcp"
 }
 
 
@@ -547,6 +548,19 @@ def get_auto_update_to_latest_version(conf=__conf__):
     return conf.get_switch("AutoUpdate.UpdateToLatestVersion", default)
 
 
+def get_protocol_endpoint_discovery(conf=__conf__):
+    return conf.get("Protocol.EndpointDiscovery", "dhcp")
+
+
+def get_dhcp_discovery_enabled(conf=__conf__):
+    """
+    Determines how the agent will discover the wireserver endpoint.
+    If set to 'dhcp', the agent will use DHCP to get the wireserver endpoint.
+    Otherwise, the agent will use the known wireserver endpoint (168.63.129.16).
+    """
+    return get_protocol_endpoint_discovery(conf) == "dhcp"
+
+
 def get_cgroup_check_period(conf=__conf__):
     """
     How often to perform checks on cgroups (are the processes in the cgroups as expected,
@@ -619,6 +633,7 @@ def get_enable_agent_memory_usage_check(conf=__conf__):
     """
     return conf.get_switch("Debug.EnableAgentMemoryUsageCheck", False)
 
+
 def get_enable_fast_track(conf=__conf__):
     """
     If True, the agent use FastTrack when retrieving goal states
@@ -692,3 +707,11 @@ def get_log_collector_initial_delay(conf=__conf__):
     NOTE: This option is experimental and may be removed in later versions of the Agent.
     """
     return conf.get_int("Debug.LogCollectorInitialDelay", 5 * 60)
+
+def get_enable_rsm_downgrade(conf=__conf__):
+    """
+    If False, the agent will not downgrade to a lower version when a lower version is requested in the goal state.
+
+    Todo: Flag will be removed once we have a fix for rsm downgrade scenario.
+    """
+    return conf.get_switch("Debug.EnableRsmDowngrade", False)

azurelinuxagent/common/dhcp.py

@@ -20,6 +20,7 @@
 import time
 
 import azurelinuxagent.common.logger as logger
+from azurelinuxagent.common import conf
 from azurelinuxagent.common.exception import DhcpError
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.utils.restutil import KNOWN_WIRESERVER_IP
@@ -151,9 +152,15 @@ def send_dhcp_req(self):
         """
         Check if DHCP is available
         """
-        dhcp_available =  self.osutil.is_dhcp_available()
-        if not dhcp_available:
-            logger.info("send_dhcp_req: DHCP not available")
+        dhcp_available = self.osutil.is_dhcp_available()
+        # If user has DHCP disabled for their VM then the dhcp request will fail. The user can configure the agent to
+        # use the known wire server ip instead.
+        use_dhcp = conf.get_dhcp_discovery_enabled()
+        if not dhcp_available or not use_dhcp:
+            if not use_dhcp:
+                logger.info("send_dhcp_req: DHCP usage for endpoint discovery is disabled (Protocol.EndpointDiscovery={0}). Will use known wireserver endpoint.".format(conf.get_protocol_endpoint_discovery()))
+            elif not dhcp_available:
+                logger.info("send_dhcp_req: DHCP not available")
             self.endpoint = KNOWN_WIRESERVER_IP
             return