diff --git a/conda/conda-reqs.txt b/conda/conda-reqs.txt index a16b7500..e165c433 100644 --- a/conda/conda-reqs.txt +++ b/conda/conda-reqs.txt @@ -5,7 +5,7 @@ azure-identity>=1.16.1 azure-keyvault-secrets>=4.0.0 azure-kusto-data<7.0.0,>=4.4.0 azure-mgmt-compute>=4.6.2 -azure-mgmt-core>=1.2.1 +azure-mgmt-core>=1.6.0 azure-mgmt-keyvault>=2.0.0 azure-mgmt-monitor azure-mgmt-network>=2.7.0 @@ -31,7 +31,6 @@ matplotlib>=3.0.0 msal_extensions>=0.3.0 msal>=1.12.0 msrest>=0.6.0 -msrestazure>=0.6.0 networkx>=2.2 numpy>=1.15.4 pandas>=1.4.0, <3.0.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index 39146563..31d6acef 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -265,7 +265,6 @@ "msal", "msal_extensions", "msrest", - "msrestazure", "msrest.authentication", "nest_asyncio", "networkx", diff --git a/msticpy/auth/azure_auth_core.py b/msticpy/auth/azure_auth_core.py index 1f48022f..d41dfe3e 100644 --- a/msticpy/auth/azure_auth_core.py +++ b/msticpy/auth/azure_auth_core.py @@ -16,8 +16,7 @@ from enum import Enum from typing import Any, ClassVar -from azure.common.credentials import get_cli_profile -from azure.core.credentials import TokenCredential +from azure.core.credentials import AccessTokenInfo, TokenCredential from azure.core.exceptions import ClientAuthenticationError from azure.identity import ( AzureCliCredential, @@ -31,7 +30,6 @@ ManagedIdentityCredential, VisualStudioCodeCredential, ) -from dateutil import parser from .._version import VERSION from ..common.exceptions import MsticpyAzureConfigError @@ -536,13 +534,10 @@ class AzureCliStatus(Enum): def check_cli_credentials() -> tuple[AzureCliStatus, str | None]: """Check to see if there is a CLI session with a valid AAD token.""" try: - cli_profile = get_cli_profile() - raw_token = cli_profile.get_raw_token() - bearer_token = None - if isinstance(raw_token, tuple) and len(raw_token) == 3 and len(raw_token[0]) == 3: - bearer_token = raw_token[0][2] - if parser.parse(bearer_token.get("expiresOn", datetime.min)) < datetime.now(): - raise ValueError("AADSTS70043: The refresh token has expired") + cli_profile: AzureCliCredential = AzureCliCredential() + token: AccessTokenInfo = cli_profile.get_token_info("/.default") + if token.expires_on < datetime.now().timestamp(): + raise ValueError("AADSTS70043: The refresh token has expired") return AzureCliStatus.CLI_OK, "Azure CLI credentials available." except ImportError: diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index bf74dc31..ee93260f 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -36,8 +36,7 @@ ) try: - from azure.common.exceptions import CloudError - from azure.core.exceptions import ClientAuthenticationError + from azure.core.exceptions import ClientAuthenticationError, HttpResponseError from azure.mgmt.network import NetworkManagementClient from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.subscription import SubscriptionClient @@ -198,7 +197,7 @@ def connect( Raises ------ - CloudError + ClientAuthenticationError If no valid credentials are found or if subscription client can't be created See Also @@ -220,7 +219,7 @@ def connect( ) if not self.credentials: err_msg: str = "Could not obtain credentials." - raise CloudError(err_msg) + raise ClientAuthenticationError(err_msg) if only_interactive_cred(self.credentials.modern) and not silent: logger.warning("Check your default browser for interactive sign-in prompt.") @@ -231,7 +230,7 @@ def connect( ) if not self.sub_client: err_msg = "Could not create a Subscription client." - raise CloudError(err_msg) + raise ClientAuthenticationError(err_msg) logger.info("Connected to Azure Subscription Client") self.connected = True @@ -454,7 +453,7 @@ def get_resources( "2019-08-01", ).properties - except CloudError: + except HttpResponseError: props = self.resource_client.resources.get_by_id( resource.id, self._get_api(resource_id=resource.id, sub_id=sub_id), @@ -1023,7 +1022,7 @@ def _check_client(self: Self, client_name: str, sub_id: str | None = None) -> No if getattr(self, client_name) is None: err_msg = "Could not create client" - raise CloudError(err_msg) + raise ClientAuthenticationError(err_msg) def _legacy_auth(self: Self, client_name: str, sub_id: str | None = None) -> None: """ diff --git a/msticpy/context/azure/sentinel_analytics.py b/msticpy/context/azure/sentinel_analytics.py index 398a0987..b341ad7d 100644 --- a/msticpy/context/azure/sentinel_analytics.py +++ b/msticpy/context/azure/sentinel_analytics.py @@ -14,7 +14,13 @@ import httpx import pandas as pd -from azure.common.exceptions import CloudError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, + ResourceNotModifiedError, +) from IPython.display import display from typing_extensions import Self @@ -190,7 +196,11 @@ def create_analytic_rule( # pylint: disable=too-many-arguments, too-many-locals ------ MsticpyUserError If template provided isn't found. - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If the API returns an error. """ @@ -250,7 +260,18 @@ def create_analytic_rule( # pylint: disable=too-many-arguments, too-many-locals timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Analytic Created.") return response.json().get("name") @@ -305,7 +326,10 @@ def delete_analytic_rule( Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + HttpResponseError If the API returns an error. """ @@ -323,7 +347,16 @@ def delete_analytic_rule( timeout=get_http_timeout(), ) if response.is_error: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Analytic Deleted.") def list_analytic_templates(self) -> pd.DataFrame: @@ -337,7 +370,11 @@ def list_analytic_templates(self) -> pd.DataFrame: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If a valid result is not returned. """ diff --git a/msticpy/context/azure/sentinel_bookmarks.py b/msticpy/context/azure/sentinel_bookmarks.py index 770a2933..1900835b 100644 --- a/msticpy/context/azure/sentinel_bookmarks.py +++ b/msticpy/context/azure/sentinel_bookmarks.py @@ -14,7 +14,13 @@ import httpx import pandas as pd -from azure.common.exceptions import CloudError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, + ResourceNotModifiedError, +) from IPython.display import display from typing_extensions import Self @@ -79,7 +85,11 @@ def create_bookmark( # noqa:PLR0913 Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If API returns an error. """ @@ -112,7 +122,20 @@ def create_bookmark( # noqa:PLR0913 if response.is_success: logger.info("Bookmark created.") return response.json().get("name") - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = ( + f"Received HTTP return code {response.status_code}: {response.text}" + ) + raise HttpResponseError(err_msg) def delete_bookmark( self: Self, @@ -128,7 +151,11 @@ def delete_bookmark( Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If the API returns an error. """ @@ -147,8 +174,21 @@ def delete_bookmark( ) if response.is_success: logger.info("Bookmark deleted.") - else: - raise CloudError(response=response) + return + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = ( + f"Received HTTP return code {response.status_code}: {response.text}" + ) + raise HttpResponseError(err_msg) def _get_bookmark_id(self: Self, bookmark: str) -> str: """ diff --git a/msticpy/context/azure/sentinel_core.py b/msticpy/context/azure/sentinel_core.py index 6491b19b..6dbe64af 100644 --- a/msticpy/context/azure/sentinel_core.py +++ b/msticpy/context/azure/sentinel_core.py @@ -471,7 +471,11 @@ def list_data_connectors(self: Self) -> pd.DataFrame: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If a valid result is not returned. """ diff --git a/msticpy/context/azure/sentinel_incidents.py b/msticpy/context/azure/sentinel_incidents.py index dce3f57e..330e92da 100644 --- a/msticpy/context/azure/sentinel_incidents.py +++ b/msticpy/context/azure/sentinel_incidents.py @@ -14,7 +14,13 @@ import httpx import pandas as pd -from azure.common.exceptions import CloudError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, + ResourceNotModifiedError, +) from IPython.display import display from typing_extensions import Self @@ -73,7 +79,11 @@ def get_incident( # noqa:PLR0913 Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If incident could not be retrieved. """ @@ -83,7 +93,18 @@ def get_incident( # noqa:PLR0913 incident_url, ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) incident_df: pd.DataFrame = _azs_api_result_to_df(response) @@ -271,7 +292,11 @@ def update_incident( Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If incident could not be updated. """ @@ -299,11 +324,22 @@ def update_incident( timeout=get_http_timeout(), ) if response.status_code not in (200, 201): - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Incident updated.") return response.json().get("name") - def create_incident( # pylint: disable=too-many-arguments, too-many-locals #noqa:PLR0913 + def create_incident( # pylint: disable=too-many-arguments, too-many-locals, too-many-branches #noqa:PLR0913 self: Self, title: str, severity: str, @@ -345,7 +381,11 @@ def create_incident( # pylint: disable=too-many-arguments, too-many-locals #noq Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If the API returns an error """ @@ -379,7 +419,18 @@ def create_incident( # pylint: disable=too-many-arguments, too-many-locals #noq timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) if bookmarks: for mark in bookmarks: relation_id: UUID = uuid4() @@ -456,7 +507,11 @@ def post_comment( Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If message could not be posted. """ @@ -475,7 +530,20 @@ def post_comment( timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = ( + f"Received HTTP return code {response.status_code}: {response.text}" + ) + raise HttpResponseError(err_msg) logger.info("Comment posted.") return response.json().get("name") @@ -492,7 +560,11 @@ def add_bookmark_to_incident(self: Self, incident: str, bookmark: str) -> str: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If API returns error """ @@ -519,7 +591,20 @@ def add_bookmark_to_incident(self: Self, incident: str, bookmark: str) -> str: timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = ( + f"Received HTTP return code {response.status_code}: {response.text}" + ) + raise HttpResponseError(err_msg) logger.info("Bookmark added to incident.") return response.json().get("name") @@ -539,7 +624,11 @@ def list_incidents(self: Self, params: dict | None = None) -> pd.DataFrame: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If incidents could not be retrieved. """ diff --git a/msticpy/context/azure/sentinel_search.py b/msticpy/context/azure/sentinel_search.py index 20af405d..7084ef75 100644 --- a/msticpy/context/azure/sentinel_search.py +++ b/msticpy/context/azure/sentinel_search.py @@ -13,7 +13,13 @@ from uuid import uuid4 import httpx -from azure.common.exceptions import CloudError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, + ResourceNotModifiedError, +) from typing_extensions import Self from ..._version import VERSION @@ -61,7 +67,11 @@ def create_search( # noqa: PLR0913 Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If there is an error creating the search job. """ @@ -93,7 +103,21 @@ def create_search( # noqa: PLR0913 timeout=60, ) if not search_create_response.is_success: - raise CloudError(response=search_create_response) + match search_create_response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = ( + f"Received HTTP return code {search_create_response.status_code}: " + f"{search_create_response.text}" + ) + raise HttpResponseError(err_msg) logger.info("Search job created with for %s_SRCH.", search_name) def check_search_status(self: Self, search_name: str) -> bool: @@ -112,7 +136,11 @@ def check_search_status(self: Self, search_name: str) -> bool: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If error in checking the search job status. """ @@ -128,7 +156,21 @@ def check_search_status(self: Self, search_name: str) -> bool: headers=get_api_headers(self._token), ) if not search_check_response.is_success: - raise CloudError(response=search_check_response) + match search_check_response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = ( + f"Received HTTP return code {search_check_response.status_code}: " + f"{search_check_response.text}" + ) + raise HttpResponseError(err_msg) check_result: str = search_check_response.json()["properties"]["provisioningState"] logger.info("%s_SRCH status is '%s'", search_name, check_result) @@ -145,7 +187,11 @@ def delete_search(self: Self, search_name: str) -> None: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If an error occurs when attempting to delete the search """ @@ -161,5 +207,19 @@ def delete_search(self: Self, search_name: str) -> None: headers=get_api_headers(self._token), ) if not search_delete_response.is_success: - raise CloudError(response=search_delete_response) + match search_delete_response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = ( + f"Received HTTP return code {search_delete_response.status_code}: " + f"{search_delete_response.text}" + ) + raise HttpResponseError(err_msg) logger.info("%s_SRCH set for deletion.", search_name) diff --git a/msticpy/context/azure/sentinel_ti.py b/msticpy/context/azure/sentinel_ti.py index 99a61d6b..04bf04bb 100644 --- a/msticpy/context/azure/sentinel_ti.py +++ b/msticpy/context/azure/sentinel_ti.py @@ -12,7 +12,13 @@ from typing import TYPE_CHECKING, Any import httpx -from azure.common.exceptions import CloudError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, + ResourceNotModifiedError, +) from typing_extensions import Self from ..._version import VERSION @@ -162,7 +168,11 @@ def create_indicator( # pylint:disable=too-many-arguments, too-many-locals #noq ------ MsticpyUserError If invalid ioc_type or confidence value provided - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If API call fails """ @@ -217,7 +227,18 @@ def create_indicator( # pylint:disable=too-many-arguments, too-many-locals #noq timeout=get_http_timeout(), ) if response.status_code not in (200, 201): - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) if not silent: logger.info("Indicator created.") @@ -255,7 +276,13 @@ def bulk_create_indicators( confidence=confidence, silent=True, ) - except CloudError: + except ( + ClientAuthenticationError, + ResourceNotFoundError, + ResourceExistsError, + ResourceNotModifiedError, + HttpResponseError, + ): logger.exception( "Error creating indicator %s", row[1][indicator_column], @@ -278,7 +305,11 @@ def get_indicator(self: Self, indicator_id: str) -> dict: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If API call fails. """ @@ -295,7 +326,18 @@ def get_indicator(self: Self, indicator_id: str) -> dict: timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) return response.json() def update_indicator( # pylint:disable=too-many-arguments,too-many-locals #noqa:PLR0913 @@ -342,7 +384,11 @@ def update_indicator( # pylint:disable=too-many-arguments,too-many-locals #noqa Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If API call fails """ @@ -380,7 +426,18 @@ def update_indicator( # pylint:disable=too-many-arguments,too-many-locals #noqa timeout=get_http_timeout(), ) if response.status_code not in (200, 201): - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Indicator updated.") def add_tag(self: Self, indicator_id: str, tag: str) -> None: @@ -413,7 +470,11 @@ def delete_indicator(self: Self, indicator_id: str) -> None: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If API call fails """ @@ -430,10 +491,21 @@ def delete_indicator(self: Self, indicator_id: str) -> None: timeout=get_http_timeout(), ) if response.status_code not in (200, 204): - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Indicator deleted.") - def query_indicators( # pylint:disable=too-many-arguments, too-many-locals #noqa:PLR0913 + def query_indicators( # pylint:disable=too-many-arguments, too-many-locals, too-many-branches #noqa:PLR0913 self: Self, *, include_disabled: bool = False, @@ -485,7 +557,11 @@ def query_indicators( # pylint:disable=too-many-arguments, too-many-locals #noq Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If API call fails """ @@ -524,7 +600,18 @@ def query_indicators( # pylint:disable=too-many-arguments, too-many-locals #noq timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) return _azs_api_result_to_df(response) diff --git a/msticpy/context/azure/sentinel_utils.py b/msticpy/context/azure/sentinel_utils.py index c759030e..23a1f2fc 100644 --- a/msticpy/context/azure/sentinel_utils.py +++ b/msticpy/context/azure/sentinel_utils.py @@ -14,7 +14,13 @@ import httpx import pandas as pd -from azure.common.exceptions import CloudError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, + ResourceNotModifiedError, +) from azure.mgmt.core import tools as az_tools # pylint: disable=import-error, no-name-in-module @@ -88,7 +94,7 @@ def _get_items(self: Self, url: str, params: dict | None = None) -> httpx.Respon timeout=get_http_timeout(), ) - def _list_items( # noqa:PLR0913 + def _list_items( # noqa:PLR0913 #pylint: disable=too-many-locals self: Self, item_type: str, api_version: str = "2020-01-01", @@ -120,7 +126,11 @@ def _list_items( # noqa:PLR0913 Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If a valid result is not returned. """ @@ -134,7 +144,18 @@ def _list_items( # noqa:PLR0913 if response.is_success: results_df: pd.DataFrame = _azs_api_result_to_df(response) else: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) j_resp: dict[str, Any] = response.json() results: list[pd.DataFrame] = [results_df] # If nextLink in response, go get that data as well @@ -341,7 +362,9 @@ def parse_resource_id(res_id: str) -> dict[str, Any]: """Extract components from workspace resource ID.""" if not res_id.startswith("/"): res_id = f"/{res_id}" - res_id_parts: dict[str, str] = cast(dict[str, str], az_tools.parse_resource_id(res_id)) + res_id_parts: dict[str, str] = cast( + dict[str, str], az_tools.parse_resource_id(res_id) + ) workspace_name: str | None = None if ( res_id_parts.get("namespace") == "Microsoft.OperationalInsights" diff --git a/msticpy/context/azure/sentinel_watchlists.py b/msticpy/context/azure/sentinel_watchlists.py index 614f586c..d9d77429 100644 --- a/msticpy/context/azure/sentinel_watchlists.py +++ b/msticpy/context/azure/sentinel_watchlists.py @@ -13,7 +13,13 @@ import httpx import pandas as pd -from azure.common.exceptions import CloudError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, + ResourceNotModifiedError, +) from typing_extensions import Self from ..._version import VERSION @@ -45,7 +51,11 @@ def list_watchlists(self: Self) -> pd.DataFrame: Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If a valid result is not returned. """ @@ -94,7 +104,11 @@ def create_watchlist( # noqa: PLR0913 ------ MsticpyUserError Raised if the watchlist name already exists. - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If there is an issue creating the watchlist. """ @@ -126,7 +140,18 @@ def create_watchlist( # noqa: PLR0913 timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Watchlist created.") return response.json().get("name") @@ -150,7 +175,11 @@ def list_watchlist_items( Raises ------ - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If a valid result is not returned. """ @@ -161,7 +190,7 @@ def list_watchlist_items( appendix=watchlist_name_str, ) - def add_watchlist_item( + def add_watchlist_item( # noqa: PLR0912 # pylint:disable=too-many-branches self: Self, watchlist_name: str, item: dict | pd.Series | pd.DataFrame, @@ -187,7 +216,11 @@ def add_watchlist_item( If the specified Watchlist does not exist. MsticpyUserError If the item already exists in the Watchlist and overwrite is set to False - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If the API returns an error. """ @@ -253,7 +286,18 @@ def add_watchlist_item( timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Items added to %s", watchlist_name) @@ -273,7 +317,11 @@ def delete_watchlist( ------ MsticpyUserError If Watchlist does not exist. - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If the API returns an error. """ @@ -294,7 +342,18 @@ def delete_watchlist( timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Watchlist %s deleted", watchlist_name) def delete_watchlist_item( @@ -316,7 +375,11 @@ def delete_watchlist_item( ------ MsticpyUserError If the specified Watchlist does not exist. - CloudError + ClientAuthenticationError + ResourceNotFoundError + ResourceExistsError + ResourceNotModifiedError + HttpResponseError If the API returns an error. """ @@ -340,7 +403,18 @@ def delete_watchlist_item( timeout=get_http_timeout(), ) if not response.is_success: - raise CloudError(response=response) + match response.status_code: + case httpx.codes.UNAUTHORIZED: + raise ClientAuthenticationError() + case httpx.codes.NOT_FOUND: + raise ResourceNotFoundError() + case httpx.codes.CONFLICT: + raise ResourceExistsError() + case httpx.codes.NOT_MODIFIED: + raise ResourceNotModifiedError() + case _: + err_msg = f"Received HTTP return code {response.status_code}: {response.text}" + raise HttpResponseError(err_msg) logger.info("Item deleted from %s", watchlist_name) diff --git a/msticpy/context/azure/sentinel_workspaces.py b/msticpy/context/azure/sentinel_workspaces.py index f914c59f..d413de47 100644 --- a/msticpy/context/azure/sentinel_workspaces.py +++ b/msticpy/context/azure/sentinel_workspaces.py @@ -14,7 +14,7 @@ from urllib import parse import httpx -from msrestazure import tools as az_tools +from azure.mgmt.core import tools as az_tools from typing_extensions import Self from msticpy.context.azure.sentinel_utils import SentinelUtilsMixin @@ -334,7 +334,10 @@ def _extract_resource_id( raw_res_id: str = uri_match.groupdict()["res_id"] raw_res_id = parse.unquote(raw_res_id) - res_components: dict[str, Any] = az_tools.parse_resource_id(raw_res_id) + res_components: dict[str, Any] = { + key: str(value) + for key, value in az_tools.parse_resource_id(raw_res_id).items() + } try: resource_id: str = cls._normalize_resource_id(res_components) except KeyError: diff --git a/msticpy/data/storage/azure_blob_storage.py b/msticpy/data/storage/azure_blob_storage.py index c966fb30..0439c9d6 100644 --- a/msticpy/data/storage/azure_blob_storage.py +++ b/msticpy/data/storage/azure_blob_storage.py @@ -11,8 +11,8 @@ from typing import Any import pandas as pd -from azure.common.exceptions import CloudError from azure.core.exceptions import ( + ClientAuthenticationError, ResourceExistsError, ResourceNotFoundError, ServiceRequestError, @@ -22,6 +22,7 @@ from ..._version import VERSION from ...auth.azure_auth import az_connect from ...auth.azure_auth_core import AzCredentials, AzureCloudConfig +from ...common.exceptions import MsticpyParameterError, MsticpyResourceError __version__ = VERSION __author__ = "Pete Bryan" @@ -53,13 +54,13 @@ def connect( """Authenticate with the SDK.""" self.credentials = az_connect(auth_methods=auth_methods, silent=silent) if not self.credentials: - raise CloudError("Could not obtain credentials.") + raise ClientAuthenticationError("Could not obtain credentials.") if not self.connection_string: self.abs_client = BlobServiceClient(self.abs_site, self.credentials.modern) else: - self.abs_client = BlobServiceClient.from_connection_string(self.connection_string) - if not self.abs_client: - raise CloudError("Could not create a Blob Storage client.") + self.abs_client = BlobServiceClient.from_connection_string( + self.connection_string, + ) self.connected = True def containers(self) -> pd.DataFrame: @@ -67,7 +68,7 @@ def containers(self) -> pd.DataFrame: try: container_list = self.abs_client.list_containers() # type:ignore except ServiceRequestError as err: - raise CloudError( + raise MsticpyParameterError( "Unable to connect check the Azure Blob Store account name" ) from err return ( @@ -97,7 +98,9 @@ def create_container(self, container_name: str, **kwargs) -> pd.DataFrame: container_name, **kwargs ) except ResourceExistsError as err: - raise CloudError(f"Container {container_name} already exists.") from err + raise MsticpyParameterError( + f"Container {container_name} already exists.", + ) from err properties = new_container.get_container_properties() return _parse_returned_items([properties], ["encryption_scope", "lease"]) @@ -146,13 +149,15 @@ def upload_to_blob( ) upload = blob_client.upload_blob(blob, overwrite=overwrite) except ResourceNotFoundError as err: - raise CloudError( + raise MsticpyParameterError( "Unknown container, check container name or create it first." ) from err if not upload["error_code"]: print("Upload complete") else: - raise CloudError(f"There was a problem uploading the blob: {upload['error_code']}") + raise MsticpyResourceError( + f"There was a problem uploading the blob: {upload['error_code']}", + ) return True def get_blob(self, container_name: str, blob_name: str) -> bytes: @@ -176,7 +181,9 @@ def get_blob(self, container_name: str, blob_name: str) -> bytes: container=container_name, blob=blob_name ) if not blob_client.exists(): - raise CloudError(f"The blob {blob_name} does not exist in {container_name}") + raise MsticpyResourceError( + f"The blob {blob_name} does not exist in {container_name}", + ) data_stream = blob_client.download_blob() return data_stream.content_as_bytes() @@ -204,7 +211,9 @@ def delete_blob(self, container_name: str, blob_name: str) -> bool: if blob_client.exists(): blob_client.delete_blob(delete_snapshots="include") else: - raise CloudError(f"The blob {blob_name} does not exist in {container_name}") + raise MsticpyResourceError( + f"The blob {blob_name} does not exist in {container_name}", + ) return True diff --git a/mypy.ini b/mypy.ini index fa8f6109..1117c823 100644 --- a/mypy.ini +++ b/mypy.ini @@ -69,9 +69,6 @@ ignore_missing_imports = True [mypy-msrest.*] ignore_missing_imports = True -[mypy-msrestazure.*] -ignore_missing_imports = True - [mypy-seaborn.*] ignore_missing_imports = True diff --git a/requirements-all.txt b/requirements-all.txt index 2be60514..83f728a6 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,7 +5,7 @@ azure-identity>=1.16.1 azure-keyvault-secrets>=4.0.0 azure-kusto-data<7.0.0,>=4.4.0 azure-mgmt-compute>=4.6.2 -azure-mgmt-core>=1.2.1 +azure-mgmt-core>=1.6.0 azure-mgmt-keyvault>=2.0.0 azure-mgmt-monitor>=2.0.0 azure-mgmt-network>=2.7.0 @@ -34,7 +34,6 @@ mo-sql-parsing<12.0.0,>=11 msal>=1.12.0 msal_extensions>=0.3.0 msrest>=0.6.0 -msrestazure>=0.6.0 nest_asyncio>=1.4.0 networkx>=2.2 numpy>=1.15.4 diff --git a/requirements.txt b/requirements.txt index 88b835cf..ed502f3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ azure-core>=1.24.0 azure-identity>=1.16.1 azure-keyvault-secrets>=4.0.0 azure-kusto-data>=4.4.0, <7.0.0 +azure-mgmt-core>=1.6.0 azure-mgmt-keyvault>=2.0.0 azure-mgmt-subscription>=3.0.0 azure-monitor-query>=1.0.0, <=3.0.0 @@ -24,7 +25,6 @@ lxml>=4.6.5 msal>=1.12.0 msal_extensions>=0.3.0 msrest>=0.6.0 -msrestazure>=0.6.0 nest_asyncio>=1.4.0 networkx>=2.2 numpy>=1.15.4 # pandas diff --git a/setup.py b/setup.py index 82f61071..a0801bce 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def _combine_extras(extras: list) -> list: "sumologic": ["sumologic-sdk>=0.1.11", "openpyxl>=3.0"], "azure": [ "azure-mgmt-compute>=4.6.2", - "azure-mgmt-core>=1.2.1", + "azure-mgmt-core>=1.6.0", "azure-mgmt-monitor>=2.0.0", "azure-mgmt-network>=2.7.0", "azure-mgmt-resource>=16.1.0", diff --git a/tests/auth/test_azure_auth_core.py b/tests/auth/test_azure_auth_core.py index ed4e559e..15ba0694 100644 --- a/tests/auth/test_azure_auth_core.py +++ b/tests/auth/test_azure_auth_core.py @@ -83,7 +83,7 @@ def test_azure_cloud_config(mp_config_file): _TOKEN = { "tokenType": "Bearer", "expiresIn": 3000, - "expiresOn": str(datetime.now() + timedelta(0.1)), + "expiresOn": (datetime.now() + timedelta(0.1)).timestamp(), "resource": "https://management.core.windows.net/", "accessToken": "_b64_token_string_", "refreshToken": "_b64_token_string2_", @@ -92,7 +92,7 @@ def test_azure_cloud_config(mp_config_file): _CLI_TESTS = [ (({}, None), AzureCliStatus.CLI_OK), ( - ({"expiresOn": str(datetime.now() - timedelta(0.1))}, None), + ({"expiresOn": (datetime.now() - timedelta(0.1)).timestamp()}, None), AzureCliStatus.CLI_TOKEN_EXPIRED, ), (({}, ImportError), AzureCliStatus.CLI_NOT_INSTALLED), @@ -123,8 +123,15 @@ def get_raw_token(self): """Return raw token.""" return (*_TOKEN_WRAPPER, self.token), None, None + def get_token_info(self, scope): + """Return raw token.""" + del scope + token: MagicMock = MagicMock() + token.expires_on = self.token["expiresOn"] + return token + -@patch(check_cli_credentials.__module__ + ".get_cli_profile") +@patch(check_cli_credentials.__module__ + ".AzureCliCredential") @pytest.mark.parametrize("test, expected", _CLI_TESTS, ids=_test_ids(_CLI_TESTS)) def test_check_cli_credentials(get_cli_profile, test, expected): # sourcery skip: use-fstring-for-concatenation diff --git a/tests/data/drivers/test_openobserve_driver.py b/tests/data/drivers/test_openobserve_driver.py index 4bfc1644..ee73becf 100644 --- a/tests/data/drivers/test_openobserve_driver.py +++ b/tests/data/drivers/test_openobserve_driver.py @@ -7,7 +7,7 @@ # pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument from datetime import datetime, timedelta, timezone from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock import pandas as pd import pytest @@ -22,6 +22,8 @@ _OPEN_OBSERVE_NOT_LOADED = True try: + import httpx + from msticpy.data.drivers.openobserve_driver import OpenObserveDriver _OPEN_OBSERVE_NOT_LOADED = False @@ -31,44 +33,21 @@ UTC = timezone.utc OO_HOST = OO_USER = OO_PASS = "MOCK_INPUT" - -def mock_post(*args, **kwargs): - """MockResponse function for openobserve calls of requests.post.""" - url = args[0] - - class MockResponse: - """MockResponse class for openobserve calls of requests.post.""" - - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - - def json(self): - return self.json_data - - if "/api/default/_search" in url: - return MockResponse( - { - "took": 155, - "hits": [ - { - "_p": "F", - "_timestamp": 1674213225158000, - "log": ( - "[2023-01-20T11:13:45Z INFO actix_web::middleware::logger] " - '10.2.80.192 "POST /api/demo/_bulk HTTP/1.1" 200 68 "-" ' - '"go-resty/2.7.0 (https://github.com/go-resty/resty)" 0.001074', - ), - "stream": "stderr", - } - ], - "total": 27179431, - "from": 0, - "size": 1, - "scan_size": 28943, - }, - 200, - ) +_SEARCH_RESULT_DF = pd.DataFrame( + [ + { + "_p": "F", + "_timestamp": 1674213225158000, + "log": ( + "[2023-01-20T11:13:45Z INFO actix_web::middleware::logger] " + '10.2.80.192 "POST /api/demo/_bulk HTTP/1.1" 200 68 "-" ' + '"go-resty/2.7.0 (https://github.com/go-resty/resty)"' + " 0.001074" + ), + "stream": "stderr", + } + ] +) @pytest.mark.skipif(_OPEN_OBSERVE_NOT_LOADED, reason="OpenObserve driver not installed") @@ -105,24 +84,31 @@ def test_openobserve_connect_errors(): openobserve_driver.connect( connection_str="invalid", user="***", password="***" ) # nosec - - with pytest.raises(MsticpyConnectionError) as mp_ex: - openobserve_driver.query('select * from "default"', days=1) - check.is_in( - ( - "Failed to search job: Invalid URL 'invalid/api/default/_search': " - "No scheme supplied. Perhaps you meant https://invalid/api/default/_search?" + openobserve_driver.service.search2df = MagicMock( + side_effect=httpx.UnsupportedProtocol( + "Request URL is missing an 'http://' or 'https://' protocol." ), - mp_ex.value.args, ) + with pytest.raises( + MsticpyConnectionError, + match="Communication error connecting to OpenObserve:", + ): + openobserve_driver.query('select * from "default"', days=1) openobserve_driver = OpenObserveDriver() openobserve_driver.connect( - connection_str="https://nonexistent.example.com", user="***", password="***" + connection_str="https://nonexistent.example.com", + user="***", + password="***", ) # nosec + openobserve_driver.service.search2df = MagicMock( + side_effect=httpx.ConnectError( + "[Errno -5] No address associated with hostname" + ), + ) with pytest.raises( MsticpyConnectionError, - match="Max retries exceeded with url:", + match="Communication error connecting to OpenObserve:", ): openobserve_driver.query('select * from "default"', days=1) @@ -139,11 +125,13 @@ def test_openobserve_query_no_connect(): @pytest.mark.skipif(_OPEN_OBSERVE_NOT_LOADED, reason="OpenObserve driver not installed") -@patch("requests.post", side_effect=mock_post) -def test_openobserve_query(mock_post): +def test_openobserve_query(): """Check queries with different outcomes.""" openobserve_drv = OpenObserveDriver() openobserve_drv.connect(connection_str=OO_HOST, user=OO_USER, password=OO_PASS) + openobserve_drv.service.search2df = MagicMock( + return_value=_SEARCH_RESULT_DF.copy() + ) end = datetime.now(UTC) start = end - timedelta(1) @@ -155,11 +143,13 @@ def test_openobserve_query(mock_post): @pytest.mark.skipif(_OPEN_OBSERVE_NOT_LOADED, reason="OpenObserve driver not installed") -@patch("requests.post", side_effect=mock_post) -def test_openobserve_query_params(mock_post): +def test_openobserve_query_params(): """Check queries with different parameters.""" openobserve_drv = OpenObserveDriver() openobserve_drv.connect(connection_str=OO_HOST, user=OO_USER, password=OO_PASS) + openobserve_drv.service.search2df = MagicMock( + return_value=_SEARCH_RESULT_DF.copy() + ) df_result = openobserve_drv.query("RecordSuccess", days=1) check.is_instance(df_result, pd.DataFrame) @@ -171,11 +161,13 @@ def test_openobserve_query_params(mock_post): @pytest.mark.skipif(_OPEN_OBSERVE_NOT_LOADED, reason="OpenObserve driver not installed") -@patch("requests.post", side_effect=mock_post) -def test_openobserve_query_export(mock_post, tmpdir): +def test_openobserve_query_export(tmpdir): """Check queries with different parameters.""" openobserve_drv = OpenObserveDriver() openobserve_drv.connect(connection_str=OO_HOST, user=OO_USER, password=OO_PASS) + openobserve_drv.service.search2df = MagicMock( + return_value=_SEARCH_RESULT_DF.copy() + ) ext = "csv" exp_file = f"openobserve_test.{ext}" f_path = tmpdir.join(exp_file)